diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5852c2bc5..ccffefcff 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -34,6 +34,15 @@ You are assisting in this repository. The following rules are **mandatory** and ## Testing & Validation +0. **Lint only with the pinned toolchain, memory-capped** + + - `cla-backend-go` lint = golangci-lint **v1.64.8 built with and run under the CI Go toolchain** (`go 1.25.x`, `go.mod` / `build-pr.yml`). + Any other combination (newer Go, newer `GOTOOLCHAIN`) re-type-checks the module per linter, exceeds 30 GB RSS and OOM-kills the + whole session. Before linting: `GOTOOLCHAIN=go1.25.13 golangci-lint version` must report that Go version. + - Run it on Linux only, one at a time, with `--concurrency 4` (`make lint LINT_ARGS="--concurrency 4"`) inside a memory-capped cgroup + (`systemd-run --user --scope -p MemoryMax=16G -p MemorySwapMax=0 …`). Never replace a shared `~/go/bin/golangci-lint` while other + sessions run; install a pinned copy elsewhere and use `LINT_TOOL=`. + 5. **Always test changes** - Whenever you modify code, **run tests** relevant to the change to ensure correctness. diff --git a/.github/workflows/org-import-sweep.yml b/.github/workflows/org-import-sweep.yml new file mode 100644 index 000000000..96d0f08cd --- /dev/null +++ b/.github/workflows/org-import-sweep.yml @@ -0,0 +1,251 @@ +--- +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +# M3 organization import (lfx-self-serve #2750). Runbook: cla-backend-go/cmd/org_import/README.md. +# Manual runs choose stage/mode/routes. Scheduled runs are register-only and do nothing unless the +# repository variable ORG_IMPORT_SWEEP_DEV / ORG_IMPORT_SWEEP_PROD is set to dry-run or apply. +# Every run writes run.log + CSV reports (artifact), copies run.log to CloudWatch Logs +# /easycla/org-import/ and e-mails the full decision record to SSM cla-org-import-report-emails-. + +name: Org import sweep + +on: + workflow_dispatch: + inputs: + stage: + description: Stage + type: choice + options: [dev, prod] + default: dev + mode: + description: dry-run writes nothing; apply performs the plan + type: choice + options: [dry-run, apply] + default: dry-run + routes: + description: Routes to process + type: choice + options: [register, rewrite, 'register,rewrite'] + default: register + tranche: + description: Max groups to act on (0 = all) + type: string + default: '0' + ids: + description: Optional comma-separated external ids (old or new) + type: string + default: '' + mapping: + description: Optional mapping CSV for the rewrite route (rows separated by newlines or '|'), header old_id,new_id,action,approved + type: string + default: '' + decisions: + description: Optional duplicate-review decisions CSV (rows separated by newlines or '|'), header decision,old_ids,target_sfid,reviewer,note + type: string + default: '' + shared_domains: + description: Optional shared-domain list (one domain per line or '|'-separated); empty = built-in list + type: string + default: '' + notify: + description: E-mail the full run report to SSM cla-org-import-report-emails- + type: boolean + default: true + schedule: + - cron: '0 6 * * *' + +permissions: + id-token: write + contents: read + actions: read + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - id: plan + env: + EVENT: ${{ github.event_name }} + IN_STAGE: ${{ inputs.stage }} + IN_MODE: ${{ inputs.mode }} + IN_ROUTES: ${{ inputs.routes }} + IN_TRANCHE: ${{ inputs.tranche }} + IN_IDS: ${{ inputs.ids }} + IN_NOTIFY: ${{ inputs.notify }} + SWEEP_DEV: ${{ vars.ORG_IMPORT_SWEEP_DEV }} + SWEEP_PROD: ${{ vars.ORG_IMPORT_SWEEP_PROD }} + run: | + set -euo pipefail + role() { + case "$1" in + dev) echo 'arn:aws:iam::395594542180:role/github-actions-deploy' ;; + prod) echo 'arn:aws:iam::716487311010:role/github-actions-deploy' ;; + *) echo "unknown stage $1" >&2; exit 1 ;; + esac + } + entry() { + jq -cn --arg stage "$1" --arg mode "$2" --arg routes "$3" --arg tranche "$4" --arg ids "$5" --arg notify "$6" --arg role "$(role "$1")" \ + '{stage:$stage, mode:$mode, routes:$routes, tranche:$tranche, ids:$ids, notify:$notify, role:$role}' + } + entries=() + if [ "$EVENT" = workflow_dispatch ]; then + entries+=("$(entry "$IN_STAGE" "$IN_MODE" "$IN_ROUTES" "${IN_TRANCHE:-0}" "$IN_IDS" "${IN_NOTIFY:-true}")") + else + for pair in "dev:${SWEEP_DEV:-off}" "prod:${SWEEP_PROD:-off}"; do + stage="${pair%%:*}"; mode="${pair#*:}" + case "$mode" in + dry-run|apply) entries+=("$(entry "$stage" "$mode" register 0 '' true)") ;; + *) echo "scheduled sweep for $stage is off" ;; + esac + done + fi + matrix="$(printf '%s\n' "${entries[@]:-}" | jq -cs 'map(select(. != null))')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "$matrix" + + sweep: + needs: plan + if: needs.plan.outputs.matrix != '[]' + runs-on: ubuntu-latest + timeout-minutes: 180 + environment: ${{ matrix.stage }} + concurrency: + group: org-import-${{ matrix.stage }} + cancel-in-progress: false + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + AWS_REGION: us-east-1 + STAGE: ${{ matrix.stage }} + steps: + - uses: actions/checkout@v4 + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Setup python (swagger tooling) + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: cla-backend-go/swagger/requirements.txt + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + role-to-assume: ${{ matrix.role }} + aws-region: us-east-1 + - name: Cache Go modules + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Configure Git to clone private Github repos + run: git config --global url."https://${TOKEN_USER}:${TOKEN}@github.com".insteadOf "https://github.com" + env: + TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }} + TOKEN_USER: ${{ secrets.PERSONAL_ACCESS_TOKEN_USER_GITHUB }} + + - name: Add OS Tools + run: sudo apt update && sudo apt-get install file -y + + - name: Go Setup + working-directory: cla-backend-go + run: make clean setup + + - name: Go Dependencies + working-directory: cla-backend-go + run: make deps + + - name: Go Swagger Generate + working-directory: cla-backend-go + run: make swagger + + - name: Build org-import + working-directory: cla-backend-go + run: make build-org-import-linux + + - name: Restore state and operator files + working-directory: cla-backend-go + env: + GH_TOKEN: ${{ github.token }} + MAPPING: ${{ github.event_name == 'workflow_dispatch' && inputs.mapping || '' }} + DECISIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.decisions || '' }} + SHARED_DOMAINS: ${{ github.event_name == 'workflow_dispatch' && inputs.shared_domains || '' }} + run: | + set -euo pipefail + mkdir -p org-import-out + run_id="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=org-import-state-${STAGE}&per_page=10" \ + --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .workflow_run.id // empty')" + if [ -n "$run_id" ]; then + # a state artifact exists: failing to restore it must fail the job, never start from an empty journal + gh run download "$run_id" -n "org-import-state-${STAGE}" -D org-import-out + echo "restored state from run $run_id ($(wc -l < org-import-out/state.jsonl) journal lines)" + else + echo "no state artifact for ${STAGE}: starting with an empty journal" + fi + lines() { printf '%s\n' "$1" | tr '|' '\n' | sed '/^[[:space:]]*$/d'; } + if [ -n "$MAPPING" ]; then + lines "$MAPPING" > org-import-out/mapping.csv + echo "mapping rows: $(($(wc -l < org-import-out/mapping.csv) - 1))" + fi + if [ -n "$DECISIONS" ]; then + lines "$DECISIONS" > org-import-out/decisions.csv + echo "decision rows: $(($(wc -l < org-import-out/decisions.csv) - 1))" + fi + if [ -n "$SHARED_DOMAINS" ]; then + lines "$SHARED_DOMAINS" > org-import-out/shared_domains.txt + echo "shared domains: $(wc -l < org-import-out/shared_domains.txt)" + fi + + - name: Run org-import ingest (${{ matrix.stage }} ${{ matrix.mode }} ${{ matrix.routes }}) + working-directory: cla-backend-go + env: + MODE: ${{ matrix.mode }} + ROUTES: ${{ matrix.routes }} + TRANCHE: ${{ matrix.tranche }} + IDS: ${{ matrix.ids }} + NOTIFY: ${{ matrix.notify }} + run: | + set -euo pipefail + args=(ingest --routes "$ROUTES" --out-dir org-import-out --state org-import-out/state.jsonl) + [ "${TRANCHE:-0}" != "0" ] && args+=(--tranche "$TRANCHE") + [ -n "$IDS" ] && args+=(--ids "$IDS") + [ -s org-import-out/mapping.csv ] && args+=(--mapping org-import-out/mapping.csv) + [ -s org-import-out/decisions.csv ] && args+=(--decisions org-import-out/decisions.csv) + [ -s org-import-out/shared_domains.txt ] && args+=(--shared-domains org-import-out/shared_domains.txt) + [ "$NOTIFY" = false ] && args+=(--no-email) + [ "$MODE" = apply ] && args+=(--apply --yes) + echo "bin/org-import ${args[*]}" + bin/org-import "${args[@]}" + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: org-import-out-${{ matrix.stage }}-${{ github.run_id }} + path: cla-backend-go/org-import-out + if-no-files-found: ignore + retention-days: 90 + + - name: Upload state + if: always() && matrix.mode == 'apply' + uses: actions/upload-artifact@v4 + with: + name: org-import-state-${{ matrix.stage }} + path: cla-backend-go/org-import-out/state.jsonl + if-no-files-found: ignore + retention-days: 90 diff --git a/.gitignore b/.gitignore index 3c25f9b75..09a17bfcb 100755 --- a/.gitignore +++ b/.gitignore @@ -291,3 +291,4 @@ spans*.json .venv copilot-*.md /e2e-tests/ +org-import-out/ diff --git a/CLAUDE.md b/CLAUDE.md index 7ad0e5a70..94bd49e2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ make setup # one-time: install swagger, golangci-lint, goimports; sets make swagger # regenerate API models/clients from swagger specs into gen/ (see below) make build-mac # build local binary -> bin/cla-mac (build-linux for Linux) make test # go test -v ./... with coverage -make lint # golangci-lint (v1.64.8, config .golangci.yaml) + license header check +make lint # golangci-lint (v1.64.8, config .golangci.yaml) + license header check — see "Lint safely" below make fmt # gofmt + goimports make mock # regenerate mocks via tools/regenmocks.sh make all-mac # full pipeline: clean swagger deps fmt build test lint (all-linux on Linux) @@ -39,6 +39,15 @@ make all-mac # full pipeline: clean swagger deps fmt build test lint (all Run a single test: `go test -v ./signatures/ -run TestName` +### Lint safely (memory) + +golangci-lint v1.64.8 must be the binary built with the CI Go toolchain (`go 1.25.x`, see `go.mod`/`build-pr.yml`) and run under that +toolchain. A v1.64.8 binary built with a newer Go, or run with a newer `GOTOOLCHAIN`, type-checks every package again per linter and grows +past 30 GB RSS on this module — it OOM-kills the whole shell/tmux session, not just the linter. Rules: Linux only; pinned toolchain +(`GOTOOLCHAIN=go1.25.13 golangci-lint version` must print the same Go version); `LINT_ARGS="--concurrency 4"` on shared machines; run it in +its own memory-capped cgroup (`systemd-run --user --scope -p MemoryMax=16G -p MemorySwapMax=0 make lint`); one lint at a time; never overwrite +a shared `~/go/bin/golangci-lint` while other sessions may use it — install a pinned copy elsewhere and point `LINT_TOOL` at it. + Run locally (points at a real AWS environment — see below): build, set env, then `./bin/cla-mac` (from `make build-mac`) or `./bin/cla` (from `make build-linux`). Health checks at `http://localhost:8080/v3/ops/health` and `/v4/ops/health`. Set `GH_ORG_VALIDATION=false` to bypass GitHub auth checks for local curl/Postman testing. ### Swagger code generation (important) diff --git a/cla-backend-go/Makefile b/cla-backend-go/Makefile index a603a9fb4..22f620f0b 100644 --- a/cla-backend-go/Makefile +++ b/cla-backend-go/Makefile @@ -13,6 +13,7 @@ GITLAB_REPO_CHECK_BIN = gitlab-repository-check-lambda FUNCTIONAL_TESTS_BIN = functional-tests USER_SUBSCRIBE_BIN = user-subscribe-lambda REPOSITORY_UPDATE_BIN = repository-update-tool +ORG_IMPORT_BIN = org-import MAKEFILE_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) GOPRIVATE=github.com/LF-Engineering/* BUILD_TIME=$(shell sh -c 'date -u +%FT%T%z') @@ -35,6 +36,8 @@ ifeq "$(shell uname -s)" "Linux" endif LINT_TOOL=$(shell go env GOPATH)/bin/golangci-lint +# extra golangci-lint flags, e.g. LINT_ARGS="--concurrency 4" on shared/low-memory machines +LINT_ARGS ?= # LINT_VERSION=v1.51.2 LINT_VERSION=v1.64.8 SWAGGER_DIR=$(ROOT_DIR)/swagger @@ -328,6 +331,17 @@ build-gitlab-repository-check-lambda-mac: deps build-prep env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(LDFLAGS) -o $(BIN_DIR)/$(GITLAB_REPO_CHECK_BIN)-mac cmd/gitlab_repository_check/main.go @chmod +x $(BIN_DIR)/$(GITLAB_REPO_CHECK_BIN)-mac +build-org-import: build-org-import-linux +build-org-import-linux: deps build-prep + @echo "==> Building the org import tool (Linux amd64)..." + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(LDFLAGS) -o $(BIN_DIR)/$(ORG_IMPORT_BIN) cmd/org_import/main.go + @chmod +x $(BIN_DIR)/$(ORG_IMPORT_BIN) + +build-org-import-mac: deps build-prep + @echo "==> Building the org import tool (Mac OSX)..." + env CGO_ENABLED=0 GOOS=darwin GOARCH=$(BUILD_ARCH) go build $(LDFLAGS) -o $(BIN_DIR)/$(ORG_IMPORT_BIN)-mac cmd/org_import/main.go + @chmod +x $(BIN_DIR)/$(ORG_IMPORT_BIN)-mac + build-functional-tests: build-functional-tests-linux build-functional-tests-linux: deps build-prep @echo "==> Building Functional Tests for Linux amd64 binary..." @@ -351,5 +365,5 @@ build-repository-update-mac: deps build-prep @chmod +x $(BIN_DIR)/$(REPOSITORY_UPDATE_BIN)-mac lint: - @cd $(MAKEFILE_DIR) && $(LINT_TOOL) version && echo "==> Running lint..." && $(LINT_TOOL) run --timeout 30m --exclude="this method will not auto-escape HTML. Verify data is well formed" --allow-parallel-runners --config=.golangci.yaml ./... && echo "==> Lint check passed." + @cd $(MAKEFILE_DIR) && $(LINT_TOOL) version && echo "==> Running lint..." && $(LINT_TOOL) run --timeout 30m --exclude="this method will not auto-escape HTML. Verify data is well formed" --allow-parallel-runners --config=.golangci.yaml $(LINT_ARGS) ./... && echo "==> Lint check passed." @cd $(MAKEFILE_DIR) && ./check-headers.sh diff --git a/cla-backend-go/cmd/org_import/README.md b/cla-backend-go/cmd/org_import/README.md new file mode 100644 index 000000000..0abeab035 --- /dev/null +++ b/cla-backend-go/cmd/org_import/README.md @@ -0,0 +1,357 @@ +# org_import — EasyCLA organization import/sync (M3) + +Copyright The Linux Foundation and each contributor to CommunityBridge. + +SPDX-License-Identifier: CC-BY-4.0 + +Tickets: linuxfoundation/lfx-self-serve#2750 (import), #2751 (no company rows created by reads), #2749 (cleanup). +Design: `docs/easycla-ss-migration/m3-b2b-org-import.md` (PR #5223). Cleanup runbook: `docs/easycla-ss-migration/m3-org-cleanup.md`. + +## 1. What it does + +For every EasyCLA company with an **active CCLA** (`signature_type=ccla`, `signature_reference_type=company`, +`signature_signed=true`, `signature_approved=true`) the tool makes the organization usable by LFX Self Serve: + +| Route | When | What happens (apply mode) | +|---|---|---| +| `register` | `company_external_id` is a live Salesforce account id (`001…`, 15/18 chars) | `POST /b2b_orgs {"sfid": id}` on the LFX v2 member-service (idempotent) | +| `rewrite` | id is `lf…` (legacy console) or a dead `001…` account | needs a new Salesforce id from `--mapping` (or `--use-apex`): copy ACS grants old→new, rewrite `company_external_id` on every row of the group (old id kept in `previous_company_external_id`), re-key activity events, delete old grants, register the new id | +| `manual` | empty/invalid id, ambiguous/unapproved mapping, collision on the target Account without a decision (§4.1), no/shared website on the Apex path | nothing; listed with `manual_reason` and a suggested action | + +A **group** is every company row sharing one `company_external_id` (signing entities of one organization); it is +eligible when at least one row has an active CCLA. Internal `company_id`s never change. + +Never done by the tool: creating or deleting EasyCLA rows, merging companies, deleting Salesforce accounts or +org-service organizations, deleting roles, touching groups without an active CCLA, writing anything without `--apply`. +Rows sharing one `company_external_id` are one group and are never merged; merging duplicate companies is a post-import +runbook (linuxfoundation/lfx-self-serve#2056), the pre-import duplicate review (#3085) only feeds the decisions file (§4.1). + +Safety rules: +- Default is **dry run**. `--apply` prompts you to type the stage name (`--yes` skips the prompt, for automation). +- Every rewrite step is idempotent and recorded in `--state` (required for a rewrite `--apply`; the first journal line precedes the first + write and a journal write failure stops the group); re-running the same command converges. +- The row rewrite is a conditional write (`company_external_id = `); a concurrent change stops that group only. +- Grants are copied before rows are rewritten and old grants deleted last, each old grant re-checked (and copied if missing) right before + its deletion: managers never lose access. +- An apply that could not finish is refused before its first write: rewrites without `--state`, or any register/rewrite without member-service. +- `register` is stateless and needs no Salesforce input — it is the only route ever scheduled (§6). + +## 2. Prerequisites + +- Go 1.25+, `cd cla-backend-go && make build-org-import-linux` (Linux) or `make build-org-import-mac` → `bin/org-import[-mac]`. +- AWS credentials for the stage account (`dev.md`, "assume role" recipe; MFA). Either `AWS_PROFILE=lfproduct-dev AWS_SDK_LOAD_CONFIG=1` + or exported `AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN`. `AWS_REGION` defaults to `us-east-1`. +- `STAGE=dev|prod` (required). +- IAM: read of SSM `cla-*-{stage}` parameters; DynamoDB `Scan/Query/GetItem` on `cla-{stage}-companies`, `cla-{stage}-signatures`, + `cla-{stage}-events`; `UpdateItem` on `cla-{stage}-companies` and `cla-{stage}-events` (rewrite route only). Same tables the API uses. +- SSM parameters read (all pre-existing except the first two): + - `cla-member-service-base-url-{stage}`, `cla-member-service-auth0-audience-{stage}` — member-service; when missing the tool warns, + every SFID group is `live=unverified` → `pending`/`crm_unverified` (org-service is **not** a liveness source) and an `--apply` with + anything to register or rewrite is refused with `member-service is not configured` before writing. + Values (from `lfx-v2-argocd/values/{stage}/lfx-platform.yaml`): dev `https://lfx-api.dev.v2.cluster.linuxfound.info` / + `https://lfx-api.dev.v2.cluster.linuxfound.info/`, prod `https://lfx-api.v2.cluster.lfx.dev` / `https://lfx-api.v2.cluster.lfx.dev/`. + Create as plain `String` parameters (`aws ssm put-parameter --type String …`); dev already has them, prod does not yet. + - `cla-auth0-platform-*-{stage}` — M2M token (org-service, ACS, member-service). + - `cla-api-gateway-url-{stage}`, `cla-acs-api-key-{stage}` — org-service and ACS. + - `cla-salesforce-apex-base-url-{stage}`, `cla-salesforce-apex-token-{stage}` — only with `--use-apex`. + - `cla-org-import-report-emails-{stage}` — optional, comma-separated recipients of the run report e-mail (§8); absent ⇒ no e-mail, noted in run.log. + `cla-ses-sender-email-address-{stage}` (pre-existing) is the sender. +- Reports (§8) additionally need `logs:CreateLogGroup/CreateLogStream/PutLogEvents` on `/easycla/org-import/{stage}` and `ses:SendRawEmail`; + `--no-aws-log` / `--no-email` switch them off (use both for a read-only look at prod from a laptop). +- Member-service access (ops, one-time per stage): (1) an Auth0 **client grant** for EasyCLA's M2M client (`cla-auth0-platform-client-id-{stage}`) + on the lfx-api resource server (the audience above) — without it the token request fails with + `authorization failed (403): Client "…" is not authorized to access resource server` (current state on dev); (2) the client must be in the + `global_org_admin` team (member-service Heimdall ruleset) for `POST /b2b_orgs` and have `auditor` for `GET /b2b_orgs/{id}` (dry-run liveness). + Check: `STAGE=dev bin/org-import ingest --routes register --ids ` must print `live=live`; `live=error` means the GET + is not permitted (never treated as dead) and the run exits 1. +- Nothing sensitive is ever written: report files contain company names/ids only; tokens stay in memory. + +## 3. Commands + +``` +org_import audit [--out-dir ./org-import-out] [report flags] +org_import ingest [--apply] [--yes] [--tranche N] [--ids id1,id2] [--mapping map.csv] [--decisions decisions.csv] + [--shared-domains domains.txt] [--state state.jsonl] [--routes register,rewrite] [--skip-wait] + [--use-apex] [--wait-max 40m] [--out-dir ./org-import-out] [report flags] +report flags: [--email-to a@x,b@y] [--no-email] [--no-aws-log] [--aws-log-group /easycla/org-import/] +``` + +| Flag | Meaning | +|---|---| +| `--out-dir` | CSV reports directory (default `./org-import-out`, gitignored) | +| `--apply` | perform writes; without it nothing is written anywhere except report files | +| `--yes` | skip the "type the stage name" confirmation (automation) | +| `--tranche N` | act on at most N groups (planning still classifies everything) | +| `--ids` | only these groups; accepts old **or** new external ids (comma-separated) | +| `--mapping` | Salesforce mapping CSV (§4); required for the rewrite route unless `--use-apex` | +| `--decisions` | duplicate-review decisions CSV (§4.1): collapse/distinct verdicts for old ids landing on one Account | +| `--shared-domains` | file replacing the built-in shared-domain list (§4.2; one domain per line, `#` comments) | +| `--state` | append-only JSONL; one record per group per step; unfinished groups are replayed first on the next run. Required for `--apply` with the rewrite route (exit 2 otherwise) | +| `--routes` | `register`, `rewrite` or both (default both) | +| `--skip-wait` | check the new ids in org-service once instead of polling up to `--wait-max` | +| `--use-apex` | resolve new ids via the Salesforce Apex endpoint (`ORG_IMPORT_USE_APEX=true` equivalent); refused when the SSM params are missing | +| `--wait-max` | org-service propagation wait for new Salesforce accounts (default 40m, poll 30s) | +| `--email-to` | report recipients, overrides SSM `cla-org-import-report-emails-{stage}` | +| `--no-email` / `--no-aws-log` | do not send the report e-mail / do not copy run.log to CloudWatch Logs | +| `--aws-log-group` | CloudWatch log group (default `/easycla/org-import/{stage}`), created when missing | + +Env: `STAGE` (required), `AWS_REGION` (default `us-east-1`), `LOG_LEVEL` (default `warn`; `debug` shows every AWS/HTTP call; `error` hides the +expected org-service "not found" warnings printed for every dead/unknown id). + +Every run writes `/run.log` (everything printed, truncated per run) plus copies of the inputs it used +(`input-mapping.csv`, `input-decisions.csv`, `input-shared_domains.txt`, `input-state.jsonl` = the journal before the run; a `--state` outside +the out-dir is copied in after the run as `state.jsonl`) and ends with the report step (§8); report failures are logged and never change the exit +code. The apply commands in the report reference those copies as `"${RECORD:-}/input-*.csv"` and `"${RECORD:-}/state.jsonl"`: +run them where the dry run ran, or extract the e-mailed zip / Actions artifact elsewhere and `export RECORD=` first. + +### 3.1 `audit` (reads only) + +```bash +cd cla-backend-go && make build-org-import-linux +export AWS_PROFILE=lfproduct-dev AWS_SDK_LOAD_CONFIG=1 STAGE=dev +bin/org-import audit --out-dir ./org-import-out/dev-$(date -u +%F) +``` + +Output line: `audit stage=dev companies=N eligible_groups=M MISSING_SFID=a INVALID_SFID_FORMAT=b SFID_OK=c SFID_DANGLING_OR_DELETED=d UNKNOWN=e register=r rewrite=w manual=m duplicates=k` +— the five tiers are 1:1 with `utils/audit_company_reachability.sh`; route counts are per row (only rows in eligible groups have a route). + +Files (`--out-dir`): +- `audit.csv` — `company_id, company_name, signing_entity_name, company_external_id, id_shape(001|lf|empty|other), active_ccla, ccla_count, ecla_count, org_service(200|404|err), website, duplicate_sfid_group, route, manual_reason, tier`. + `ecla_count` is filled only for active rows that are manual, duplicate or unresolvable and for every row of `possible_duplicates.csv` (cheap); `-1` = count failed. +- `unresolvable.csv` — active rows with empty/invalid ids or dead `001…` ids (#2749 input). +- `possible_duplicates.csv` — candidate targets for the #3085 review: rows sharing an id with the same (or empty) signing entity name, and rows + with the same normalized company name **or** the same org-service website domain under different ids (#2056 input). Shared/missing domains + (§4.2) never group; a set reached by both name and domain is listed once. Columns: `group, company_id, company_name, signing_entity_name, + company_external_id, id_shape, domain, active_ccla, ccla_count, ecla_count` (`ecla_count` empty = not measured, `-1` = count failed). + Distinct signing entities under one id are **not** duplicates. + +Runtime: full companies scan + one CCLA query + one org-service GET per distinct id (4 in parallel) — ~30 s on dev, ~10 min on prod (3.7k companies). + +### 3.2 `ingest` dry run + +```bash +STAGE=dev bin/org-import ingest # all routes, no mapping: rewrite candidates are "pending" +STAGE=dev bin/org-import ingest --routes register # what the sweep does +STAGE=dev bin/org-import ingest --routes rewrite --mapping map.csv --state state.jsonl +``` + +Annotated sample: + +``` +org_import ingest stage=dev mode=dry-run eligible_groups=4 register=1 rewrite=1 pending=1 skipped=0 +register 0014100000Te0G7AAJ shape=001 rows=2 live=live | Live Corp; Live Corp / Live Corp Asia +rewrite lfbd1c2b3a4d5e6f7a8 shape=lf rows=1 new_id=0014100000NewNewNe action=matched domain=legacy.example | Legacy Ltd +rewrite 0014100000DeadDead1 shape=001 rows=1 live=dead reason=no_mapping | Dead Corp <- pending: needs a mapping row +manual c0ffee00-... shape=empty rows=1 reason=empty_external_id | Empty Inc +PLAN register 0014100000Te0G7AAJ: POST /b2b_orgs {sfid:0014100000Te0G7AAJ} (rows: c-live,c-live-sub) +PLAN rewrite lfbd1c2b3a4d5e6f7a8 -> 0014100000NewNewNe (matched): wait org-service; copy ACS grants; rewrite 1 row(s) c-lf; re-key events; delete old grants; POST /b2b_orgs {sfid:0014100000NewNewNe} +TARGETS (rewrite destinations grouped by Account): +target 0014100000NewNewNe groups=1 old_ids=lfbd1c2b3a4d5e6f7a8 existing_rows=0 status=ok +MANUAL ACTIONS (1): +c0ffee00-... [manual] empty_external_id: Fill company_external_id (…) or leave for cleanup (#2749); the tool never guesses an id. +stage=dev mode=dry-run eligible=4 registered=0 rewritten=0 pending=1 manual=1 failed=0 +``` + +- `live=live|dead|unverified|error` — member-service `GET /b2b_orgs/{id}`; `unverified` = member-service not configured for the stage + (the group stays `pending`/`crm_unverified`, nothing is written); `error` is never treated as dead. +- `pending` — rewrite candidates without a resolved new id (no mapping row, or a `register` POST answered 404 = dead account). They appear in `to_salesforce.csv`. +- `skipped` — eligible groups excluded by `--routes`, `--tranche`, or already `done` in `--state`. +- Summary line: `eligible` = groups after `--ids`; `registered`/`rewritten` = groups completed in apply mode; `manual`; `failed` = groups with an + error — the run exits 1 in dry-run and apply mode alike (a dry run with `live=error` is not a clean dry run). + +Runtime: ~10 s on dev, ~7 min on prod (one liveness GET per group). + +Files (all rewritten after apply with the final state): +- `plan.csv` — `key, old_id, id_shape, route, manual_reason, live, org_service, website, domain, shared_domain, new_id, action, decision, reviewer, company_ids, company_names, error`. +- `manual_actions.csv` — one row per manual/pending/failed group with `reason` and `suggested_action` (what a human must do next). +- `targets.csv` — rewrite destinations grouped by Account: `target_sfid, groups, old_ids, company_ids, company_names, existing_rows, decision, reviewer, status(ok|needs_decision|distinct_conflict|target_forms_differ)`. +- `to_salesforce.csv` — `old_id, name, website, ccla_signed_date, domain, shared_domain` — the hand-off to sales ops (§4). + +Manual reasons: `empty_external_id`, `invalid_id_shape`, `mapping_ambiguous`, `mapping_not_approved`, `mapping_same_id` (also the 15/18-char form of the same Account), +`sfid_alias_forms` (rows of one Account carry both its 15- and 18-char id — normalize them to one form first; §4), `target_forms_differ` +(ids landing on one Account use both forms — normalize the mapping/rows first; a decision does not lift it), +`target_collision` (several old ids → one Account, or the Account already has EasyCLA rows, and no decision covers it — §4.1), `distinct_conflict` +(a `distinct` decision spans two ids resolved to the same Account), `missing_website` / `shared_domain` (Apex path only, §4.2), +`apex_match_needs_approval`, `apex_error`; pending reasons: `no_mapping`, `dead_account`, `crm_unverified` (no member-service for the stage). +With `--use-apex` a dry-run `created` has no Account id yet (`new_id=`): the id is assigned by the real call at apply and a +different answer at apply time (`changed between dry run`) fails the group before any write; a failed resolution is never replayed as approved. + +### 3.3 `ingest --apply` + +```bash +# register only, 10 groups, interactive confirmation +STAGE=prod bin/org-import ingest --routes register --tranche 10 --apply +# rewrite tranche with mapping + state +STAGE=prod bin/org-import ingest --routes rewrite --mapping map.csv --state state.jsonl --tranche 10 --apply +# one group, by old or new id +STAGE=prod bin/org-import ingest --ids lfbd1c2b3a4d5e6f7a8 --mapping map.csv --state state.jsonl --apply +``` + +Order inside one run: all `register` POSTs → (Apex real calls) → **one** shared wait until org-service serves every new id (≤ `--wait-max`) +→ per rewrite group: `copy_grants` → `rewrite_rows` → `rekey_events` → `delete_old_grants` → `register` → `done` +→ events recheck: every previously completed group in the journal is listed again by its old id (the events indexes are eventually consistent, +and a writer that read the company row before `rewrite_rows` may still add events under the old id) and stragglers are re-keyed +(`events recheck old -> new: N listed, M re-keyed`; dry run lists only). It runs with the `rewrite` route only, honours `--ids` and the tranche +budget left after the planned groups (skipped rechecks are counted; groups rewritten by this run are rechecked from the next run on), is never +journaled (so it repeats every run) and fails — instead of re-keying — a group whose recorded rows are gone or no longer carry the recorded new id. +A failing step stops that group (state records it), the run continues with the next group and exits 1 with `FAILED …` lines. + +Expected runtime: register = seconds per group; rewrite = the org-service wait (new Salesforce accounts take up to ~40 minutes to appear; +existing accounts are immediate) plus seconds per group. + +## 4. Mapping file + +CSV with header `old_id,new_id,action,approved`: + +``` +old_id,new_id,action,approved +lfbd1c2b3a4d5e6f7a8,0014100000NewNewNe,matched,true +0014100000DeadDead1,0014100000Fresh001,created,true +lf000000000000000001,,ambiguous,false +``` + +- `action`: `matched` (existing account found by domain/name — **requires `approved=true`**, a human checked it), `created` (new account created for this org, accepted as is), `ambiguous` (several candidates — stays manual). +- `new_id` must be a 15/18-char alphanumeric Salesforce id (`ambiguous` rows may leave it empty); `new_id == old_id` is refused (`mapping_same_id`). +- Ids are compared **per Account**: the 15-char id and its 18-char form (first 15 chars + checksum) name the same target, so co-targeting + mappings and rows already carrying the other form collide the same way; a source group whose Account appears in the inventory in both + forms is `sfid_alias_forms` (manual) until the rows are normalized to one form, and a destination that would end up carrying both forms + is `target_forms_differ` (manual; no decision lifts it). Journal keys, CAS values and `old_id` stay the exact stored value. +- One row per `old_id`; two old ids pointing to the same `new_id` (or a `new_id` some EasyCLA row already carries) → `target_collision` until a + `collapse` decision covers them (§4.1); `distinct` decisions cannot share an Account. +- Loader errors are fatal and name the line: `missing column`, `expected N columns`, `approved must be true|false`, `empty old_id`, `duplicate old_id`, `new_id … is not a Salesforce account id`, `action must be matched|created|ambiguous`. +- Rows with an empty or malformed `company_external_id` (`empty_external_id` / `invalid_id_shape` in `unresolvable.csv`, `manual_actions.csv`) + have no organization id to key on, so `old_id` is the row's **`company_id`** instead (`,001…,matched,true`). Such a row is a + group of its own — blank or garbage values are never grouped —, the write is conditional on that one row still carrying its current value + (`attribute_not_exists` / the malformed string), no `previous_company_external_id` is recorded for a blank value, ACS grants and events are + not moved (nothing is keyed by the old value) and the new id is registered as usual. Two such rows on one Account are a `target_collision` + until a `collapse` decision names both company ids (§4.1). `state.jsonl` records them under `key` = `company_id`. + +How it is produced (sales ops, outside the tool): take `to_salesforce.csv` from a dry run, domain-match `website` against live accounts, +create the rest with Data Loader (unique external-id field = `old_id`, non-member record type, origin marker), export `old_id,new_id,action,approved`. +Mark `approved=true` only after a person confirmed each `matched` pair. Keep the file out of git (`org-import-out/` is ignored). + +`--use-apex` replaces the file for `created` results: the tool does a `dryRun` call per group first, then the real call, and stops the group when +action/id differ between the two; `matched` results still need an approved mapping row with the same id. An approved/`created` mapping row wins +over Apex for its old id, and groups without a website or with a shared domain (§4.2) never reach Apex. + +### 4.1 Decisions file (`--decisions`, lfx-self-serve#3085) + +Outcome of the human duplicate review. CSV with header `decision,old_ids,target_sfid,reviewer,note`; `old_ids` separated by `;` or spaces: + +``` +decision,old_ids,target_sfid,reviewer,note +collapse,lfaaaa000000000000001;lfbbbb000000000000002,0014100000NewNewNe,michal,same company (Acme Inc / Acme GmbH) +distinct,lfcccc000000000000003;lfdddd000000000000004,,michal,different companies despite the shared domain +``` + +- `collapse` — the listed old ids (and any EasyCLA rows already on `target_sfid`) are one organization: the Account may receive all of them. + Without it every Account that would end up with more than one old id — including one that already has EasyCLA rows — is `target_collision`. +- `distinct` — the listed old ids are different organizations; ≥2 ids, no target. If two of them still resolve to one Account the groups become + `distinct_conflict` (fix the mapping). `reviewer` is required; an old id may appear in one decision only. +- Decisions never change the mapping: the Account comes from `--mapping` / Apex; the decision only approves the collision. + +### 4.2 Shared domains (`--shared-domains`) + +Groups whose website domain is in the shared list (`github.com`, `nowebsite.com`, `gmail.com`, `googlemail.com`, `yahoo.com`, `hotmail.com`, +`outlook.com`, `live.com`, `icloud.com`, `protonmail.com`, `qq.com`, `163.com`) or who have no website are `manual` (`shared_domain` / +`missing_website`) instead of being domain-matched by Apex, and `audit` never groups them by domain in `possible_duplicates.csv`. A file +replaces the whole list (one domain per line, `#` comments). Mapping rows are explicit human decisions and are not gated. + +## 5. Tranche protocol (prod) + +1. `audit` → compare tier counts with the last `utils/audit_company_reachability.sh` numbers; read `unresolvable.csv` / `possible_duplicates.csv`. +2. `ingest` dry run, all routes → read `plan.csv`; hand `to_salesforce.csv` to sales ops. +3. `ingest --routes register --tranche 10 --apply` → verify (below) → `--tranche 100` → rest. +4. With the mapping file: `ingest --routes rewrite --mapping map.csv --state state.jsonl --tranche 10 --apply` → verify → 100 → rest. +5. Re-run the same command: it must report nothing new (`registered=0 rewritten=0`, groups `skipped` as done). +6. After the last rewrite tranche, run `ingest --routes rewrite --mapping map.csv --state state.jsonl --apply` once more without `--tranche` + and check the `events recheck: … 0 event(s) listed, … 0 failed, 0 skipped` line; repeat later if it re-keyed anything. + +Verification per tranche (pick 3 groups): +- `GET /b2b_orgs/{new id}` → 200 (member-service). +- org-service `GET /organizations/{new id}` → 200. +- ACS `GET /users/rolescopes/organization?orgid={new id}&scopetype=all` with `X-LFX-CACHE: false` → same users/roles as the old id had; old id → empty. +- `GET /v4/company/external/{new id}/cla-groups` lists the CCLA; `GET /v4/company/{companyID}/project/{projectSFID}/events` still shows the pre-rewrite history. +- DynamoDB row: `company_external_id = new`, `previous_company_external_id = old`. +- Corporate Console: a CLA manager of the company logs in, sees the company and its CCLA; Self Serve org lens shows the organization. + +## 6. Sweep (`register` route only) + +Manual: `STAGE= bin/org-import ingest --routes register [--apply --yes]` — stateless, idempotent, minutes. +It **never** rewrites and never touches ACS or Salesforce; new `lf…` (legacy console) companies only show up as `pending` — moving them is the +manual rewrite tranche of §5 (a named owner is required, see the design doc). + +GitHub Actions `.github/workflows/org-import-sweep.yml`: +- Actions tab → "Org import sweep" → Run workflow: `stage` (`dev|prod`), `mode` (`dry-run|apply`), `routes` (default `register`), + `tranche`, optional `ids`, optional `mapping` / `decisions` / `shared_domains` (the files of §4 pasted as text; rows separated by newlines or + `|` — e.g. `old_id,new_id,action,approved|lf…,001…,matched,true`), `notify` (default true; false ⇒ `--no-email`). Without a mapping, rewrite + candidates are only reported as pending. Same from a shell (the report e-mail of every dry run prints this line ready to paste): + `gh workflow run org-import-sweep.yml -f stage=dev -f mode=dry-run -f routes=register,rewrite -f mapping="$(tr '\n' '|' < map.csv)"`. +- State: apply runs upload `state.jsonl` as artifact `org-import-state-`; the next run of the same stage restores the newest one first, + so crashed rewrite groups are replayed; a state artifact that exists but cannot be downloaded fails the job (never start from an empty + journal). Artifacts expire after 90 days: before that, copy `state.jsonl` from the newest `org-import-out--*` artifact and keep it + (it is also uploaded there). Without the journal, finished groups are re-classified as `register` on their new id, but a group that crashed + mid-rewrite is not recognised as such — see §7 before re-running. +- One run per stage at a time (`concurrency: org-import-`): a manual dispatch queues behind a scheduled run instead of racing it. +- Schedule (`0 6 * * *`) is gated per stage by repository variables `ORG_IMPORT_SWEEP_DEV` / `ORG_IMPORT_SWEEP_PROD` ∈ `off|dry-run|apply`; + unset or `off` (default) ⇒ the scheduled job does nothing. Scheduled runs are always `--routes register`. Manual runs ignore the variables. +- Promotion: ≥10 clean manual runs (dry-run, then apply) → set the variable to `dry-run`, read the artifacts for a week → `apply`. + Rollback: set `off` (no deploy, no code change). The `prod` environment's protection rules (reviewers) apply to every run. +- Each run uploads `org-import-out/` (run.log and the CSVs of §3.2) as artifact `org-import-out--`, copies run.log to CloudWatch + Logs and e-mails the report (§8) — scheduled runs included. +- IAM: the `github-actions-deploy` role of each account must allow the DynamoDB/SSM access listed in §2 (it deploys the API, so it already does). + +## 7. Failures and recovery + +| Where | Message | Meaning / action | +|---|---|---| +| setup | `loading SSM config` / `STAGE is not set` | wrong account/profile or missing stage | +| planning | `live=error` | member-service GET failed (403 = missing `auditor`, network) — fix access; nothing is classified dead | +| planning | `live=unverified` / `crm_unverified` | no member-service params for the stage; groups stay pending, nothing is registered | +| apply | `rewrite apply requires --state` / `member-service is not configured` | refused before the first write; every planned group is reported as failed | +| any step | `state file …: cannot record` | the journal could not be written; the group stops (rows are never rewritten before their `start` line) | +| resolve | `changed between dry run` / `non-account id` | the live Apex answer differs from the plan; nothing written; re-run (not replayed) | +| planning | `mapping line N: …` | fix the CSV | +| planning | `state: unfinished group … has no valid new_id` | hand-edit the state file only if you know why; otherwise stop | +| register | `account … does not exist in Salesforce (rewrite candidate)` | POST 404: the id is dead; group is now `pending`/`dead_account` → mapping | +| register | `member-service is not configured` | SSM params missing for this stage | +| wait | `org-service does not serve … yet` | new Salesforce account not propagated within `--wait-max`; re-run later (state replays the group) | +| wait | `org-service serves target … as "…"` | org-service returns the account under another id than the mapping/decisions; nothing written; fix the input to the id org-service returns and re-run | +| copy_grants | `granting … on …` | org-service create failed; nothing else was changed; re-run | +| rewrite_rows | `… conflict, group stopped` | a row's external id was changed by someone else meanwhile; investigate that row before re-running | +| rekey_events | `event …` | UpdateItem failed; re-run (already re-keyed events are skipped) | +| events recheck | `FAILED events recheck old -> new: company … carries … / is gone` | a completed group's row was changed or deleted since; nothing is re-keyed for it — investigate the row (the journal is not changed) | +| events recheck | `FAILED events recheck old -> new: listing events … / event …` | Query/UpdateItem failed; the other groups are still rechecked; re-run | +| delete_old_grants | `deleting … grant …` | rows are already rewritten and new grants exist; re-run to finish | + +Resume: re-run the same command with the same `--state`; unfinished groups are replayed first (pinned by their `company_ids`, so they are found +even after their rows already carry the new id). `--ids ` narrows a run to one group. + +Revert a rewrite by hand (only if really needed): for each row, `aws dynamodb update-item --table-name cla-{stage}-companies --key '{"company_id":{"S":""}}' +--update-expression 'SET company_external_id = :o REMOVE previous_company_external_id' --expression-attribute-values '{":o":{"S":""}}'`, +then re-create the ACS grants on the old id (org-service `CreateRolescopes`, by username) and remove them from the new id. Events keep the new key +(run `events.RekeyRepository` the other way round if required). Stop and contact the EasyCLA maintainers before reverting more than one group. +`register` cannot be reverted through the API: member-service has no `DELETE /b2b_orgs/{id}`; a wrongly registered organization must be removed +by the member-service owners. Hence the register route is only ever applied to Salesforce-live ids, and never without member-service. + +## 8. Run report (e-mail + CloudWatch Logs) + +After every `audit`/`ingest` run (dry-run or apply, success or failure) the tool: +1. copies `run.log` to CloudWatch Logs group `/easycla/org-import/{stage}` (created when missing), stream `---`; +2. e-mails `cla-org-import-report-emails-{stage}` (or `--email-to`) from `cla-ses-sender-email-address-{stage}` via SES: subject + `[EasyCLA org-import][] : (OK|FAILED)`; body = run header (stage, mode, arguments, runner, build revision, + Actions run/artifact links, CloudWatch stream), the **Manual actions** table with suggested actions, the **Targets** table, the full plan (≤2000 + rows inline) or audit tier counts, the ready-to-paste local and `gh workflow run` apply commands for a dry run, and the run.log tail; attachments = + every CSV of the out-dir, run.log and a zip of the out-dir (≤6 MiB). To stay under SES's 10 MiB the largest text attachment is gzip-compressed + (`run.log.gz`, full content) or, when not text, dropped — the zip last; every such change, an input or journal copy that failed, and a zip + that could not be built or is over 6 MiB (it is the only carrier of the `input-*` copies and the journal) is announced at the top of the e-mail + ("Delivery incomplete …") and in run.log; the complete record is always the out-dir / Actions artifact / CloudWatch stream. + `audit` also prints one `audit row company_id=… route=… tier=…` line per company row into run.log. + +Missing recipients parameter, SES or CloudWatch errors are written to run.log only; the exit code reflects the import itself. + +## 9. Cleanup pointers + +`unresolvable.csv` and `possible_duplicates.csv` feed `docs/easycla-ss-migration/m3-org-cleanup.md` (#2749, #2056): human review, nothing is deleted +or merged by the tool. Duplicate companies (same organization under several ids) are collapsed onto one Account by the import via the decisions +file (§4.1); deleting or re-pointing their EasyCLA rows is post-import work (#2056). diff --git a/cla-backend-go/cmd/org_import/main.go b/cla-backend-go/cmd/org_import/main.go new file mode 100644 index 000000000..2876b0287 --- /dev/null +++ b/cla-backend-go/cmd/org_import/main.go @@ -0,0 +1,536 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +// org_import is the M3 organization import/sync CLI (lfx-self-serve #2750). See README.md. +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "runtime/debug" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/ses" + "github.com/aws/aws-sdk-go/service/ssm" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/orgimport" + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/approvals" + member_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/member-service" + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" +) + +const usageText = `usage: + org_import audit [--out-dir ./org-import-out] [report flags] + org_import ingest [--apply] [--yes] [--tranche N] [--ids id1,id2] [--mapping map.csv] [--decisions decisions.csv] + [--shared-domains domains.txt] [--state state.jsonl] [--routes register,rewrite] [--skip-wait] + [--use-apex] [--out-dir ./org-import-out] [report flags] + report flags: [--email-to a@x,b@y] [--no-email] [--no-aws-log] [--aws-log-group /easycla/org-import/] + +Environment: STAGE=dev|prod (required), AWS credentials for that account (AWS_PROFILE/AWS_SDK_LOAD_CONFIG=1 +or exported keys), AWS_REGION (default us-east-1), LOG_LEVEL (default warn), ORG_IMPORT_USE_APEX=true. +Without --apply nothing is written to EasyCLA, ACS, Salesforce or member-service; every run still writes the +report files and run.log under --out-dir, copies run.log to CloudWatch Logs and e-mails the full decision +record to SSM cla-org-import-report-emails- (or --email-to). +` + +type combinedRepo struct { + users.UserRepository + company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository +} + +type env struct { + deps orgimport.Deps + cfg config.Config + sess *session.Session +} + +type reportFlags struct { + emailTo string + noEmail bool + noAWSLog bool + logGroup string +} + +const ( + cmdAudit = "audit" + cmdIngest = "ingest" + trueString = "true" + unknownValue = "unknown" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdin)) +} + +func run(args []string, stdin io.Reader) int { + if len(args) < 1 { + fmt.Fprint(os.Stderr, usageText) + return 2 + } + if os.Getenv("LOG_LEVEL") == "" { + _ = os.Setenv("LOG_LEVEL", "warn") + } + cmd := args[0] + fs := flag.NewFlagSet("org_import "+cmd, flag.ContinueOnError) + fs.Usage = func() { fmt.Fprint(os.Stderr, usageText); fs.PrintDefaults() } + outDir := fs.String("out-dir", "./org-import-out", "directory for CSV reports and run.log (gitignored)") + apply := fs.Bool("apply", false, "perform writes (default: dry run)") + yes := fs.Bool("yes", false, "skip the confirmation prompt with --apply") + tranche := fs.Int("tranche", 0, "process at most N groups (0 = all)") + ids := fs.String("ids", "", "comma-separated company_external_id list (old or new ids)") + mapping := fs.String("mapping", "", "Salesforce mapping CSV old_id,new_id,action,approved") + decisions := fs.String("decisions", "", "duplicate-review decisions CSV decision,old_ids,target_sfid,reviewer,note (lfx-self-serve #3085)") + sharedDomains := fs.String("shared-domains", "", "shared-domain list, one domain per line (default: built-in list)") + state := fs.String("state", "", "append-only JSONL state file for rewrite tranches") + routes := fs.String("routes", "register,rewrite", "routes to process: register,rewrite") + skipWait := fs.Bool("skip-wait", false, "check new ids in org-service once instead of waiting up to 40 minutes") + useApex := fs.Bool("use-apex", os.Getenv("ORG_IMPORT_USE_APEX") == trueString, "resolve new ids through the Salesforce Apex endpoint (needs cla-salesforce-apex-* SSM params)") + waitMax := fs.Duration("wait-max", 40*time.Minute, "maximum org-service propagation wait") + var rf reportFlags + fs.StringVar(&rf.emailTo, "email-to", "", "report recipients (comma-separated); default SSM cla-org-import-report-emails-") + fs.BoolVar(&rf.noEmail, "no-email", false, "do not e-mail the report") + fs.BoolVar(&rf.noAWSLog, "no-aws-log", false, "do not copy run.log to CloudWatch Logs") + fs.StringVar(&rf.logGroup, "aws-log-group", "", "CloudWatch Logs group (default /easycla/org-import/)") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + if cmd != cmdAudit && cmd != cmdIngest { + fmt.Fprint(os.Stderr, usageText) + return 2 + } + stage := os.Getenv("STAGE") + if stage == "" { + fmt.Fprintln(os.Stderr, "STAGE is not set") + return 2 + } + if os.Getenv("AWS_REGION") == "" { + _ = os.Setenv("AWS_REGION", "us-east-1") + } + if rf.logGroup == "" { + rf.logGroup = "/easycla/org-import/" + stage + } + + var err error + opts := orgimport.Options{ + Stage: stage, Apply: *apply, Tranche: *tranche, Mapping: *mapping, Decisions: *decisions, SharedDomains: *sharedDomains, + State: *state, SkipWait: *skipWait, UseApex: *useApex, OutDir: *outDir, WaitMax: *waitMax, + } + if *ids != "" { + opts.IDs = strings.Split(*ids, ",") + } + if opts.Routes, err = parseRoutes(*routes); err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + if !*apply && *yes { + fmt.Fprintln(os.Stderr, "note: --yes has no effect without --apply") + } + if *apply && *state == "" && cmd == cmdIngest && hasRoute(opts.Routes, orgimport.RouteRewrite) { + fmt.Fprintf(os.Stderr, "%v; use --routes register for a register-only apply\n", orgimport.ErrStateRequired) + return 2 + } + + if err = os.MkdirAll(*outDir, 0o750); err != nil { + fmt.Fprintf(os.Stderr, "cannot create %s: %v\n", *outDir, err) + return 2 + } + logPath := filepath.Join(*outDir, "run.log") + logFile, err := os.Create(filepath.Clean(logPath)) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot create %s: %v\n", logPath, err) + return 2 + } + out := io.MultiWriter(os.Stdout, logFile) + errOut := io.MultiWriter(os.Stderr, logFile) + log.GetLogger().SetOutput(io.MultiWriter(os.Stderr, logFile)) + notices := snapshotInputs(*outDir, opts, out, errOut) + + info := orgimport.RunInfo{ + Stage: stage, Command: cmd, Apply: *apply, Args: args, Start: time.Now().UTC(), Runner: runnerName(), + Repository: os.Getenv("GITHUB_REPOSITORY"), Revision: buildRevision(), OutDir: *outDir, Notices: notices, + Workflow: orgimport.WorkflowInputs{Routes: *routes, Tranche: fmt.Sprint(*tranche), IDs: *ids, Mapping: *mapping, Decisions: *decisions, SharedDomains: *sharedDomains}, + } + if runID := os.Getenv("GITHUB_RUN_ID"); runID != "" { + info.RunURL = fmt.Sprintf("%s/%s/actions/runs/%s", os.Getenv("GITHUB_SERVER_URL"), os.Getenv("GITHUB_REPOSITORY"), runID) + info.Artifact = fmt.Sprintf("org-import-out-%s-%s", stage, runID) + } + fmt.Fprintf(out, "org-import %s %s stage=%s start=%s runner=%s revision=%s args=%s\n", cmd, info.Mode(), stage, info.Start.Format(time.RFC3339), info.Runner, info.Revision, strings.Join(args, " ")) + + ctx := context.Background() + var plan *orgimport.Plan + var audit *orgimport.AuditResult + var e env + code := func() int { + var err error + e, err = wire(ctx, stage, opts.UseApex) + if err != nil { + fmt.Fprintf(errOut, "setup failed: %v\n", err) + info.Err = err.Error() + return 2 + } + e.deps.Out = out + if cmd == cmdAudit { + if audit, err = orgimport.Audit(ctx, e.deps, opts); err != nil { + fmt.Fprintf(errOut, "audit failed: %v\n", err) + info.Err = err.Error() + return 1 + } + info.Summary = fmt.Sprintf("%d company rows audited", len(audit.Rows)) + fmt.Fprintf(out, "reports written to %s (audit.csv, unresolvable.csv, possible_duplicates.csv, run.log)\n", *outDir) + return 0 + } + if plan, err = orgimport.BuildPlan(ctx, e.deps, opts); err != nil { + fmt.Fprintf(errOut, "planning failed: %v\n", err) + info.Err = err.Error() + return 1 + } + plan.Print(out) + if *apply && !*yes && !confirm(stage, out, stdin) { + fmt.Fprintf(out, "aborted: nothing was written\n") + info.Err = "aborted at the confirmation prompt" + return 2 + } + summary, err := orgimport.Execute(ctx, e.deps, opts, plan) + fmt.Fprintf(out, "%s\n", summary.String()) + info.Summary = summary.String() + if err != nil { + fmt.Fprintf(errOut, "ingest finished with errors: %v\n", err) + info.Err = err.Error() + return 1 + } + fmt.Fprintf(out, "reports written to %s (plan.csv, manual_actions.csv, targets.csv, to_salesforce.csv, run.log)\n", *outDir) + return 0 + }() + + info.End = time.Now().UTC() + info.Notices = append(info.Notices, captureJournal(*outDir, opts.State, out, errOut)...) + report(ctx, e, info, plan, audit, rf, logPath, out, errOut) + _ = logFile.Close() + return code +} + +// report ships run.log to CloudWatch Logs and e-mails the decision record; failures never change the exit code. +func report(ctx context.Context, e env, info orgimport.RunInfo, plan *orgimport.Plan, audit *orgimport.AuditResult, rf reportFlags, logPath string, out, errOut io.Writer) { + runLog, err := os.ReadFile(filepath.Clean(logPath)) + if err != nil { + fmt.Fprintf(errOut, "report: cannot read %s: %v\n", logPath, err) + } + if e.sess == nil { + fmt.Fprintf(errOut, "report: AWS session unavailable, skipping CloudWatch Logs and e-mail\n") + return + } + if !rf.noAWSLog { + info.LogGroup = rf.logGroup + info.LogStream = strings.NewReplacer(":", "-", "*", "-").Replace(fmt.Sprintf("%s-%s-%s-%s", info.Start.Format("2006-01-02T15-04-05Z"), info.Command, info.Mode(), info.Runner)) + shipper := &orgimport.LogShipper{Client: cloudwatchlogs.New(e.sess), Group: info.LogGroup, Stream: info.LogStream} + n, shipErr := shipper.Ship(ctx, runLog) + if shipErr != nil { + fmt.Fprintf(errOut, "report: CloudWatch Logs failed (%s/%s): %v\n", info.LogGroup, info.LogStream, shipErr) + info.LogGroup, info.LogStream = "", "" + } else { + fmt.Fprintf(out, "run.log copied to CloudWatch Logs %s stream %s (%d events)\n", info.LogGroup, info.LogStream, n) + } + } + if rf.noEmail { + fmt.Fprintf(out, "report e-mail disabled (--no-email)\n") + return + } + recipients := orgimport.ParseRecipients(rf.emailTo) + if len(recipients) == 0 { + value, found, ssmErr := readSSM(ctx, ssm.New(e.sess), fmt.Sprintf("cla-org-import-report-emails-%s", info.Stage), false) + if ssmErr != nil { + fmt.Fprintf(errOut, "report: %v\n", ssmErr) + } + if !found { + fmt.Fprintf(out, "report e-mail skipped: SSM cla-org-import-report-emails-%s is not set and --email-to is empty\n", info.Stage) + return + } + recipients = orgimport.ParseRecipients(value) + } + if len(recipients) == 0 || e.cfg.SenderEmailAddress == "" { + fmt.Fprintf(out, "report e-mail skipped: recipients=%d sender=%q\n", len(recipients), e.cfg.SenderEmailAddress) + return + } + rep := orgimport.BuildReport(info, plan, audit, runLog) + raw, notice, err := rep.Deliverable(e.cfg.SenderEmailAddress, recipients) + if err != nil { + fmt.Fprintf(errOut, "report: building e-mail failed: %v\n", err) + return + } + if notice != "" { + fmt.Fprintf(errOut, "report: %s\n", notice) + } + id, err := orgimport.Mailer{Client: ses.New(e.sess)}.Send(ctx, e.cfg.SenderEmailAddress, recipients, raw) + if err != nil { + fmt.Fprintf(errOut, "report: SES send failed: %v\n", err) + return + } + fmt.Fprintf(out, "report e-mailed to %s (%d bytes, %d attachments built, SES message id %s)\n", strings.Join(recipients, ","), len(raw), len(rep.Attachments), id) +} + +// parseRoutes turns the --routes value into the route list. +func parseRoutes(spec string) ([]orgimport.Route, error) { + var routes []orgimport.Route + for _, r := range strings.Split(spec, ",") { + switch strings.TrimSpace(r) { + case "register": + routes = append(routes, orgimport.RouteRegister) + case "rewrite": + routes = append(routes, orgimport.RouteRewrite) + case "": + default: + return nil, fmt.Errorf("unknown route %q (register|rewrite)", r) + } + } + return routes, nil +} + +func hasRoute(routes []orgimport.Route, want orgimport.Route) bool { + for _, r := range routes { + if r == want { + return true + } + } + return false +} + +// snapshotInputs copies the effective non-secret inputs (mapping, decisions, shared domains and the +// state file as it was before the run) into outDir so the artifact and the report zip are a complete +// record; every file it could not capture is returned as a notice for the report. +func snapshotInputs(outDir string, opts orgimport.Options, out, errOut io.Writer) []string { + var notices []string + for _, in := range []struct{ name, src string }{ + {orgimport.RecordMapping, opts.Mapping}, {orgimport.RecordDecisions, opts.Decisions}, + {orgimport.RecordSharedDomains, opts.SharedDomains}, {orgimport.RecordStateBefore, opts.State}, + } { + if in.src == "" { + continue + } + dst := filepath.Join(outDir, in.name) + data, err := os.ReadFile(filepath.Clean(in.src)) + if err != nil { + if os.IsNotExist(err) && in.name == orgimport.RecordStateBefore { + fmt.Fprintf(out, "input %s: %s does not exist yet (fresh state)\n", in.name, in.src) + continue + } + notices = append(notices, fmt.Sprintf("input %s: cannot read %s: %v", in.name, in.src, err)) + fmt.Fprintf(errOut, "%s\n", notices[len(notices)-1]) + continue + } + if err = os.WriteFile(dst, data, 0o600); err != nil { + notices = append(notices, fmt.Sprintf("input %s: cannot write %s: %v", in.name, dst, err)) + fmt.Fprintf(errOut, "%s\n", notices[len(notices)-1]) + continue + } + fmt.Fprintf(out, "input %s: copied %s (%d bytes) to %s\n", in.name, in.src, len(data), dst) + } + return notices +} + +// captureJournal copies the final journal into outDir as state.jsonl (input-state.jsonl keeps the +// before-run copy) so the record can resume the run; a journal already at that path is left alone. +func captureJournal(outDir, statePath string, out, errOut io.Writer) []string { + if statePath == "" { + return nil + } + dst := filepath.Join(outDir, orgimport.RecordState) + if absSrc, err := filepath.Abs(statePath); err == nil { + if absDst, absErr := filepath.Abs(dst); absErr == nil && absSrc == absDst { + return nil + } + } + data, err := os.ReadFile(filepath.Clean(statePath)) + if err != nil { + if os.IsNotExist(err) { + fmt.Fprintf(out, "journal %s: nothing was journaled\n", statePath) + return nil + } + msg := fmt.Sprintf("journal %s: cannot read: %v", statePath, err) + fmt.Fprintf(errOut, "%s\n", msg) + return []string{msg} + } + if err = os.WriteFile(dst, data, 0o600); err != nil { + msg := fmt.Sprintf("journal %s: cannot write copy %s: %v", statePath, dst, err) + fmt.Fprintf(errOut, "%s\n", msg) + return []string{msg} + } + fmt.Fprintf(out, "journal %s: copied (%d bytes) to %s\n", statePath, len(data), dst) + return nil +} + +func runnerName() string { + if id := os.Getenv("GITHUB_RUN_ID"); id != "" { + return "gha-" + id + } + user := os.Getenv("USER") + if user == "" { + user = unknownValue + } + host, err := os.Hostname() + if err != nil || host == "" { + host = "localhost" + } + return user + "@" + host +} + +// set by the Makefile (-X main.commit=… -X main.branch=…); a package build stamps VCS info instead +var ( + commit string + branch string +) + +func buildRevision() string { + if commit != "" { + if branch != "" { + return commit + " (" + branch + ")" + } + return commit + } + if bi, ok := debug.ReadBuildInfo(); ok { + rev, modified := "", "" + for _, s := range bi.Settings { + switch s.Key { + case "vcs.revision": + rev = s.Value + case "vcs.modified": + if s.Value == trueString { + modified = "-dirty" + } + } + } + if rev != "" { + return rev + modified + } + } + if sha := os.Getenv("GITHUB_SHA"); sha != "" { + return sha + } + return unknownValue +} + +func confirm(stage string, out io.Writer, stdin io.Reader) bool { + fmt.Fprintf(out, "APPLY mode: the plan above will be executed against %s. Type the stage name to continue: ", stage) + line, err := bufio.NewReader(stdin).ReadString('\n') + if err != nil && line == "" { + return false + } + return strings.TrimSpace(line) == stage +} + +func wire(ctx context.Context, stage string, useApex bool) (env, error) { + awsSession := session.Must(session.NewSession(&aws.Config{})) + e := env{sess: awsSession} + cfg, err := config.LoadConfig("", awsSession, stage) + if err != nil { + return e, fmt.Errorf("loading SSM config: %w", err) + } + e.cfg = cfg + token.Init(cfg.Auth0Platform.ClientID, cfg.Auth0Platform.ClientSecret, cfg.Auth0Platform.URL, cfg.Auth0Platform.Audience) + + usersRepo := users.NewRepository(awsSession, stage) + companyRepo := company.NewRepository(awsSession, stage) + projectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) + repositoriesRepo := repositories.NewRepository(awsSession, stage) + gerritRepo := gerrits.NewRepository(awsSession, stage) + projectRepo := repository.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) + eventsRepo := events.NewRepository(awsSession, stage) + githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage) + approvalRepo := approvals.NewRepository(stage, awsSession, fmt.Sprintf("cla-%s-approvals", stage)) + eventsService := events.NewService(eventsRepo, combinedRepo{usersRepo, companyRepo, projectRepo, projectClaGroupRepo}) + signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo, gerrits.NewService(gerritRepo), approvalRepo) + + organization_service.InitClient(cfg.APIGatewayURL, eventsService) + acs_service.InitClient(cfg.APIGatewayURL, cfg.AcsAPIKey) + + e.deps = orgimport.Deps{ + Companies: companyRepo, + Signatures: signaturesRepo, + ECLAs: orgimport.DynamoECLACounter{DB: dynamodb.New(awsSession), Table: fmt.Sprintf("cla-%s-signatures", stage)}, + Events: events.NewRekeyRepository(awsSession, stage), + Orgs: orgimport.OrgServiceAdapter{Client: organization_service.GetClient()}, + ACS: acs_service.GetClient(), + } + + members, err := member_service.NewClient(member_service.Config{ + BaseURL: cfg.MemberService.BaseURL, + Audience: cfg.MemberService.Audience, + OAuthTokenURL: cfg.Auth0Platform.URL, + ClientID: cfg.Auth0Platform.ClientID, + ClientSecret: cfg.Auth0Platform.ClientSecret, + }) + switch { + case err == nil: + e.deps.Members = members + case errors.Is(err, member_service.ErrNotConfigured): + log.Warnf("member-service not configured (cla-member-service-base-url-%s / cla-member-service-auth0-audience-%s): liveness is unverified, SFID groups stay pending (crm_unverified) and nothing is registered or rewritten for them", stage, stage) + default: + return e, err + } + + if useApex { + ssmClient := ssm.New(awsSession) + baseURL, err := requireSSM(ctx, ssmClient, fmt.Sprintf("cla-salesforce-apex-base-url-%s", stage), false) + if err != nil { + return e, err + } + apexToken, err := requireSSM(ctx, ssmClient, fmt.Sprintf("cla-salesforce-apex-token-%s", stage), true) + if err != nil { + return e, err + } + apex, err := orgimport.NewApexClient(baseURL, apexToken) + if err != nil { + return e, err + } + e.deps.Apex = apex + } + return e, nil +} + +func requireSSM(ctx context.Context, client *ssm.SSM, key string, decrypt bool) (string, error) { + value, found, err := readSSM(ctx, client, key, decrypt) + if err != nil { + return "", err + } + if !found { + return "", fmt.Errorf("%w: %s", orgimport.ErrApexUnavailable, key) + } + return value, nil +} + +func readSSM(ctx context.Context, client *ssm.SSM, key string, decrypt bool) (string, bool, error) { + out, err := client.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(key), WithDecryption: aws.Bool(decrypt)}) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ssm.ErrCodeParameterNotFound { + return "", false, nil + } + return "", false, fmt.Errorf("reading SSM %s: %w", key, err) + } + return strings.TrimSpace(aws.StringValue(out.Parameter.Value)), true, nil +} diff --git a/cla-backend-go/cmd/org_import/main_test.go b/cla-backend-go/cmd/org_import/main_test.go new file mode 100644 index 000000000..4f8665976 --- /dev/null +++ b/cla-backend-go/cmd/org_import/main_test.go @@ -0,0 +1,113 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/linuxfoundation/easycla/cla-backend-go/orgimport" +) + +func TestRunUsageErrors(t *testing.T) { + t.Setenv("STAGE", "") + assert.Equal(t, 2, run(nil, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"bogus"}, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"ingest", "--nope"}, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"ingest"}, strings.NewReader("")), "STAGE is required") + t.Setenv("STAGE", "dev") + assert.Equal(t, 2, run([]string{"ingest", "--routes", "delete"}, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"ingest", "--apply", "--yes", "--out-dir", t.TempDir()}, strings.NewReader("")), "rewrite apply needs --state") + assert.Equal(t, 2, run([]string{"ingest", "--apply", "--yes", "--routes", "rewrite", "--out-dir", t.TempDir()}, strings.NewReader(""))) +} + +func TestSnapshotInputs(t *testing.T) { + dir := t.TempDir() + mapping := filepath.Join(dir, "m.csv") + require.NoError(t, os.WriteFile(mapping, []byte("old_id,new_id,action,approved\n"), 0o600)) + outDir := filepath.Join(dir, "out") + require.NoError(t, os.MkdirAll(outDir, 0o750)) + var out, errOut bytes.Buffer + notices := snapshotInputs(outDir, orgimport.Options{Mapping: mapping, Decisions: filepath.Join(dir, "missing.csv"), State: filepath.Join(dir, "state.jsonl")}, &out, &errOut) + data, err := os.ReadFile(filepath.Clean(filepath.Join(outDir, "input-mapping.csv"))) + require.NoError(t, err) + assert.Equal(t, "old_id,new_id,action,approved\n", string(data)) + assert.Contains(t, errOut.String(), "input-decisions.csv: cannot read") + require.Len(t, notices, 1, "an uncaptured input is a report notice") + assert.Contains(t, notices[0], "input input-decisions.csv: cannot read") + assert.Contains(t, out.String(), "input-state.jsonl") + assert.Contains(t, out.String(), "does not exist yet") + _, err = os.Stat(filepath.Join(outDir, "input-state.jsonl")) + assert.True(t, os.IsNotExist(err)) +} + +func TestCaptureJournal(t *testing.T) { + dir := t.TempDir() + outDir := filepath.Join(dir, "out") + require.NoError(t, os.MkdirAll(outDir, 0o750)) + var out, errOut bytes.Buffer + + assert.Nil(t, captureJournal(outDir, "", &out, &errOut), "no journal configured") + assert.Nil(t, captureJournal(outDir, filepath.Join(dir, "never.jsonl"), &out, &errOut)) + assert.Contains(t, out.String(), "nothing was journaled") + _, err := os.Stat(filepath.Join(outDir, "state.jsonl")) + assert.True(t, os.IsNotExist(err)) + + // an external journal is copied in full as state.jsonl after the run + external := filepath.Join(dir, "journal.jsonl") + require.NoError(t, os.WriteFile(external, []byte("{\"step\":\"start\"}\n{\"step\":\"done\"}\n"), 0o600)) + assert.Nil(t, captureJournal(outDir, external, &out, &errOut)) + data, err := os.ReadFile(filepath.Clean(filepath.Join(outDir, "state.jsonl"))) + require.NoError(t, err) + assert.Equal(t, "{\"step\":\"start\"}\n{\"step\":\"done\"}\n", string(data)) + assert.Contains(t, out.String(), "copied (33 bytes)") + + // the record's own journal is not copied onto itself + inRecord := filepath.Join(outDir, "state.jsonl") + before, err := os.Stat(inRecord) + require.NoError(t, err) + assert.Nil(t, captureJournal(outDir, filepath.Join(outDir, "..", "out", "state.jsonl"), &out, &errOut)) + after, err := os.Stat(inRecord) + require.NoError(t, err) + assert.Equal(t, before.ModTime(), after.ModTime()) + + // an unreadable journal is a report notice + if os.Geteuid() != 0 { + locked := filepath.Join(dir, "locked.jsonl") + require.NoError(t, os.WriteFile(locked, []byte("{}\n"), 0o000)) + notices := captureJournal(outDir, locked, &out, &errOut) + require.Len(t, notices, 1) + assert.Contains(t, notices[0], "journal "+locked+": cannot read") + assert.Contains(t, errOut.String(), notices[0]) + } +} + +func TestConfirm(t *testing.T) { + var out bytes.Buffer + assert.True(t, confirm("dev", &out, strings.NewReader("dev\n"))) + assert.Contains(t, out.String(), "Type the stage name") + assert.False(t, confirm("dev", &out, strings.NewReader("prod\n"))) + assert.False(t, confirm("dev", &out, strings.NewReader(""))) + assert.True(t, confirm("prod", &out, strings.NewReader("prod")), "last line without newline") +} + +func TestRunnerAndRevision(t *testing.T) { + t.Setenv("GITHUB_RUN_ID", "42") + assert.Equal(t, "gha-42", runnerName()) + t.Setenv("GITHUB_RUN_ID", "") + t.Setenv("USER", "lukasz") + assert.True(t, strings.HasPrefix(runnerName(), "lukasz@")) + assert.NotEmpty(t, buildRevision()) + commit, branch = "abc123", "unicron-x" + defer func() { commit, branch = "", "" }() + assert.Equal(t, "abc123 (unicron-x)", buildRevision(), "Makefile -X values win over VCS stamping") + branch = "" + assert.Equal(t, "abc123", buildRevision()) +} diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index c272b7410..7ba361ffa 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -508,7 +508,7 @@ func server(localMode bool) http.Handler { version.Configure(api, Version, Commit, Branch, BuildDate) v2Version.Configure(v2API, Version, Commit, Branch, BuildDate) events.Configure(api, eventsService) - v2Events.Configure(v2API, eventsService, v1CompanyRepo, v1ProjectClaGroupRepo, v1ProjectService) + v2Events.Configure(v2API, eventsService, v1CompanyService, v1ProjectClaGroupRepo, v1ProjectService) v2Metrics.Configure(v2API, v2MetricsService, v1CompanyRepo) github_organizations.Configure(api, githubOrganizationsService, eventsService) v2GithubOrganizations.Configure(v2API, v2GithubOrganizationsService, eventsService) diff --git a/cla-backend-go/company/handlers.go b/cla-backend-go/company/handlers.go index e90018996..9422a3d6d 100644 --- a/cla-backend-go/company/handlers.go +++ b/cla-backend-go/company/handlers.go @@ -23,7 +23,6 @@ import ( "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/company" log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/linuxfoundation/easycla/cla-backend-go/user" - orgService "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" "github.com/go-openapi/runtime/middleware" ) @@ -73,21 +72,10 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv api.CompanyGetCompanyByExternalIDHandler = company.GetCompanyByExternalIDHandlerFunc(func(params company.GetCompanyByExternalIDParams) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint - // Check for Salesforce org - orgClient := orgService.GetClient() - org, getErr := orgClient.GetOrganization(ctx, params.CompanySFID) - - if getErr != nil { - msg := fmt.Sprintf("Failed to get salesforce org for ID: %s ", params.CompanySFID) - log.Warn(msg) - return company.NewGetCompanyByExternalIDBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Code: "400", - Message: msg, - }) - } - companyModel, err := service.GetCompanyByExternalID(ctx, params.CompanySFID) + // Persisted row when it exists, otherwise a non-persisted virtual company backed by the Salesforce org (#2751) + companyModel, err := service.ResolveCompany(ctx, params.CompanySFID) if err != nil { - msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to get associated salesforce Organization: %s using SFID: %s, error: %v", org.Name, params.CompanySFID, err) + msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to get associated salesforce Organization using SFID: %s, error: %v", params.CompanySFID, err) log.Warnf("%s", msg) return company.NewGetCompanyByExternalIDBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", diff --git a/cla-backend-go/company/mocks/mock_repo.go b/cla-backend-go/company/mocks/mock_repo.go index c3d59cd2a..695e12cb2 100644 --- a/cla-backend-go/company/mocks/mock_repo.go +++ b/cla-backend-go/company/mocks/mock_repo.go @@ -1,5 +1,6 @@ // Copyright The Linux Foundation and each contributor to CommunityBridge. // SPDX-License-Identifier: MIT +// // Code generated by MockGen. DO NOT EDIT. // Source: company/repository.go @@ -69,6 +70,21 @@ func (mr *MockIRepositoryMockRecorder) ApproveCompanyAccessRequest(ctx, companyI return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveCompanyAccessRequest", reflect.TypeOf((*MockIRepository)(nil).ApproveCompanyAccessRequest), ctx, companyInviteID) } +// ClearCompanySanctionStatusIfSSS mocks base method. +func (m *MockIRepository) ClearCompanySanctionStatusIfSSS(ctx context.Context, companyID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClearCompanySanctionStatusIfSSS", ctx, companyID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ClearCompanySanctionStatusIfSSS indicates an expected call of ClearCompanySanctionStatusIfSSS. +func (mr *MockIRepositoryMockRecorder) ClearCompanySanctionStatusIfSSS(ctx, companyID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearCompanySanctionStatusIfSSS", reflect.TypeOf((*MockIRepository)(nil).ClearCompanySanctionStatusIfSSS), ctx, companyID) +} + // CreateCompany mocks base method. func (m *MockIRepository) CreateCompany(ctx context.Context, in *models.Company) (*models.Company, error) { m.ctrl.T.Helper() @@ -112,6 +128,22 @@ func (mr *MockIRepositoryMockRecorder) DeleteCompanyBySFID(ctx, companySFID inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCompanyBySFID", reflect.TypeOf((*MockIRepository)(nil).DeleteCompanyBySFID), ctx, companySFID) } +// EnsureCompanyForExternalID mocks base method. +func (m *MockIRepository) EnsureCompanyForExternalID(ctx context.Context, externalID, companyName, signingEntityName string) (*models.Company, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureCompanyForExternalID", ctx, externalID, companyName, signingEntityName) + ret0, _ := ret[0].(*models.Company) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// EnsureCompanyForExternalID indicates an expected call of EnsureCompanyForExternalID. +func (mr *MockIRepositoryMockRecorder) EnsureCompanyForExternalID(ctx, externalID, companyName, signingEntityName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureCompanyForExternalID", reflect.TypeOf((*MockIRepository)(nil).EnsureCompanyForExternalID), ctx, externalID, companyName, signingEntityName) +} + // GetCompanies mocks base method. func (m *MockIRepository) GetCompanies(ctx context.Context) (*models.Companies, error) { m.ctrl.T.Helper() @@ -262,6 +294,21 @@ func (mr *MockIRepositoryMockRecorder) GetCompanyInviteRequests(ctx, companyID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompanyInviteRequests", reflect.TypeOf((*MockIRepository)(nil).GetCompanyInviteRequests), ctx, companyID, status) } +// GetCompanyRecord mocks base method. +func (m *MockIRepository) GetCompanyRecord(ctx context.Context, companyID string) (*company.DBModel, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCompanyRecord", ctx, companyID) + ret0, _ := ret[0].(*company.DBModel) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCompanyRecord indicates an expected call of GetCompanyRecord. +func (mr *MockIRepositoryMockRecorder) GetCompanyRecord(ctx, companyID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompanyRecord", reflect.TypeOf((*MockIRepository)(nil).GetCompanyRecord), ctx, companyID) +} + // GetCompanyUserInviteRequests mocks base method. func (m *MockIRepository) GetCompanyUserInviteRequests(ctx context.Context, companyID, userID string) (*company.Invite, error) { m.ctrl.T.Helper() @@ -350,31 +397,30 @@ func (mr *MockIRepositoryMockRecorder) UpdateCompanyAccessList(ctx, companyID, c return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanyAccessList", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanyAccessList), ctx, companyID, companyACL) } -// UpdateCompanySanctionStatus mocks base method. -func (m *MockIRepository) UpdateCompanySanctionStatus(ctx context.Context, companyID string, sanctioned bool, origin string) error { +// UpdateCompanyExternalID mocks base method. +func (m *MockIRepository) UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateCompanySanctionStatus", ctx, companyID, sanctioned, origin) + ret := m.ctrl.Call(m, "UpdateCompanyExternalID", ctx, companyID, oldExternalID, newExternalID) ret0, _ := ret[0].(error) return ret0 } -// UpdateCompanySanctionStatus indicates an expected call of UpdateCompanySanctionStatus. -func (mr *MockIRepositoryMockRecorder) UpdateCompanySanctionStatus(ctx, companyID, sanctioned, origin interface{}) *gomock.Call { +// UpdateCompanyExternalID indicates an expected call of UpdateCompanyExternalID. +func (mr *MockIRepositoryMockRecorder) UpdateCompanyExternalID(ctx, companyID, oldExternalID, newExternalID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanySanctionStatus", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanySanctionStatus), ctx, companyID, sanctioned, origin) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanyExternalID", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanyExternalID), ctx, companyID, oldExternalID, newExternalID) } -// ClearCompanySanctionStatusIfSSS mocks base method. -func (m *MockIRepository) ClearCompanySanctionStatusIfSSS(ctx context.Context, companyID string) (bool, error) { +// UpdateCompanySanctionStatus mocks base method. +func (m *MockIRepository) UpdateCompanySanctionStatus(ctx context.Context, companyID string, sanctioned bool, origin string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClearCompanySanctionStatusIfSSS", ctx, companyID) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "UpdateCompanySanctionStatus", ctx, companyID, sanctioned, origin) + ret0, _ := ret[0].(error) + return ret0 } -// ClearCompanySanctionStatusIfSSS indicates an expected call of ClearCompanySanctionStatusIfSSS. -func (mr *MockIRepositoryMockRecorder) ClearCompanySanctionStatusIfSSS(ctx, companyID interface{}) *gomock.Call { +// UpdateCompanySanctionStatus indicates an expected call of UpdateCompanySanctionStatus. +func (mr *MockIRepositoryMockRecorder) UpdateCompanySanctionStatus(ctx, companyID, sanctioned, origin interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearCompanySanctionStatusIfSSS", reflect.TypeOf((*MockIRepository)(nil).ClearCompanySanctionStatusIfSSS), ctx, companyID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanySanctionStatus", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanySanctionStatus), ctx, companyID, sanctioned, origin) } diff --git a/cla-backend-go/company/mocks/mock_service.go b/cla-backend-go/company/mocks/mock_service.go index d103990b0..ba764b9ea 100644 --- a/cla-backend-go/company/mocks/mock_service.go +++ b/cla-backend-go/company/mocks/mock_service.go @@ -1,5 +1,6 @@ // Copyright The Linux Foundation and each contributor to CommunityBridge. // SPDX-License-Identifier: MIT +// // Code generated by MockGen. DO NOT EDIT. // Source: company/service.go @@ -83,21 +84,6 @@ func (mr *MockIServiceMockRecorder) ApproveCompanyAccessRequest(ctx, companyInvi return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveCompanyAccessRequest", reflect.TypeOf((*MockIService)(nil).ApproveCompanyAccessRequest), ctx, companyInviteID) } -// CreateOrgFromExternalID mocks base method. -func (m *MockIService) CreateOrgFromExternalID(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateOrgFromExternalID", ctx, signingEntityName, companySFID) - ret0, _ := ret[0].(*models.Company) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateOrgFromExternalID indicates an expected call of CreateOrgFromExternalID. -func (mr *MockIServiceMockRecorder) CreateOrgFromExternalID(ctx, signingEntityName, companySFID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrgFromExternalID", reflect.TypeOf((*MockIService)(nil).CreateOrgFromExternalID), ctx, signingEntityName, companySFID) -} - // GetCompanies mocks base method. func (m *MockIService) GetCompanies(ctx context.Context) (*models.Companies, error) { m.ctrl.T.Helper() @@ -263,6 +249,21 @@ func (mr *MockIServiceMockRecorder) RejectCompanyAccessRequest(ctx, companyInvit return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RejectCompanyAccessRequest", reflect.TypeOf((*MockIService)(nil).RejectCompanyAccessRequest), ctx, companyInviteID) } +// ResolveCompany mocks base method. +func (m *MockIService) ResolveCompany(ctx context.Context, companyIDOrSFID string) (*models.Company, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveCompany", ctx, companyIDOrSFID) + ret0, _ := ret[0].(*models.Company) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveCompany indicates an expected call of ResolveCompany. +func (mr *MockIServiceMockRecorder) ResolveCompany(ctx, companyIDOrSFID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveCompany", reflect.TypeOf((*MockIService)(nil).ResolveCompany), ctx, companyIDOrSFID) +} + // SearchCompanyByName mocks base method. func (m *MockIService) SearchCompanyByName(ctx context.Context, companyName, nextKey string) (*models.Companies, error) { m.ctrl.T.Helper() @@ -292,55 +293,3 @@ func (mr *MockIServiceMockRecorder) SearchOrganizationByName(ctx, orgName, websi mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchOrganizationByName", reflect.TypeOf((*MockIService)(nil).SearchOrganizationByName), ctx, orgName, websiteName, includeSigningEntityName, filter) } - -// getPreferredNameAndEmail mocks base method. -func (m *MockIService) getPreferredNameAndEmail(ctx context.Context, lfid string) (string, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "getPreferredNameAndEmail", ctx, lfid) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// getPreferredNameAndEmail indicates an expected call of getPreferredNameAndEmail. -func (mr *MockIServiceMockRecorder) getPreferredNameAndEmail(ctx, lfid interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getPreferredNameAndEmail", reflect.TypeOf((*MockIService)(nil).getPreferredNameAndEmail), ctx, lfid) -} - -// sendRequestAccessEmail mocks base method. -func (m *MockIService) sendRequestAccessEmail(ctx context.Context, companyModel *models.Company, requesterName, requesterEmail, recipientName, recipientAddress string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "sendRequestAccessEmail", ctx, companyModel, requesterName, requesterEmail, recipientName, recipientAddress) -} - -// sendRequestAccessEmail indicates an expected call of sendRequestAccessEmail. -func (mr *MockIServiceMockRecorder) sendRequestAccessEmail(ctx, companyModel, requesterName, requesterEmail, recipientName, recipientAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "sendRequestAccessEmail", reflect.TypeOf((*MockIService)(nil).sendRequestAccessEmail), ctx, companyModel, requesterName, requesterEmail, recipientName, recipientAddress) -} - -// sendRequestApprovedEmailToRecipient mocks base method. -func (m *MockIService) sendRequestApprovedEmailToRecipient(ctx context.Context, companyModel *models.Company, recipientName, recipientAddress string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "sendRequestApprovedEmailToRecipient", ctx, companyModel, recipientName, recipientAddress) -} - -// sendRequestApprovedEmailToRecipient indicates an expected call of sendRequestApprovedEmailToRecipient. -func (mr *MockIServiceMockRecorder) sendRequestApprovedEmailToRecipient(ctx, companyModel, recipientName, recipientAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "sendRequestApprovedEmailToRecipient", reflect.TypeOf((*MockIService)(nil).sendRequestApprovedEmailToRecipient), ctx, companyModel, recipientName, recipientAddress) -} - -// sendRequestRejectedEmailToRecipient mocks base method. -func (m *MockIService) sendRequestRejectedEmailToRecipient(ctx context.Context, companyModel *models.Company, recipientName, recipientAddress string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "sendRequestRejectedEmailToRecipient", ctx, companyModel, recipientName, recipientAddress) -} - -// sendRequestRejectedEmailToRecipient indicates an expected call of sendRequestRejectedEmailToRecipient. -func (mr *MockIServiceMockRecorder) sendRequestRejectedEmailToRecipient(ctx, companyModel, recipientName, recipientAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "sendRequestRejectedEmailToRecipient", reflect.TypeOf((*MockIService)(nil).sendRequestRejectedEmailToRecipient), ctx, companyModel, recipientName, recipientAddress) -} diff --git a/cla-backend-go/company/models.go b/cla-backend-go/company/models.go index 73f58a57c..7d3fab470 100644 --- a/cla-backend-go/company/models.go +++ b/cla-backend-go/company/models.go @@ -29,6 +29,9 @@ type DBModel struct { SanctionOrigin string `dynamodbav:"sanction_origin" json:"sanction_origin,omitempty"` SanctionedDate string `dynamodbav:"sanctioned_date" json:"sanctioned_date,omitempty"` Version string `dynamodbav:"version" json:"version"` + + // PreviousCompanyExternalID is set only by the org import tool when it rewrites company_external_id. + PreviousCompanyExternalID string `dynamodbav:"previous_company_external_id,omitempty" json:"previous_company_external_id,omitempty"` } // Invite data model diff --git a/cla-backend-go/company/repository.go b/cla-backend-go/company/repository.go index bfcb8313f..3c7f10e14 100644 --- a/cla-backend-go/company/repository.go +++ b/cla-backend-go/company/repository.go @@ -57,6 +57,9 @@ type IRepository interface { //nolint UpdateCompanySanctionStatus(ctx context.Context, companyID string, sanctioned bool, origin string) error ClearCompanySanctionStatusIfSSS(ctx context.Context, companyID string) (bool, error) IsCCLAEnabledForCompany(ctx context.Context, companyID string) (bool, error) + EnsureCompanyForExternalID(ctx context.Context, externalID, companyName, signingEntityName string) (*models.Company, bool, error) + GetCompanyRecord(ctx context.Context, companyID string) (*DBModel, error) + UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error } type repository struct { diff --git a/cla-backend-go/company/repository_external_id.go b/cla-backend-go/company/repository_external_id.go new file mode 100644 index 000000000..799734ed7 --- /dev/null +++ b/cla-backend-go/company/repository_external_id.go @@ -0,0 +1,244 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package company + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/gofrs/uuid" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" +) + +// ErrExternalIDConditionFailed is returned by UpdateCompanyExternalID when the row no longer carries the expected external ID. +var ErrExternalIDConditionFailed = errors.New("company external id changed concurrently") + +// ErrEnsureCompanyConflict is returned by EnsureCompanyForExternalID when the deterministic company id is already taken by a row of another organization. +var ErrEnsureCompanyConflict = errors.New("company id already used by a different organization") + +// ErrEmptyExternalID is returned when an external ID is required but empty. +var ErrEmptyExternalID = errors.New("company external id is empty") + +const ensureCompanyNote = "created at CCLA signing" + +// canonicalSigningEntity returns the identity component of a signing entity: empty for the +// parent record (no name, or the name equals the company name), otherwise the normalized name. +func canonicalSigningEntity(companyName, signingEntityName string) string { + entity := strings.ToLower(strings.TrimSpace(signingEntityName)) + if entity == "" || entity == strings.ToLower(strings.TrimSpace(companyName)) { + return "" + } + return entity +} + +// deterministicCompanyID derives a stable, UUIDv4-shaped company id from the external ID and the +// canonical signing entity so that concurrent first creations for the same organization collide on +// the primary key instead of producing two rows. +func deterministicCompanyID(externalID, canonicalEntity string) string { + sum := sha256.Sum256([]byte("easycla-company|" + externalID + "|" + canonicalEntity)) + var id uuid.UUID + copy(id[:], sum[:16]) + id[6] = (id[6] & 0x0f) | 0x40 + id[8] = (id[8] & 0x3f) | 0x80 + return id.String() +} + +// pickCompanyForEntity returns the row matching the canonical signing entity, or nil. +func pickCompanyForEntity(rows []*models.Company, canonicalEntity string) *models.Company { + for _, row := range rows { + if row != nil && canonicalSigningEntity(row.CompanyName, row.SigningEntityName) == canonicalEntity { + return row + } + } + return nil +} + +// EnsureCompanyForExternalID returns the company row for (externalID, signing entity), creating it +// when it does not exist. Existing rows always win and keep their ids; a new row gets the +// deterministic id and a conditional put, and on a conditional failure the winning row is read back +// and returned when it belongs to the same organization. The returned flag reports a creation. +func (repo repository) EnsureCompanyForExternalID(ctx context.Context, externalID, companyName, signingEntityName string) (*models.Company, bool, error) { + f := logrus.Fields{ + "functionName": "company.repository.EnsureCompanyForExternalID", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companySFID": externalID, + "companyName": companyName, + "signingEntityName": signingEntityName, + } + externalID = strings.TrimSpace(externalID) + if externalID == "" { + return nil, false, ErrEmptyExternalID + } + companyName = strings.TrimSpace(companyName) + if companyName == "" { + return nil, false, errors.New("company name is empty") + } + canonical := canonicalSigningEntity(companyName, signingEntityName) + + var existing *models.Company + if canonical == "" { + rows, err := repo.GetCompaniesByExternalID(ctx, externalID, false) + if err != nil { + if _, notFound := err.(*utils.CompanyNotFound); !notFound { + return nil, false, err + } + } else if len(rows) > 0 { + existing = rows[0] + } + } else { + rows, err := repo.GetCompaniesByExternalID(ctx, externalID, true) + if err != nil { + if _, notFound := err.(*utils.CompanyNotFound); !notFound { + return nil, false, err + } + } else { + existing = pickCompanyForEntity(rows, canonical) + } + } + if existing != nil { + log.WithFields(f).Debugf("reusing existing company %s", existing.CompanyID) + return existing, false, nil + } + + _, now := utils.CurrentTime() + record := &DBModel{ + CompanyID: deterministicCompanyID(externalID, canonical), + CompanyName: companyName, + SigningEntityName: companyName, + CompanyExternalID: externalID, + Created: now, + Updated: now, + Note: ensureCompanyNote, + Version: "v1", + } + if canonical != "" { + record.SigningEntityName = strings.TrimSpace(signingEntityName) + } + f["companyID"] = record.CompanyID + + av, err := dynamodbattribute.MarshalMap(record) + if err != nil { + return nil, false, err + } + _, err = repo.dynamoDBClient.PutItem(&dynamodb.PutItemInput{ + Item: av, + TableName: aws.String(repo.companyTableName), + ConditionExpression: aws.String("attribute_not_exists(company_id)"), + }) + if err == nil { + log.WithFields(f).Info("company created") + created, convErr := record.toModel() + return created, true, convErr + } + if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != dynamodb.ErrCodeConditionalCheckFailedException { + log.WithFields(f).WithError(err).Warn("problem creating company") + return nil, false, err + } + + winner, err := repo.getCompanyRecord(ctx, record.CompanyID, true) + if err != nil { + return nil, false, err + } + if winner.CompanyExternalID != externalID || canonicalSigningEntity(winner.CompanyName, winner.SigningEntityName) != canonical { + log.WithFields(f).Warnf("company id collision with external id %s / entity %q", winner.CompanyExternalID, winner.SigningEntityName) + return nil, false, ErrEnsureCompanyConflict + } + log.WithFields(f).Debug("company created concurrently - returning the winning row") + model, err := winner.toModel() + return model, false, err +} + +// GetCompanyRecord returns the raw company row (including previous_company_external_id) by id. +func (repo repository) GetCompanyRecord(ctx context.Context, companyID string) (*DBModel, error) { + return repo.getCompanyRecord(ctx, companyID, false) +} + +func (repo repository) getCompanyRecord(ctx context.Context, companyID string, consistent bool) (*DBModel, error) { + f := logrus.Fields{ + "functionName": "company.repository.getCompanyRecord", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyID": companyID, + } + if companyID == "" { + return nil, &utils.CompanyNotFound{Message: "company_id cannot be empty", CompanyID: companyID} + } + out, err := repo.dynamoDBClient.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(repo.companyTableName), + Key: map[string]*dynamodb.AttributeValue{"company_id": {S: aws.String(companyID)}}, + ConsistentRead: aws.Bool(consistent), + }) + if err != nil { + log.WithFields(f).WithError(err).Warn("error fetching company record") + return nil, err + } + if len(out.Item) == 0 { + return nil, &utils.CompanyNotFound{Message: "no company matching company record", CompanyID: companyID} + } + record := &DBModel{} + if err = dynamodbattribute.UnmarshalMap(out.Item, record); err != nil { + return nil, err + } + return record, nil +} + +// UpdateCompanyExternalID rewrites company_external_id from oldExternalID to newExternalID for one +// existing row, remembering the old value in previous_company_external_id. The write is conditional +// on the row still carrying exactly oldExternalID (a blank oldExternalID means the row must have no +// external id or exactly that blank value, and no previous value is recorded); a missing row or any +// other value returns ErrExternalIDConditionFailed and nothing is written. +func (repo repository) UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error { + f := logrus.Fields{ + "functionName": "company.repository.UpdateCompanyExternalID", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyID": companyID, + "oldExternalID": oldExternalID, + "newExternalID": newExternalID, + } + if strings.TrimSpace(companyID) == "" || strings.TrimSpace(newExternalID) == "" { + return fmt.Errorf("company id and new external id are required") + } + update, condition := "SET #E = :new, #P = :old, #M = :m", "#E = :old" + names := map[string]*string{ + "#E": aws.String("company_external_id"), + "#P": aws.String("previous_company_external_id"), + "#M": aws.String("date_modified"), + } + if strings.TrimSpace(oldExternalID) == "" { + update, condition = "SET #E = :new, #M = :m", "attribute_exists(#K) AND (attribute_not_exists(#E) OR #E = :old)" + delete(names, "#P") + names["#K"] = aws.String("company_id") + } + _, now := utils.CurrentTime() + _, err := repo.dynamoDBClient.UpdateItem(&dynamodb.UpdateItemInput{ + TableName: aws.String(repo.companyTableName), + Key: map[string]*dynamodb.AttributeValue{"company_id": {S: aws.String(companyID)}}, + UpdateExpression: aws.String(update), + ConditionExpression: aws.String(condition), + ExpressionAttributeNames: names, + ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ + ":new": {S: aws.String(newExternalID)}, + ":old": {S: aws.String(oldExternalID)}, + ":m": {S: aws.String(now)}, + }, + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == dynamodb.ErrCodeConditionalCheckFailedException { + return ErrExternalIDConditionFailed + } + log.WithFields(f).WithError(err).Warn("error updating company external id") + return err + } + log.WithFields(f).Info("company external id rewritten") + return nil +} diff --git a/cla-backend-go/company/repository_external_id_test.go b/cla-backend-go/company/repository_external_id_test.go new file mode 100644 index 000000000..d5028aa0d --- /dev/null +++ b/cla-backend-go/company/repository_external_id_test.go @@ -0,0 +1,533 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package company + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const fnAttributeExists = "attribute_exists" + +// fakeCompaniesTable is a minimal in-memory DynamoDB endpoint for the companies table: GetItem, +// conditional PutItem (attribute_not_exists), Query on external-company-index and the conditional +// UpdateItem used by UpdateCompanyExternalID. +type fakeCompaniesTable struct { + mu sync.Mutex + items map[string]map[string]interface{} + hiddenFromIndex map[string]bool // simulates GSI propagation lag + puts int + conditionFailures int + lastCondition string + failPuts bool +} + +type fakeAttr struct { + S *string + BOOL *bool +} + +var fakePlaceholderPair = regexp.MustCompile(`(#[0-9A-Za-z_]+) = (:[0-9A-Za-z_]+)`) + +func (f *fakeCompaniesTable) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + f.mu.Lock() + defer f.mu.Unlock() + switch r.Header.Get("X-Amz-Target") { + case "DynamoDB_20120810.GetItem": + var req struct { + Key map[string]fakeAttr + } + fakeCompanyDecode(w, body, &req) + resp := map[string]interface{}{} + if item, ok := f.items[*req.Key["company_id"].S]; ok { + resp["Item"] = item + } + fakeCompanyJSON(w, resp) + case "DynamoDB_20120810.PutItem": + var req struct { + Item map[string]interface{} + ConditionExpression string + } + fakeCompanyDecode(w, body, &req) + id := fakeCompanyString(req.Item, "company_id") + if f.failPuts { + fakeCompanyError(w, "InternalServerError", "injected") + return + } + if _, exists := f.items[id]; exists && strings.Contains(req.ConditionExpression, "attribute_not_exists(company_id)") { + f.conditionFailures++ + fakeCompanyError(w, "ConditionalCheckFailedException", "exists") + return + } + f.puts++ + f.items[id] = req.Item + fakeCompanyJSON(w, map[string]interface{}{}) + case "DynamoDB_20120810.Query": + var req struct { + IndexName string + KeyConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeAttr + } + fakeCompanyDecode(w, body, &req) + m := fakePlaceholderPair.FindStringSubmatch(req.KeyConditionExpression) + attr, value := req.ExpressionAttributeNames[m[1]], *req.ExpressionAttributeValues[m[2]].S + matched := []map[string]interface{}{} + for id, item := range f.items { + if !f.hiddenFromIndex[id] && fakeCompanyString(item, attr) == value { + matched = append(matched, item) + } + } + fakeCompanyJSON(w, map[string]interface{}{"Items": matched, "Count": len(matched), "ScannedCount": len(matched)}) + case "DynamoDB_20120810.UpdateItem": + var req struct { + Key map[string]fakeAttr + UpdateExpression string + ConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeAttr + } + fakeCompanyDecode(w, body, &req) + if msg := fakeValidateExpressions(req.UpdateExpression+" "+req.ConditionExpression, req.ExpressionAttributeNames, req.ExpressionAttributeValues); msg != "" { + fakeCompanyError(w, "ValidationException", msg) + return + } + item, ok := f.items[*req.Key["company_id"].S] + if !ok { + // DynamoDB evaluates the condition against the missing item and upserts when it passes + item = map[string]interface{}{} + } + f.lastCondition = req.ConditionExpression + if req.ConditionExpression != "" { + values := map[string]string{} + for k, v := range req.ExpressionAttributeValues { + if v.S != nil { + values[k] = *v.S + } + } + cond := &fakeCondition{expr: req.ConditionExpression, item: item, names: req.ExpressionAttributeNames, values: values} + if !cond.eval() { + f.conditionFailures++ + fakeCompanyError(w, "ConditionalCheckFailedException", "condition failed") + return + } + } + if !ok { + item["company_id"] = map[string]interface{}{"S": *req.Key["company_id"].S} + } + for _, pair := range fakePlaceholderPair.FindAllStringSubmatch(strings.TrimPrefix(req.UpdateExpression, "SET "), -1) { + item[req.ExpressionAttributeNames[pair[1]]] = map[string]interface{}{"S": *req.ExpressionAttributeValues[pair[2]].S} + } + f.items[*req.Key["company_id"].S] = item + fakeCompanyJSON(w, map[string]interface{}{}) + default: + http.Error(w, "unsupported operation "+r.Header.Get("X-Amz-Target"), http.StatusBadRequest) + } +} + +// fakeValidateExpressions mirrors DynamoDB's rejection of unused or undefined expression placeholders. +func fakeValidateExpressions(exprs string, names map[string]string, values map[string]fakeAttr) string { + for alias := range names { + if !strings.Contains(exprs, alias) { + return "Value provided in ExpressionAttributeNames unused in expressions: keys: {" + alias + "}" + } + } + for alias := range values { + if !strings.Contains(exprs, alias) { + return "Value provided in ExpressionAttributeValues unused in expressions: keys: {" + alias + "}" + } + } + for _, tok := range strings.FieldsFunc(exprs, func(r rune) bool { return r == ' ' || r == '(' || r == ')' || r == ',' }) { + switch { + case strings.HasPrefix(tok, "#"): + if _, ok := names[tok]; !ok { + return "An expression attribute name used in the document path is not defined; attribute name: " + tok + } + case strings.HasPrefix(tok, ":"): + if _, ok := values[tok]; !ok { + return "An expression attribute value used in expression is not defined; attribute value: " + tok + } + } + } + return "" +} + +// fakeCondition evaluates the condition-expression subset the repository uses: +// attribute_exists(#X), attribute_not_exists(#X), #X = :v, AND, OR and parentheses. +type fakeCondition struct { + expr string + item map[string]interface{} + names map[string]string + values map[string]string + toks []string + pos int +} + +func (c *fakeCondition) eval() bool { + c.toks = strings.Fields(strings.NewReplacer("(", " ( ", ")", " ) ").Replace(c.expr)) + return c.parseOr() +} + +func (c *fakeCondition) parseOr() bool { + v := c.parseAnd() + for c.pos < len(c.toks) && strings.EqualFold(c.toks[c.pos], "OR") { + c.pos++ + v = c.parseAnd() || v + } + return v +} + +func (c *fakeCondition) parseAnd() bool { + v := c.parseAtom() + for c.pos < len(c.toks) && strings.EqualFold(c.toks[c.pos], "AND") { + c.pos++ + v = c.parseAtom() && v + } + return v +} + +func (c *fakeCondition) parseAtom() bool { + tok := c.toks[c.pos] + c.pos++ + switch { + case tok == "(": + v := c.parseOr() + c.pos++ // ")" + return v + case tok == fnAttributeExists || tok == "attribute_not_exists": + c.pos++ // "(" + _, exists := c.item[c.names[c.toks[c.pos]]] + c.pos += 2 // name, ")" + return exists == (tok == fnAttributeExists) + default: + c.pos++ // "=" + value := c.toks[c.pos] + c.pos++ + attr, exists := c.item[c.names[tok]] + if !exists { + return false + } + s, ok := attr.(map[string]interface{})["S"].(string) + return ok && s == c.values[value] + } +} + +func fakeCompanyString(item map[string]interface{}, name string) string { + if attr, ok := item[name].(map[string]interface{}); ok { + if s, ok := attr["S"].(string); ok { + return s + } + } + return "" +} + +func fakeCompanyJSON(w http.ResponseWriter, payload interface{}) { + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func fakeCompanyDecode(w http.ResponseWriter, body []byte, v interface{}) { + if err := json.Unmarshal(body, v); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } +} + +func fakeCompanyError(w http.ResponseWriter, code, message string) { + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + w.WriteHeader(http.StatusBadRequest) + fakeCompanyJSON(w, map[string]string{"__type": "com.amazonaws.dynamodb.v20120810#" + code, "message": message}) +} + +func fakeCompanyItem(id, name, entity, externalID string) map[string]interface{} { + return map[string]interface{}{ + "company_id": map[string]interface{}{"S": id}, + "company_name": map[string]interface{}{"S": name}, + "signing_entity_name": map[string]interface{}{"S": entity}, + "company_external_id": map[string]interface{}{"S": externalID}, + "date_created": map[string]interface{}{"S": "2024-01-01T00:00:00Z"}, + "date_modified": map[string]interface{}{"S": "2024-01-01T00:00:00Z"}, + } +} + +func newCompanyRepo(t *testing.T, table *fakeCompaniesTable) (repository, *fakeCompaniesTable) { + if table.items == nil { + table.items = map[string]map[string]interface{}{} + } + server := httptest.NewServer(table) + t.Cleanup(server.Close) + awsSession, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(server.URL), + Credentials: credentials.NewStaticCredentials("test", "test", ""), + DisableSSL: aws.Bool(true), + MaxRetries: aws.Int(0), + }) + require.NoError(t, err) + return repository{stage: "test", dynamoDBClient: dynamodb.New(awsSession), companyTableName: "cla-test-companies"}, table +} + +func TestCanonicalSigningEntity(t *testing.T) { + assert.Equal(t, "", canonicalSigningEntity("Acme Inc", "")) + assert.Equal(t, "", canonicalSigningEntity("Acme Inc", " acme inc ")) + assert.Equal(t, "acme gmbh", canonicalSigningEntity("Acme Inc", " Acme GmbH")) +} + +func TestDeterministicCompanyID(t *testing.T) { + id := deterministicCompanyID("0014100000Te1TUAAZ", "") + parsed, err := uuid.FromString(id) + require.NoError(t, err) + assert.Equal(t, byte(4), parsed.Version()) + assert.Equal(t, uuid.VariantRFC4122, parsed.Variant()) + assert.Regexp(t, `^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[89ab][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$`, id) + assert.Equal(t, id, deterministicCompanyID("0014100000Te1TUAAZ", ""), "stable") + assert.NotEqual(t, id, deterministicCompanyID("0014100000Te1TUAAZ", "acme gmbh"), "entity is part of the identity") + assert.NotEqual(t, id, deterministicCompanyID("0014100000Te1TUAAY", ""), "external id is part of the identity") +} + +func TestEnsureCompanyForExternalID(t *testing.T) { + const sfid = "0014100000Te1TUAAZ" + + t.Run("empty external id is rejected without a write", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{}) + _, created, err := repo.EnsureCompanyForExternalID(context.Background(), " ", "Acme", "") + assert.ErrorIs(t, err, ErrEmptyExternalID) + assert.False(t, created) + assert.Equal(t, 0, table.puts) + }) + + t.Run("existing parent row is reused with its original id", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "legacy-id": fakeCompanyItem("legacy-id", "Acme Inc", "Acme Inc", sfid), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, "legacy-id", comp.CompanyID) + assert.Equal(t, 0, table.puts) + }) + + t.Run("existing named signing entity row is reused", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "parent": fakeCompanyItem("parent", "Acme Inc", "Acme Inc", sfid), + "child": fakeCompanyItem("child", "Acme Inc", "Acme GmbH", sfid), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "acme gmbh ") + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, "child", comp.CompanyID) + assert.Equal(t, 0, table.puts) + }) + + t.Run("a same-name row with another external id is never reused", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "other": fakeCompanyItem("other", "Acme Inc", "Acme Inc", "0014100000Other00"), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.NoError(t, err) + assert.True(t, created) + assert.Equal(t, deterministicCompanyID(sfid, ""), comp.CompanyID) + assert.Equal(t, sfid, comp.CompanyExternalID) + assert.Equal(t, "Acme Inc", comp.SigningEntityName) + assert.Equal(t, ensureCompanyNote, comp.Note) + assert.Equal(t, 1, table.puts) + assert.Len(t, table.items, 2) + }) + + t.Run("named entity row is created with the requested entity name", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "parent": fakeCompanyItem("parent", "Acme Inc", "Acme Inc", sfid), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "Acme GmbH") + require.NoError(t, err) + assert.True(t, created) + assert.Equal(t, deterministicCompanyID(sfid, "acme gmbh"), comp.CompanyID) + assert.Equal(t, "Acme GmbH", comp.SigningEntityName) + }) + + t.Run("concurrent creation converges on the winning row", func(t *testing.T) { + // the winner's row exists on the primary key but the GSI has not caught up yet + winnerID := deterministicCompanyID(sfid, "") + repo, table := newCompanyRepo(t, &fakeCompaniesTable{ + items: map[string]map[string]interface{}{winnerID: fakeCompanyItem(winnerID, "Acme Inc", "Acme Inc", sfid)}, + hiddenFromIndex: map[string]bool{winnerID: true}, + }) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, winnerID, comp.CompanyID) + assert.Equal(t, 1, table.conditionFailures) + assert.Equal(t, 0, table.puts) + }) + + t.Run("a foreign row on the deterministic key is a conflict", func(t *testing.T) { + winnerID := deterministicCompanyID(sfid, "") + table := &fakeCompaniesTable{items: map[string]map[string]interface{}{ + winnerID: fakeCompanyItem(winnerID, "Someone Else", "Someone Else", "0014100000Other00"), + }} + repo, _ := newCompanyRepo(t, table) + _, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + assert.ErrorIs(t, err, ErrEnsureCompanyConflict) + assert.False(t, created) + assert.Equal(t, 1, table.conditionFailures) + assert.Equal(t, 0, table.puts) + }) + + t.Run("a write failure is returned", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{failPuts: true}) + _, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.Error(t, err) + assert.False(t, created) + }) +} + +func TestUpdateCompanyExternalID(t *testing.T) { + t.Run("rewrites and remembers the previous id", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "lf-old"), + }}) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "lf-old", "0014100000New0000")) + record, err := repo.GetCompanyRecord(context.Background(), "c1") + require.NoError(t, err) + assert.Equal(t, "0014100000New0000", record.CompanyExternalID) + assert.Equal(t, "lf-old", record.PreviousCompanyExternalID) + assert.NotEqual(t, "2024-01-01T00:00:00Z", record.Updated) + assert.Equal(t, 0, table.conditionFailures) + }) + + t.Run("a row that no longer carries the old id fails the condition", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "0014100000New0000"), + }}) + err := repo.UpdateCompanyExternalID(context.Background(), "c1", "lf-old", "0014100000New0000") + assert.ErrorIs(t, err, ErrExternalIDConditionFailed) + assert.Equal(t, 1, table.conditionFailures) + }) + + t.Run("a blank old id fills a row without an external id and records no previous value", func(t *testing.T) { + missing := fakeCompanyItem("c1", "Blank Inc", "Blank Inc", "") + delete(missing, "company_external_id") + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": missing, + "c2": fakeCompanyItem("c2", "Blank Two", "Blank Two", ""), + }}) + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "c2", " ", "0014100000New0001"), ErrExternalIDConditionFailed, "blanks are pinned exactly too") + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "", "0014100000New0000")) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c2", "", "0014100000New0001")) + for id, want := range map[string]string{"c1": "0014100000New0000", "c2": "0014100000New0001"} { + record, err := repo.GetCompanyRecord(context.Background(), id) + require.NoError(t, err) + assert.Equal(t, want, record.CompanyExternalID) + assert.Equal(t, "", record.PreviousCompanyExternalID) + } + assert.Equal(t, 1, table.conditionFailures) + assert.Contains(t, table.lastCondition, "attribute_exists(#K) AND (attribute_not_exists(#E) OR #E = :old)") + }) + + t.Run("a blank old id never overwrites an existing external id", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "lf-old"), + }}) + err := repo.UpdateCompanyExternalID(context.Background(), "c1", "", "0014100000New0000") + assert.ErrorIs(t, err, ErrExternalIDConditionFailed) + assert.Equal(t, 1, table.conditionFailures) + record, err := repo.GetCompanyRecord(context.Background(), "c1") + require.NoError(t, err) + assert.Equal(t, "lf-old", record.CompanyExternalID) + }) + + t.Run("a deleted row is never recreated", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{}) + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "gone", "", "0014100000New0000"), ErrExternalIDConditionFailed) + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "gone", " ", "0014100000New0000"), ErrExternalIDConditionFailed) + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "gone", "lf-old", "0014100000New0000"), ErrExternalIDConditionFailed) + assert.Equal(t, 3, table.conditionFailures) + assert.Empty(t, table.items) + }) + + t.Run("whitespace-only and malformed stored values are pinned exactly", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Space Inc", "Space Inc", " "), + "c2": fakeCompanyItem("c2", "Garbage Inc", "Garbage Inc", "N/A"), + }}) + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "", "0014100000New0000"), ErrExternalIDConditionFailed, "blank is not the stored space") + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "c2", "n/a", "0014100000New0001"), ErrExternalIDConditionFailed) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c1", " ", "0014100000New0000")) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c2", "N/A", "0014100000New0001")) + assert.Equal(t, 2, table.conditionFailures) + c1, err := repo.GetCompanyRecord(context.Background(), "c1") + require.NoError(t, err) + assert.Equal(t, "0014100000New0000", c1.CompanyExternalID) + assert.Equal(t, "", c1.PreviousCompanyExternalID, "a blank source records no previous value") + c2, err := repo.GetCompanyRecord(context.Background(), "c2") + require.NoError(t, err) + assert.Equal(t, "0014100000New0001", c2.CompanyExternalID) + assert.Equal(t, "N/A", c2.PreviousCompanyExternalID) + }) + + t.Run("a concurrently changed value fails the condition and changes nothing", func(t *testing.T) { + item := fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "0014100000Other00") + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{"c1": item}}) + assert.ErrorIs(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "lf-old", "0014100000New0000"), ErrExternalIDConditionFailed) + record, err := repo.GetCompanyRecord(context.Background(), "c1") + require.NoError(t, err) + assert.Equal(t, "0014100000Other00", record.CompanyExternalID) + assert.Equal(t, "", record.PreviousCompanyExternalID) + assert.Equal(t, "2024-01-01T00:00:00Z", record.Updated) + assert.Equal(t, 1, table.conditionFailures) + }) + + t.Run("every expression placeholder is used and defined", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Blank", "Blank", ""), + "c2": fakeCompanyItem("c2", "Legacy", "Legacy", "lf-old"), + }}) + checked := 0 + repo.dynamoDBClient.Handlers.Build.PushBack(func(req *request.Request) { + input, ok := req.Params.(*dynamodb.UpdateItemInput) + if !ok { + return + } + checked++ + exprs := *input.UpdateExpression + " " + *input.ConditionExpression + for alias := range input.ExpressionAttributeNames { + assert.Contains(t, exprs, alias, "unused ExpressionAttributeNames entry") + } + for alias := range input.ExpressionAttributeValues { + assert.Contains(t, exprs, alias, "unused ExpressionAttributeValues entry") + } + }) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "", "0014100000New0000")) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c2", "lf-old", "0014100000New0001")) + assert.Equal(t, 2, checked) + }) + + t.Run("blank company or new id is rejected", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{}) + assert.Error(t, repo.UpdateCompanyExternalID(context.Background(), "", "old", "new")) + assert.Error(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "old", "")) + }) +} diff --git a/cla-backend-go/company/resolve_company_test.go b/cla-backend-go/company/resolve_company_test.go new file mode 100644 index 000000000..77f70743d --- /dev/null +++ b/cla-backend-go/company/resolve_company_test.go @@ -0,0 +1,320 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package company_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/linuxfoundation/easycla/cla-backend-go/company" + mock_company "github.com/linuxfoundation/easycla/cla-backend-go/company/mocks" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + organizationService "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + resolvePlatformHost = "platform.resolve.invalid" + resolveAuthHost = "auth.resolve.invalid" + resolveSFID = "0014100000Te0yqQAB" + resolveShortSFID = "0014100000Te0yq" + resolveLegacyOrgID = "lfbd1c2b3a4d5e6f7a8" + resolveCompanyID = "9b8e7d66-40a5-4cde-9f00-3e1d1a2b3c4d" + resolveMissingID = "2f1c6c1e-6a2d-4c3b-9d7e-8a5b4c3d2e1f" +) + +// resolveHTTP answers the token endpoint and the organization-service routes the tests need; +// anything else fails the test (no network) +type resolveHTTP struct { + t *testing.T + orgs map[string]string + searchBody string + calls []string +} + +func (h *resolveHTTP) RoundTrip(r *http.Request) (*http.Response, error) { + h.calls = append(h.calls, r.Method+" "+r.URL.Path) + response := &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Request: r} + switch { + case r.URL.Host == resolveAuthHost && r.URL.Path == "/oauth/token": + response.Body = io.NopCloser(strings.NewReader(`{"access_token":"unit-test-token","token_type":"Bearer","expires_in":3600}`)) + case r.URL.Host == resolvePlatformHost && r.URL.Path == "/organization-service/v1/orgs/search": + response.Body = io.NopCloser(strings.NewReader(h.searchBody)) + case r.URL.Host == resolvePlatformHost && strings.HasPrefix(r.URL.Path, "/organization-service/v1/orgs/"): + body, ok := h.orgs[strings.TrimPrefix(r.URL.Path, "/organization-service/v1/orgs/")] + if !ok { + response.StatusCode = http.StatusNotFound + body = `{"Message":"not found"}` + } + response.Body = io.NopCloser(strings.NewReader(body)) + default: + h.t.Errorf("unexpected HTTP request (no network allowed): %s %s", r.Method, r.URL) + return nil, fmt.Errorf("unexpected HTTP request: %s %s", r.Method, r.URL) + } + return response, nil +} + +func setupResolveHTTP(t *testing.T, orgs map[string]string, searchBody string) *resolveHTTP { + t.Helper() + transport := &resolveHTTP{t: t, orgs: orgs, searchBody: searchBody} + oldTransport, oldClient := http.DefaultTransport, http.DefaultClient + http.DefaultTransport = transport + http.DefaultClient = &http.Client{Transport: transport} + t.Cleanup(func() { + http.DefaultTransport, http.DefaultClient = oldTransport, oldClient + }) + token.Init("test-client", "test-secret", "https://"+resolveAuthHost+"/oauth/token", "test-audience") + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := token.GetToken(); err == nil { + break + } + require.True(t, time.Now().Before(deadline), "mock token initialization did not complete") + time.Sleep(10 * time.Millisecond) + } + organizationService.InitClient("https://"+resolvePlatformHost, nil) + return transport +} + +func newResolveService(repo company.IRepository) company.IService { + return company.NewService(repo, "https://corporate.invalid", nil, nil) +} + +func TestResolveCompanyOrder(t *testing.T) { + transport := setupResolveHTTP(t, map[string]string{ + resolveSFID: `{"ID":"` + resolveSFID + `","Name":"Acme Corp","SigningEntityName":["Acme Labs"]}`, + resolveLegacyOrgID: `{"ID":"` + resolveLegacyOrgID + `","Name":"Legacy Ltd"}`, + resolveShortSFID: `{"ID":"` + resolveSFID + `","Name":"Acme Corp","SigningEntityName":["Acme Labs"]}`, + }, "") + orgCalls := func() int { + n := 0 + for _, c := range transport.calls { + if strings.Contains(c, "/organization-service/v1/orgs/") { + n++ + } + } + return n + } + notFoundByID := &utils.CompanyNotFound{CompanyID: resolveSFID} + notFoundBySFID := &utils.CompanyNotFound{CompanySFID: resolveSFID} + dynamoErr := errors.New("dynamodb unavailable") + persisted := &models.Company{CompanyID: resolveCompanyID, CompanyExternalID: resolveSFID, CompanyName: "Acme", IsSanctioned: true} + + t.Run("internal id wins", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveCompanyID).Return(persisted, nil) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveCompanyID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + }) + t.Run("external id row wins over the organization", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveSFID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(persisted, nil) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveSFID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + assert.True(t, got.IsSanctioned) + }) + t.Run("virtual company when no row exists", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveSFID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, notFoundBySFID) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveSFID) + + require.NoError(t, err) + assert.Equal(t, &models.Company{CompanyID: resolveSFID, CompanyExternalID: resolveSFID, CompanyName: "Acme Corp", SigningEntityName: "Acme Corp"}, got) + }) + t.Run("legacy organization id falls back to the organization service", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveLegacyOrgID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveLegacyOrgID).Return(nil, notFoundBySFID) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveLegacyOrgID) + + require.NoError(t, err) + assert.Equal(t, company.VirtualCompany(resolveLegacyOrgID, "Legacy Ltd"), got) + }) + t.Run("internal id without a row is not found and never asks the organization service", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + missing := &utils.CompanyNotFound{CompanyID: resolveMissingID} + repo.EXPECT().GetCompany(gomock.Any(), resolveMissingID).Return(nil, missing) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveMissingID).Return(nil, missing) + before := orgCalls() + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveMissingID) + + var notFound *utils.CompanyNotFound + require.ErrorAs(t, err, ¬Found) + assert.Same(t, missing, err) + assert.Nil(t, got) + assert.Equal(t, before, orgCalls(), "a UUID cannot name an organization") + }) + t.Run("unknown organization is not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), "0014100000Unknown0").Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), "0014100000Unknown0").Return(nil, notFoundBySFID) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), "0014100000Unknown0") + + var notFound *utils.CompanyNotFound + require.ErrorAs(t, err, ¬Found) + assert.Nil(t, got) + }) + t.Run("repository errors propagate", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveSFID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, dynamoErr) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveSFID) + + require.ErrorIs(t, err, dynamoErr) + assert.Nil(t, got) + }) + t.Run("row stored under the id the organization service returns wins over a virtual company", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveShortSFID).Return(nil, &utils.CompanyNotFound{CompanyID: resolveShortSFID}) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveShortSFID).Return(nil, &utils.CompanyNotFound{CompanySFID: resolveShortSFID}) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(persisted, nil) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveShortSFID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + }) + t.Run("virtual company carries the id the organization service returns", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveShortSFID).Return(nil, &utils.CompanyNotFound{CompanyID: resolveShortSFID}) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveShortSFID).Return(nil, &utils.CompanyNotFound{CompanySFID: resolveShortSFID}) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, notFoundBySFID) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveShortSFID) + + require.NoError(t, err) + assert.Equal(t, company.VirtualCompany(resolveSFID, "Acme Corp"), got) + }) + t.Run("repository errors on the returned id propagate", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveShortSFID).Return(nil, &utils.CompanyNotFound{CompanyID: resolveShortSFID}) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveShortSFID).Return(nil, &utils.CompanyNotFound{CompanySFID: resolveShortSFID}) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, dynamoErr) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveShortSFID) + + require.ErrorIs(t, err, dynamoErr) + assert.Nil(t, got) + }) +} + +func TestVirtualCompany(t *testing.T) { + got := company.VirtualCompany(resolveSFID, "Acme Corp") + assert.Equal(t, resolveSFID, got.CompanyID) + assert.Equal(t, resolveSFID, got.CompanyExternalID) + assert.Equal(t, "Acme Corp", got.CompanyName) + assert.Equal(t, "Acme Corp", got.SigningEntityName) + assert.False(t, got.IsSanctioned) +} + +func TestGetCompanyByExternalIDDoesNotCreate(t *testing.T) { + t.Run("no row", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + notFound := &utils.CompanyNotFound{CompanySFID: resolveSFID} + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, notFound) + + got, err := newResolveService(repo).GetCompanyByExternalID(context.Background(), resolveSFID) + + require.ErrorIs(t, err, notFound) + assert.Nil(t, got) + }) + t.Run("persisted row", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + persisted := &models.Company{CompanyID: resolveCompanyID, CompanyExternalID: resolveSFID, CompanyName: "Acme"} + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(persisted, nil) + + got, err := newResolveService(repo).GetCompanyByExternalID(context.Background(), resolveSFID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + }) +} + +func TestSearchOrganizationByNameDoesNotCreate(t *testing.T) { + transport := setupResolveHTTP(t, nil, `{"Data":[{"ID":"`+resolveSFID+`","Name":"Acme Corp","Link":"https://acme.invalid","SigningEntityName":["Acme Labs"]}],"Metadata":{"TotalSize":1,"Offset":0,"PageSize":1000}}`) + for _, includeSigningEntityName := range []bool{false, true} { + t.Run(fmt.Sprintf("includeSigningEntityName=%t", includeSigningEntityName), func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, &utils.CompanyNotFound{CompanySFID: resolveSFID}) + + result, err := newResolveService(repo).SearchOrganizationByName(context.Background(), "Acme", "", includeSigningEntityName, "") + + require.NoError(t, err) + require.Len(t, result.List, 1) + assert.Equal(t, resolveSFID, result.List[0].OrganizationID) + assert.False(t, *result.List[0].CclaEnabled) + if includeSigningEntityName { + assert.Equal(t, []string{"Acme Labs"}, result.List[0].SigningEntityNames) + } + }) + } + t.Run("persisted company reports its CCLA state", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(&models.Company{CompanyID: resolveCompanyID, CompanyExternalID: resolveSFID, CompanyName: "Acme"}, nil) + repo.EXPECT().IsCCLAEnabledForCompany(gomock.Any(), resolveCompanyID).Return(true, nil) + + result, err := newResolveService(repo).SearchOrganizationByName(context.Background(), "Acme", "", false, "") + + require.NoError(t, err) + require.Len(t, result.List, 1) + assert.Equal(t, resolveSFID, result.List[0].OrganizationID) + assert.True(t, *result.List[0].CclaEnabled) + }) + for _, call := range transport.calls { + if strings.Contains(call, "/organization-service/") { + assert.True(t, strings.HasPrefix(call, "GET "), "the search must only read from the organization service: %s", call) + } + } +} diff --git a/cla-backend-go/company/service.go b/cla-backend-go/company/service.go index ab8116356..55124425e 100644 --- a/cla-backend-go/company/service.go +++ b/cla-backend-go/company/service.go @@ -40,11 +40,10 @@ const ( // IService interface defining the functions for the company service type IService interface { // nolint - CreateOrgFromExternalID(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) - GetCompanies(ctx context.Context) (*models.Companies, error) GetCompany(ctx context.Context, companyID string) (*models.Company, error) GetCompanyByExternalID(ctx context.Context, companySFID string) (*models.Company, error) + ResolveCompany(ctx context.Context, companyIDOrSFID string) (*models.Company, error) GetCompaniesByExternalID(ctx context.Context, companySFID string, includeChildCompanies bool) ([]*models.Company, error) GetCompanyBySigningEntityName(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) SearchCompanyByName(ctx context.Context, companyName string, nextKey string) (*models.Companies, error) @@ -641,19 +640,77 @@ func (s service) GetCompanyByExternalID(ctx context.Context, companySFID string) } log.WithFields(f).Debug("Searching company by external ID...") comp, err := s.repo.GetCompanyByExternalID(ctx, companySFID) + if err != nil { + return nil, err + } + log.WithFields(f).Debugf("Loaded and returning company: %+v...", comp) + return comp, nil +} + +// ResolveCompany returns the persisted company for an internal ID or a Salesforce ID; when no row +// exists it returns a non-persisted virtual company (CompanyID = CompanyExternalID = SFID) built +// from the organization service. Only "not found" outcomes fall through (never for an internal +// UUID, which cannot name an organization); other errors propagate. A row stored under the ID the +// organization service returns for the reference (its authoritative spelling) wins over a virtual company. +func (s service) ResolveCompany(ctx context.Context, companyIDOrSFID string) (*models.Company, error) { + f := logrus.Fields{ + "functionName": "company.service.ResolveCompany", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyIDOrSFID": companyIDOrSFID, + } + comp, err := s.repo.GetCompany(ctx, companyIDOrSFID) if err == nil { - log.WithFields(f).Debugf("Loaded and returning company: %+v...", comp) return comp, nil } - - if _, ok := err.(*utils.CompanyNotFound); ok { - comp, err = s.CreateOrgFromExternalID(ctx, "", companySFID) - if err != nil { - return comp, err - } + if _, ok := err.(*utils.CompanyNotFound); !ok { + return nil, err + } + comp, err = s.repo.GetCompanyByExternalID(ctx, companyIDOrSFID) + if err == nil { return comp, nil } - return nil, err + if _, ok := err.(*utils.CompanyNotFound); !ok { + return nil, err + } + if utils.IsUUIDv4(companyIDOrSFID) { + log.WithFields(f).Debug("no company row for an internal id - not found") + return nil, err + } + orgClient := organization_service.GetClient() + if orgClient == nil { + return nil, err + } + org, orgErr := orgClient.GetOrganization(ctx, companyIDOrSFID) + if orgErr != nil { + if _, ok := orgErr.(*organizations.GetOrgNotFound); ok { + log.WithFields(f).Debug("no company row and no organization - not found") + return nil, &utils.CompanyNotFound{Message: "no company or organization matching the id", CompanyID: companyIDOrSFID} + } + log.WithFields(f).WithError(orgErr).Warn("problem loading organization") + return nil, orgErr + } + if org.ID != "" && org.ID != companyIDOrSFID { + comp, err = s.repo.GetCompanyByExternalID(ctx, org.ID) + if err == nil { + return comp, nil + } + if _, ok := err.(*utils.CompanyNotFound); !ok { + return nil, err + } + } + log.WithFields(f).Debugf("no company row - returning virtual company for organization %s", org.Name) + return VirtualCompany(org.ID, org.Name), nil +} + +// VirtualCompany is the non-persisted view of an organization that has no EasyCLA row yet. +func VirtualCompany(companySFID, companyName string) *models.Company { + return &models.Company{ + CompanyID: companySFID, + CompanyExternalID: companySFID, + CompanyName: companyName, + SigningEntityName: companyName, + IsSanctioned: false, + } } func (s service) GetCompaniesByExternalID(ctx context.Context, companySFID string, includeChildCompanies bool) ([]*models.Company, error) { @@ -744,13 +801,6 @@ func (s service) SearchOrganizationByName(ctx context.Context, orgName string, w var signingEntityNames []string if len(org.SigningEntityName) > 0 { signingEntityNames = utils.TrimSpaceFromItems(org.SigningEntityName) - for _, signingEntityName := range signingEntityNames { - // Auto-create the internal record, if needed - _, err = s.CreateOrgFromExternalID(ctx, signingEntityName, org.ID) - if err != nil { - log.WithFields(f).WithError(err).Warnf("Unable to create organization from external ID: %s using signing entity name: %s", org.ID, signingEntityName) - } - } resultsChannel <- &models.Org{ OrganizationID: org.ID, OrganizationName: org.Name, @@ -798,144 +848,3 @@ func (s service) SearchOrganizationByName(ctx context.Context, orgName string, w return result, nil } - -// CreateOrgFromExternalID creates a new EasyCLA company from the external SF Organization ID -func (s service) CreateOrgFromExternalID(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) { - f := logrus.Fields{ - "functionName": "company.service.CreateOrgFromExternalID", - utils.XREQUESTID: ctx.Value(utils.XREQUESTID), - "companySFID": companySFID, - "signingEntityName": signingEntityName, - } - - var companyModel *models.Company - var lookupErr error - - // Lookup the company in our database...does it exist? - companyModel, lookupErr = s.GetCompanyBySigningEntityName(ctx, signingEntityName, companySFID) - if lookupErr != nil { - log.WithFields(f).WithError(lookupErr).Debug("problem locating internal company record by signing entity name and SFID - must not exist yet") - } - - // Already exists - no need to create in our own database - if companyModel != nil { - return companyModel, nil - } - - osc := organization_service.GetClient() - log.WithFields(f).Debugf("Searching organization by company SFID in the organization service...") - org, err := osc.GetOrganization(ctx, companySFID) - if err != nil { - log.WithFields(f).WithError(err).Warn("getting organization details failed") - return nil, err - } - - // Add some fields to the logger - f["companyName"] = org.Name - f["companyStatus"] = org.Status - - // Query the platform user service to locate the company admin - log.WithFields(f).Debugf("getting company-admin information...") - companyAdmin, err := getCompanyAdmin(ctx, companySFID) - if err != nil { - log.WithFields(f).WithError(err).Warnf("unable to load company admin information for company: %s", companySFID) - } - - var claUser *models.User - if companyAdmin != nil { - f["company-admin"] = companyAdmin - log.WithFields(f).Debugf("loaded company admin: %+v", companyAdmin) - - log.WithFields(f).Debugf("getting user information from cla") - claUser, err = s.userService.GetUserByLFUserName(companyAdmin.LfUsername) - if err != nil { - log.WithFields(f).WithError(err).Warnf("problem loading user by username: %s", companyAdmin.LfUsername) - return nil, err - } - - if claUser == nil { - // create cla-user - log.WithFields(f).Debugf("cla user not found. creating cla user.") - claUser, err = s.userService.CreateUser(companyAdmin, nil) - if err != nil { - log.WithFields(f).WithError(err).Warn("creating cla user failed") - return nil, err - } - } - } else { - log.WithFields(f).Debug("unable to load company admin from companySFID - admin not found") - } - - additionalNote := "" - if signingEntityName == "" { - additionalNote = fmt.Sprintf("signing entity name not set - using organization name: %s", org.Name) - log.WithFields(f).Debugf("%s", additionalNote) - signingEntityName = org.Name - } - - _, now := utils.CurrentTime() - newComp := &models.Company{ - CompanyExternalID: org.ID, - CompanyName: org.Name, - SigningEntityName: signingEntityName, - IsSanctioned: false, - Note: fmt.Sprintf("%s - Created based on SF Organization Service record - %s", now, additionalNote), - } - if companyAdmin != nil { - newComp.CompanyACL = []string{companyAdmin.LfUsername} - } - if claUser != nil { - newComp.CompanyManagerID = claUser.UserID - } - - f["company"] = newComp - log.WithFields(f).Debugf("creating cla company record") - // create company - comp, err := s.repo.CreateCompany(ctx, newComp) - if err != nil { - log.WithFields(f).WithError(err).Warnf("creating cla company failed") - return nil, err - } - - log.WithFields(f).Debugf("Created company %s with Signing Entity Name: %s with ID: %s", - comp.CompanyName, signingEntityName, comp.CompanyID) - return comp, nil -} - -// getCompanyAdmin is helper function which queries org-service to get first company-admin -func getCompanyAdmin(ctx context.Context, companySFID string) (*models.User, error) { - f := logrus.Fields{ - "functionName": "company.service.getCompanyAdmin", - utils.XREQUESTID: ctx.Value(utils.XREQUESTID), - "companySFID": companySFID, - } - osc := organization_service.GetClient() - result, err := osc.ListOrgUserAdminScopes(ctx, companySFID, nil) - if err != nil { - if _, ok := err.(*organizations.ListOrgUsrAdminScopesNotFound); !ok { - log.WithFields(f).Warnf("getting company-admin failed. error = %s", err.Error()) - return nil, err - } - } - if result != nil { - for _, usc := range result.Userroles { - for _, rs := range usc.RoleScopes { - if rs.RoleName == "company-admin" { - companyAdmin := &models.User{ - LfEmail: strfmt.Email(usc.Contact.EmailAddress), - LfUsername: usc.Contact.Username, - UserExternalID: usc.Contact.ID, - Username: usc.Contact.Name, - } - log.WithFields(f).WithField("company-admin", companyAdmin).Debug("company-admin found") - return companyAdmin, nil - } - } - } - } - log.WithFields(f).Warnf("no company-admin found") - return nil, &utils.CompanyAdminNotFound{ - CompanySFID: companySFID, - Err: nil, - } -} diff --git a/cla-backend-go/config/config.go b/cla-backend-go/config/config.go index f0df045e0..1797e3c2e 100644 --- a/cla-backend-go/config/config.go +++ b/cla-backend-go/config/config.go @@ -104,6 +104,18 @@ type Config struct { // SelfServe holds the LFX Self Serve trusted-caller configuration SelfServe SelfServe `json:"self_serve"` + + // MemberService holds the LFX v2 member-service client configuration (org import tool only) + MemberService MemberService `json:"member_service"` +} + +// MemberService holds the LFX v2 member-service configuration used by the org import tool to +// register organizations as B2B orgs. It reuses the shared Auth0Platform M2M credentials; only the +// base URL (cla-member-service-base-url-{stage}) and audience (cla-member-service-auth0-audience-{stage}) +// are configured here, both optional: when empty the register route is unavailable. +type MemberService struct { + BaseURL string `json:"base_url"` + Audience string `json:"audience"` } // SelfServe holds the LFX Self Serve trusted-caller configuration: the Auth0 azp (client ID) diff --git a/cla-backend-go/config/ssm.go b/cla-backend-go/config/ssm.go index c087bbad3..bf26d7ddc 100644 --- a/cla-backend-go/config/ssm.go +++ b/cla-backend-go/config/ssm.go @@ -279,6 +279,9 @@ func loadSSMConfig(awsSession *session.Session, stage string) Config { //nolint loadOptionalSelfServeConfig(ssmClient, stage, &config, f) + config.MemberService.BaseURL = getOptionalSSMString(ssmClient, fmt.Sprintf("cla-member-service-base-url-%s", stage), f) + config.MemberService.Audience = getOptionalSSMString(ssmClient, fmt.Sprintf("cla-member-service-auth0-audience-%s", stage), f) + return config } @@ -350,9 +353,9 @@ func getOptionalSSMString(ssmClient *ssm.SSM, key string, f logrus.Fields) strin }) if err != nil { if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ssm.ErrCodeParameterNotFound { - log.WithFields(f).Debugf("optional SSM key %s not provisioned - sanctions screening disabled until it is set", key) + log.WithFields(f).Debugf("optional SSM key %s not provisioned - the feature depending on it stays disabled until it is set", key) } else { - log.WithFields(f).WithError(err).Warnf("unable to read optional SSM key %s - sanctions screening disabled", key) + log.WithFields(f).WithError(err).Warnf("unable to read optional SSM key %s - the feature depending on it stays disabled", key) } return "" } diff --git a/cla-backend-go/events/repository_rekey.go b/cla-backend-go/events/repository_rekey.go new file mode 100644 index 000000000..917ea109e --- /dev/null +++ b/cla-backend-go/events/repository_rekey.go @@ -0,0 +1,187 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package events + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/expression" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" +) + +// ErrEventNotFound is returned by RekeyEventCompanySFID when the event does not exist. +var ErrEventNotFound = errors.New("event not found") + +// ErrEventCompanySFIDMismatch is returned when an event carries a company SFID that is neither the old nor the new one. +var ErrEventCompanySFIDMismatch = errors.New("event carries an unexpected company sfid") + +// ErrEventRekeyConflict is returned when the event changed between the read and the conditional write; the call can be retried. +var ErrEventRekeyConflict = errors.New("event changed concurrently") + +// company SFID prefixed composite index keys maintained by AddDataToEvent +var companySFIDCompositeAttributes = []string{"company_sfid_foundation_sfid", "company_sfid_project_id", "company_sfid_cla_group_id"} + +// RekeyRepository is the org-import view of the events table: locate the events of a company SFID and move them to a new SFID. +type RekeyRepository interface { + ListEventIDsByCompanySFID(ctx context.Context, companySFID string) ([]string, error) + ListEventIDsByCompanySFIDCLAGroup(ctx context.Context, companySFID, claGroupID string) ([]string, error) + RekeyEventCompanySFID(ctx context.Context, eventID, oldSFID, newSFID string) (bool, error) +} + +// NewRekeyRepository creates the org-import events repository +func NewRekeyRepository(awsSession *session.Session, stage string) RekeyRepository { + return &repository{ + stage: stage, + dynamoDBClient: dynamodb.New(awsSession), + eventsTable: fmt.Sprintf("cla-%s-events", stage), + } +} + +// ListEventIDsByCompanySFID returns the ids of all events with event_company_sfid = companySFID +func (repo *repository) ListEventIDsByCompanySFID(ctx context.Context, companySFID string) ([]string, error) { + if strings.TrimSpace(companySFID) == "" { + return nil, errors.New("company sfid is required") + } + return repo.listEventIDs(ctx, EventCompanySFIDEventDataLowerIndex, expression.Key("event_company_sfid").Equal(expression.Value(companySFID))) +} + +// ListEventIDsByCompanySFIDCLAGroup returns the ids of all events with company_sfid_cla_group_id = companySFID#claGroupID +func (repo *repository) ListEventIDsByCompanySFIDCLAGroup(ctx context.Context, companySFID, claGroupID string) ([]string, error) { + if strings.TrimSpace(companySFID) == "" || strings.TrimSpace(claGroupID) == "" { + return nil, errors.New("company sfid and cla group id are required") + } + return repo.listEventIDs(ctx, CompanySFIDClaGroupIDEpochIndex, expression.Key("company_sfid_cla_group_id").Equal(expression.Value(companySFID+"#"+claGroupID))) +} + +func (repo *repository) listEventIDs(ctx context.Context, indexName string, keyCondition expression.KeyConditionBuilder) ([]string, error) { + f := logrus.Fields{ + "functionName": "v1.events.repository.listEventIDs", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "indexName": indexName, + } + expr, err := expression.NewBuilder().WithKeyCondition(keyCondition).WithProjection(expression.NamesList(expression.Name("event_id"))).Build() + if err != nil { + return nil, err + } + input := &dynamodb.QueryInput{ + TableName: aws.String(repo.eventsTable), + IndexName: aws.String(indexName), + KeyConditionExpression: expr.KeyCondition(), + ProjectionExpression: expr.Projection(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + } + var ids []string + for { + out, queryErr := repo.dynamoDBClient.Query(input) + if queryErr != nil { + log.WithFields(f).WithError(queryErr).Warn("error listing events") + return nil, queryErr + } + for _, item := range out.Items { + if id := item["event_id"]; id != nil && id.S != nil { + ids = append(ids, *id.S) + } + } + if len(out.LastEvaluatedKey) == 0 { + return ids, nil + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } +} + +// RekeyEventCompanySFID moves one event from oldSFID to newSFID: event_company_sfid and the +// company_sfid_* composite keys that still start with oldSFID are rewritten with a conditional +// update. An event already carrying newSFID is left alone (false, nil); any other company SFID is +// an ErrEventCompanySFIDMismatch. +func (repo *repository) RekeyEventCompanySFID(ctx context.Context, eventID, oldSFID, newSFID string) (bool, error) { + f := logrus.Fields{ + "functionName": "v1.events.repository.RekeyEventCompanySFID", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "eventID": eventID, + "oldSFID": oldSFID, + "newSFID": newSFID, + } + if strings.TrimSpace(eventID) == "" || strings.TrimSpace(oldSFID) == "" || strings.TrimSpace(newSFID) == "" || oldSFID == newSFID { + return false, errors.New("event id and distinct old/new company sfids are required") + } + key := map[string]*dynamodb.AttributeValue{"event_id": {S: aws.String(eventID)}} + out, err := repo.dynamoDBClient.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(repo.eventsTable), + Key: key, + ConsistentRead: aws.Bool(true), + }) + if err != nil { + log.WithFields(f).WithError(err).Warn("error reading event") + return false, err + } + if len(out.Item) == 0 { + return false, ErrEventNotFound + } + + names := map[string]*string{} + values := map[string]*dynamodb.AttributeValue{} + var sets, conditions []string + rewrite := func(attribute, current, updated string) { + placeholder := fmt.Sprintf("#a%d", len(sets)) + names[placeholder] = aws.String(attribute) + values[":new"+placeholder[1:]] = &dynamodb.AttributeValue{S: aws.String(updated)} + values[":cur"+placeholder[1:]] = &dynamodb.AttributeValue{S: aws.String(current)} + sets = append(sets, fmt.Sprintf("%s = :new%s", placeholder, placeholder[1:])) + conditions = append(conditions, fmt.Sprintf("%s = :cur%s", placeholder, placeholder[1:])) + } + if current := stringAttribute(out.Item, "event_company_sfid"); current != "" && current != newSFID { + if current != oldSFID { + return false, ErrEventCompanySFIDMismatch + } + rewrite("event_company_sfid", current, newSFID) + } + for _, attribute := range companySFIDCompositeAttributes { + current := stringAttribute(out.Item, attribute) + if current == "" || strings.HasPrefix(current, newSFID+"#") { + continue + } + if !strings.HasPrefix(current, oldSFID+"#") { + return false, ErrEventCompanySFIDMismatch + } + rewrite(attribute, current, newSFID+strings.TrimPrefix(current, oldSFID)) + } + if len(sets) == 0 { + return false, nil + } + + _, err = repo.dynamoDBClient.UpdateItem(&dynamodb.UpdateItemInput{ + TableName: aws.String(repo.eventsTable), + Key: key, + UpdateExpression: aws.String("SET " + strings.Join(sets, ", ")), + ConditionExpression: aws.String(strings.Join(conditions, " AND ")), + ExpressionAttributeNames: names, + ExpressionAttributeValues: values, + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == dynamodb.ErrCodeConditionalCheckFailedException { + return false, ErrEventRekeyConflict + } + log.WithFields(f).WithError(err).Warn("error rekeying event") + return false, err + } + log.WithFields(f).Debugf("event rekeyed (%d attributes)", len(sets)) + return true, nil +} + +func stringAttribute(item map[string]*dynamodb.AttributeValue, name string) string { + if attr := item[name]; attr != nil && attr.S != nil { + return *attr.S + } + return "" +} diff --git a/cla-backend-go/events/repository_rekey_test.go b/cla-backend-go/events/repository_rekey_test.go new file mode 100644 index 000000000..e66728ae3 --- /dev/null +++ b/cla-backend-go/events/repository_rekey_test.go @@ -0,0 +1,313 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package events + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeEventsTable is a minimal in-memory DynamoDB endpoint for the events table: GetItem, Query on +// a single-attribute key (paginated pageSize items at a time) and conditional UpdateItem. +type fakeEventsTable struct { + mu sync.Mutex + items map[string]map[string]string + pageSize int + queries int + updates int + conditionFailures int + beforeUpdate func(items map[string]map[string]string) +} + +type fakeEventAttr struct { + S *string +} + +var fakeEventPair = regexp.MustCompile(`(#[0-9A-Za-z_]+) = (:[0-9A-Za-z_]+)`) + +func (f *fakeEventsTable) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + f.mu.Lock() + defer f.mu.Unlock() + switch r.Header.Get("X-Amz-Target") { + case "DynamoDB_20120810.GetItem": + var req struct{ Key map[string]fakeEventAttr } + fakeEventDecode(w, body, &req) + resp := map[string]interface{}{} + if item, ok := f.items[*req.Key["event_id"].S]; ok { + resp["Item"] = fakeEventWire(item) + } + fakeEventJSON(w, resp) + case "DynamoDB_20120810.Query": + f.queries++ + var req struct { + KeyConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeEventAttr + ExclusiveStartKey map[string]fakeEventAttr + } + fakeEventDecode(w, body, &req) + m := fakeEventPair.FindStringSubmatch(req.KeyConditionExpression) + attr, value := req.ExpressionAttributeNames[m[1]], *req.ExpressionAttributeValues[m[2]].S + var ids []string + for id, item := range f.items { + if item[attr] == value { + ids = append(ids, id) + } + } + sort.Strings(ids) + if start, ok := req.ExclusiveStartKey["event_id"]; ok { + for i, id := range ids { + if id == *start.S { + ids = ids[i+1:] + break + } + } + } + resp := map[string]interface{}{} + if f.pageSize > 0 && len(ids) > f.pageSize { + ids = ids[:f.pageSize] + resp["LastEvaluatedKey"] = map[string]interface{}{"event_id": map[string]string{"S": ids[len(ids)-1]}} + } + page := []map[string]interface{}{} + for _, id := range ids { + page = append(page, map[string]interface{}{"event_id": map[string]string{"S": id}}) + } + resp["Items"], resp["Count"] = page, len(page) + fakeEventJSON(w, resp) + case "DynamoDB_20120810.UpdateItem": + var req struct { + Key map[string]fakeEventAttr + UpdateExpression string + ConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeEventAttr + } + fakeEventDecode(w, body, &req) + if f.beforeUpdate != nil { + f.beforeUpdate(f.items) + } + item := f.items[*req.Key["event_id"].S] + for _, pair := range fakeEventPair.FindAllStringSubmatch(req.ConditionExpression, -1) { + if item[req.ExpressionAttributeNames[pair[1]]] != *req.ExpressionAttributeValues[pair[2]].S { + f.conditionFailures++ + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + w.WriteHeader(http.StatusBadRequest) + fakeEventJSON(w, map[string]string{"__type": "com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException", "message": "condition failed"}) + return + } + } + for _, pair := range fakeEventPair.FindAllStringSubmatch(strings.TrimPrefix(req.UpdateExpression, "SET "), -1) { + item[req.ExpressionAttributeNames[pair[1]]] = *req.ExpressionAttributeValues[pair[2]].S + } + f.updates++ + fakeEventJSON(w, map[string]interface{}{}) + default: + http.Error(w, "unsupported operation "+r.Header.Get("X-Amz-Target"), http.StatusBadRequest) + } +} + +func fakeEventWire(item map[string]string) map[string]interface{} { + wire := map[string]interface{}{} + for k, v := range item { + wire[k] = map[string]string{"S": v} + } + return wire +} + +func fakeEventJSON(w http.ResponseWriter, payload interface{}) { + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func fakeEventDecode(w http.ResponseWriter, body []byte, v interface{}) { + if err := json.Unmarshal(body, v); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } +} + +func newRekeyRepo(t *testing.T, table *fakeEventsTable) RekeyRepository { + server := httptest.NewServer(table) + t.Cleanup(server.Close) + awsSession, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(server.URL), + Credentials: credentials.NewStaticCredentials("test", "test", ""), + DisableSSL: aws.Bool(true), + MaxRetries: aws.Int(0), + }) + require.NoError(t, err) + return NewRekeyRepository(awsSession, "test") +} + +func fakeEvent(id, sfid, foundation, project, claGroup string) map[string]string { + item := map[string]string{"event_id": id, "event_type": "TestEvent", "event_data_lower": "data " + id} + if sfid != "" { + item["event_company_sfid"] = sfid + } + if foundation != "" { + item["company_sfid_foundation_sfid"] = sfid + "#" + foundation + } + if project != "" { + item["company_sfid_project_id"] = sfid + "#" + project + } + if claGroup != "" { + item["company_sfid_cla_group_id"] = sfid + "#" + claGroup + } + return item +} + +func TestListEventIDsByCompanySFID(t *testing.T) { + table := &fakeEventsTable{pageSize: 2, items: map[string]map[string]string{ + "e1": fakeEvent("e1", "lf-old", "f1", "p1", "g1"), + "e2": fakeEvent("e2", "lf-old", "", "", "g1"), + "e3": fakeEvent("e3", "lf-old", "f1", "", ""), + "e4": fakeEvent("e4", "0014100000Other00", "f1", "p1", "g1"), + "e5": fakeEvent("e5", "", "", "", ""), + }} + repo := newRekeyRepo(t, table) + + ids, err := repo.ListEventIDsByCompanySFID(context.Background(), "lf-old") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"e1", "e2", "e3"}, ids) + assert.Equal(t, 2, table.queries, "pagination follows LastEvaluatedKey") + + ids, err = repo.ListEventIDsByCompanySFIDCLAGroup(context.Background(), "lf-old", "g1") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"e1", "e2"}, ids) + + ids, err = repo.ListEventIDsByCompanySFID(context.Background(), "unknown") + require.NoError(t, err) + assert.Empty(t, ids) + + _, err = repo.ListEventIDsByCompanySFID(context.Background(), " ") + assert.Error(t, err) + _, err = repo.ListEventIDsByCompanySFIDCLAGroup(context.Background(), "lf-old", "") + assert.Error(t, err) +} + +func TestRekeyEventCompanySFID(t *testing.T) { + const oldSFID, newSFID = "lf-old", "0014100000New0000" + + t.Run("rewrites the sfid and every composite key, keeps the rest", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e1": fakeEvent("e1", oldSFID, "f1", "p1", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, map[string]string{ + "event_id": "e1", + "event_type": "TestEvent", + "event_data_lower": "data e1", + "event_company_sfid": newSFID, + "company_sfid_foundation_sfid": newSFID + "#f1", + "company_sfid_project_id": newSFID + "#p1", + "company_sfid_cla_group_id": newSFID + "#g1", + }, table.items["e1"]) + assert.Equal(t, 1, table.updates) + }) + + t.Run("only the attributes present are rewritten", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e2": fakeEvent("e2", oldSFID, "", "", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e2", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newSFID, table.items["e2"]["event_company_sfid"]) + assert.Equal(t, newSFID+"#g1", table.items["e2"]["company_sfid_cla_group_id"]) + _, hasFoundation := table.items["e2"]["company_sfid_foundation_sfid"] + assert.False(t, hasFoundation) + }) + + t.Run("is idempotent", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e1": fakeEvent("e1", newSFID, "f1", "p1", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, 0, table.updates) + }) + + t.Run("finishes a half-rekeyed event", func(t *testing.T) { + item := fakeEvent("e1", newSFID, "", "", "") + item["company_sfid_cla_group_id"] = oldSFID + "#g1" + table := &fakeEventsTable{items: map[string]map[string]string{"e1": item}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newSFID+"#g1", table.items["e1"]["company_sfid_cla_group_id"]) + }) + + t.Run("refuses an event of another company", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e4": fakeEvent("e4", "0014100000Other00", "f1", "p1", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e4", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventCompanySFIDMismatch) + assert.False(t, changed) + assert.Equal(t, 0, table.updates) + }) + + t.Run("refuses a composite key of another company", func(t *testing.T) { + item := fakeEvent("e6", oldSFID, "", "", "") + item["company_sfid_project_id"] = "0014100000Other00#p1" + table := &fakeEventsTable{items: map[string]map[string]string{"e6": item}} + repo := newRekeyRepo(t, table) + _, err := repo.RekeyEventCompanySFID(context.Background(), "e6", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventCompanySFIDMismatch) + assert.Equal(t, 0, table.updates) + }) + + t.Run("missing event", func(t *testing.T) { + repo := newRekeyRepo(t, &fakeEventsTable{items: map[string]map[string]string{}}) + _, err := repo.RekeyEventCompanySFID(context.Background(), "nope", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventNotFound) + }) + + t.Run("concurrent change between read and write is a retryable conflict", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e1": fakeEvent("e1", oldSFID, "f1", "", "")}} + table.beforeUpdate = func(items map[string]map[string]string) { + items["e1"]["company_sfid_foundation_sfid"] = oldSFID + "#f2" + } + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventRekeyConflict) + assert.False(t, changed) + assert.Equal(t, 1, table.conditionFailures) + + table.beforeUpdate = nil + changed, err = repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newSFID+"#f2", table.items["e1"]["company_sfid_foundation_sfid"]) + }) + + t.Run("invalid arguments", func(t *testing.T) { + repo := newRekeyRepo(t, &fakeEventsTable{items: map[string]map[string]string{}}) + _, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, oldSFID) + assert.Error(t, err) + _, err = repo.RekeyEventCompanySFID(context.Background(), "", oldSFID, newSFID) + assert.Error(t, err) + }) +} diff --git a/cla-backend-go/go.mod b/cla-backend-go/go.mod index 1ff5c0253..b7c04faa8 100644 --- a/cla-backend-go/go.mod +++ b/cla-backend-go/go.mod @@ -59,32 +59,32 @@ require ( github.com/verdverm/frisby v0.0.0-20170604211311-b16556248a9a github.com/xanzy/go-gitlab v0.50.1 go.uber.org/ratelimit v0.1.0 - golang.org/x/crypto v0.52.0 // indirect - golang.org/x/image v0.39.0 // indirect - golang.org/x/net v0.55.0 - golang.org/x/oauth2 v0.35.0 - golang.org/x/sync v0.20.0 - golang.org/x/sys v0.45.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/image v0.45.0 // indirect + golang.org/x/net v0.58.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e google.golang.org/protobuf v1.36.11 // indirect ) require ( github.com/akamensky/base58 v0.0.0-20210829145138-ce8bf8802e8f - github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 github.com/bradleyfalzon/ghinstallation/v2 v2.2.0 github.com/golang-jwt/jwt/v4 v4.5.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 - go.opentelemetry.io/otel v1.43.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 - go.opentelemetry.io/otel/sdk v1.43.0 - go.opentelemetry.io/otel/trace v1.43.0 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 + go.opentelemetry.io/otel/sdk v1.45.0 + go.opentelemetry.io/otel/trace v1.45.0 ) require ( github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/smithy-go v1.20.2 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -95,7 +95,7 @@ require ( github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -106,7 +106,7 @@ require ( github.com/google/go-github/v50 v50.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -135,16 +135,16 @@ require ( github.com/subosito/gotenv v1.4.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect - go.mongodb.org/mongo-driver v1.10.1 // indirect + go.mongodb.org/mongo-driver v1.17.7 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/grpc v1.80.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/grpc v1.83.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/cla-backend-go/go.sum b/cla-backend-go/go.sum index b5a23debc..f02088da7 100644 --- a/cla-backend-go/go.sum +++ b/cla-backend-go/go.sum @@ -70,10 +70,10 @@ github.com/aws/aws-lambda-go v1.22.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XO github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.36.27 h1:wc3xLJJHog2SwiqlLnrLUuct/n+dBjB45QhuZw2psVE= github.com/aws/aws-sdk-go v1.36.27/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= -github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= -github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= @@ -146,8 +146,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -386,8 +386,8 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -588,7 +588,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -628,8 +627,8 @@ go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4= -go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8= +go.mongodb.org/mongo-driver v1.17.7 h1:a9w+U3Vt67eYzcfq3k/OAv284/uUUkL0uP75VE5rCOU= +go.mongodb.org/mongo-driver v1.17.7/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -640,22 +639,22 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -683,8 +682,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -699,8 +698,8 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190823064033-3a9bac650e44/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww= -golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -770,8 +769,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -782,8 +781,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -798,8 +797,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -856,8 +855,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -873,8 +872,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1008,10 +1007,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1028,8 +1027,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/cla-backend-go/orgimport/account_forms_test.go b/cla-backend-go/orgimport/account_forms_test.go new file mode 100644 index 000000000..5b65f354f --- /dev/null +++ b/cla-backend-go/orgimport/account_forms_test.go @@ -0,0 +1,306 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// 15-char forms of the 18-char fixture ids (same Salesforce Account, checksum suffix dropped) +var ( + liveSFID15 = liveSFID[:15] + targetSFID15 = targetSFID[:15] +) + +func TestAccountKey(t *testing.T) { + assert.Equal(t, targetSFID15, accountKey(targetSFID)) + assert.Equal(t, targetSFID15, accountKey(targetSFID15)) + assert.NotEqual(t, accountKey(targetSFID), accountKey(targetSFID2), "distinct Accounts stay distinct") + assert.Equal(t, lfID, accountKey(lfID), "legacy ids are compared as stored") + assert.Equal(t, "", accountKey("")) + assert.Equal(t, "0064100000Te0G7AAJ", accountKey("0064100000Te0G7AAJ"), "non-Account ids are never shortened") + assert.Equal(t, ShapeSFID, ShapeOf(targetSFID15)) +} + +func TestMappingSameAccountFormIsSameID(t *testing.T) { + m, err := ParseMapping(strings.NewReader("old_id,new_id,action,approved\n" + deadSFID + "," + deadSFID[:15] + ",matched,true\n" + lfID + "," + targetSFID + ",matched,true\n")) + require.NoError(t, err) + _, _, reason := m.Resolve(deadSFID) + assert.Equal(t, ReasonMappingSameID, reason, "the 15-char form of the old id is the same Account") + newID, _, reason := m.Resolve(lfID) + assert.Empty(t, reason) + assert.Equal(t, targetSFID, newID) + assert.Contains(t, Suggest(ReasonMappingSameID), "same Account") +} + +func TestCollisionAcrossSFIDForms(t *testing.T) { + fx, dir := collisionFixture(t) + fx.platform.sfAccounts[targetSFID15] = true + // two old ids mapped to the two forms of one Account collide like any co-targets + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID15+",created,true") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + require.Len(t, plan.Targets, 1, "both forms are one destination") + tgt := plan.Targets[0] + assert.Equal(t, []string{"lf-a", "lf-b"}, tgt.OldIDs()) + assert.Equal(t, ReasonTargetFormsDiffer, tgt.Status) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteManual, g.Route, id) + assert.Equal(t, ReasonTargetFormsDiffer, g.ManualReason, id) + } + // a collapse decision does not make the tool write two forms of one id + decisions := writeDecisions(t, dir, "collapse,lf-a;lf-b,"+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, ReasonTargetFormsDiffer, groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, ReasonTargetFormsDiffer, groupByKey(plan.Groups, "lf-b").ManualReason) + assert.Empty(t, plan.Rewrite) + + // mapping normalized to one form: an ordinary collision that the decision (in either form) approves + mapping = writeMapping(t, dir, "lf-a,"+targetSFID15+",matched,true", "lf-b,"+targetSFID15+",created,true") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, "needs_decision", targetByID(plan.Targets, targetSFID15).Status) + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + require.Len(t, plan.Rewrite, 2) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteRewrite, g.Route, id) + assert.Equal(t, targetSFID15, g.NewID, id, "the mapped form is written as given") + require.NotNil(t, g.Decision, id) + } + assert.Equal(t, "ok", targetByID(plan.Targets, targetSFID15).Status) + + // a mapping row outside the tranche pointing at the other form is a co-target too + mapping = writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-z,"+targetSFID15+",created,true") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, ReasonTargetFormsDiffer, groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, []string{"lf-a", "lf-z"}, targetByID(plan.Targets, targetSFID).OldIDs()) + mapping = writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-z,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-c").Route, "a different Account is not a collision") + assert.Contains(t, Suggest(ReasonTargetFormsDiffer), "one form") +} + +func TestCollisionWithExistingRowInOtherForm(t *testing.T) { + fx, dir := collisionFixture(t) + fx.company("c-existing", "Acme Holdings", "", targetSFID15) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + a := groupByKey(plan.Groups, "lf-a") + assert.Equal(t, RouteManual, a.Route) + assert.Equal(t, ReasonTargetFormsDiffer, a.ManualReason, "a row carrying the 15-char form of the target is on the same Account") + tgt := targetByID(plan.Targets, targetSFID) + require.NotNil(t, tgt) + require.Len(t, tgt.Existing, 1) + assert.Equal(t, []string{"lf-a", targetSFID15}, tgt.OldIDs()) + assert.Equal(t, ReasonTargetFormsDiffer, tgt.Status) + assert.True(t, plan.inv.ExternalIDCollision(a, targetSFID)) + + // mapping normalized to the form the existing row carries: a plain collision, lifted by a collapse + // decision that may name the Account in either form + mapping = writeMapping(t, dir, "lf-a,"+targetSFID15+",matched,true") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + decisions := writeDecisions(t, dir, "collapse,lf-a;"+targetSFID15+","+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + assert.Equal(t, targetSFID15, groupByKey(plan.Groups, "lf-a").NewID) + assert.Equal(t, "ok", targetByID(plan.Targets, targetSFID15).Status) +} + +func TestSFIDAliasFormsAreManual(t *testing.T) { + fx := newFixture() + fx.company("c-18", "Acme", "", liveSFID, "cg-1") + fx.company("c-15", "Acme", "Acme Labs", liveSFID15, "cg-2") + fx.company("c-other", "Other", "", liveSFID2, "cg-1") + fx.platform.sfAccounts[liveSFID] = true + fx.platform.sfAccounts[liveSFID15] = true + fx.platform.sfAccounts[liveSFID2] = true + dir := t.TempDir() + opts := Options{Stage: "dev", Apply: true, OutDir: dir} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + for key, alias := range map[string]string{liveSFID: liveSFID15, liveSFID15: liveSFID} { + g := groupByKey(plan.Groups, key) + require.NotNil(t, g, key) + assert.Equal(t, RouteManual, g.Route, key) + assert.Equal(t, ReasonSFIDAliasForms, g.ManualReason, key) + assert.Equal(t, []string{alias}, g.Aliases, key) + assert.Equal(t, key, g.OldID, key, "the stored id is kept as is") + assert.Empty(t, g.Live, key) + } + other := groupByKey(plan.Groups, liveSFID2) + assert.Equal(t, RouteRegister, other.Route, "a different Account is unaffected") + assert.Empty(t, other.Aliases) + for _, c := range fx.platform.calls { + assert.False(t, strings.HasPrefix(c, "get-b2b:"+liveSFID15), "no liveness call before the rows are normalized: %s", c) + } + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 2, sum.Manual) + assert.Equal(t, 1, sum.Registered) + assert.Equal(t, []string{liveSFID2}, fx.platform.registered, "neither form of the aliased Account is registered") + assert.Empty(t, fx.companies.updates) + var buf bytes.Buffer + plan.Print(&buf) + assert.Contains(t, buf.String(), "reason="+ReasonSFIDAliasForms+" aliases="+liveSFID15) + manual := readCSV(t, filepath.Join(dir, "manual_actions.csv")) + require.Len(t, manual, 3) + assert.Contains(t, Suggest(ReasonSFIDAliasForms), "normalize") + + // the audit reports the same gate + res, err := Audit(context.Background(), fx.deps(), Options{Stage: "dev", OutDir: t.TempDir()}) + require.NoError(t, err) + assert.Equal(t, ReasonSFIDAliasForms, groupByKey(res.Groups, liveSFID).ManualReason) + assert.Equal(t, ReasonSFIDAliasForms, groupByKey(res.Groups, liveSFID15).ManualReason) + assert.Equal(t, 2, res.Routes[RouteManual]) +} + +func TestSFIDAliasSiblingWithoutCCLAIsManual(t *testing.T) { + // only the 18-char rows have an active CCLA; the 15-char sibling would otherwise be left behind + fx := newFixture() + fx.company("c-18", "Acme", "", deadSFID, "cg-1") + fx.company("c-15", "Acme", "Acme Labs", deadSFID[:15]) + dir := t.TempDir() + mapping := writeMapping(t, dir, deadSFID+","+targetSFID+",matched,true") + fx.platform.sfAccounts[targetSFID] = true + fx.platform.orgs[targetSFID] = &Org{ID: targetSFID} + opts := Options{Stage: "dev", Apply: true, Mapping: mapping, State: filepath.Join(dir, "state.jsonl")} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + require.Len(t, plan.Groups, 1, "a row without an active CCLA is not a group of its own") + g := plan.Groups[0] + assert.Equal(t, RouteManual, g.Route) + assert.Equal(t, ReasonSFIDAliasForms, g.ManualReason) + assert.Equal(t, []string{deadSFID[:15]}, g.Aliases) + assert.Empty(t, plan.Rewrite) + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 1, sum.Manual) + assert.Empty(t, fx.companies.updates, "neither representation is rewritten on its own") + assert.Empty(t, fx.platform.registered) +} + +func TestDuplicateEntityParentKey(t *testing.T) { + assert.True(t, hasDuplicateEntity([]*Row{ + {CompanyID: "a", CompanyName: "Acme", SigningEntityName: ""}, + {CompanyID: "b", CompanyName: "Acme", SigningEntityName: "Acme"}, + }), "an entity name equal to the company name is the parent row, like an empty one") + assert.True(t, hasDuplicateEntity([]*Row{ + {CompanyID: "a", CompanyName: "Acme", SigningEntityName: "acme "}, + {CompanyID: "b", CompanyName: "Acme", SigningEntityName: ""}, + })) + assert.False(t, hasDuplicateEntity([]*Row{ + {CompanyID: "a", CompanyName: "Acme", SigningEntityName: "Acme"}, + {CompanyID: "b", CompanyName: "Acme", SigningEntityName: "Acme GmbH"}, + }), "distinct signing entities stay distinct") + assert.True(t, hasDuplicateEntity([]*Row{ + {CompanyID: "a", CompanyName: "Acme", SigningEntityName: ""}, + {CompanyID: "b", CompanyName: "Acme GmbH", SigningEntityName: "Acme GmbH"}, + }), "two parent-style rows of one Account are reported even when their names differ (review aid, never fewer flags)") + + fx := newFixture() + fx.company("c-parent", "Acme", "Acme", liveSFID, "cg-1") + fx.company("c-dup", "Acme", "", liveSFID) + fx.company("c-sub", "Acme", "Acme GmbH", liveSFID) + inv, err := LoadInventory(context.Background(), fx.deps()) + require.NoError(t, err) + acme := groupByKey(inv.EligibleGroups(), liveSFID) + require.NotNil(t, acme) + assert.True(t, acme.Duplicate) + assert.Empty(t, acme.Aliases) +} + +func TestCopyGrantsRefusesGrantWithoutUsername(t *testing.T) { + fx, mapping, statePath := rewriteFixture(t) + fx.platform.addGrant(lfID, "", "role-mgr", "") + opts := Options{Stage: "dev", Apply: true, Mapping: mapping, State: statePath, Routes: []Route{RouteRewrite}} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.Error(t, err) + assert.Equal(t, 1, sum.Failed) + require.Error(t, groupByKey(plan.Groups, lfID).Err) + assert.Contains(t, groupByKey(plan.Groups, lfID).Err.Error(), "has no username") + assert.Empty(t, fx.companies.updates, "the group stops before the row cutover") + assert.Equal(t, lfID, fx.companies.rows["c-parent"].CompanyExternalID) + assert.Len(t, mustListOrgGrants(t, fx, lfID), 4, "nothing is deleted on the old org") + assert.Empty(t, fx.events.rekeyed) + st, err := LoadState(statePath) + require.NoError(t, err) + assert.False(t, st.Done(lfID)) + assert.Equal(t, StepGrants, st.Last[lfID].Step) + assert.Equal(t, statusFailed, st.Last[lfID].Status) + assert.Contains(t, st.Last[lfID].Err, "has no username") +} + +func TestCleanupRefusesToDeleteGrantWithoutUsername(t *testing.T) { + fx, mapping, statePath := rewriteFixture(t) + // the grant appears on the old org after the copy step (an ACS write racing the import) + lists := 0 + fx.platform.listHook = func(orgID string) { + if orgID == lfID { + if lists++; lists == 2 { + fx.platform.addGrant(lfID, "", "role-mgr", "cg-1") + } + } + } + opts := Options{Stage: "dev", Apply: true, Mapping: mapping, State: statePath, Routes: []Route{RouteRewrite}} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.Error(t, err) + assert.Equal(t, 0, sum.Rewritten) + require.Error(t, groupByKey(plan.Groups, lfID).Err) + assert.Contains(t, groupByKey(plan.Groups, lfID).Err.Error(), "has no username") + assert.Equal(t, targetSFID, fx.companies.rows["c-parent"].CompanyExternalID, "rows were cut over before cleanup") + var kept []string + for _, g := range mustListOrgGrants(t, fx, lfID) { + kept = append(kept, g.Username+"/"+g.RoleID) + } + assert.Contains(t, kept, "/role-mgr", "the uncopiable grant is never deleted") + st, err := LoadState(statePath) + require.NoError(t, err) + assert.False(t, st.Done(lfID), "no false done record") + assert.Equal(t, StepCleanup, st.Last[lfID].Step) + + // once the grant is fixed (a username is known) the replay converges + fx.platform.listHook = nil + for i := range fx.platform.grants[lfID] { + if fx.platform.grants[lfID][i].Username == "" { + fx.platform.grants[lfID][i].Username = "late-user" + } + } + plan, err = BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + sum, err = Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 1, sum.Rewritten) + assert.Empty(t, mustListOrgGrants(t, fx, lfID)) + var users []string + for _, g := range mustListOrgGrants(t, fx, targetSFID) { + users = append(users, g.Username) + } + assert.Contains(t, users, "late-user") + st, err = LoadState(statePath) + require.NoError(t, err) + assert.True(t, st.Done(lfID)) +} diff --git a/cla-backend-go/orgimport/adapters.go b/cla-backend-go/orgimport/adapters.go new file mode 100644 index 000000000..dc65492e6 --- /dev/null +++ b/cla-backend-go/orgimport/adapters.go @@ -0,0 +1,86 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" + + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service/client/organizations" +) + +// OrgServiceAdapter maps the shared org-service client onto OrgService. +type OrgServiceAdapter struct { + Client *organization_service.Client +} + +// GetOrganization returns ErrOrgNotFound on 404. +func (a OrgServiceAdapter) GetOrganization(ctx context.Context, orgID string) (*Org, error) { + org, err := a.Client.GetOrganization(ctx, orgID) + if err != nil { + var notFound *organizations.GetOrgNotFound + if errors.As(err, ¬Found) { + return nil, ErrOrgNotFound + } + return nil, err + } + if org == nil { + return nil, ErrOrgNotFound + } + return &Org{ID: org.ID, Name: org.Name, Website: org.Link, SigningEntityNames: org.SigningEntityName}, nil +} + +// CreateUserRoleScope grants a role scope by LFID username; an existing grant is not an error. +func (a OrgServiceAdapter) CreateUserRoleScope(ctx context.Context, username, organizationID, objectType, objectID, roleID string) error { + return a.Client.CreateOrgUserRoleScopeByUsername(ctx, username, organizationID, objectType, objectID, roleID) +} + +// DeleteUserRoleScope removes one grant (ACS grant id); a missing grant is success. +func (a OrgServiceAdapter) DeleteUserRoleScope(ctx context.Context, organizationID, roleID, grantID, username string) error { + email := "" + err := a.Client.DeleteOrgUserRoleOrgScopeProjectOrg(ctx, organizationID, roleID, grantID, &username, &email) + if err != nil { + var notFound *organizations.DeleteOrgUsrRoleScopesNotFound + if errors.As(err, ¬Found) { + return nil + } + return err + } + return nil +} + +// DynamoECLACounter counts rows of the signature-user-ccla-company-index partition of a company. +type DynamoECLACounter struct { + DB dynamodbiface.DynamoDBAPI + Table string +} + +// CountECLAs sums Count over all pages. +func (c DynamoECLACounter) CountECLAs(ctx context.Context, companyID string) (int, error) { + input := &dynamodb.QueryInput{ + TableName: aws.String(c.Table), + IndexName: aws.String("signature-user-ccla-company-index"), + KeyConditionExpression: aws.String("signature_user_ccla_company_id = :id"), + ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{":id": {S: aws.String(companyID)}}, + Select: aws.String(dynamodb.SelectCount), + } + total := 0 + for { + out, err := c.DB.QueryWithContext(ctx, input) + if err != nil { + return 0, fmt.Errorf("counting ECLAs of %s: %w", companyID, err) + } + total += int(aws.Int64Value(out.Count)) + if len(out.LastEvaluatedKey) == 0 { + return total, nil + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } +} diff --git a/cla-backend-go/orgimport/apex.go b/cla-backend-go/orgimport/apex.go new file mode 100644 index 000000000..3470a347f --- /dev/null +++ b/cla-backend-go/orgimport/apex.go @@ -0,0 +1,97 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// ApexRequest is the design §4 find-or-create payload. +type ApexRequest struct { + Name string `json:"name"` + Website string `json:"website,omitempty"` + Source string `json:"source"` + ExternalKey string `json:"externalKey"` + CCLASignedDate string `json:"cclaSignedDate,omitempty"` + DryRun bool `json:"dryRun"` +} + +// ApexResult is the Apex response. +type ApexResult struct { + ID string `json:"id"` + Action string `json:"action"` +} + +// ApexClient calls the Salesforce Apex REST endpoint (off unless --use-apex and both SSM params exist). +type ApexClient struct { + BaseURL string + Token string + HTTPClient *http.Client +} + +// NewApexClient returns ErrApexUnavailable when the endpoint is not configured. +func NewApexClient(baseURL, token string) (*ApexClient, error) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" || strings.TrimSpace(token) == "" { + return nil, ErrApexUnavailable + } + return &ApexClient{BaseURL: baseURL, Token: strings.TrimSpace(token), HTTPClient: &http.Client{Timeout: 60 * time.Second}}, nil +} + +// FindOrCreate posts the request; the action is validated and the id must be a Salesforce account +// id, except that a dry run reporting "created" may carry no id (the Account does not exist yet). +func (c *ApexClient) FindOrCreate(ctx context.Context, req ApexRequest) (*ApexResult, error) { + if req.Source == "" { + req.Source = "EasyCLA" + } + body, err := json.Marshal(req) + if err != nil { + return nil, err + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/services/apexrest/lfx/account", bytes.NewReader(body)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Authorization", "Bearer "+c.Token) + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "application/json") + resp, err := c.HTTPClient.Do(httpReq) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + _ = resp.Body.Close() + if readErr != nil { + return nil, readErr + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, fmt.Errorf("apex: %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + var res ApexResult + if err = json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("apex: decoding response: %w", err) + } + res.Action = strings.ToLower(strings.TrimSpace(res.Action)) + res.ID = strings.TrimSpace(res.ID) + switch res.Action { + case ActionMatched, ActionCreated: + if res.Action == ActionCreated && req.DryRun && res.ID == "" { + break + } + if !IsSFID(res.ID) { + return nil, fmt.Errorf("apex: action %s returned non-account id %q", res.Action, res.ID) + } + case ActionAmbiguous: + default: + return nil, fmt.Errorf("apex: unknown action %q", res.Action) + } + return &res, nil +} diff --git a/cla-backend-go/orgimport/audit.go b/cla-backend-go/orgimport/audit.go new file mode 100644 index 000000000..365984737 --- /dev/null +++ b/cla-backend-go/orgimport/audit.go @@ -0,0 +1,387 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "encoding/csv" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" +) + +// Tiers mirror utils/audit_company_reachability.sh so counts can be cross-checked. +const ( + TierMissing = "MISSING_SFID" + TierInvalid = "INVALID_SFID_FORMAT" + TierOK = "SFID_OK" + TierDangling = "SFID_DANGLING_OR_DELETED" + TierUnknown = "UNKNOWN" +) + +// AuditRow is one line of audit.csv. +type AuditRow struct { + Row *Row + Shape IDShape + ECLACount int + ECLACounted bool + OrgStatus string + Website string + Duplicate bool + Route Route + ManualReason string + Tier string +} + +// AuditResult is the audit output; Duplicates are the candidate-target sets (rows sharing an id, +// a normalized name or a non-shared website domain under different ids). +type AuditResult struct { + Rows []*AuditRow + Groups []*Group + Tiers map[string]int + Routes map[Route]int + Duplicates [][]*Row + rowByID map[string]*AuditRow +} + +// Audit classifies every company row (reads only) and writes audit.csv, unresolvable.csv and +// possible_duplicates.csv into opts.OutDir. +func Audit(ctx context.Context, deps Deps, opts Options) (*AuditResult, error) { + deps = withDefaults(deps) + shared, err := LoadSharedDomains(opts.SharedDomains) + if err != nil { + return nil, err + } + inv, err := LoadInventory(ctx, deps) + if err != nil { + return nil, err + } + groups := inv.EligibleGroups() + for _, g := range groups { + classify(ctx, deps, g, nil, nil, Options{}) + } + groupByRow := map[string]*Group{} + for _, g := range groups { + for _, r := range g.Rows { + groupByRow[r.CompanyID] = g + } + } + + orgs := lookupOrgs(ctx, deps, inv) + res := &AuditResult{Tiers: map[string]int{}, Routes: map[Route]int{}, rowByID: map[string]*AuditRow{}} + byName, byDomain := map[string][]*Row{}, map[string][]*Row{} + for _, row := range inv.Rows { + ar := &AuditRow{Row: row, Shape: ShapeOf(row.ExternalID)} + if o := orgs[row.ExternalID]; o != nil { + ar.OrgStatus = o.status + if o.org != nil { + ar.Website = o.org.Website + } + } + ar.Tier = tier(row.ExternalID, ar.OrgStatus) + if g := groupByRow[row.CompanyID]; g != nil { + ar.Duplicate, ar.Route, ar.ManualReason = g.Duplicate, g.Route, g.ManualReason + if g.Err != nil { + ar.ManualReason = "error: " + g.Err.Error() + } + } + res.Tiers[ar.Tier]++ + if ar.Route != "" { + res.Routes[ar.Route]++ + } + res.Rows = append(res.Rows, ar) + res.rowByID[row.CompanyID] = ar + byName[canonicalEntity(row.CompanyName)] = append(byName[canonicalEntity(row.CompanyName)], row) + if d, gate := shared.Shared(ar.Website); gate == "" { + byDomain[d] = append(byDomain[d], row) + } + } + res.Groups = groups + + res.collectDuplicates(byName, byDomain) + res.countECLAs(ctx, deps.ECLAs) + + if opts.OutDir != "" { + if err = writeAuditReports(opts.OutDir, res); err != nil { + return nil, err + } + } + for _, ar := range res.Rows { + r := ar.Row + fmt.Fprintf(deps.Out, "audit row company_id=%s name=%q external_id=%q shape=%s active_ccla=%t ccla=%d ecla=%d org_service=%s route=%s reason=%q tier=%s\n", + r.CompanyID, firstNonEmpty(r.SigningEntityName, r.CompanyName), r.RawExternalID, ar.Shape, r.ActiveCCLA, r.CCLACount, ar.ECLACount, ar.OrgStatus, ar.Route, ar.ManualReason, ar.Tier) + } + fmt.Fprintf(deps.Out, "audit stage=%s companies=%d eligible_groups=%d", opts.Stage, len(res.Rows), len(groups)) + for _, t := range []string{TierMissing, TierInvalid, TierOK, TierDangling, TierUnknown} { + fmt.Fprintf(deps.Out, " %s=%d", t, res.Tiers[t]) + } + fmt.Fprintf(deps.Out, " register=%d rewrite=%d manual=%d duplicates=%d\n", res.Routes[RouteRegister], res.Routes[RouteRewrite], res.Routes[RouteManual], len(res.Duplicates)) + return res, nil +} + +func (ar *AuditRow) unresolvable() bool { + return ar.Shape == ShapeEmpty || ar.Shape == ShapeOther || (ar.Shape == ShapeSFID && ar.OrgStatus == "404") +} + +// collectDuplicates builds the candidate-target sets: rows sharing an id (with the same or empty +// signing entity), then rows with the same normalized name or the same non-shared domain under +// different ids. +func (res *AuditResult) collectDuplicates(byName, byDomain map[string][]*Row) { + for _, g := range res.Groups { + if g.Duplicate { + res.addDuplicates(g.Rows) + } + } + for _, sets := range []map[string][]*Row{byName, byDomain} { + keys := make([]string, 0, len(sets)) + for k := range sets { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if rows := sets[k]; k != "" && len(rows) >= 2 && distinctExternalIDs(rows) { + res.addDuplicates(rows) + } + } + } +} + +// countECLAs fills ECLACount for the active manual/duplicate/unresolvable rows and for every row of a +// candidate set (inactive ones included); -1 marks a failed count. +func (res *AuditResult) countECLAs(ctx context.Context, counter ECLACounter) { + if counter == nil { + return + } + reported := map[string]bool{} + for _, set := range res.Duplicates { + for _, r := range set { + reported[r.CompanyID] = true + } + } + for _, ar := range res.Rows { + if reported[ar.Row.CompanyID] || (ar.Row.ActiveCCLA && (ar.Route == RouteManual || ar.Duplicate || ar.unresolvable())) { + ar.ECLACounted = true + if n, cErr := counter.CountECLAs(ctx, ar.Row.CompanyID); cErr == nil { + ar.ECLACount = n + } else { + ar.ECLACount = -1 + } + } + } +} + +// addDuplicates appends a candidate set once: the same rows reached by name and by domain are one set. +func (res *AuditResult) addDuplicates(rows []*Row) { + ids := make([]string, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.CompanyID) + } + sort.Strings(ids) + key := strings.Join(ids, "\x00") + for _, set := range res.Duplicates { + other := make([]string, 0, len(set)) + for _, r := range set { + other = append(other, r.CompanyID) + } + sort.Strings(other) + if strings.Join(other, "\x00") == key { + return + } + } + res.Duplicates = append(res.Duplicates, rows) +} + +func distinctExternalIDs(rows []*Row) bool { + seen := map[string]bool{} + for _, r := range rows { + seen[r.ExternalID] = true + } + return len(seen) > 1 +} + +func tier(externalID, orgStatus string) string { + if externalID == "" { + return TierMissing + } + if len(externalID) != 15 && len(externalID) != 18 || !alnum(externalID) { + return TierInvalid + } + switch orgStatus { + case "200": + return TierOK + case "404": + return TierDangling + default: + return TierUnknown + } +} + +func alnum(s string) bool { + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +type orgLookup struct { + org *Org + status string +} + +// lookupOrgs resolves every distinct 15/18-char external id in org-service (4 workers). +func lookupOrgs(ctx context.Context, deps Deps, inv *Inventory) map[string]*orgLookup { + ids := make([]string, 0, len(inv.ByExternal)) + for id := range inv.ByExternal { + if (len(id) == 15 || len(id) == 18) && alnum(id) { + ids = append(ids, id) + } + } + sort.Strings(ids) + out := make(map[string]*orgLookup, len(ids)) + var mu sync.Mutex + var wg sync.WaitGroup + work := make(chan string) + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for id := range work { + l := &orgLookup{} + org, err := deps.Orgs.GetOrganization(ctx, id) + switch { + case err == nil: + l.org, l.status = org, "200" + case errors.Is(err, ErrOrgNotFound): + l.status = "404" + default: + l.status = orgStatusErr + } + mu.Lock() + out[id] = l + mu.Unlock() + } + }() + } + for _, id := range ids { + work <- id + } + close(work) + wg.Wait() + return out +} + +func writeAuditReports(dir string, res *AuditResult) error { + if err := os.MkdirAll(dir, 0o750); err != nil { + return err + } + rows := [][]string{{"company_id", "company_name", "signing_entity_name", "company_external_id", "id_shape", "active_ccla", "ccla_count", "ecla_count", "org_service", "website", "duplicate_sfid_group", "route", "manual_reason", "tier"}} + var unresolvable [][]string + unresolvable = append(unresolvable, []string{"company_id", "company_name", "signing_entity_name", "company_external_id", "id_shape", "ccla_count", "ecla_count", "org_service", "manual_reason"}) + for _, ar := range res.Rows { + r := ar.Row + rows = append(rows, []string{r.CompanyID, r.CompanyName, r.SigningEntityName, r.ExternalID, string(ar.Shape), strconv.FormatBool(r.ActiveCCLA), strconv.Itoa(r.CCLACount), strconv.Itoa(ar.ECLACount), ar.OrgStatus, ar.Website, strconv.FormatBool(ar.Duplicate), string(ar.Route), ar.ManualReason, ar.Tier}) + if r.ActiveCCLA && ar.unresolvable() { + unresolvable = append(unresolvable, []string{r.CompanyID, r.CompanyName, r.SigningEntityName, r.ExternalID, string(ar.Shape), strconv.Itoa(r.CCLACount), strconv.Itoa(ar.ECLACount), ar.OrgStatus, ar.ManualReason}) + } + } + dups := [][]string{{"group", "company_id", "company_name", "signing_entity_name", "company_external_id", "id_shape", "domain", "active_ccla", "ccla_count", "ecla_count"}} + for i, set := range res.Duplicates { + for _, r := range set { + shape, domain, ecla := string(ShapeOf(r.ExternalID)), "", "" + if ar := res.rowByID[r.CompanyID]; ar != nil { + shape, domain = string(ar.Shape), Domain(ar.Website) + if ar.ECLACounted { + ecla = strconv.Itoa(ar.ECLACount) + } + } + dups = append(dups, []string{strconv.Itoa(i + 1), r.CompanyID, r.CompanyName, r.SigningEntityName, r.ExternalID, shape, domain, strconv.FormatBool(r.ActiveCCLA), strconv.Itoa(r.CCLACount), ecla}) + } + } + for name, data := range map[string][][]string{"audit.csv": rows, "unresolvable.csv": unresolvable, "possible_duplicates.csv": dups} { + if err := writeCSV(filepath.Join(dir, name), data); err != nil { + return err + } + } + return nil +} + +func writeIngestReports(dir string, plan *Plan) error { + if err := os.MkdirAll(dir, 0o750); err != nil { + return err + } + rows := [][]string{{"key", "old_id", "id_shape", "route", "manual_reason", "live", "org_service", "website", "domain", "shared_domain", "new_id", "action", "decision", "reviewer", "company_ids", "company_names", "error"}} + toSF := [][]string{{"old_id", "name", "website", "ccla_signed_date", "domain", "shared_domain"}} + manual := [][]string{{"key", "old_id", "route", "reason", "suggested_action", "live", "org_service", "website", "domain", "shared_domain", "new_id", "action", "company_ids", "company_names", "error"}} + for _, g := range plan.Groups { + names := strings.Join(g.Names(), ";") + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + domain, shared := plan.domainOf(g) + decision, reviewer := "", "" + if g.Decision != nil { + decision, reviewer = g.Decision.Kind, g.Decision.Reviewer + } + rows = append(rows, []string{g.Key, g.OldID, string(g.Shape), string(g.Route), g.ManualReason, g.Live, g.OrgStatus, g.Website(), domain, shared, g.NewID, g.Action, decision, reviewer, strings.Join(g.CompanyIDs(), ";"), names, errText}) + if g.ManualReason == ReasonNoMapping || g.ManualReason == ReasonDeadAccount { + req := apexRequest(g, true) + toSF = append(toSF, []string{g.OldID, req.Name, req.Website, req.CCLASignedDate, domain, shared}) + } + } + for _, a := range plan.ManualActions() { + g := a.Group + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + domain, shared := plan.domainOf(g) + manual = append(manual, []string{g.Key, g.OldID, string(g.Route), a.Reason, a.Suggested, g.Live, g.OrgStatus, g.Website(), domain, shared, g.NewID, g.Action, strings.Join(g.CompanyIDs(), ";"), strings.Join(g.Names(), ";"), errText}) + } + targets := [][]string{{"target_sfid", "groups", "old_ids", "company_ids", "company_names", "existing_rows", "decision", "reviewer", "status"}} + for _, t := range plan.Targets { + var ids, names []string + for _, g := range t.Groups { + ids = append(ids, g.CompanyIDs()...) + names = append(names, g.Names()...) + } + decision, reviewer := "", "" + if t.Decision != nil { + decision, reviewer = t.Decision.Kind, t.Decision.Reviewer + } + targets = append(targets, []string{t.SFID, strconv.Itoa(len(t.Groups)), strings.Join(t.OldIDs(), ";"), strings.Join(ids, ";"), strings.Join(names, ";"), strconv.Itoa(len(t.Existing)), decision, reviewer, t.Status}) + } + for name, data := range map[string][][]string{"plan.csv": rows, "to_salesforce.csv": toSF, "manual_actions.csv": manual, "targets.csv": targets} { + if err := writeCSV(filepath.Join(dir, name), data); err != nil { + return err + } + } + return nil +} + +// domainOf returns the group's website domain and "true"/"false" for the shared-domain flag. +func (p *Plan) domainOf(g *Group) (string, string) { + domain, reason := p.shared.Shared(g.Website()) + return domain, strconv.FormatBool(reason == ReasonSharedDomain) +} + +func writeCSV(path string, rows [][]string) error { + f, err := os.Create(filepath.Clean(path)) + if err != nil { + return err + } + w := csv.NewWriter(f) + if err = w.WriteAll(rows); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/cla-backend-go/orgimport/awsreport.go b/cla-backend-go/orgimport/awsreport.go new file mode 100644 index 000000000..3686b2bb2 --- /dev/null +++ b/cla-backend-go/orgimport/awsreport.go @@ -0,0 +1,131 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ses" + "github.com/aws/aws-sdk-go/service/ses/sesiface" +) + +const ( + // CloudWatch Logs limits: 256 KB per event (26 bytes overhead), 1 MB and 10,000 events per batch. + cwMaxEventBytes = 256*1024 - 26 + cwMaxBatchBytes = 1024*1024 - 26*10000 + cwMaxBatchCount = 10000 + cwEventOverhead = 26 +) + +// LogShipper copies the run log into a CloudWatch Logs stream (group and stream are created on demand). +type LogShipper struct { + Client cloudwatchlogsiface.CloudWatchLogsAPI + Group string + Stream string + Now func() time.Time +} + +// Ship writes data line by line as log events and returns the number of events written. +func (s *LogShipper) Ship(ctx context.Context, data []byte) (int, error) { + if s.Client == nil { + return 0, ErrNotConfigured + } + now := time.Now + if s.Now != nil { + now = s.Now + } + if _, err := s.Client.CreateLogGroupWithContext(ctx, &cloudwatchlogs.CreateLogGroupInput{LogGroupName: aws.String(s.Group)}); err != nil && !isAlreadyExists(err) { + return 0, fmt.Errorf("create log group %s: %w", s.Group, err) + } + if _, err := s.Client.CreateLogStreamWithContext(ctx, &cloudwatchlogs.CreateLogStreamInput{LogGroupName: aws.String(s.Group), LogStreamName: aws.String(s.Stream)}); err != nil && !isAlreadyExists(err) { + return 0, fmt.Errorf("create log stream %s: %w", s.Stream, err) + } + ts := now().UnixMilli() + var events []*cloudwatchlogs.InputLogEvent + for _, line := range splitEvents(data) { + events = append(events, &cloudwatchlogs.InputLogEvent{Message: aws.String(line), Timestamp: aws.Int64(ts)}) + } + var token *string + sent := 0 + for len(events) > 0 { + n, size := 0, 0 + for n < len(events) && n < cwMaxBatchCount { + size += len(*events[n].Message) + cwEventOverhead + if size > cwMaxBatchBytes && n > 0 { + break + } + n++ + } + out, err := s.Client.PutLogEventsWithContext(ctx, &cloudwatchlogs.PutLogEventsInput{ + LogGroupName: aws.String(s.Group), LogStreamName: aws.String(s.Stream), LogEvents: events[:n], SequenceToken: token, + }) + if err != nil { + return sent, fmt.Errorf("put log events: %w", err) + } + if out != nil { + token = out.NextSequenceToken + } + sent += n + events = events[n:] + } + return sent, nil +} + +// splitEvents splits the log into lines, chunking lines longer than the event limit; empty lines are kept +// as a single space so line numbering matches run.log. +func splitEvents(data []byte) []string { + var out []string + for _, line := range bytes.Split(bytes.TrimRight(data, "\n"), []byte("\n")) { + if len(line) == 0 { + out = append(out, " ") + continue + } + for len(line) > cwMaxEventBytes { + out = append(out, string(line[:cwMaxEventBytes])) + line = line[cwMaxEventBytes:] + } + out = append(out, string(line)) + } + return out +} + +func isAlreadyExists(err error) bool { + var aerr awserr.Error + return errors.As(err, &aerr) && aerr.Code() == cloudwatchlogs.ErrCodeResourceAlreadyExistsException +} + +// Mailer sends the report through SES SendRawEmail. +type Mailer struct { + Client sesiface.SESAPI +} + +// Send delivers a raw MIME message and returns the SES message id. +func (m Mailer) Send(ctx context.Context, from string, to []string, raw []byte) (string, error) { + if m.Client == nil { + return "", ErrNotConfigured + } + if len(to) == 0 { + return "", errors.New("no recipients") + } + if len(raw) > MaxRawEmailBytes { + return "", fmt.Errorf("message is %d bytes, SES limit is %d", len(raw), MaxRawEmailBytes) + } + out, err := m.Client.SendRawEmailWithContext(ctx, &ses.SendRawEmailInput{ + Source: aws.String(from), + Destinations: aws.StringSlice(to), + RawMessage: &ses.RawMessage{Data: raw}, + }) + if err != nil { + return "", err + } + return aws.StringValue(out.MessageId), nil +} diff --git a/cla-backend-go/orgimport/decisions.go b/cla-backend-go/orgimport/decisions.go new file mode 100644 index 000000000..c5d38e40c --- /dev/null +++ b/cla-backend-go/orgimport/decisions.go @@ -0,0 +1,357 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bufio" + "encoding/csv" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" +) + +// Decision kinds recorded by the duplicate review (lfx-self-serve #3085). +const ( + DecisionCollapse = "collapse" + DecisionDistinct = "distinct" +) + +// Decision is one reviewed group: collapse (all old ids land on TargetSFID) or distinct (separate Accounts). +type Decision struct { + Kind string + OldIDs []string + Target string + Reviewer string + Note string +} + +// Decisions indexes the review file by old id. +type Decisions struct { + ByOldID map[string]*Decision + List []*Decision +} + +// LoadDecisions reads and validates the decisions CSV (header required). +func LoadDecisions(path string) (*Decisions, error) { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + return ParseDecisions(f) +} + +// ParseDecisions parses `decision,old_ids,target_sfid,reviewer,note` rows; old_ids are separated by `;` or spaces. +func ParseDecisions(r io.Reader) (*Decisions, error) { + cr := csv.NewReader(r) + cr.TrimLeadingSpace = true + cr.FieldsPerRecord = -1 + records, err := cr.ReadAll() + if err != nil { + return nil, fmt.Errorf("decisions: %w", err) + } + if len(records) == 0 { + return nil, fmt.Errorf("decisions: empty file") + } + col := map[string]int{} + for i, h := range records[0] { + col[strings.ToLower(strings.TrimSpace(h))] = i + } + for _, name := range []string{"decision", "old_ids", "target_sfid", "reviewer"} { + if _, ok := col[name]; !ok { + return nil, fmt.Errorf("decisions: missing column %q (need decision,old_ids,target_sfid,reviewer[,note])", name) + } + } + field := func(rec []string, name string) string { + i, ok := col[name] + if !ok || i >= len(rec) { + return "" + } + return strings.TrimSpace(rec[i]) + } + d := &Decisions{ByOldID: map[string]*Decision{}} + for n, rec := range records[1:] { + line := n + 2 + if len(rec) == 0 || (len(rec) == 1 && strings.TrimSpace(rec[0]) == "") { + continue + } + dec := &Decision{Kind: strings.ToLower(field(rec, "decision")), Target: field(rec, "target_sfid"), Reviewer: field(rec, "reviewer"), Note: field(rec, "note")} + dec.OldIDs = append(dec.OldIDs, strings.FieldsFunc(field(rec, "old_ids"), func(r rune) bool { return r == ';' || r == ' ' || r == '\t' })...) + if dec.Reviewer == "" { + return nil, fmt.Errorf("decisions line %d: reviewer is required", line) + } + if len(dec.OldIDs) == 0 { + return nil, fmt.Errorf("decisions line %d: old_ids is empty", line) + } + switch dec.Kind { + case DecisionCollapse: + if !IsSFID(dec.Target) { + return nil, fmt.Errorf("decisions line %d: collapse needs a Salesforce account id in target_sfid", line) + } + case DecisionDistinct: + if dec.Target != "" { + return nil, fmt.Errorf("decisions line %d: distinct must not set target_sfid", line) + } + if len(dec.OldIDs) < 2 { + return nil, fmt.Errorf("decisions line %d: distinct needs at least two old ids", line) + } + default: + return nil, fmt.Errorf("decisions line %d: decision must be collapse|distinct", line) + } + for _, id := range dec.OldIDs { + if prev, dup := d.ByOldID[id]; dup { + return nil, fmt.Errorf("decisions line %d: %s already appears in a %s decision", line, id, prev.Kind) + } + d.ByOldID[id] = dec + } + d.List = append(d.List, dec) + } + return d, nil +} + +// Collapse reports whether a recorded collapse decision lets oldID land on target. +func (d *Decisions) Collapse(oldID, target string) *Decision { + if d == nil { + return nil + } + dec := d.ByOldID[oldID] + if dec != nil && dec.Kind == DecisionCollapse && accountKey(dec.Target) == accountKey(target) { + return dec + } + return nil +} + +// Distinct reports whether a recorded distinct decision separates oldID from any id in others. +func (d *Decisions) Distinct(oldID string, others []string) *Decision { + if d == nil { + return nil + } + dec := d.ByOldID[oldID] + if dec == nil || dec.Kind != DecisionDistinct { + return nil + } + for _, o := range others { + if o != oldID && d.ByOldID[o] == dec { + return dec + } + } + return nil +} + +// Target groups the plan by destination Account (dry-run report of design §5 step 3). +type Target struct { + SFID string + Groups []*Group + Existing []*Row + Mapped []string + Decision *Decision + Status string +} + +// OldIDs lists the old ids landing on the target: tranche groups, rows already carrying it and +// mapping rows outside the tranche. +func (t *Target) OldIDs() []string { + seen := map[string]bool{} + var ids []string + add := func(id string) { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + for _, g := range t.Groups { + add(g.Key) + } + for _, r := range t.Existing { + add(r.ExternalID) + } + for _, id := range t.Mapped { + add(id) + } + return ids +} + +// forms lists the exact id strings (15- or 18-char) that would end up on the target. +func (t *Target) forms(mapping *Mapping) []string { + var forms []string + add := func(id string) { + if id != "" && !containsString(forms, id) { + forms = append(forms, id) + } + } + for _, g := range t.Groups { + add(g.NewID) + } + for _, r := range t.Existing { + add(r.ExternalID) + } + if mapping != nil { + for _, id := range t.Mapped { + add(mapping.Rows[id].NewID) + } + } + return forms +} + +// applyDecisions is the approval-aware collision pass: a rewrite group may share its destination with +// other groups, other mapping rows or rows already carrying the id only under a recorded collapse decision. +// Destinations are compared per Account (15- and 18-char forms of one id are the same target). +func applyDecisions(groups []*Group, inv *Inventory, mapping *Mapping, decisions *Decisions) []*Target { + byTarget := map[string]*Target{} + for _, g := range groups { + if g.Route != RouteRewrite || g.NewID == "" || g.Err != nil { + continue + } + t := byTarget[accountKey(g.NewID)] + if t == nil { + t = &Target{SFID: g.NewID} + t.Existing = append(t.Existing, inv.rowsCarrying(g.NewID)...) + byTarget[accountKey(g.NewID)] = t + } + t.Groups = append(t.Groups, g) + } + targets := make([]*Target, 0, len(byTarget)) + for _, t := range byTarget { + targets = append(targets, t) + } + sort.Slice(targets, func(i, j int) bool { return targets[i].SFID < targets[j].SFID }) + for _, t := range targets { + t.Status = statusOK + if mapping != nil { + seen := t.OldIDs() + for oldID, row := range mapping.Rows { + if row.NewID != "" && accountKey(row.NewID) == accountKey(t.SFID) && !containsString(seen, oldID) { + t.Mapped = append(t.Mapped, oldID) + } + } + sort.Strings(t.Mapped) + } + ids := t.OldIDs() + if len(ids) < 2 { + continue + } + if forms := t.forms(mapping); len(forms) > 1 { + // the tool never writes a second form of one Account id: the mapping (or the existing rows) must agree first + for _, g := range t.Groups { + if !g.Replayed { + g.Route, g.ManualReason = RouteManual, ReasonTargetFormsDiffer + } + } + t.Status = ReasonTargetFormsDiffer + continue + } + for _, g := range t.Groups { + if g.Replayed { + continue + } + if dec := decisions.Distinct(g.Key, ids); dec != nil { + g.Route, g.ManualReason = RouteManual, ReasonDistinctConflict + t.Decision, t.Status = dec, ReasonDistinctConflict + continue + } + dec := decisions.Collapse(g.Key, t.SFID) + if dec == nil { + g.Route, g.ManualReason = RouteManual, "target_collision" + if t.Status == statusOK { + t.Status = "needs_decision" + } + continue + } + g.Decision, t.Decision = dec, dec + } + } + return targets +} + +func containsString(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// SharedDomains is the list of domains that never match an Account automatically (design §4). +type SharedDomains map[string]bool + +// DefaultSharedDomains is the built-in list; the #3085 review file replaces it (--shared-domains). +var DefaultSharedDomains = []string{ + "github.com", "nowebsite.com", + "gmail.com", "googlemail.com", "yahoo.com", "hotmail.com", "outlook.com", "live.com", "icloud.com", "protonmail.com", "qq.com", "163.com", +} + +// LoadSharedDomains reads one domain per line (`#` comments); an empty path yields the default list. +func LoadSharedDomains(path string) (SharedDomains, error) { + set := SharedDomains{} + if path == "" { + for _, d := range DefaultSharedDomains { + set[d] = true + } + return set, nil + } + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if i := strings.IndexByte(line, '#'); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + continue + } + set[normalizeDomain(line)] = true + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("shared domains: %w", err) + } + return set, nil +} + +// Domain extracts the registrable host of a website value (scheme, path and www. stripped). +func Domain(website string) string { + website = strings.TrimSpace(strings.ToLower(website)) + if website == "" { + return "" + } + if !strings.Contains(website, "://") { + website = "http://" + website + } + u, err := url.Parse(website) + if err != nil || u.Hostname() == "" { + return normalizeDomain(website) + } + return normalizeDomain(u.Hostname()) +} + +func normalizeDomain(host string) string { + host = strings.TrimSpace(strings.ToLower(host)) + host = strings.TrimPrefix(host, "http://") + host = strings.TrimPrefix(host, "https://") + if i := strings.IndexAny(host, "/:?"); i >= 0 { + host = host[:i] + } + host = strings.TrimSuffix(host, ".") + return strings.TrimPrefix(host, "www.") +} + +// Shared reports whether website has no domain or a listed shared domain; the reason is the manual reason. +func (s SharedDomains) Shared(website string) (string, string) { + d := Domain(website) + switch { + case d == "": + return "", "missing_website" + case s[d]: + return d, ReasonSharedDomain + } + return d, "" +} diff --git a/cla-backend-go/orgimport/decisions_test.go b/cla-backend-go/orgimport/decisions_test.go new file mode 100644 index 000000000..bb4a0648a --- /dev/null +++ b/cla-backend-go/orgimport/decisions_test.go @@ -0,0 +1,369 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeDecisions(t *testing.T, dir string, rows ...string) string { + t.Helper() + path := filepath.Join(dir, "decisions.csv") + require.NoError(t, os.WriteFile(path, []byte("decision,old_ids,target_sfid,reviewer,note\n"+strings.Join(rows, "\n")+"\n"), 0o600)) + return path +} + +func TestParseDecisions(t *testing.T) { + d, err := ParseDecisions(strings.NewReader("decision,old_ids,target_sfid,reviewer,note\n" + + "collapse,lf-a; lf-b," + targetSFID + ",michal,same legal entity\n" + + "DISTINCT,lf-c;lf-d,,michal,\n")) + require.NoError(t, err) + require.Len(t, d.List, 2) + assert.Equal(t, []string{"lf-a", "lf-b"}, d.List[0].OldIDs) + assert.NotNil(t, d.Collapse("lf-a", targetSFID)) + assert.Nil(t, d.Collapse("lf-a", targetSFID2), "collapse is bound to its target") + assert.Nil(t, d.Collapse("lf-c", targetSFID)) + assert.NotNil(t, d.Distinct("lf-c", []string{"lf-d", "lf-x"})) + assert.Nil(t, d.Distinct("lf-c", []string{"lf-a"}), "distinct only separates ids of the same decision") + assert.Nil(t, d.Distinct("lf-a", []string{"lf-b"})) + assert.Nil(t, (*Decisions)(nil).Collapse("lf-a", targetSFID)) + assert.Nil(t, (*Decisions)(nil).Distinct("lf-a", nil)) + spaced, err := ParseDecisions(strings.NewReader("decision,old_ids,target_sfid,reviewer\ncollapse,lf-a lf-b\t" + targetSFID + "," + targetSFID + ",michal\n")) + require.NoError(t, err) + assert.Equal(t, []string{"lf-a", "lf-b", targetSFID}, spaced.List[0].OldIDs, "old ids may be separated by spaces (workflow input rows use '|')") + + for _, bad := range []string{ + "", + "decision,old_ids,target_sfid\ncollapse,lf-a," + targetSFID, + "decision,old_ids,target_sfid,reviewer\ncollapse,lf-a," + targetSFID + ",", + "decision,old_ids,target_sfid,reviewer\ncollapse,lf-a,not-an-sfid,michal", + "decision,old_ids,target_sfid,reviewer\ncollapse,," + targetSFID + ",michal", + "decision,old_ids,target_sfid,reviewer\ndistinct,lf-a,,michal", + "decision,old_ids,target_sfid,reviewer\ndistinct,lf-a;lf-b," + targetSFID + ",michal", + "decision,old_ids,target_sfid,reviewer\nmerge,lf-a;lf-b," + targetSFID + ",michal", + "decision,old_ids,target_sfid,reviewer\ncollapse,lf-a," + targetSFID + ",michal\ndistinct,lf-a;lf-b,,michal", + } { + _, err = ParseDecisions(strings.NewReader(bad)) + assert.Error(t, err, bad) + } + _, err = LoadDecisions(filepath.Join(t.TempDir(), "missing.csv")) + assert.Error(t, err) +} + +func TestSharedDomains(t *testing.T) { + def, err := LoadSharedDomains("") + require.NoError(t, err) + assert.True(t, def["github.com"]) + assert.True(t, def["nowebsite.com"]) + assert.Equal(t, "legacy.example", Domain("https://www.legacy.example/about?x=1")) + assert.Equal(t, "legacy.example", Domain("Legacy.Example")) + assert.Equal(t, "github.com", Domain("github.com/some-org")) + assert.Equal(t, "", Domain(" ")) + d, reason := def.Shared("https://github.com/acme") + assert.Equal(t, "github.com", d) + assert.Equal(t, "shared_domain", reason) + _, reason = def.Shared("") + assert.Equal(t, "missing_website", reason) + d, reason = def.Shared("https://acme.example") + assert.Equal(t, "acme.example", d) + assert.Empty(t, reason) + + path := filepath.Join(t.TempDir(), "shared.txt") + require.NoError(t, os.WriteFile(path, []byte("# review list\nWWW.Example.ORG # comment\nhttps://foo.test/\n\n"), 0o600)) + custom, err := LoadSharedDomains(path) + require.NoError(t, err) + assert.Equal(t, SharedDomains{"example.org": true, "foo.test": true}, custom) + assert.False(t, custom["github.com"], "a review list replaces the built-in list") + _, err = LoadSharedDomains(filepath.Join(t.TempDir(), "missing")) + assert.Error(t, err) +} + +func collisionFixture(t *testing.T) (*fixture, string) { + t.Helper() + fx := newFixture() + fx.company("c-a", "Acme Inc", "", "lf-a", "cg-1") + fx.company("c-b", "Acme GmbH", "", "lf-b", "cg-1") + fx.company("c-c", "Acme Labs", "", "lf-c", "cg-2") + fx.platform.sfAccounts[targetSFID] = true + fx.platform.orgs[targetSFID] = &Org{ID: targetSFID, Name: "Acme"} + fx.platform.orgs["lf-a"] = &Org{ID: "lf-a", Name: "Acme Inc", Website: "https://acme.example"} + fx.platform.orgs["lf-b"] = &Org{ID: "lf-b", Name: "Acme GmbH", Website: "https://acme.example/de"} + fx.platform.orgs["lf-c"] = &Org{ID: "lf-c", Name: "Acme Labs", Website: "https://labs.acme.example"} + dir := t.TempDir() + return fx, dir +} + +func TestCollisionNeedsDecision(t *testing.T) { + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + opts := Options{Stage: "dev", Mapping: mapping, OutDir: filepath.Join(dir, "out")} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteManual, g.Route, id) + assert.Equal(t, "target_collision", g.ManualReason, id) + assert.Equal(t, targetSFID, g.NewID, id, "the resolved target stays on the group for the report") + } + c := groupByKey(plan.Groups, "lf-c") + assert.Equal(t, RouteRewrite, c.Route) + assert.Equal(t, targetSFID2, c.NewID) + require.Len(t, plan.Targets, 2) + tgt := targetByID(plan.Targets, targetSFID) + assert.Equal(t, "needs_decision", tgt.Status) + assert.Equal(t, []string{"lf-a", "lf-b"}, tgt.OldIDs()) + assert.Equal(t, "ok", targetByID(plan.Targets, targetSFID2).Status) + assert.Empty(t, plan.Rewrite[0].ManualReason) + require.Len(t, plan.Rewrite, 1) + + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 2, sum.Manual) + targets := readCSV(t, filepath.Join(dir, "out", "targets.csv")) + require.Len(t, targets, 3) + assert.Equal(t, []string{targetSFID, "2", "lf-a;lf-b", "c-a;c-b", "Acme Inc;Acme GmbH", "0", "", "", "needs_decision"}, targets[1]) + assert.Equal(t, targetSFID2, targets[2][0]) + assert.Equal(t, "ok", targets[2][8]) +} + +func TestCollisionCollapseDecision(t *testing.T) { + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + decisions := writeDecisions(t, dir, "collapse,lf-a;lf-b,"+targetSFID+",michal,one legal entity") + opts := Options{Stage: "dev", Mapping: mapping, Decisions: decisions, OutDir: filepath.Join(dir, "out")} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + require.Len(t, plan.Rewrite, 3) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteRewrite, g.Route, id) + assert.Empty(t, g.ManualReason, id) + assert.Equal(t, targetSFID, g.NewID, id) + require.NotNil(t, g.Decision, id) + assert.Equal(t, "michal", g.Decision.Reviewer) + } + tgt := targetByID(plan.Targets, targetSFID) + assert.Equal(t, "ok", tgt.Status) + assert.Equal(t, DecisionCollapse, tgt.Decision.Kind) + + // a collapse recorded for another target does not approve this one + other := writeDecisions(t, dir, "collapse,lf-a;lf-b,"+targetSFID2+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: other}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-b").ManualReason) + + // a collapse covering only one of the ids approves only that id + partial := writeDecisions(t, dir, "collapse,lf-a,"+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: partial}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-b").ManualReason) + assert.Equal(t, "needs_decision", targetByID(plan.Targets, targetSFID).Status) + + // dry run of the approved collapse writes the decision into the plan report + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 1, sum.Manual) + rows := readCSV(t, filepath.Join(dir, "out", "plan.csv")) + byKey := map[string][]string{} + for _, r := range rows[1:] { + byKey[r[0]] = r + } + assert.Equal(t, "collapse", byKey["lf-a"][12]) + assert.Equal(t, "michal", byKey["lf-a"][13]) + assert.Equal(t, "acme.example", byKey["lf-a"][8]) + assert.Equal(t, "false", byKey["lf-a"][9]) + var buf bytes.Buffer + plan.Print(&buf) + assert.Contains(t, buf.String(), targetSFID+" status=needs_decision old_ids=lf-a;lf-b") +} + +func TestCollisionDistinctDecision(t *testing.T) { + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true") + decisions := writeDecisions(t, dir, "distinct,lf-a;lf-b,,michal,separate entities") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteManual, g.Route, id) + assert.Equal(t, "distinct_conflict", g.ManualReason, id) + } + assert.Equal(t, "distinct_conflict", targetByID(plan.Targets, targetSFID).Status) + assert.Contains(t, Suggest("distinct_conflict"), "distinct decision") + + // distinct ids landing on different accounts are fine + mapping = writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID2+",created,true") + fx.platform.orgs[targetSFID2] = &Org{ID: targetSFID2} + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-b").Route) +} + +func TestCollisionWithExistingRowsAndMappingCoTargets(t *testing.T) { + fx, dir := collisionFixture(t) + fx.company("c-existing", "Acme Holdings", "", targetSFID) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + a := groupByKey(plan.Groups, "lf-a") + assert.Equal(t, "target_collision", a.ManualReason, "rows already carrying the target need a collapse decision") + tgt := targetByID(plan.Targets, targetSFID) + assert.Equal(t, []string{"lf-a", targetSFID}, tgt.OldIDs()) + require.Len(t, tgt.Existing, 1) + + decisions := writeDecisions(t, dir, "collapse,lf-a;"+targetSFID+","+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + + // a mapping row outside the tranche landing on the same target is a co-target too + fx, dir = collisionFixture(t) + mapping = writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-z,"+targetSFID+",created,true") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, []string{"lf-a", "lf-z"}, targetByID(plan.Targets, targetSFID).OldIDs()) + decisions = writeDecisions(t, dir, "collapse,lf-a;lf-z,"+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) +} + +func TestReplayedGroupSkipsDecisionPass(t *testing.T) { + fx, mapping, statePath := rewriteFixture(t) + fx.company("c-existing", "Legacy Holdings", "", targetSFID) + require.NoError(t, os.WriteFile(statePath, []byte(`{"ts":"2026-09-29T11:00:00Z","old_id":"`+lfID+`","new_id":"`+targetSFID+`","company_ids":["c-parent","c-sub"],"step":"rows","status":"ok"}`+"\n"), 0o600)) + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, State: statePath}) + require.NoError(t, err) + g := groupByKey(plan.Groups, lfID) + require.NotNil(t, g) + assert.True(t, g.Replayed) + assert.Equal(t, RouteRewrite, g.Route, "an unfinished rewrite resumes regardless of collisions") + assert.Empty(t, g.ManualReason) +} + +const legacyGitHubSite = "https://github.com/legacy-ltd" + +func TestApexSharedDomainGate(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":%q,"action":%q}`, targetSFID, ActionCreated) + })) + defer srv.Close() + apex, err := NewApexClient(srv.URL, "apex-token") + require.NoError(t, err) + + fx, mapping, _ := rewriteFixture(t) + fx.platform.orgs[lfID].Website = legacyGitHubSite + deps := fx.deps() + deps.Apex = apex + dir := t.TempDir() + opts := Options{Stage: "dev", UseApex: true, OutDir: dir} + plan, err := BuildPlan(context.Background(), deps, opts) + require.NoError(t, err) + g := groupByKey(plan.Groups, lfID) + assert.Equal(t, RouteManual, g.Route) + assert.Equal(t, "shared_domain", g.ManualReason) + assert.Equal(t, 0, calls, "shared domains never reach Apex") + _, err = Execute(context.Background(), deps, opts, plan) + require.NoError(t, err) + rows := readCSV(t, filepath.Join(dir, "plan.csv")) + var lfRow []string + for _, r := range rows[1:] { + if r[0] == lfID { + lfRow = r + } + } + require.NotNil(t, lfRow) + assert.Equal(t, legacyGitHubSite, lfRow[7]) + assert.Equal(t, "github.com", lfRow[8]) + assert.Equal(t, "true", lfRow[9]) + manual := readCSV(t, filepath.Join(dir, "manual_actions.csv")) + require.Len(t, manual, 2) + assert.Equal(t, "shared_domain", manual[1][3]) + assert.Contains(t, manual[1][4], "never matched automatically") + + fx.platform.orgs[lfID].Website = "" + plan, err = BuildPlan(context.Background(), deps, Options{UseApex: true}) + require.NoError(t, err) + assert.Equal(t, "missing_website", groupByKey(plan.Groups, lfID).ManualReason) + assert.Equal(t, 0, calls) + + // an approved mapping row is the operator's resolution and bypasses the gate and Apex + plan, err = BuildPlan(context.Background(), deps, Options{UseApex: true, Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, lfID).Route) + assert.Equal(t, targetSFID, groupByKey(plan.Groups, lfID).NewID) + assert.Equal(t, 0, calls) + + // a review list replaces the built-in list + fx.platform.orgs[lfID].Website = legacyGitHubSite + shared := filepath.Join(dir, "shared.txt") + require.NoError(t, os.WriteFile(shared, []byte("nowebsite.com\n"), 0o600)) + plan, err = BuildPlan(context.Background(), deps, Options{UseApex: true, SharedDomains: shared}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, lfID).Route) + assert.Equal(t, 1, calls) + _, err = BuildPlan(context.Background(), deps, Options{UseApex: true, SharedDomains: filepath.Join(dir, "missing")}) + assert.Error(t, err) + _, err = BuildPlan(context.Background(), deps, Options{Decisions: filepath.Join(dir, "missing")}) + assert.Error(t, err) +} + +func TestManualActionsAndSuggestions(t *testing.T) { + fx := newFixture() + fx.company("c-empty", "Empty Inc", "", "", "cg-1") + fx.company("c-lf", "Legacy Ltd", "", lfID, "cg-1") + fx.company("c-dead", "Dead Corp", "", deadSFID, "cg-1") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev"}) + require.NoError(t, err) + actions := plan.ManualActions() + require.Len(t, actions, 3) + reasons := map[string]string{} + for _, a := range actions { + reasons[a.Group.Key] = a.Reason + assert.NotEmpty(t, a.Suggested) + assert.Contains(t, a.String(), a.Reason) + } + assert.Equal(t, map[string]string{"c-empty": "empty_external_id", lfID: "no_mapping", deadSFID: "no_mapping"}, reasons) + assert.Contains(t, Suggest("empty_external_id"), "m3-org-cleanup.md") + assert.Contains(t, Suggest("error: boom"), "re-run") + assert.Contains(t, SuggestError(errors.New("liveness check failed for 001: member-service: authorization failed (403): Client \"x\" is not authorized to access resource server \"y\". You need to create a \"client-grant\" associated to this API.")), "client grant") + assert.Contains(t, SuggestError(errors.New("member-service: authorization failed (403): forbidden")), "Heimdall") + assert.Contains(t, SuggestError(ErrNotConfigured), "cla-member-service-base-url") + assert.Contains(t, SuggestError(errors.New("boom")), "re-run") + assert.Contains(t, Suggest("something_new"), "run.log") + + // a failed group is listed with its error + fx.platform.failGetB2B = true + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev"}) + require.NoError(t, err) + var failed *ManualAction + for _, a := range plan.ManualActions() { + if a.Group.Key == deadSFID { + cp := a + failed = &cp + } + } + require.NotNil(t, failed) + assert.Equal(t, "error", failed.Reason) + assert.Contains(t, failed.String(), "liveness check failed") +} diff --git a/cla-backend-go/orgimport/eligible.go b/cla-backend-go/orgimport/eligible.go new file mode 100644 index 000000000..164f72ca0 --- /dev/null +++ b/cla-backend-go/orgimport/eligible.go @@ -0,0 +1,228 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/linuxfoundation/easycla/cla-backend-go/utils" +) + +const sfidPrefix = "001" + +// ShapeOf classifies an external id: 15/18-char alphanumeric starting with 001 → SFID, lf… → lf. +func ShapeOf(id string) IDShape { + id = strings.TrimSpace(id) + switch { + case id == "": + return ShapeEmpty + case IsSFID(id): + return ShapeSFID + case strings.HasPrefix(id, "lf"): + return ShapeLF + default: + return ShapeOther + } +} + +// IsSFID reports whether id looks like a Salesforce Account ID. +func IsSFID(id string) bool { + if len(id) != 15 && len(id) != 18 || !strings.HasPrefix(id, sfidPrefix) { + return false + } + for _, c := range id { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +func canonicalEntity(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +// accountKey identifies one Salesforce Account regardless of the 15- or 18-char form of its id +// (the last three characters of an 18-char id are a checksum of the first fifteen). +func accountKey(id string) string { + if IsSFID(id) { + return id[:15] + } + return id +} + +// Inventory is the companies table joined with the active-CCLA set. +type Inventory struct { + Rows []*Row + ByID map[string]*Row + ByExternal map[string][]*Row + ByAccount map[string][]*Row // Salesforce-shaped external ids keyed by accountKey +} + +// LoadInventory scans companies and active (signed+approved, company-referenced) CCLAs. +func LoadInventory(ctx context.Context, deps Deps) (*Inventory, error) { + cclas, err := deps.Signatures.GetCCLASignatures(ctx, aws.Bool(true), aws.Bool(true)) + if err != nil { + return nil, fmt.Errorf("listing active CCLAs: %w", err) + } + active := map[string][]string{} + signedOn := map[string]string{} + for _, s := range cclas { + if s == nil || s.SignatureReferenceID == "" || s.SignatureReferenceType != utils.SignatureReferenceTypeCompany || + s.SignatureType != utils.SignatureTypeCCLA || !s.SignatureSigned || !s.SignatureApproved { + continue + } + active[s.SignatureReferenceID] = append(active[s.SignatureReferenceID], s.SignatureProjectID) + if d := firstNonEmpty(s.SignedOn, s.DateCreated); d != "" && (signedOn[s.SignatureReferenceID] == "" || d < signedOn[s.SignatureReferenceID]) { + signedOn[s.SignatureReferenceID] = d + } + } + companies, err := deps.Companies.GetCompanies(ctx) + if err != nil { + return nil, fmt.Errorf("scanning companies: %w", err) + } + inv := &Inventory{ByID: map[string]*Row{}, ByExternal: map[string][]*Row{}, ByAccount: map[string][]*Row{}} + for i := range companies.Companies { + c := companies.Companies[i] + row := &Row{ + CompanyID: c.CompanyID, + CompanyName: c.CompanyName, + SigningEntityName: c.SigningEntityName, + ExternalID: strings.TrimSpace(c.CompanyExternalID), + RawExternalID: c.CompanyExternalID, + } + if groups, ok := active[c.CompanyID]; ok { + row.ActiveCCLA = true + row.CCLACount = len(groups) + row.CLAGroupIDs = distinct(groups) + row.CCLASignedOn = signedOn[c.CompanyID] + } + inv.Rows = append(inv.Rows, row) + inv.ByID[row.CompanyID] = row + if row.ExternalID != "" { + inv.ByExternal[row.ExternalID] = append(inv.ByExternal[row.ExternalID], row) + if IsSFID(row.ExternalID) { + inv.ByAccount[accountKey(row.ExternalID)] = append(inv.ByAccount[accountKey(row.ExternalID)], row) + } + } + } + sort.Slice(inv.Rows, func(i, j int) bool { return inv.Rows[i].CompanyID < inv.Rows[j].CompanyID }) + return inv, nil +} + +// EligibleGroups returns one group per external id that has at least one active-CCLA row (all +// sibling rows included); rows with an empty or malformed external id form single-row groups keyed +// by company id (a blank or garbage value never identifies one organization). +func (inv *Inventory) EligibleGroups() []*Group { + var groups []*Group + seen := map[string]bool{} + for _, row := range inv.Rows { + if !row.ActiveCCLA { + continue + } + if shape := ShapeOf(row.ExternalID); shape == ShapeEmpty || shape == ShapeOther { + groups = append(groups, &Group{OldID: row.RawExternalID, Key: row.CompanyID, Shape: shape, Rows: []*Row{row}}) + continue + } + if seen[row.ExternalID] { + continue + } + seen[row.ExternalID] = true + rows := append([]*Row(nil), inv.ByExternal[row.ExternalID]...) + sort.Slice(rows, func(i, j int) bool { return rows[i].CompanyID < rows[j].CompanyID }) + g := &Group{OldID: row.ExternalID, Key: row.ExternalID, Shape: ShapeOf(row.ExternalID), Rows: rows} + g.Duplicate = hasDuplicateEntity(rows) + g.Aliases = inv.aliasForms(row.ExternalID) + groups = append(groups, g) + } + sort.Slice(groups, func(i, j int) bool { return groups[i].Key < groups[j].Key }) + return groups +} + +// aliasForms lists the other exact forms (15- vs 18-char) of the same Account carried by inventory rows. +func (inv *Inventory) aliasForms(externalID string) []string { + if !IsSFID(externalID) { + return nil + } + var forms []string + for _, r := range inv.ByAccount[accountKey(externalID)] { + if r.ExternalID != externalID && !containsString(forms, r.ExternalID) { + forms = append(forms, r.ExternalID) + } + } + sort.Strings(forms) + return forms +} + +// hasDuplicateEntity is true when two rows of one external id share the same (or empty) entity name; +// a signing entity equal to the company name is the parent row (the console writes it that way). +func hasDuplicateEntity(rows []*Row) bool { + seen := map[string]bool{} + for _, r := range rows { + k := canonicalEntity(r.SigningEntityName) + if k == canonicalEntity(r.CompanyName) { + k = "" + } + if seen[k] { + return true + } + seen[k] = true + } + return false +} + +// rowsCarrying returns the inventory rows whose external id is newID (in either 15- or 18-char form). +func (inv *Inventory) rowsCarrying(newID string) []*Row { + if IsSFID(newID) { + return inv.ByAccount[accountKey(newID)] + } + return inv.ByExternal[newID] +} + +// ExternalIDCollision reports whether a row outside the group already carries newID. +func (inv *Inventory) ExternalIDCollision(g *Group, newID string) bool { + for _, r := range inv.rowsCarrying(newID) { + if !containsRow(g.Rows, r.CompanyID) { + return true + } + } + return false +} + +func containsRow(rows []*Row, companyID string) bool { + for _, r := range rows { + if r.CompanyID == companyID { + return true + } + } + return false +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func distinct(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + } + sort.Strings(out) + return out +} diff --git a/cla-backend-go/orgimport/manual.go b/cla-backend-go/orgimport/manual.go new file mode 100644 index 000000000..1a4e86515 --- /dev/null +++ b/cla-backend-go/orgimport/manual.go @@ -0,0 +1,99 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "fmt" + "strings" +) + +// ManualAction is one row of manual_actions.csv: a group the tool will not process on its own. +type ManualAction struct { + Group *Group + Reason string + Suggested string +} + +// suggestions maps a manual/pending reason to what the operator should do (README §7, m3-org-cleanup.md). +var suggestions = map[string]string{ + "empty_external_id": "Row has no company_external_id: find or create the Account with sales ops (m3-org-cleanup.md, #2749), then add a mapping row company_id,new_id,matched,true and re-run.", + "invalid_id_shape": "company_external_id is neither a Salesforce id nor lf…: find the Account with sales ops (m3-org-cleanup.md, #2749), then add a mapping row company_id,new_id,matched,true and re-run.", + "no_mapping": "Rewrite candidate without a target: add a mapping row old_id,new_id,action,approved (data in to_salesforce.csv) or run with --use-apex.", + "dead_account": "Salesforce account no longer exists: treat as rewrite — add a mapping row (data in to_salesforce.csv) or run with --use-apex.", + "mapping_ambiguous": "Several Accounts share the domain: pick the right Account with sales ops and set action=matched,approved=true in the mapping.", + "mapping_not_approved": "Matched Account is not approved: confirm it is the same legal entity, then set approved=true.", + "apex_match_needs_approval": "Apex matched an existing Account: approve it with a mapping row old_id,new_id,matched,true (or reject with a different target).", + "mapping_same_id": "new_id equals old_id (or is the 15/18-char form of the same Account): fix the mapping row.", + "target_collision": "Several old ids land on this Account (or it already has EasyCLA rows): record a collapse (one Account) or distinct (separate Accounts) decision with a reviewer in the decisions file (#3085), then re-run.", + "distinct_conflict": "A distinct decision separates these ids but they resolve to the same Account: fix the mapping/Apex target or the decision.", + ReasonTargetFormsDiffer: "Ids landing on this Account use both its 15- and 18-char form: normalize the mapping new_id (and any EasyCLA row already carrying the Account) to one form, then re-run.", + ReasonSFIDAliasForms: "Rows of one Account carry both its 15- and 18-char id: normalize company_external_id to one form (m3-org-cleanup.md, #2749) so the rows form one group, then re-run; nothing is registered or rewritten meanwhile.", + "missing_website": "Org-service has no website for the organization: automatic domain match is impossible — add a mapping row after sales ops name the Account.", + "shared_domain": "Website domain is shared (github.com, nowebsite.com, mail provider): never matched automatically — add a mapping row after sales ops name the Account.", + "apex_error": "Apex call failed: re-run; if it persists check cla-salesforce-apex-* and the endpoint.", + ReasonCRMUnverified: "Account liveness cannot be verified without member-service: set SSM cla-member-service-base-url- / cla-member-service-auth0-audience- and the Auth0 client grant + Heimdall roles (README §2), then re-run.", +} + +// Suggest returns the operator guidance for a reason. +func Suggest(reason string) string { + if s, ok := suggestions[reason]; ok { + return s + } + if strings.HasPrefix(reason, LiveError) { + return "Transient error: re-run; if it persists inspect run.log." + } + return "Review run.log for this group." +} + +// SuggestError maps a group error to an action; access errors get the ops item instead of "re-run". +func SuggestError(err error) string { + msg := err.Error() + switch { + case strings.Contains(msg, "client-grant"), strings.Contains(msg, "not authorized to access resource server"): + return "EasyCLA's Auth0 M2M client has no client grant for the member-service audience (SSM cla-member-service-auth0-audience-): ops must add it (README §2), then re-run." + case strings.Contains(msg, "(403)"), strings.Contains(msg, "(401)"): + return "member-service refused the call: the client needs auditor (GET /b2b_orgs) and global_org_admin (POST /b2b_orgs) in the member-service Heimdall ruleset (README §2), then re-run." + case strings.Contains(msg, ErrNotConfigured.Error()): + return "SSM cla-member-service-base-url- / cla-member-service-auth0-audience- are missing for this stage (README §2)." + } + return Suggest(LiveError) +} + +// ManualActions lists every group that needs a human: manual, pending and failed groups. +func (p *Plan) ManualActions() []ManualAction { + var out []ManualAction + for _, g := range p.Groups { + var reason, suggested string + switch { + case g.Err != nil: + reason = g.ManualReason + if reason == "" { + reason = LiveError + } + suggested = SuggestError(g.Err) + case g.Route == RouteManual, g.Pending(): + reason = g.ManualReason + suggested = Suggest(reason) + default: + continue + } + out = append(out, ManualAction{Group: g, Reason: reason, Suggested: suggested}) + } + return out +} + +func (a ManualAction) String() string { + g := a.Group + s := fmt.Sprintf("%s [%s] %s", g.Key, a.Reason, a.Suggested) + if g.NewID != "" { + s += " target=" + g.NewID + } + if d := g.Domain(); d != "" { + s += " domain=" + d + } + if g.Err != nil { + s += " error=" + g.Err.Error() + } + return s + " | rows=" + strings.Join(g.CompanyIDs(), ";") +} diff --git a/cla-backend-go/orgimport/mapping.go b/cla-backend-go/orgimport/mapping.go new file mode 100644 index 000000000..44b2d42ed --- /dev/null +++ b/cla-backend-go/orgimport/mapping.go @@ -0,0 +1,127 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const trueString = "true" + +// Mapping actions produced by Salesforce (mapping file or Apex). +const ( + ActionMatched = "matched" + ActionCreated = "created" + ActionAmbiguous = "ambiguous" +) + +// MappingRow is one old_id,new_id,action,approved line. +type MappingRow struct { + OldID string + NewID string + Action string + Approved bool +} + +// Mapping is the validated Salesforce mapping file; Targets counts old ids per new id (feeds the +// approval-aware collision pass). +type Mapping struct { + Rows map[string]MappingRow + Targets map[string]int +} + +// LoadMapping reads and validates a mapping CSV (header required). +func LoadMapping(path string) (*Mapping, error) { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + return ParseMapping(f) +} + +// ParseMapping parses a mapping CSV. +func ParseMapping(r io.Reader) (*Mapping, error) { + cr := csv.NewReader(r) + cr.TrimLeadingSpace = true + records, err := cr.ReadAll() + if err != nil { + return nil, fmt.Errorf("mapping: %w", err) + } + if len(records) == 0 { + return nil, fmt.Errorf("mapping: empty file") + } + header := records[0] + col := map[string]int{} + for i, h := range header { + col[strings.ToLower(strings.TrimSpace(h))] = i + } + for _, name := range []string{"old_id", "new_id", "action", "approved"} { + if _, ok := col[name]; !ok { + return nil, fmt.Errorf("mapping: missing column %q (need old_id,new_id,action,approved)", name) + } + } + m := &Mapping{Rows: map[string]MappingRow{}, Targets: map[string]int{}} + for n, rec := range records[1:] { + line := n + 2 + if len(rec) < len(header) { + return nil, fmt.Errorf("mapping line %d: expected %d columns", line, len(header)) + } + row := MappingRow{ + OldID: strings.TrimSpace(rec[col["old_id"]]), + NewID: strings.TrimSpace(rec[col["new_id"]]), + Action: strings.ToLower(strings.TrimSpace(rec[col["action"]])), + } + switch strings.ToLower(strings.TrimSpace(rec[col["approved"]])) { + case trueString, "yes", "1": + row.Approved = true + case "false", "no", "0", "": + default: + return nil, fmt.Errorf("mapping line %d: approved must be true|false", line) + } + if row.OldID == "" { + return nil, fmt.Errorf("mapping line %d: empty old_id", line) + } + if _, dup := m.Rows[row.OldID]; dup { + return nil, fmt.Errorf("mapping line %d: duplicate old_id %s", line, row.OldID) + } + switch row.Action { + case ActionMatched, ActionCreated: + if !IsSFID(row.NewID) { + return nil, fmt.Errorf("mapping line %d: new_id %q is not a Salesforce account id", line, row.NewID) + } + m.Targets[row.NewID]++ + case ActionAmbiguous: + default: + return nil, fmt.Errorf("mapping line %d: action must be matched|created|ambiguous", line) + } + m.Rows[row.OldID] = row + } + return m, nil +} + +// Resolve returns the new id for oldID or the manual reason why it cannot be used. +func (m *Mapping) Resolve(oldID string) (newID, action, reason string) { + if m == nil { + return "", "", ReasonNoMapping + } + row, ok := m.Rows[oldID] + if !ok { + return "", "", ReasonNoMapping + } + switch { + case row.Action == ActionAmbiguous: + return "", row.Action, ReasonMappingAmbiguous + case row.Action == ActionMatched && !row.Approved: + return "", row.Action, "mapping_not_approved" + case accountKey(row.NewID) == accountKey(oldID): + return "", row.Action, ReasonMappingSameID + } + return row.NewID, row.Action, "" +} diff --git a/cla-backend-go/orgimport/orgimport.go b/cla-backend-go/orgimport/orgimport.go new file mode 100644 index 000000000..a18ee2065 --- /dev/null +++ b/cla-backend-go/orgimport/orgimport.go @@ -0,0 +1,307 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +// Package orgimport implements the M3 organization import/sync tool (lfx-self-serve #2750): +// registration of EasyCLA companies with an active CCLA as B2B orgs and the rewrite of dead or +// legacy (lf…) external IDs to live Salesforce account IDs. It never creates, merges or deletes +// EasyCLA rows. +package orgimport + +import ( + "context" + "errors" + "io" + "strconv" + "strings" + "time" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + member_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/member-service" +) + +// Route is the import route of a company group. +type Route string + +// Routes. +const ( + RouteRegister Route = "register" + RouteRewrite Route = "rewrite" + RouteManual Route = "manual" +) + +// IDShape classifies a company_external_id. +type IDShape string + +// ID shapes. +const ( + ShapeSFID IDShape = "001" + ShapeLF IDShape = "lf" + ShapeEmpty IDShape = "empty" + ShapeOther IDShape = "other" +) + +// Liveness values (Group.Live), run modes and the manual/pending reasons shared by several files. +const ( + LiveLive = "live" + LiveDead = "dead" + LiveError = "error" + LiveUnverified = "unverified" + + ModeApply = "apply" + ModeDryRun = "dry-run" + + ReasonNoMapping = "no_mapping" + ReasonDeadAccount = "dead_account" + ReasonMappingAmbiguous = "mapping_ambiguous" + ReasonMappingSameID = "mapping_same_id" + ReasonSharedDomain = "shared_domain" + ReasonDistinctConflict = "distinct_conflict" + ReasonCRMUnverified = "crm_unverified" + ReasonSFIDAliasForms = "sfid_alias_forms" + ReasonTargetFormsDiffer = "target_forms_differ" + + orgStatusErr = "err" +) + +// Step names recorded in the state file. +const ( + StepStart = "start" + StepResolve = "resolve" + StepWait = "wait" + StepGrants = "copy_grants" + StepRows = "rewrite_rows" + StepEvents = "rekey_events" + StepCleanup = "delete_old_grants" + StepRegister = "register" + StepDone = "done" +) + +// Sentinel errors. +var ( + ErrOrgNotFound = errors.New("organization not found") + ErrNotConfigured = errors.New("member-service is not configured (cla-member-service-* SSM parameters missing)") + ErrApexUnavailable = errors.New("apex endpoint is not configured (cla-salesforce-apex-* SSM parameters missing)") + ErrStateRequired = errors.New("rewrite apply requires --state (the resume journal; keep it across runs)") +) + +// CompanyStore is the subset of company.IRepository the tool uses. +type CompanyStore interface { + GetCompanies(ctx context.Context) (*models.Companies, error) + GetCompanyRecord(ctx context.Context, companyID string) (*company.DBModel, error) + UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error +} + +// CCLALister lists CCLA signature rows (signatures.SignatureRepository satisfies it). +type CCLALister interface { + GetCCLASignatures(ctx context.Context, signed, approved *bool) ([]*signatures.ItemSignature, error) +} + +// ECLACounter counts employee acknowledgements referencing a company (audit only). +type ECLACounter interface { + CountECLAs(ctx context.Context, companyID string) (int, error) +} + +// EventRekeyer re-keys company-scoped events (events.RekeyRepository satisfies it). +type EventRekeyer interface { + ListEventIDsByCompanySFID(ctx context.Context, companySFID string) ([]string, error) + ListEventIDsByCompanySFIDCLAGroup(ctx context.Context, companySFID, claGroupID string) ([]string, error) + RekeyEventCompanySFID(ctx context.Context, eventID, oldSFID, newSFID string) (bool, error) +} + +// Org is the org-service view of an organization. +type Org struct { + ID string + Name string + Website string + SigningEntityNames []string +} + +// OrgService is the org-service surface the tool uses; GetOrganization returns ErrOrgNotFound on 404. +type OrgService interface { + GetOrganization(ctx context.Context, orgID string) (*Org, error) + CreateUserRoleScope(ctx context.Context, username, organizationID, objectType, objectID, roleID string) error + DeleteUserRoleScope(ctx context.Context, organizationID, roleID, grantID, username string) error +} + +// ACSService lists every grant scoped to an organization. +type ACSService interface { + ListOrgGrants(ctx context.Context, orgSFID string) ([]acs_service.OrgGrant, error) +} + +// MemberService registers B2B orgs; nil when the SSM parameters are absent. +type MemberService interface { + GetB2BOrg(ctx context.Context, uid string) (*member_service.B2BOrg, error) + RegisterB2BOrg(ctx context.Context, sfid string) (*member_service.B2BOrg, error) +} + +// ApexService is the Salesforce find-or-create contract (design §4), used only with --use-apex. +type ApexService interface { + FindOrCreate(ctx context.Context, req ApexRequest) (*ApexResult, error) +} + +// Deps are the external dependencies; Members, Events, ECLAs and Apex may be nil. +type Deps struct { + Companies CompanyStore + Signatures CCLALister + ECLAs ECLACounter + Events EventRekeyer + Orgs OrgService + ACS ACSService + Members MemberService + Apex ApexService + Out io.Writer + Now func() time.Time + Sleep func(context.Context, time.Duration) error +} + +// Options are the ingest/audit flags. +type Options struct { + Stage string + Apply bool + Tranche int + IDs []string + Mapping string + Decisions string + SharedDomains string + State string + Routes []Route + SkipWait bool + UseApex bool + OutDir string + WaitMax time.Duration + WaitPoll time.Duration +} + +// Row is one companies-table row; ExternalID is trimmed for classification, RawExternalID is the +// exact stored value pinned by the conditional rewrite. +type Row struct { + CompanyID string + CompanyName string + SigningEntityName string + ExternalID string + RawExternalID string + ActiveCCLA bool + CCLACount int + CLAGroupIDs []string + CCLASignedOn string +} + +// Group is the import unit: every row sharing one company_external_id. +type Group struct { + OldID string + Key string + Shape IDShape + Rows []*Row + Aliases []string // other 15/18-char forms of the same Account present in the inventory + Route Route + ManualReason string + Duplicate bool + NewID string + Action string + Org *Org + OrgStatus string + Live string + Decision *Decision + Replayed bool + ViaApex bool + Err error +} + +// Summary is the one-line run result. +type Summary struct { + Stage string + Mode string + Eligible int + Registered int + Rewritten int + Pending int + Manual int + Failed int +} + +func (s Summary) String() string { + return "stage=" + s.Stage + " mode=" + s.Mode + + " eligible=" + strconv.Itoa(s.Eligible) + " registered=" + strconv.Itoa(s.Registered) + + " rewritten=" + strconv.Itoa(s.Rewritten) + " pending=" + strconv.Itoa(s.Pending) + " manual=" + strconv.Itoa(s.Manual) + " failed=" + strconv.Itoa(s.Failed) +} + +// Pending is a rewrite candidate without a resolved new id (mapping row missing or account found +// dead) or a register candidate whose Account could not be verified in the CRM; a group whose +// Account is created by Apex at apply time is not pending. +func (g *Group) Pending() bool { + if g.Route == RouteRegister { + return g.ManualReason == ReasonCRMUnverified + } + return g.Route == RouteRewrite && g.NewID == "" && !g.ApexCreates() +} + +// ApexCreates reports a rewrite whose new Account does not exist yet: Apex creates it (and assigns +// the id) during apply. +func (g *Group) ApexCreates() bool { + return g.ViaApex && g.Action == ActionCreated && g.NewID == "" +} + +// RowTargeted is a single-row group whose company_external_id is empty or malformed: it is selected +// by its company id (Key) and nothing in ACS, org-service or the events table is keyed by the old value. +func (g *Group) RowTargeted() bool { + return g.Key != g.OldID +} + +// pinnedOld is the exact stored value the row's conditional rewrite pins: the raw value while it +// still carries the group's old id, otherwise the old id itself (an already rewritten replayed row +// then fails the condition and is verified by the read-back). +func (g *Group) pinnedOld(row *Row) string { + if strings.TrimSpace(row.RawExternalID) == strings.TrimSpace(g.OldID) { + return row.RawExternalID + } + return g.OldID +} + +// Website is the org-service website of the group's organization ("" when unknown). +func (g *Group) Website() string { + if g.Org == nil { + return "" + } + return strings.TrimSpace(g.Org.Website) +} + +// Domain is the normalized website domain ("" when unknown). +func (g *Group) Domain() string { + return Domain(g.Website()) +} + +// Names returns the signing entity (or company) name of each row. +func (g *Group) Names() []string { + names := make([]string, 0, len(g.Rows)) + for _, r := range g.Rows { + names = append(names, firstNonEmpty(r.SigningEntityName, r.CompanyName)) + } + return names +} + +// CompanyIDs returns the internal ids of the group's rows. +func (g *Group) CompanyIDs() []string { + ids := make([]string, 0, len(g.Rows)) + for _, r := range g.Rows { + ids = append(ids, r.CompanyID) + } + return ids +} + +// CLAGroupIDs returns the distinct CLA group ids of the group's active CCLAs. +func (g *Group) CLAGroupIDs() []string { + seen := map[string]bool{} + var out []string + for _, r := range g.Rows { + for _, id := range r.CLAGroupIDs { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + } + return out +} diff --git a/cla-backend-go/orgimport/recheck.go b/cla-backend-go/orgimport/recheck.go new file mode 100644 index 000000000..123bd0d33 --- /dev/null +++ b/cla-backend-go/orgimport/recheck.go @@ -0,0 +1,109 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "errors" + "fmt" + "strings" +) + +// recheckEvents revisits every previously completed rewrite group of the journal: the event listings +// of step 7b read eventually consistent indexes and a writer that read the company row before step 7a +// can still add events under the old id, so the old id is listed again (and, in apply mode, the +// stragglers re-keyed) on every rewrite-route run. Groups rewritten by this run are left to the next +// one (their step 7b just ran and the inventory of a replayed group is stale). Nothing is journaled, +// so the recheck repeats until it reports 0 listed. --ids and the tranche budget left after the +// planned groups bound the selection; a recorded row that is gone or no longer carries the recorded +// new id fails its recheck instead of re-keying events towards a stale destination. Returns the +// number of failed rechecks. +func (r *runner) recheckEvents(ctx context.Context, budget int) int { + if r.deps.Events == nil || r.plan.state == nil || r.plan.inv == nil { + return 0 + } + want := map[string]bool{} + for _, id := range r.opts.IDs { + want[strings.TrimSpace(id)] = true + } + planned := map[string]bool{} + for _, g := range r.plan.Rewrite { + planned[g.Key] = true + } + var recs []StateRecord + for _, rec := range r.plan.state.Last { + if rec.Step != StepDone || rec.Status != statusOK || rec.Key != "" || rec.NewID == "" || rec.OldID == rec.NewID || planned[rec.OldID] { + continue + } + if len(want) > 0 && !want[rec.OldID] && !want[rec.NewID] { + continue + } + recs = append(recs, rec) + } + if len(recs) == 0 { + return 0 + } + sortRecords(recs) + checked, listed, rekeyed, failed, skipped := 0, 0, 0, 0, 0 + for _, rec := range recs { + if r.opts.Tranche > 0 && budget <= 0 { + skipped++ + continue + } + budget-- + g, err := r.doneGroup(rec) + var n, m int + if err == nil { + n, m, err = r.recheckGroup(ctx, g) + } + if err != nil { + failed++ + fmt.Fprintf(r.deps.Out, "FAILED events recheck %s -> %s: %v\n", rec.OldID, rec.NewID, err) + continue + } + checked++ + listed += n + rekeyed += m + } + fmt.Fprintf(r.deps.Out, "events recheck: %d previously completed group(s) checked, %d event(s) listed, %d re-keyed, %d failed, %d skipped (tranche budget)\n", + checked, listed, rekeyed, failed, skipped) + return failed +} + +// doneGroup rebuilds a completed group from its journal record and the current inventory. +func (r *runner) doneGroup(rec StateRecord) (*Group, error) { + g := &Group{OldID: rec.OldID, Key: rec.OldID, Shape: ShapeOf(rec.OldID), Route: RouteRewrite, NewID: rec.NewID} + for _, id := range rec.CompanyIDs { + row := r.plan.inv.ByID[id] + if row == nil { + return nil, fmt.Errorf("company %s is gone", id) + } + if row.ExternalID != rec.NewID { + return nil, fmt.Errorf("company %s carries %q, not the recorded %s", id, row.RawExternalID, rec.NewID) + } + g.Rows = append(g.Rows, row) + } + if len(g.Rows) == 0 { + return nil, errors.New("no company ids recorded") + } + return g, nil +} + +// recheckGroup lists the events still keyed by the group's old id; a dry run only reports them. +func (r *runner) recheckGroup(ctx context.Context, g *Group) (listed, rekeyed int, err error) { + if !r.plan.Apply { + ids, listErr := r.listOldEvents(ctx, g) + if listErr != nil { + return 0, 0, listErr + } + fmt.Fprintf(r.deps.Out, "events recheck %s -> %s: %d listed (dry run)\n", g.OldID, g.NewID, len(ids)) + return len(ids), 0, nil + } + listed, rekeyed, err = r.rekeyListed(ctx, g) + if err != nil { + return listed, rekeyed, err + } + fmt.Fprintf(r.deps.Out, "events recheck %s -> %s: %d listed, %d re-keyed\n", g.OldID, g.NewID, listed, rekeyed) + return listed, rekeyed, nil +} diff --git a/cla-backend-go/orgimport/report.go b/cla-backend-go/orgimport/report.go new file mode 100644 index 000000000..9979d2108 --- /dev/null +++ b/cla-backend-go/orgimport/report.go @@ -0,0 +1,636 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "archive/zip" + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "html" + "io" + "mime" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +const ( + // MaxRawEmailBytes is the SES SendRawEmail limit (v1 API). + MaxRawEmailBytes = 10 * 1024 * 1024 + // maxZipAttachment keeps the whole message under the SES limit with base64 overhead. + maxZipAttachment = 6 * 1024 * 1024 + // maxInlinePlanRows caps the inline plan table; plan.csv is attached in full. + maxInlinePlanRows = 2000 + // maxLogTailBytes caps the inline run.log tail; run.log is attached in full. + maxLogTailBytes = 96 * 1024 + + contentTypeZip = "application/zip" +) + +// RunInfo describes one CLI run for the report e-mail and the AWS log header. +type RunInfo struct { + Stage string + Command string + Apply bool + Args []string + Start time.Time + End time.Time + Runner string + RunURL string + Repository string + Artifact string + Revision string + OutDir string + LogGroup string + LogStream string + Summary string + Err string + Workflow WorkflowInputs + // Notices are record-capture problems from the CLI (input snapshot or journal copy failures); + // they reach the recipient through the delivery notice. + Notices []string +} + +// Record file names inside OutDir that the apply commands reference: the captured operator files and +// the final journal, so a command from the e-mail or the Actions artifact applies the reviewed selection. +const ( + RecordMapping = "input-mapping.csv" + RecordDecisions = "input-decisions.csv" + RecordSharedDomains = "input-shared_domains.txt" + RecordStateBefore = "input-state.jsonl" + RecordState = "state.jsonl" +) + +// WorkflowInputs mirrors the org-import-sweep.yml dispatch inputs for the "how to apply" command. +type WorkflowInputs struct { + Routes string + Tranche string + IDs string + Mapping string + Decisions string + SharedDomains string +} + +// Mode is dry-run or apply. +func (i RunInfo) Mode() string { + if i.Apply { + return ModeApply + } + return ModeDryRun +} + +// Failed reports whether the run ended with an error. +func (i RunInfo) Failed() bool { return i.Err != "" } + +// Attachment is one e-mail attachment. +type Attachment struct { + Name string + ContentType string + Data []byte +} + +// Report is the complete decision record of a run: subject, HTML and text bodies, attachments. +// Notices are record problems (capture failures, attachments left out) that Deliverable puts at the +// top of both bodies; a report with notices is never delivered as an apparently complete record. +type Report struct { + Subject string + HTML string + Text string + Attachments []Attachment + Notices []string +} + +// BuildReport renders the run as a self-contained decision record. plan or audit may be nil. +func BuildReport(info RunInfo, plan *Plan, audit *AuditResult, runLog []byte) Report { + status := "OK" + if info.Failed() { + status = "FAILED" + } + summary := info.Summary + if summary == "" { + summary = "no summary" + } + r := Report{Subject: fmt.Sprintf("[EasyCLA org-import][%s] %s %s: %s (%s)", info.Stage, info.Command, info.Mode(), summary, status)} + + var h, t strings.Builder + h.WriteString("") + fmt.Fprintf(&h, "

EasyCLA org-import: %s %s on %s — %s

", esc(info.Command), esc(info.Mode()), esc(info.Stage), status) + fmt.Fprintf(&t, "EasyCLA org-import: %s %s on %s - %s\n\n", info.Command, info.Mode(), info.Stage, status) + + header := [][2]string{ + {"Stage", info.Stage}, {"Command", info.Command}, {"Mode", info.Mode()}, {"Result", status}, + {"Summary", summary}, + {"Started (UTC)", info.Start.UTC().Format(time.RFC3339)}, {"Finished (UTC)", info.End.UTC().Format(time.RFC3339)}, + {"Duration", info.End.Sub(info.Start).Round(time.Second).String()}, + {"Runner", info.Runner}, {"Arguments", strings.Join(info.Args, " ")}, + {"Build revision", info.Revision}, {"Output directory", info.OutDir}, + } + if info.RunURL != "" { + header = append(header, [2]string{"GitHub Actions run", info.RunURL}) + } + if info.Artifact != "" { + header = append(header, [2]string{"Actions artifact", info.Artifact}) + } + if info.LogGroup != "" { + header = append(header, [2]string{"CloudWatch Logs", info.LogGroup + " / " + info.LogStream}) + } + if info.Err != "" { + header = append(header, [2]string{"Error", info.Err}) + } + h.WriteString("") + for _, kv := range header { + v := esc(kv[1]) + if kv[0] == "GitHub Actions run" { + v = fmt.Sprintf("%s", v, v) + } + fmt.Fprintf(&h, "", esc(kv[0]), v) + fmt.Fprintf(&t, "%-20s %s\n", kv[0]+":", kv[1]) + } + h.WriteString("
%s%s
") + t.WriteString("\n") + + if plan != nil { + writePlanSections(&h, &t, info, plan) + } + if audit != nil { + writeAuditSections(&h, &t, audit) + } + + tail := logTail(runLog) + fmt.Fprintf(&h, "

run.log%s

%s
", tailNote(runLog, tail), esc(string(tail))) + fmt.Fprintf(&t, "== run.log%s ==\n%s\n", tailNote(runLog, tail), string(tail)) + + r.Attachments, r.Notices = attachments(info.OutDir, runLog) + r.Notices = append(append([]string(nil), info.Notices...), r.Notices...) + if len(r.Attachments) > 0 { + h.WriteString("

Attachments

    ") + t.WriteString("\n== Attachments ==\n") + for _, a := range r.Attachments { + fmt.Fprintf(&h, "
  • %s (%d bytes)
    • ", esc(a.Name), len(a.Data)) + fmt.Fprintf(&t, "- %s (%d bytes)\n", a.Name, len(a.Data)) + } + h.WriteString("
") + } + h.WriteString("") + r.HTML, r.Text = h.String(), t.String() + return r +} + +func writePlanSections(h, t *strings.Builder, info RunInfo, plan *Plan) { + actions := plan.ManualActions() + fmt.Fprintf(h, "

Manual actions (%d)

", len(actions)) + fmt.Fprintf(t, "== Manual actions (%d) ==\n", len(actions)) + if len(actions) == 0 { + h.WriteString("

None: every eligible group is registered, rewritten or ready.

") + t.WriteString("None.\n") + } else { + h.WriteString("

Each row needs a human before the tool can act on it; the suggested action says how.

") + writeTable(h, t, []string{"key", "old id", "route", "reason", "suggested action", "live", "org-service", "website", "domain", "new id", "action", "company ids", "company names", "error"}, manualRows(plan, actions)) + } + + if len(plan.Targets) > 0 { + fmt.Fprintf(h, "

Targets (%d)

Rewrite destinations grouped by Salesforce Account. needs_decision requires a collapse or distinct decision in the decisions file (lfx-self-serve #3085).

", len(plan.Targets)) + fmt.Fprintf(t, "\n== Targets (%d) ==\n", len(plan.Targets)) + var rows [][]string + for _, tg := range plan.Targets { + decision, reviewer := "", "" + if tg.Decision != nil { + decision, reviewer = tg.Decision.Kind, tg.Decision.Reviewer + } + var ids, names []string + for _, g := range tg.Groups { + ids = append(ids, g.CompanyIDs()...) + names = append(names, g.Names()...) + } + rows = append(rows, []string{tg.SFID, strconv.Itoa(len(tg.Groups)), strings.Join(tg.OldIDs(), "; "), strings.Join(ids, "; "), strings.Join(names, "; "), strconv.Itoa(len(tg.Existing)), decision, reviewer, tg.Status}) + } + writeTable(h, t, []string{"target", "groups", "old ids", "company ids", "company names", "existing rows", "decision", "reviewer", "status"}, rows) + } + + fmt.Fprintf(h, "

Plan (%d groups: %d register, %d rewrite)

", len(plan.Groups), len(plan.Register), len(plan.Rewrite)) + fmt.Fprintf(t, "\n== Plan (%d groups: %d register, %d rewrite) ==\n", len(plan.Groups), len(plan.Register), len(plan.Rewrite)) + var rows [][]string + for i, g := range plan.Groups { + if i == maxInlinePlanRows { + fmt.Fprintf(h, "

Only the first %d groups are shown inline; plan.csv (attached) has all %d.

", maxInlinePlanRows, len(plan.Groups)) + fmt.Fprintf(t, "(only the first %d groups shown; plan.csv has all %d)\n", maxInlinePlanRows, len(plan.Groups)) + break + } + domain, shared := plan.domainOf(g) + decision := "" + if g.Decision != nil { + decision = g.Decision.Kind + " by " + g.Decision.Reviewer + } + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + rows = append(rows, []string{g.Key, string(g.Shape), string(g.Route), g.ManualReason, g.Live, g.OrgStatus, g.Website(), domain, shared, g.NewID, g.Action, decision, strings.Join(g.CompanyIDs(), "; "), strings.Join(g.Names(), "; "), errText}) + } + writeTable(h, t, []string{"key", "shape", "route", "reason", "live", "org-service", "website", "domain", "shared", "new id", "action", "decision", "company ids", "company names", "error"}, rows) + + if !info.Apply { + h.WriteString("

How to apply this plan

After reviewing the record above, re-run the same command in apply mode against this record: the operator files are the captured input-* copies and the journal is state.jsonl. RECORD defaults to the output directory of this run; when applying elsewhere, extract the attached zip (or the Actions artifact) and export RECORD= that directory first. Locally:

") + t.WriteString("\n== How to apply this plan ==\nRe-run against this record (captured input-* copies and state.jsonl). RECORD defaults to this run's output directory; elsewhere, extract the attached zip / Actions artifact and export RECORD= first. Locally:\n") + for _, c := range ApplyCommands(info) { + fmt.Fprintf(h, "
%s
", esc(c)) + fmt.Fprintf(t, " %s\n", c) + } + } +} + +func manualRows(plan *Plan, actions []ManualAction) [][]string { + var rows [][]string + for _, a := range actions { + g := a.Group + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + domain, _ := plan.domainOf(g) + rows = append(rows, []string{g.Key, g.OldID, string(g.Route), a.Reason, a.Suggested, g.Live, g.OrgStatus, g.Website(), domain, g.NewID, g.Action, strings.Join(g.CompanyIDs(), "; "), strings.Join(g.Names(), "; "), errText}) + } + return rows +} + +func writeAuditSections(h, t *strings.Builder, audit *AuditResult) { + fmt.Fprintf(h, "

Audit (%d company rows)

", len(audit.Rows)) + fmt.Fprintf(t, "\n== Audit (%d company rows) ==\n", len(audit.Rows)) + var rows [][]string + for _, k := range sortedKeys(audit.Tiers) { + rows = append(rows, []string{"tier", k, strconv.Itoa(audit.Tiers[k])}) + } + routes := make([]string, 0, len(audit.Routes)) + for r := range audit.Routes { + routes = append(routes, string(r)) + } + sort.Strings(routes) + for _, r := range routes { + rows = append(rows, []string{"route", r, strconv.Itoa(audit.Routes[Route(r)])}) + } + rows = append(rows, []string{"possible duplicate groups", "", strconv.Itoa(len(audit.Duplicates))}) + writeTable(h, t, []string{"kind", "value", "count"}, rows) + h.WriteString("

audit.csv, unresolvable.csv and possible_duplicates.csv are attached (zip).

") +} + +// ApplyCommands returns the exact local and GitHub Actions commands that re-run this plan in apply +// mode from the record: operator file arguments point at the captured input-* copies, the journal at +// state.jsonl, both under RECORD (default: this run's output directory); every other option is kept. +func ApplyCommands(info RunInfo) []string { + var args []string + hasState := false + for i := 0; i < len(info.Args); i++ { + a := info.Args[i] + switch a { + case "--apply", "-apply", "--yes", "-yes", "--no-email", "-no-email", "--no-aws-log", "-no-aws-log": + continue + } + name, inline := strings.TrimLeft(a, "-"), false + if eq := strings.Index(name, "="); eq >= 0 && strings.HasPrefix(a, "-") { + name, inline = name[:eq], true + } + if file := recordFiles[name]; file != "" && strings.HasPrefix(a, "-") { + if !inline { + i++ + } + if name == "state" { + hasState = true + } + args = append(args, "--"+name, recordPath(info.OutDir, file)) + continue + } + args = append(args, shellQuote(a)) + } + if !hasState { + args = append(args, "--state", recordPath(info.OutDir, RecordState)) + } + local := fmt.Sprintf("STAGE=%s ./bin/org-import %s --apply --yes", info.Stage, strings.Join(args, " ")) + repo := info.Repository + if repo == "" { + repo = "linuxfoundation/easycla" + } + w := info.Workflow + gh := fmt.Sprintf("gh workflow run org-import-sweep.yml -R %s -f stage=%s -f mode=apply", repo, info.Stage) + if w.Routes != "" { + gh += " -f routes=" + shellQuote(w.Routes) + } + if w.Tranche != "" && w.Tranche != "0" { + gh += " -f tranche=" + shellQuote(w.Tranche) + } + if w.IDs != "" { + gh += " -f ids=" + shellQuote(w.IDs) + } + for _, f := range [][3]string{{"mapping", w.Mapping, RecordMapping}, {"decisions", w.Decisions, RecordDecisions}, {"shared_domains", w.SharedDomains, RecordSharedDomains}} { + if f[1] != "" { + gh += fmt.Sprintf(" -f %s=\"$(tr '\\n' '|' < %s)\"", f[0], recordPath(info.OutDir, f[2])) + } + } + return []string{local, gh} +} + +// recordFiles maps the operator-file flags to their captured copies in the record. +var recordFiles = map[string]string{"mapping": RecordMapping, "decisions": RecordDecisions, "shared-domains": RecordSharedDomains, "state": RecordState} + +// recordPath is the shell expression for a record file: "${RECORD:-}/". +func recordPath(outDir, name string) string { + def := strings.NewReplacer(`\`, `\\`, `"`, `\"`, "$", `\$`, "`", "\\`", "}", `\}`).Replace(outDir) + return `"${RECORD:-` + def + `}/` + name + `"` +} + +func shellQuote(s string) string { + if s == "" { + return "''" + } + if strings.ContainsAny(s, " \t\n'\"$`\\!*?[]{}()<>|&;#~") { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" + } + return s +} + +func writeTable(h, t *strings.Builder, head []string, rows [][]string) { + h.WriteString("") + for _, c := range head { + fmt.Fprintf(h, "", esc(c)) + } + h.WriteString("") + t.WriteString(strings.Join(head, " | ") + "\n") + for _, row := range rows { + h.WriteString("") + for _, c := range row { + fmt.Fprintf(h, "", esc(c)) + } + h.WriteString("") + t.WriteString(strings.Join(row, " | ") + "\n") + } + h.WriteString("
%s
%s
") +} + +func esc(s string) string { return html.EscapeString(s) } + +func sortedKeys(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func logTail(runLog []byte) []byte { + if len(runLog) <= maxLogTailBytes { + return runLog + } + tail := runLog[len(runLog)-maxLogTailBytes:] + if i := bytes.IndexByte(tail, '\n'); i >= 0 { + tail = tail[i+1:] + } + return tail +} + +func tailNote(runLog, tail []byte) string { + if len(tail) == len(runLog) { + return "" + } + return fmt.Sprintf(" (last %d of %d bytes; full log attached)", len(tail), len(runLog)) +} + +// attachments returns manual_actions.csv and plan.csv (or audit.csv) inline, the run log, and a zip +// of the whole output directory when it fits; a zip that cannot be built or attached is reported as a +// notice because the captured inputs and the journal travel only inside it. +func attachments(outDir string, runLog []byte) ([]Attachment, []string) { + var out []Attachment + for _, name := range []string{"manual_actions.csv", "plan.csv", "targets.csv", "to_salesforce.csv", "audit.csv", "unresolvable.csv", "possible_duplicates.csv"} { + data, err := os.ReadFile(filepath.Clean(filepath.Join(outDir, name))) + if err == nil && len(data) > 0 { + out = append(out, Attachment{Name: name, ContentType: "text/csv", Data: data}) + } + } + if len(runLog) > 0 { + out = append(out, Attachment{Name: "run.log", ContentType: "text/plain", Data: runLog}) + } + zipName := filepath.Base(outDir) + ".zip" + zipped, err := ZipDir(outDir) + switch { + case err != nil: + return out, []string{fmt.Sprintf("%s (output directory with the captured input-* files and the journal) could not be built: %v", zipName, err)} + case len(zipped) > maxZipAttachment: + return out, []string{fmt.Sprintf("%s (%d bytes, output directory with the captured input-* files and the journal) not attached: over the %d-byte attachment limit", zipName, len(zipped), maxZipAttachment)} + case len(zipped) > 0: + out = append(out, Attachment{Name: zipName, ContentType: contentTypeZip, Data: zipped}) + } + return out, nil +} + +// ZipDir zips the regular files directly under dir (reports, state, run.log). +func ZipDir(dir string) ([]byte, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, e := range entries { + if !e.Type().IsRegular() { + continue + } + data, readErr := os.ReadFile(filepath.Clean(filepath.Join(dir, e.Name()))) + if readErr != nil { + return nil, readErr + } + w, createErr := zw.Create(e.Name()) + if createErr != nil { + return nil, createErr + } + if _, err = w.Write(data); err != nil { + return nil, err + } + } + if err = zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MIME renders the report as a raw RFC 5322 message (multipart/mixed with a multipart/alternative body) +// that fits MaxRawEmailBytes; see Deliverable for what happens to oversize attachments. +func (r Report) MIME(from string, to []string) ([]byte, error) { + raw, _, err := r.Deliverable(from, to) + return raw, err +} + +// Deliverable renders the message so that it fits MaxRawEmailBytes. While it is too large, the +// largest non-zip attachment is gzip-compressed when it is text (name.gz keeps the full content) and +// dropped otherwise; the zip of the output directory goes last. Every change, and every record notice +// collected while building the report, is announced at the top of both bodies and returned as notice +// ("" when the record is complete); a message that does not fit even without attachments is an error, +// never a silently truncated record. +func (r Report) Deliverable(from string, to []string) (raw []byte, notice string, err error) { + atts := append([]Attachment(nil), r.Attachments...) + changes := append([]string(nil), r.Notices...) + for { + rep := r + if len(changes) > 0 { + notice = "Delivery incomplete: " + strings.Join(changes, "; ") + ". The complete record is in the output directory / Actions artifact and run.log in CloudWatch Logs (see the header table)." + rep.Text = notice + "\n\n" + r.Text + rep.HTML = injectNotice(r.HTML, notice) + } + msg, mimeErr := rep.mime(from, to, atts) + if mimeErr != nil { + return nil, "", mimeErr + } + if len(msg) <= MaxRawEmailBytes { + return msg, notice, nil + } + if len(atts) == 0 { + return nil, notice, fmt.Errorf("report body is %d bytes, over the %d-byte SES limit", len(msg), MaxRawEmailBytes) + } + i := largestDroppable(atts) + a := atts[i] + if gz := gzipAttachment(a); gz != nil { + atts[i] = *gz + changes = append(changes, fmt.Sprintf("%s (%d bytes) compressed to %s (%d bytes) for the SES size limit", a.Name, len(a.Data), gz.Name, len(gz.Data))) + continue + } + atts = append(atts[:i], atts[i+1:]...) + changes = append(changes, fmt.Sprintf("%s (%d bytes) dropped for the SES size limit", a.Name, len(a.Data))) + } +} + +// largestDroppable prefers the largest non-zip attachment; the zip is only touched when nothing else is left. +func largestDroppable(atts []Attachment) int { + best := -1 + for i, a := range atts { + if a.ContentType == contentTypeZip { + continue + } + if best < 0 || len(a.Data) > len(atts[best].Data) { + best = i + } + } + if best >= 0 { + return best + } + for i, a := range atts { + if best < 0 || len(a.Data) > len(atts[best].Data) { + best = i + } + } + return best +} + +// gzipAttachment returns the gzip form of a text attachment when that is clearly smaller, else nil. +func gzipAttachment(a Attachment) *Attachment { + if !strings.HasPrefix(a.ContentType, "text/") || strings.HasSuffix(a.Name, ".gz") { + return nil + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(a.Data); err != nil { + return nil + } + if err := zw.Close(); err != nil || buf.Len() >= len(a.Data)/2 { + return nil + } + return &Attachment{Name: a.Name + ".gz", ContentType: "application/gzip", Data: buf.Bytes()} +} + +// injectNotice puts the notice right after the tag (or in front of everything). +func injectNotice(htmlBody, notice string) string { + block := "

" + esc(notice) + "

" + i := strings.Index(htmlBody, "") + if j < 0 { + return block + htmlBody + } + pos := i + j + 1 + return htmlBody[:pos] + block + htmlBody[pos:] +} + +func (r Report) mime(from string, to []string, atts []Attachment) ([]byte, error) { + var buf bytes.Buffer + mixed := multipart.NewWriter(&buf) + fmt.Fprintf(&buf, "From: %s\r\n", from) + fmt.Fprintf(&buf, "To: %s\r\n", strings.Join(to, ", ")) + fmt.Fprintf(&buf, "Subject: %s\r\n", mime.QEncoding.Encode("utf-8", r.Subject)) + fmt.Fprintf(&buf, "Date: %s\r\n", time.Now().UTC().Format(time.RFC1123Z)) + buf.WriteString("MIME-Version: 1.0\r\n") + fmt.Fprintf(&buf, "Content-Type: multipart/mixed; boundary=%q\r\n\r\n", mixed.Boundary()) + + var alt bytes.Buffer + altW := multipart.NewWriter(&alt) + for _, part := range []struct{ ctype, body string }{{"text/plain; charset=utf-8", r.Text}, {"text/html; charset=utf-8", r.HTML}} { + hdr := textproto.MIMEHeader{"Content-Type": {part.ctype}, "Content-Transfer-Encoding": {"base64"}} + w, err := altW.CreatePart(hdr) + if err != nil { + return nil, err + } + if err = writeBase64(w, []byte(part.body)); err != nil { + return nil, err + } + } + if err := altW.Close(); err != nil { + return nil, err + } + bodyPart, err := mixed.CreatePart(textproto.MIMEHeader{"Content-Type": {fmt.Sprintf("multipart/alternative; boundary=%q", altW.Boundary())}}) + if err != nil { + return nil, err + } + if _, err = bodyPart.Write(alt.Bytes()); err != nil { + return nil, err + } + for _, a := range atts { + hdr := textproto.MIMEHeader{ + "Content-Type": {fmt.Sprintf("%s; name=%q", a.ContentType, a.Name)}, + "Content-Disposition": {fmt.Sprintf("attachment; filename=%q", a.Name)}, + "Content-Transfer-Encoding": {"base64"}, + } + w, partErr := mixed.CreatePart(hdr) + if partErr != nil { + return nil, partErr + } + if partErr = writeBase64(w, a.Data); partErr != nil { + return nil, partErr + } + } + if err = mixed.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func writeBase64(w io.Writer, data []byte) error { + enc := base64.StdEncoding.EncodeToString(data) + for len(enc) > 0 { + n := 76 + if n > len(enc) { + n = len(enc) + } + if _, err := io.WriteString(w, enc[:n]+"\r\n"); err != nil { + return err + } + enc = enc[n:] + } + return nil +} + +// ParseRecipients splits a comma/semicolon/whitespace separated address list. +func ParseRecipients(s string) []string { + var out []string + for _, a := range strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\t' }) { + if a = strings.TrimSpace(a); a != "" { + out = append(out, a) + } + } + return out +} diff --git a/cla-backend-go/orgimport/report_test.go b/cla-backend-go/orgimport/report_test.go new file mode 100644 index 000000000..7c61194e1 --- /dev/null +++ b/cla-backend-go/orgimport/report_test.go @@ -0,0 +1,481 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/mail" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ses" + "github.com/aws/aws-sdk-go/service/ses/sesiface" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func reportFixture(t *testing.T) (RunInfo, *Plan, string) { + t.Helper() + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + outDir := filepath.Join(dir, "out") + opts := Options{Stage: "dev", Mapping: mapping, OutDir: outDir, Routes: []Route{RouteRewrite}} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(outDir, "run.log"), []byte("line 1\nline 2\n"), 0o600)) + start := time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC) + info := RunInfo{ + Stage: "dev", Command: "ingest", Args: []string{"ingest", "--mapping", mapping, "--routes", "rewrite", "--out-dir", outDir, "--no-aws-log"}, + Start: start, End: start.Add(90 * time.Second), Runner: "lukasz@dockaws", Repository: "linuxfoundation/easycla", + RunURL: "https://github.com/linuxfoundation/easycla/actions/runs/42", Artifact: "org-import-out-dev-42", Revision: "abc123-dirty", + OutDir: outDir, LogGroup: "/easycla/org-import/dev", LogStream: "s1", Summary: sum.String(), + Workflow: WorkflowInputs{Routes: "rewrite", Tranche: "0", Mapping: mapping}, + } + return info, plan, outDir +} + +func TestBuildReport(t *testing.T) { + info, plan, outDir := reportFixture(t) + rep := BuildReport(info, plan, nil, []byte("line 1\nline 2\n")) + assert.Equal(t, "[EasyCLA org-import][dev] ingest dry-run: "+info.Summary+" (OK)", rep.Subject) + for _, want := range []string{ + "Manual actions (2)", "target_collision", "decisions file", "Targets (2)", "needs_decision", "Plan (3 groups: 0 register, 1 rewrite)", + "How to apply this plan", "STAGE=dev ./bin/org-import ingest --mapping", "--apply --yes", "gh workflow run org-import-sweep.yml -R linuxfoundation/easycla -f stage=dev -f mode=apply -f routes=rewrite", + "-f mapping="$(tr '\\n' '|' < ", "https://github.com/linuxfoundation/easycla/actions/runs/42", "org-import-out-dev-42", "abc123-dirty", "/easycla/org-import/dev / s1", + "1m30s", "line 2", "Attachments", "manual_actions.csv", "plan.csv", "targets.csv", "run.log", "out.zip", "acme.example", + } { + assert.Contains(t, rep.HTML, want, want) + } + assert.NotContains(t, rep.HTML, "--no-aws-log --apply", "report-only flags are dropped from the apply command") + for _, want := range []string{"== Manual actions (2) ==", "== Targets (2) ==", "== Plan", "== How to apply this plan ==", "== run.log ==", "line 1", `-f mapping="$(tr '\n' '|' < `} { + assert.Contains(t, rep.Text, want, want) + } + names := []string{} + for _, a := range rep.Attachments { + names = append(names, a.Name) + } + assert.Equal(t, []string{"manual_actions.csv", "plan.csv", "targets.csv", "to_salesforce.csv", "run.log", "out.zip"}, names) + zr, err := zip.NewReader(bytes.NewReader(rep.Attachments[5].Data), int64(len(rep.Attachments[5].Data))) + require.NoError(t, err) + var zipped []string + for _, f := range zr.File { + zipped = append(zipped, f.Name) + } + assert.ElementsMatch(t, []string{"plan.csv", "manual_actions.csv", "targets.csv", "to_salesforce.csv", "run.log"}, zipped) + _, err = ZipDir(filepath.Join(outDir, "missing")) + assert.Error(t, err) + + // apply run: no "how to apply", error → FAILED subject and Error row + info.Apply, info.Err, info.Summary = true, "2 group(s) failed", "" + rep = BuildReport(info, plan, nil, nil) + assert.Equal(t, "[EasyCLA org-import][dev] ingest apply: no summary (FAILED)", rep.Subject) + assert.NotContains(t, rep.HTML, "How to apply") + assert.Contains(t, rep.HTML, "Error2 group(s) failed") +} + +func TestBuildReportEscapesAndTruncates(t *testing.T) { + info, plan, _ := reportFixture(t) + info.Args = append(info.Args, "