From 06d24d11988b7db4255ee3742a095b6ba2b8e2ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Tue, 29 Sep 2026 14:48:18 +0200 Subject: [PATCH 1/8] Orgs cleanup/import, search apis pure only-read, create-org-only-when-needed #2749, #2750, #2751, SS #3085 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- .github/copilot-instructions.md | 9 + .github/workflows/org-import-sweep.yml | 244 ++++ .gitignore | 1 + CLAUDE.md | 11 +- cla-backend-go/Makefile | 16 +- cla-backend-go/cmd/org_import/README.md | 314 +++++ cla-backend-go/cmd/org_import/main.go | 444 +++++++ cla-backend-go/cmd/org_import/main_test.go | 45 + cla-backend-go/cmd/server.go | 2 +- cla-backend-go/company/handlers.go | 18 +- cla-backend-go/company/mocks/mock_repo.go | 76 +- cla-backend-go/company/mocks/mock_service.go | 83 +- cla-backend-go/company/models.go | 3 + cla-backend-go/company/repository.go | 3 + .../company/repository_external_id.go | 241 ++++ .../company/repository_external_id_test.go | 362 ++++++ .../company/resolve_company_test.go | 238 ++++ cla-backend-go/company/service.go | 210 +--- cla-backend-go/config/config.go | 12 + cla-backend-go/config/ssm.go | 7 +- cla-backend-go/events/repository_rekey.go | 187 +++ .../events/repository_rekey_test.go | 313 +++++ cla-backend-go/orgimport/adapters.go | 86 ++ cla-backend-go/orgimport/apex.go | 92 ++ cla-backend-go/orgimport/audit.go | 323 +++++ cla-backend-go/orgimport/awsreport.go | 131 ++ cla-backend-go/orgimport/decisions.go | 324 +++++ cla-backend-go/orgimport/decisions_test.go | 369 ++++++ cla-backend-go/orgimport/eligible.go | 183 +++ cla-backend-go/orgimport/manual.go | 96 ++ cla-backend-go/orgimport/mapping.go | 127 ++ cla-backend-go/orgimport/orgimport.go | 276 +++++ cla-backend-go/orgimport/report.go | 502 ++++++++ cla-backend-go/orgimport/report_test.go | 311 +++++ cla-backend-go/orgimport/run.go | 601 ++++++++++ cla-backend-go/orgimport/run_test.go | 1057 +++++++++++++++++ cla-backend-go/orgimport/state.go | 121 ++ cla-backend-go/orgimport/steps.go | 186 +++ cla-backend-go/swagger/cla.v2.yaml | 9 +- cla-backend-go/swagger/common/company.yaml | 4 +- .../swagger/common/properties/company-id.yaml | 8 + cla-backend-go/v2/acs-service/org_grants.go | 149 +++ .../v2/acs-service/org_grants_test.go | 122 ++ cla-backend-go/v2/cla_manager/handlers.go | 20 +- .../v2/cla_manager/handlers_test.go | 72 +- cla-backend-go/v2/cla_manager/service.go | 12 +- cla-backend-go/v2/company/service.go | 122 +- .../v2/company/virtual_company_test.go | 172 +++ cla-backend-go/v2/events/handlers.go | 4 +- cla-backend-go/v2/events/handlers_test.go | 106 ++ cla-backend-go/v2/member-service/client.go | 239 ++++ .../v2/member-service/client_test.go | 151 +++ .../v2/organization-service/client.go | 43 + cla-backend-go/v2/self_serve_sign/service.go | 11 +- .../v2/self_serve_sign/service_test.go | 8 +- .../v2/sign/rowless_signing_test.go | 436 +++++++ cla-backend-go/v2/sign/service.go | 143 ++- cla-backend-legacy/internal/api/handlers.go | 45 +- .../api/handlers_employee_signature_test.go | 82 ++ .../internal/store/companies.go | 69 ++ .../internal/store/companies_test.go | 30 + docs/easycla-ss-migration/m3-org-cleanup.md | 44 + 62 files changed, 9314 insertions(+), 411 deletions(-) create mode 100644 .github/workflows/org-import-sweep.yml create mode 100644 cla-backend-go/cmd/org_import/README.md create mode 100644 cla-backend-go/cmd/org_import/main.go create mode 100644 cla-backend-go/cmd/org_import/main_test.go create mode 100644 cla-backend-go/company/repository_external_id.go create mode 100644 cla-backend-go/company/repository_external_id_test.go create mode 100644 cla-backend-go/company/resolve_company_test.go create mode 100644 cla-backend-go/events/repository_rekey.go create mode 100644 cla-backend-go/events/repository_rekey_test.go create mode 100644 cla-backend-go/orgimport/adapters.go create mode 100644 cla-backend-go/orgimport/apex.go create mode 100644 cla-backend-go/orgimport/audit.go create mode 100644 cla-backend-go/orgimport/awsreport.go create mode 100644 cla-backend-go/orgimport/decisions.go create mode 100644 cla-backend-go/orgimport/decisions_test.go create mode 100644 cla-backend-go/orgimport/eligible.go create mode 100644 cla-backend-go/orgimport/manual.go create mode 100644 cla-backend-go/orgimport/mapping.go create mode 100644 cla-backend-go/orgimport/orgimport.go create mode 100644 cla-backend-go/orgimport/report.go create mode 100644 cla-backend-go/orgimport/report_test.go create mode 100644 cla-backend-go/orgimport/run.go create mode 100644 cla-backend-go/orgimport/run_test.go create mode 100644 cla-backend-go/orgimport/state.go create mode 100644 cla-backend-go/orgimport/steps.go create mode 100644 cla-backend-go/swagger/common/properties/company-id.yaml create mode 100644 cla-backend-go/v2/acs-service/org_grants.go create mode 100644 cla-backend-go/v2/acs-service/org_grants_test.go create mode 100644 cla-backend-go/v2/company/virtual_company_test.go create mode 100644 cla-backend-go/v2/events/handlers_test.go create mode 100644 cla-backend-go/v2/member-service/client.go create mode 100644 cla-backend-go/v2/member-service/client_test.go create mode 100644 cla-backend-go/v2/sign/rowless_signing_test.go create mode 100644 cla-backend-legacy/internal/api/handlers_employee_signature_test.go create mode 100644 docs/easycla-ss-migration/m3-org-cleanup.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5852c2bc5..ccffefcff 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -34,6 +34,15 @@ You are assisting in this repository. The following rules are **mandatory** and ## Testing & Validation +0. **Lint only with the pinned toolchain, memory-capped** + + - `cla-backend-go` lint = golangci-lint **v1.64.8 built with and run under the CI Go toolchain** (`go 1.25.x`, `go.mod` / `build-pr.yml`). + Any other combination (newer Go, newer `GOTOOLCHAIN`) re-type-checks the module per linter, exceeds 30 GB RSS and OOM-kills the + whole session. Before linting: `GOTOOLCHAIN=go1.25.13 golangci-lint version` must report that Go version. + - Run it on Linux only, one at a time, with `--concurrency 4` (`make lint LINT_ARGS="--concurrency 4"`) inside a memory-capped cgroup + (`systemd-run --user --scope -p MemoryMax=16G -p MemorySwapMax=0 …`). Never replace a shared `~/go/bin/golangci-lint` while other + sessions run; install a pinned copy elsewhere and use `LINT_TOOL=`. + 5. **Always test changes** - Whenever you modify code, **run tests** relevant to the change to ensure correctness. diff --git a/.github/workflows/org-import-sweep.yml b/.github/workflows/org-import-sweep.yml new file mode 100644 index 000000000..e0f534ba0 --- /dev/null +++ b/.github/workflows/org-import-sweep.yml @@ -0,0 +1,244 @@ +--- +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +# M3 organization import (lfx-self-serve #2750). Runbook: cla-backend-go/cmd/org_import/README.md. +# Manual runs choose stage/mode/routes. Scheduled runs are register-only and do nothing unless the +# repository variable ORG_IMPORT_SWEEP_DEV / ORG_IMPORT_SWEEP_PROD is set to dry-run or apply. +# Every run writes run.log + CSV reports (artifact), copies run.log to CloudWatch Logs +# /easycla/org-import/ and e-mails the full decision record to SSM cla-org-import-report-emails-. + +name: Org import sweep + +on: + workflow_dispatch: + inputs: + stage: + description: Stage + type: choice + options: [dev, prod] + default: dev + mode: + description: dry-run writes nothing; apply performs the plan + type: choice + options: [dry-run, apply] + default: dry-run + routes: + description: Routes to process + type: choice + options: [register, rewrite, 'register,rewrite'] + default: register + tranche: + description: Max groups to act on (0 = all) + type: string + default: '0' + ids: + description: Optional comma-separated external ids (old or new) + type: string + default: '' + mapping: + description: Optional mapping CSV for the rewrite route (rows separated by newlines or '|'), header old_id,new_id,action,approved + type: string + default: '' + decisions: + description: Optional duplicate-review decisions CSV (rows separated by newlines or '|'), header decision,old_ids,target_sfid,reviewer,note + type: string + default: '' + shared_domains: + description: Optional shared-domain list (one domain per line or '|'-separated); empty = built-in list + type: string + default: '' + notify: + description: E-mail the full run report to SSM cla-org-import-report-emails- + type: boolean + default: true + schedule: + - cron: '0 6 * * *' + +permissions: + id-token: write + contents: read + actions: read + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - id: plan + env: + EVENT: ${{ github.event_name }} + IN_STAGE: ${{ inputs.stage }} + IN_MODE: ${{ inputs.mode }} + IN_ROUTES: ${{ inputs.routes }} + IN_TRANCHE: ${{ inputs.tranche }} + IN_IDS: ${{ inputs.ids }} + IN_NOTIFY: ${{ inputs.notify }} + SWEEP_DEV: ${{ vars.ORG_IMPORT_SWEEP_DEV }} + SWEEP_PROD: ${{ vars.ORG_IMPORT_SWEEP_PROD }} + run: | + set -euo pipefail + role() { + case "$1" in + dev) echo 'arn:aws:iam::395594542180:role/github-actions-deploy' ;; + prod) echo 'arn:aws:iam::716487311010:role/github-actions-deploy' ;; + *) echo "unknown stage $1" >&2; exit 1 ;; + esac + } + entry() { + jq -cn --arg stage "$1" --arg mode "$2" --arg routes "$3" --arg tranche "$4" --arg ids "$5" --arg notify "$6" --arg role "$(role "$1")" \ + '{stage:$stage, mode:$mode, routes:$routes, tranche:$tranche, ids:$ids, notify:$notify, role:$role}' + } + entries=() + if [ "$EVENT" = workflow_dispatch ]; then + entries+=("$(entry "$IN_STAGE" "$IN_MODE" "$IN_ROUTES" "${IN_TRANCHE:-0}" "$IN_IDS" "${IN_NOTIFY:-true}")") + else + for pair in "dev:${SWEEP_DEV:-off}" "prod:${SWEEP_PROD:-off}"; do + stage="${pair%%:*}"; mode="${pair#*:}" + case "$mode" in + dry-run|apply) entries+=("$(entry "$stage" "$mode" register 0 '' true)") ;; + *) echo "scheduled sweep for $stage is off" ;; + esac + done + fi + matrix="$(printf '%s\n' "${entries[@]:-}" | jq -cs 'map(select(. != null))')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "$matrix" + + sweep: + needs: plan + if: needs.plan.outputs.matrix != '[]' + runs-on: ubuntu-latest + timeout-minutes: 180 + environment: ${{ matrix.stage }} + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + AWS_REGION: us-east-1 + STAGE: ${{ matrix.stage }} + steps: + - uses: actions/checkout@v4 + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Setup python (swagger tooling) + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: cla-backend-go/swagger/requirements.txt + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + role-to-assume: ${{ matrix.role }} + aws-region: us-east-1 + - name: Cache Go modules + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Configure Git to clone private Github repos + run: git config --global url."https://${TOKEN_USER}:${TOKEN}@github.com".insteadOf "https://github.com" + env: + TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }} + TOKEN_USER: ${{ secrets.PERSONAL_ACCESS_TOKEN_USER_GITHUB }} + + - name: Add OS Tools + run: sudo apt update && sudo apt-get install file -y + + - name: Go Setup + working-directory: cla-backend-go + run: make clean setup + + - name: Go Dependencies + working-directory: cla-backend-go + run: make deps + + - name: Go Swagger Generate + working-directory: cla-backend-go + run: make swagger + + - name: Build org-import + working-directory: cla-backend-go + run: make build-org-import-linux + + - name: Restore state and operator files + working-directory: cla-backend-go + env: + GH_TOKEN: ${{ github.token }} + MAPPING: ${{ github.event_name == 'workflow_dispatch' && inputs.mapping || '' }} + DECISIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.decisions || '' }} + SHARED_DOMAINS: ${{ github.event_name == 'workflow_dispatch' && inputs.shared_domains || '' }} + run: | + set -euo pipefail + mkdir -p org-import-out + run_id="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=org-import-state-${STAGE}&per_page=10" \ + --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .workflow_run.id // empty')" + if [ -n "$run_id" ]; then + gh run download "$run_id" -n "org-import-state-${STAGE}" -D org-import-out && echo "restored state from run $run_id" || echo "no state restored" + fi + lines() { printf '%s\n' "$1" | tr '|' '\n' | sed '/^[[:space:]]*$/d'; } + if [ -n "$MAPPING" ]; then + lines "$MAPPING" > org-import-out/mapping.csv + echo "mapping rows: $(($(wc -l < org-import-out/mapping.csv) - 1))" + fi + if [ -n "$DECISIONS" ]; then + lines "$DECISIONS" > org-import-out/decisions.csv + echo "decision rows: $(($(wc -l < org-import-out/decisions.csv) - 1))" + fi + if [ -n "$SHARED_DOMAINS" ]; then + lines "$SHARED_DOMAINS" > org-import-out/shared_domains.txt + echo "shared domains: $(wc -l < org-import-out/shared_domains.txt)" + fi + + - name: Run org-import ingest (${{ matrix.stage }} ${{ matrix.mode }} ${{ matrix.routes }}) + working-directory: cla-backend-go + env: + MODE: ${{ matrix.mode }} + ROUTES: ${{ matrix.routes }} + TRANCHE: ${{ matrix.tranche }} + IDS: ${{ matrix.ids }} + NOTIFY: ${{ matrix.notify }} + run: | + set -euo pipefail + args=(ingest --routes "$ROUTES" --out-dir org-import-out --state org-import-out/state.jsonl) + [ "${TRANCHE:-0}" != "0" ] && args+=(--tranche "$TRANCHE") + [ -n "$IDS" ] && args+=(--ids "$IDS") + [ -s org-import-out/mapping.csv ] && args+=(--mapping org-import-out/mapping.csv) + [ -s org-import-out/decisions.csv ] && args+=(--decisions org-import-out/decisions.csv) + [ -s org-import-out/shared_domains.txt ] && args+=(--shared-domains org-import-out/shared_domains.txt) + [ "$NOTIFY" = false ] && args+=(--no-email) + [ "$MODE" = apply ] && args+=(--apply --yes) + echo "bin/org-import ${args[*]}" + bin/org-import "${args[@]}" + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: org-import-out-${{ matrix.stage }}-${{ github.run_id }} + path: cla-backend-go/org-import-out + if-no-files-found: ignore + retention-days: 90 + + - name: Upload state + if: always() && matrix.mode == 'apply' + uses: actions/upload-artifact@v4 + with: + name: org-import-state-${{ matrix.stage }} + path: cla-backend-go/org-import-out/state.jsonl + if-no-files-found: ignore + retention-days: 90 diff --git a/.gitignore b/.gitignore index 3c25f9b75..09a17bfcb 100755 --- a/.gitignore +++ b/.gitignore @@ -291,3 +291,4 @@ spans*.json .venv copilot-*.md /e2e-tests/ +org-import-out/ diff --git a/CLAUDE.md b/CLAUDE.md index 7ad0e5a70..94bd49e2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ make setup # one-time: install swagger, golangci-lint, goimports; sets make swagger # regenerate API models/clients from swagger specs into gen/ (see below) make build-mac # build local binary -> bin/cla-mac (build-linux for Linux) make test # go test -v ./... with coverage -make lint # golangci-lint (v1.64.8, config .golangci.yaml) + license header check +make lint # golangci-lint (v1.64.8, config .golangci.yaml) + license header check — see "Lint safely" below make fmt # gofmt + goimports make mock # regenerate mocks via tools/regenmocks.sh make all-mac # full pipeline: clean swagger deps fmt build test lint (all-linux on Linux) @@ -39,6 +39,15 @@ make all-mac # full pipeline: clean swagger deps fmt build test lint (all Run a single test: `go test -v ./signatures/ -run TestName` +### Lint safely (memory) + +golangci-lint v1.64.8 must be the binary built with the CI Go toolchain (`go 1.25.x`, see `go.mod`/`build-pr.yml`) and run under that +toolchain. A v1.64.8 binary built with a newer Go, or run with a newer `GOTOOLCHAIN`, type-checks every package again per linter and grows +past 30 GB RSS on this module — it OOM-kills the whole shell/tmux session, not just the linter. Rules: Linux only; pinned toolchain +(`GOTOOLCHAIN=go1.25.13 golangci-lint version` must print the same Go version); `LINT_ARGS="--concurrency 4"` on shared machines; run it in +its own memory-capped cgroup (`systemd-run --user --scope -p MemoryMax=16G -p MemorySwapMax=0 make lint`); one lint at a time; never overwrite +a shared `~/go/bin/golangci-lint` while other sessions may use it — install a pinned copy elsewhere and point `LINT_TOOL` at it. + Run locally (points at a real AWS environment — see below): build, set env, then `./bin/cla-mac` (from `make build-mac`) or `./bin/cla` (from `make build-linux`). Health checks at `http://localhost:8080/v3/ops/health` and `/v4/ops/health`. Set `GH_ORG_VALIDATION=false` to bypass GitHub auth checks for local curl/Postman testing. ### Swagger code generation (important) diff --git a/cla-backend-go/Makefile b/cla-backend-go/Makefile index a603a9fb4..22f620f0b 100644 --- a/cla-backend-go/Makefile +++ b/cla-backend-go/Makefile @@ -13,6 +13,7 @@ GITLAB_REPO_CHECK_BIN = gitlab-repository-check-lambda FUNCTIONAL_TESTS_BIN = functional-tests USER_SUBSCRIBE_BIN = user-subscribe-lambda REPOSITORY_UPDATE_BIN = repository-update-tool +ORG_IMPORT_BIN = org-import MAKEFILE_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) GOPRIVATE=github.com/LF-Engineering/* BUILD_TIME=$(shell sh -c 'date -u +%FT%T%z') @@ -35,6 +36,8 @@ ifeq "$(shell uname -s)" "Linux" endif LINT_TOOL=$(shell go env GOPATH)/bin/golangci-lint +# extra golangci-lint flags, e.g. LINT_ARGS="--concurrency 4" on shared/low-memory machines +LINT_ARGS ?= # LINT_VERSION=v1.51.2 LINT_VERSION=v1.64.8 SWAGGER_DIR=$(ROOT_DIR)/swagger @@ -328,6 +331,17 @@ build-gitlab-repository-check-lambda-mac: deps build-prep env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(LDFLAGS) -o $(BIN_DIR)/$(GITLAB_REPO_CHECK_BIN)-mac cmd/gitlab_repository_check/main.go @chmod +x $(BIN_DIR)/$(GITLAB_REPO_CHECK_BIN)-mac +build-org-import: build-org-import-linux +build-org-import-linux: deps build-prep + @echo "==> Building the org import tool (Linux amd64)..." + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(LDFLAGS) -o $(BIN_DIR)/$(ORG_IMPORT_BIN) cmd/org_import/main.go + @chmod +x $(BIN_DIR)/$(ORG_IMPORT_BIN) + +build-org-import-mac: deps build-prep + @echo "==> Building the org import tool (Mac OSX)..." + env CGO_ENABLED=0 GOOS=darwin GOARCH=$(BUILD_ARCH) go build $(LDFLAGS) -o $(BIN_DIR)/$(ORG_IMPORT_BIN)-mac cmd/org_import/main.go + @chmod +x $(BIN_DIR)/$(ORG_IMPORT_BIN)-mac + build-functional-tests: build-functional-tests-linux build-functional-tests-linux: deps build-prep @echo "==> Building Functional Tests for Linux amd64 binary..." @@ -351,5 +365,5 @@ build-repository-update-mac: deps build-prep @chmod +x $(BIN_DIR)/$(REPOSITORY_UPDATE_BIN)-mac lint: - @cd $(MAKEFILE_DIR) && $(LINT_TOOL) version && echo "==> Running lint..." && $(LINT_TOOL) run --timeout 30m --exclude="this method will not auto-escape HTML. Verify data is well formed" --allow-parallel-runners --config=.golangci.yaml ./... && echo "==> Lint check passed." + @cd $(MAKEFILE_DIR) && $(LINT_TOOL) version && echo "==> Running lint..." && $(LINT_TOOL) run --timeout 30m --exclude="this method will not auto-escape HTML. Verify data is well formed" --allow-parallel-runners --config=.golangci.yaml $(LINT_ARGS) ./... && echo "==> Lint check passed." @cd $(MAKEFILE_DIR) && ./check-headers.sh diff --git a/cla-backend-go/cmd/org_import/README.md b/cla-backend-go/cmd/org_import/README.md new file mode 100644 index 000000000..76c074dc7 --- /dev/null +++ b/cla-backend-go/cmd/org_import/README.md @@ -0,0 +1,314 @@ +# org_import — EasyCLA organization import/sync (M3) + +Copyright The Linux Foundation and each contributor to CommunityBridge. + +SPDX-License-Identifier: CC-BY-4.0 + +Tickets: linuxfoundation/lfx-self-serve#2750 (import), #2751 (no company rows created by reads), #2749 (cleanup). +Design: `docs/easycla-ss-migration/m3-b2b-org-import.md` (PR #5223). Cleanup runbook: `docs/easycla-ss-migration/m3-org-cleanup.md`. + +## 1. What it does + +For every EasyCLA company with an **active CCLA** (`signature_type=ccla`, `signature_reference_type=company`, +`signature_signed=true`, `signature_approved=true`) the tool makes the organization usable by LFX Self Serve: + +| Route | When | What happens (apply mode) | +|---|---|---| +| `register` | `company_external_id` is a live Salesforce account id (`001…`, 15/18 chars) | `POST /b2b_orgs {"sfid": id}` on the LFX v2 member-service (idempotent) | +| `rewrite` | id is `lf…` (legacy console) or a dead `001…` account | needs a new Salesforce id from `--mapping` (or `--use-apex`): copy ACS grants old→new, rewrite `company_external_id` on every row of the group (old id kept in `previous_company_external_id`), re-key activity events, delete old grants, register the new id | +| `manual` | empty/invalid id, ambiguous/unapproved mapping, collision on the target Account without a decision (§4.1), no/shared website on the Apex path | nothing; listed with `manual_reason` and a suggested action | + +A **group** is every company row sharing one `company_external_id` (signing entities of one organization); it is +eligible when at least one row has an active CCLA. Internal `company_id`s never change. + +Never done by the tool: creating or deleting EasyCLA rows, merging companies, deleting Salesforce accounts or +org-service organizations, deleting roles, touching groups without an active CCLA, writing anything without `--apply`. +Rows sharing one `company_external_id` are one group and are never merged; merging duplicate companies is a post-import +runbook (linuxfoundation/lfx-self-serve#2056), the pre-import duplicate review (#3085) only feeds the decisions file (§4.1). + +Safety rules: +- Default is **dry run**. `--apply` prompts you to type the stage name (`--yes` skips the prompt, for automation). +- Every rewrite step is idempotent and recorded in `--state`; re-running the same command converges. +- The row rewrite is a conditional write (`company_external_id = old`); a concurrent change stops that group only. +- Grants are copied before rows are rewritten and old grants deleted last: managers never lose access. +- `register` is stateless and needs no Salesforce input — it is the only route ever scheduled (§6). + +## 2. Prerequisites + +- Go 1.25+, `cd cla-backend-go && make build-org-import-linux` (Linux) or `make build-org-import-mac` → `bin/org-import[-mac]`. +- AWS credentials for the stage account (`dev.md`, "assume role" recipe; MFA). Either `AWS_PROFILE=lfproduct-dev AWS_SDK_LOAD_CONFIG=1` + or exported `AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN`. `AWS_REGION` defaults to `us-east-1`. +- `STAGE=dev|prod` (required). +- IAM: read of SSM `cla-*-{stage}` parameters; DynamoDB `Scan/Query/GetItem` on `cla-{stage}-companies`, `cla-{stage}-signatures`, + `cla-{stage}-events`; `UpdateItem` on `cla-{stage}-companies` and `cla-{stage}-events` (rewrite route only). Same tables the API uses. +- SSM parameters read (all pre-existing except the first two): + - `cla-member-service-base-url-{stage}`, `cla-member-service-auth0-audience-{stage}` — member-service; when missing the tool warns, + dry-run liveness falls back to org-service and every `register` step fails with `member-service is not configured`. + Values (from `lfx-v2-argocd/values/{stage}/lfx-platform.yaml`): dev `https://lfx-api.dev.v2.cluster.linuxfound.info` / + `https://lfx-api.dev.v2.cluster.linuxfound.info/`, prod `https://lfx-api.v2.cluster.lfx.dev` / `https://lfx-api.v2.cluster.lfx.dev/`. + Create as plain `String` parameters (`aws ssm put-parameter --type String …`); dev already has them, prod does not yet. + - `cla-auth0-platform-*-{stage}` — M2M token (org-service, ACS, member-service). + - `cla-api-gateway-url-{stage}`, `cla-acs-api-key-{stage}` — org-service and ACS. + - `cla-salesforce-apex-base-url-{stage}`, `cla-salesforce-apex-token-{stage}` — only with `--use-apex`. + - `cla-org-import-report-emails-{stage}` — optional, comma-separated recipients of the run report e-mail (§8); absent ⇒ no e-mail, noted in run.log. + `cla-ses-sender-email-address-{stage}` (pre-existing) is the sender. +- Reports (§8) additionally need `logs:CreateLogGroup/CreateLogStream/PutLogEvents` on `/easycla/org-import/{stage}` and `ses:SendRawEmail`; + `--no-aws-log` / `--no-email` switch them off (use both for a read-only look at prod from a laptop). +- Member-service access (ops, one-time per stage): (1) an Auth0 **client grant** for EasyCLA's M2M client (`cla-auth0-platform-client-id-{stage}`) + on the lfx-api resource server (the audience above) — without it the token request fails with + `authorization failed (403): Client "…" is not authorized to access resource server` (current state on dev); (2) the client must be in the + `global_org_admin` team (member-service Heimdall ruleset) for `POST /b2b_orgs` and have `auditor` for `GET /b2b_orgs/{id}` (dry-run liveness). + Check: `STAGE=dev bin/org-import ingest --routes register --ids ` must print `live=live`; `live=error` means the GET + is not permitted (never treated as dead) and the run exits 1. +- Nothing sensitive is ever written: report files contain company names/ids only; tokens stay in memory. + +## 3. Commands + +``` +org_import audit [--out-dir ./org-import-out] [report flags] +org_import ingest [--apply] [--yes] [--tranche N] [--ids id1,id2] [--mapping map.csv] [--decisions decisions.csv] + [--shared-domains domains.txt] [--state state.jsonl] [--routes register,rewrite] [--skip-wait] + [--use-apex] [--wait-max 40m] [--out-dir ./org-import-out] [report flags] +report flags: [--email-to a@x,b@y] [--no-email] [--no-aws-log] [--aws-log-group /easycla/org-import/] +``` + +| Flag | Meaning | +|---|---| +| `--out-dir` | CSV reports directory (default `./org-import-out`, gitignored) | +| `--apply` | perform writes; without it nothing is written anywhere except report files | +| `--yes` | skip the "type the stage name" confirmation (automation) | +| `--tranche N` | act on at most N groups (planning still classifies everything) | +| `--ids` | only these groups; accepts old **or** new external ids (comma-separated) | +| `--mapping` | Salesforce mapping CSV (§4); required for the rewrite route unless `--use-apex` | +| `--decisions` | duplicate-review decisions CSV (§4.1): collapse/distinct verdicts for old ids landing on one Account | +| `--shared-domains` | file replacing the built-in shared-domain list (§4.2; one domain per line, `#` comments) | +| `--state` | append-only JSONL; one record per group per step; unfinished groups are replayed first on the next run | +| `--routes` | `register`, `rewrite` or both (default both) | +| `--skip-wait` | check the new ids in org-service once instead of polling up to `--wait-max` | +| `--use-apex` | resolve new ids via the Salesforce Apex endpoint (`ORG_IMPORT_USE_APEX=true` equivalent); refused when the SSM params are missing | +| `--wait-max` | org-service propagation wait for new Salesforce accounts (default 40m, poll 30s) | +| `--email-to` | report recipients, overrides SSM `cla-org-import-report-emails-{stage}` | +| `--no-email` / `--no-aws-log` | do not send the report e-mail / do not copy run.log to CloudWatch Logs | +| `--aws-log-group` | CloudWatch log group (default `/easycla/org-import/{stage}`), created when missing | + +Env: `STAGE` (required), `AWS_REGION` (default `us-east-1`), `LOG_LEVEL` (default `warn`; `debug` shows every AWS/HTTP call; `error` hides the +expected org-service "not found" warnings printed for every dead/unknown id). + +Every run writes `/run.log` (everything printed, truncated per run) and ends with the report step (§8); report failures are logged +and never change the exit code. + +### 3.1 `audit` (reads only) + +```bash +cd cla-backend-go && make build-org-import-linux +export AWS_PROFILE=lfproduct-dev AWS_SDK_LOAD_CONFIG=1 STAGE=dev +bin/org-import audit --out-dir ./org-import-out/dev-$(date -u +%F) +``` + +Output line: `audit stage=dev companies=N eligible_groups=M MISSING_SFID=a INVALID_SFID_FORMAT=b SFID_OK=c SFID_DANGLING_OR_DELETED=d UNKNOWN=e register=r rewrite=w manual=m duplicates=k` +— the five tiers are 1:1 with `utils/audit_company_reachability.sh`; route counts are per row (only rows in eligible groups have a route). + +Files (`--out-dir`): +- `audit.csv` — `company_id, company_name, signing_entity_name, company_external_id, id_shape(001|lf|empty|other), active_ccla, ccla_count, ecla_count, org_service(200|404|err), website, duplicate_sfid_group, route, manual_reason, tier`. + `ecla_count` is filled only for active rows that are manual, duplicate or unresolvable (cheap); `-1` = count failed. +- `unresolvable.csv` — active rows with empty/invalid ids or dead `001…` ids (#2749 input). +- `possible_duplicates.csv` — rows sharing an id with the same (or empty) signing entity name, and same company name under different ids (#2056 input). Distinct signing entities under one id are **not** duplicates. + +Runtime: full companies scan + one CCLA query + one org-service GET per distinct id (4 in parallel) — ~30 s on dev, ~10 min on prod (3.7k companies). + +### 3.2 `ingest` dry run + +```bash +STAGE=dev bin/org-import ingest # all routes, no mapping: rewrite candidates are "pending" +STAGE=dev bin/org-import ingest --routes register # what the sweep does +STAGE=dev bin/org-import ingest --routes rewrite --mapping map.csv --state state.jsonl +``` + +Annotated sample: + +``` +org_import ingest stage=dev mode=dry-run eligible_groups=4 register=1 rewrite=1 pending=1 skipped=0 +register 0014100000Te0G7AAJ shape=001 rows=2 live=live | Live Corp; Live Corp / Live Corp Asia +rewrite lfbd1c2b3a4d5e6f7a8 shape=lf rows=1 new_id=0014100000NewNewNe action=matched domain=legacy.example | Legacy Ltd +rewrite 0014100000DeadDead1 shape=001 rows=1 live=dead reason=no_mapping | Dead Corp <- pending: needs a mapping row +manual c0ffee00-... shape=empty rows=1 reason=empty_external_id | Empty Inc +PLAN register 0014100000Te0G7AAJ: POST /b2b_orgs {sfid:0014100000Te0G7AAJ} (rows: c-live,c-live-sub) +PLAN rewrite lfbd1c2b3a4d5e6f7a8 -> 0014100000NewNewNe (matched): wait org-service; copy ACS grants; rewrite 1 row(s) c-lf; re-key events; delete old grants; POST /b2b_orgs {sfid:0014100000NewNewNe} +TARGETS (rewrite destinations grouped by Account): +target 0014100000NewNewNe groups=1 old_ids=lfbd1c2b3a4d5e6f7a8 existing_rows=0 status=ok +MANUAL ACTIONS (1): +c0ffee00-... [manual] empty_external_id: Fill company_external_id (…) or leave for cleanup (#2749); the tool never guesses an id. +stage=dev mode=dry-run eligible=4 registered=0 rewritten=0 pending=1 manual=1 failed=0 +``` + +- `live=live|dead|error` — member-service `GET /b2b_orgs/{id}` (or org-service when member-service is unconfigured); `error` is never treated as dead. +- `pending` — rewrite candidates without a resolved new id (no mapping row, or a `register` POST answered 404 = dead account). They appear in `to_salesforce.csv`. +- `skipped` — eligible groups excluded by `--routes`, `--tranche`, or already `done` in `--state`. +- Summary line: `eligible` = groups after `--ids`; `registered`/`rewritten` = groups completed in apply mode; `manual`; `failed` = groups with an + error — the run exits 1 in dry-run and apply mode alike (a dry run with `live=error` is not a clean dry run). + +Runtime: ~10 s on dev, ~7 min on prod (one liveness GET per group). + +Files (all rewritten after apply with the final state): +- `plan.csv` — `key, old_id, id_shape, route, manual_reason, live, org_service, website, domain, shared_domain, new_id, action, decision, reviewer, company_ids, company_names, error`. +- `manual_actions.csv` — one row per manual/pending/failed group with `reason` and `suggested_action` (what a human must do next). +- `targets.csv` — rewrite destinations grouped by Account: `target_sfid, groups, old_ids, company_ids, company_names, existing_rows, decision, reviewer, status(ok|needs_decision|distinct_conflict)`. +- `to_salesforce.csv` — `old_id, name, website, ccla_signed_date, domain, shared_domain` — the hand-off to sales ops (§4). + +Manual reasons: `empty_external_id`, `invalid_id_shape`, `mapping_ambiguous`, `mapping_not_approved`, `mapping_same_id`, +`target_collision` (several old ids → one Account, or the Account already has EasyCLA rows, and no decision covers it — §4.1), `distinct_conflict` +(a `distinct` decision spans two ids resolved to the same Account), `missing_website` / `shared_domain` (Apex path only, §4.2), +`apex_match_needs_approval`, `apex_error`; pending reasons: `no_mapping`, `dead_account`. + +### 3.3 `ingest --apply` + +```bash +# register only, 10 groups, interactive confirmation +STAGE=prod bin/org-import ingest --routes register --tranche 10 --apply +# rewrite tranche with mapping + state +STAGE=prod bin/org-import ingest --routes rewrite --mapping map.csv --state state.jsonl --tranche 10 --apply +# one group, by old or new id +STAGE=prod bin/org-import ingest --ids lfbd1c2b3a4d5e6f7a8 --mapping map.csv --state state.jsonl --apply +``` + +Order inside one run: all `register` POSTs → (Apex real calls) → **one** shared wait until org-service serves every new id (≤ `--wait-max`) +→ per rewrite group: `copy_grants` → `rewrite_rows` → `rekey_events` → `delete_old_grants` → `register` → `done`. +A failing step stops that group (state records it), the run continues with the next group and exits 1 with `FAILED …` lines. + +Expected runtime: register = seconds per group; rewrite = the org-service wait (new Salesforce accounts take up to ~40 minutes to appear; +existing accounts are immediate) plus seconds per group. + +## 4. Mapping file + +CSV with header `old_id,new_id,action,approved`: + +``` +old_id,new_id,action,approved +lfbd1c2b3a4d5e6f7a8,0014100000NewNewNe,matched,true +0014100000DeadDead1,0014100000Fresh001,created,true +lf000000000000000001,,ambiguous,false +``` + +- `action`: `matched` (existing account found by domain/name — **requires `approved=true`**, a human checked it), `created` (new account created for this org, accepted as is), `ambiguous` (several candidates — stays manual). +- `new_id` must be a 15/18-char alphanumeric Salesforce id (`ambiguous` rows may leave it empty); `new_id == old_id` is refused (`mapping_same_id`). +- One row per `old_id`; two old ids pointing to the same `new_id` (or a `new_id` some EasyCLA row already carries) → `target_collision` until a + `collapse` decision covers them (§4.1); `distinct` decisions cannot share an Account. +- Loader errors are fatal and name the line: `missing column`, `expected N columns`, `approved must be true|false`, `empty old_id`, `duplicate old_id`, `new_id … is not a Salesforce account id`, `action must be matched|created|ambiguous`. +- Rows with an empty or malformed `company_external_id` (`empty_external_id` / `invalid_id_shape` in `unresolvable.csv`, `manual_actions.csv`) + have no organization id to key on, so `old_id` is the row's **`company_id`** instead (`,001…,matched,true`). Such a row is a + group of its own — blank or garbage values are never grouped —, the write is conditional on that one row still carrying its current value + (`attribute_not_exists` / the malformed string), no `previous_company_external_id` is recorded for a blank value, ACS grants and events are + not moved (nothing is keyed by the old value) and the new id is registered as usual. Two such rows on one Account are a `target_collision` + until a `collapse` decision names both company ids (§4.1). `state.jsonl` records them under `key` = `company_id`. + +How it is produced (sales ops, outside the tool): take `to_salesforce.csv` from a dry run, domain-match `website` against live accounts, +create the rest with Data Loader (unique external-id field = `old_id`, non-member record type, origin marker), export `old_id,new_id,action,approved`. +Mark `approved=true` only after a person confirmed each `matched` pair. Keep the file out of git (`org-import-out/` is ignored). + +`--use-apex` replaces the file for `created` results: the tool does a `dryRun` call per group first, then the real call, and stops the group when +action/id differ between the two; `matched` results still need an approved mapping row with the same id. An approved/`created` mapping row wins +over Apex for its old id, and groups without a website or with a shared domain (§4.2) never reach Apex. + +### 4.1 Decisions file (`--decisions`, lfx-self-serve#3085) + +Outcome of the human duplicate review. CSV with header `decision,old_ids,target_sfid,reviewer,note`; `old_ids` separated by `;` or spaces: + +``` +decision,old_ids,target_sfid,reviewer,note +collapse,lfaaaa000000000000001;lfbbbb000000000000002,0014100000NewNewNe,michal,same company (Acme Inc / Acme GmbH) +distinct,lfcccc000000000000003;lfdddd000000000000004,,michal,different companies despite the shared domain +``` + +- `collapse` — the listed old ids (and any EasyCLA rows already on `target_sfid`) are one organization: the Account may receive all of them. + Without it every Account that would end up with more than one old id — including one that already has EasyCLA rows — is `target_collision`. +- `distinct` — the listed old ids are different organizations; ≥2 ids, no target. If two of them still resolve to one Account the groups become + `distinct_conflict` (fix the mapping). `reviewer` is required; an old id may appear in one decision only. +- Decisions never change the mapping: the Account comes from `--mapping` / Apex; the decision only approves the collision. + +### 4.2 Shared domains (`--shared-domains`, Apex path only) + +Groups whose website domain is in the shared list (`github.com`, `nowebsite.com`, `gmail.com`, `googlemail.com`, `yahoo.com`, `hotmail.com`, +`outlook.com`, `live.com`, `icloud.com`, `protonmail.com`, `qq.com`, `163.com`) or who have no website are `manual` (`shared_domain` / +`missing_website`) instead of being domain-matched by Apex. A file replaces the whole list (one domain per line, `#` comments). Mapping rows are +explicit human decisions and are not gated. + +## 5. Tranche protocol (prod) + +1. `audit` → compare tier counts with the last `utils/audit_company_reachability.sh` numbers; read `unresolvable.csv` / `possible_duplicates.csv`. +2. `ingest` dry run, all routes → read `plan.csv`; hand `to_salesforce.csv` to sales ops. +3. `ingest --routes register --tranche 10 --apply` → verify (below) → `--tranche 100` → rest. +4. With the mapping file: `ingest --routes rewrite --mapping map.csv --state state.jsonl --tranche 10 --apply` → verify → 100 → rest. +5. Re-run the same command: it must report nothing new (`registered=0 rewritten=0`, groups `skipped` as done). + +Verification per tranche (pick 3 groups): +- `GET /b2b_orgs/{new id}` → 200 (member-service). +- org-service `GET /organizations/{new id}` → 200. +- ACS `GET /users/rolescopes/organization?orgid={new id}&scopetype=all` with `X-LFX-CACHE: false` → same users/roles as the old id had; old id → empty. +- `GET /v4/company/external/{new id}/cla-groups` lists the CCLA; `GET /v4/company/{companyID}/project/{projectSFID}/events` still shows the pre-rewrite history. +- DynamoDB row: `company_external_id = new`, `previous_company_external_id = old`. +- Corporate Console: a CLA manager of the company logs in, sees the company and its CCLA; Self Serve org lens shows the organization. + +## 6. Sweep (`register` route only) + +Manual: `STAGE= bin/org-import ingest --routes register [--apply --yes]` — stateless, idempotent, minutes. +It **never** rewrites and never touches ACS or Salesforce; new `lf…` (legacy console) companies only show up as `pending` — moving them is the +manual rewrite tranche of §5 (a named owner is required, see the design doc). + +GitHub Actions `.github/workflows/org-import-sweep.yml`: +- Actions tab → "Org import sweep" → Run workflow: `stage` (`dev|prod`), `mode` (`dry-run|apply`), `routes` (default `register`), + `tranche`, optional `ids`, optional `mapping` / `decisions` / `shared_domains` (the files of §4 pasted as text; rows separated by newlines or + `|` — e.g. `old_id,new_id,action,approved|lf…,001…,matched,true`), `notify` (default true; false ⇒ `--no-email`). Without a mapping, rewrite + candidates are only reported as pending. Same from a shell (the report e-mail of every dry run prints this line ready to paste): + `gh workflow run org-import-sweep.yml -f stage=dev -f mode=dry-run -f routes=register,rewrite -f mapping="$(tr '\n' '|' < map.csv)"`. +- State: apply runs upload `state.jsonl` as artifact `org-import-state-`; the next run of the same stage restores the newest one first, + so crashed rewrite groups are replayed. Artifacts expire after 90 days — losing state is harmless (finished groups are re-classified as + `register` on their new id, unfinished ones must be re-run with the old id via `--ids`). +- Schedule (`0 6 * * *`) is gated per stage by repository variables `ORG_IMPORT_SWEEP_DEV` / `ORG_IMPORT_SWEEP_PROD` ∈ `off|dry-run|apply`; + unset or `off` (default) ⇒ the scheduled job does nothing. Scheduled runs are always `--routes register`. Manual runs ignore the variables. +- Promotion: ≥10 clean manual runs (dry-run, then apply) → set the variable to `dry-run`, read the artifacts for a week → `apply`. + Rollback: set `off` (no deploy, no code change). The `prod` environment's protection rules (reviewers) apply to every run. +- Each run uploads `org-import-out/` (run.log and the CSVs of §3.2) as artifact `org-import-out--`, copies run.log to CloudWatch + Logs and e-mails the report (§8) — scheduled runs included. +- IAM: the `github-actions-deploy` role of each account must allow the DynamoDB/SSM access listed in §2 (it deploys the API, so it already does). + +## 7. Failures and recovery + +| Where | Message | Meaning / action | +|---|---|---| +| setup | `loading SSM config` / `STAGE is not set` | wrong account/profile or missing stage | +| planning | `live=error` | member-service GET failed (403 = missing `auditor`, network) — fix access; nothing is classified dead | +| planning | `mapping line N: …` | fix the CSV | +| planning | `state: unfinished group … has no valid new_id` | hand-edit the state file only if you know why; otherwise stop | +| register | `account … does not exist in Salesforce (rewrite candidate)` | POST 404: the id is dead; group is now `pending`/`dead_account` → mapping | +| register | `member-service is not configured` | SSM params missing for this stage | +| wait | `org-service does not serve … yet` | new Salesforce account not propagated within `--wait-max`; re-run later (state replays the group) | +| copy_grants | `granting … on …` | org-service create failed; nothing else was changed; re-run | +| rewrite_rows | `… conflict, group stopped` | a row's external id was changed by someone else meanwhile; investigate that row before re-running | +| rekey_events | `event …` | UpdateItem failed; re-run (already re-keyed events are skipped) | +| delete_old_grants | `deleting … grant …` | rows are already rewritten and new grants exist; re-run to finish | + +Resume: re-run the same command with the same `--state`; unfinished groups are replayed first (pinned by their `company_ids`, so they are found +even after their rows already carry the new id). `--ids ` narrows a run to one group. + +Revert a rewrite by hand (only if really needed): for each row, `aws dynamodb update-item --table-name cla-{stage}-companies --key '{"company_id":{"S":""}}' +--update-expression 'SET company_external_id = :o REMOVE previous_company_external_id' --expression-attribute-values '{":o":{"S":""}}'`, +then re-create the ACS grants on the old id (org-service `CreateRolescopes`, by username) and remove them from the new id. Events keep the new key +(run `events.RekeyRepository` the other way round if required). Stop and contact the EasyCLA maintainers before reverting more than one group. + +## 8. Run report (e-mail + CloudWatch Logs) + +After every `audit`/`ingest` run (dry-run or apply, success or failure) the tool: +1. copies `run.log` to CloudWatch Logs group `/easycla/org-import/{stage}` (created when missing), stream `---`; +2. e-mails `cla-org-import-report-emails-{stage}` (or `--email-to`) from `cla-ses-sender-email-address-{stage}` via SES: subject + `[EasyCLA org-import][] : (OK|FAILED)`; body = run header (stage, mode, arguments, runner, build revision, + Actions run/artifact links, CloudWatch stream), the **Manual actions** table with suggested actions, the **Targets** table, the full plan (≤2000 + rows inline) or audit tier counts, the ready-to-paste local and `gh workflow run` apply commands for a dry run, and the run.log tail; attachments = + every CSV of the out-dir, run.log and a zip of the out-dir (≤6 MiB; largest attachments dropped to stay under SES's 10 MiB). + +Missing recipients parameter, SES or CloudWatch errors are written to run.log only; the exit code reflects the import itself. + +## 9. Cleanup pointers + +`unresolvable.csv` and `possible_duplicates.csv` feed `docs/easycla-ss-migration/m3-org-cleanup.md` (#2749, #2056): human review, nothing is deleted +or merged by the tool. Duplicate companies (same organization under several ids) are collapsed onto one Account by the import via the decisions +file (§4.1); deleting or re-pointing their EasyCLA rows is post-import work (#2056). diff --git a/cla-backend-go/cmd/org_import/main.go b/cla-backend-go/cmd/org_import/main.go new file mode 100644 index 000000000..df53248cc --- /dev/null +++ b/cla-backend-go/cmd/org_import/main.go @@ -0,0 +1,444 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +// org_import is the M3 organization import/sync CLI (lfx-self-serve #2750). See README.md. +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "runtime/debug" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/ses" + "github.com/aws/aws-sdk-go/service/ssm" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/orgimport" + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/approvals" + member_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/member-service" + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" +) + +const usageText = `usage: + org_import audit [--out-dir ./org-import-out] [report flags] + org_import ingest [--apply] [--yes] [--tranche N] [--ids id1,id2] [--mapping map.csv] [--decisions decisions.csv] + [--shared-domains domains.txt] [--state state.jsonl] [--routes register,rewrite] [--skip-wait] + [--use-apex] [--out-dir ./org-import-out] [report flags] + report flags: [--email-to a@x,b@y] [--no-email] [--no-aws-log] [--aws-log-group /easycla/org-import/] + +Environment: STAGE=dev|prod (required), AWS credentials for that account (AWS_PROFILE/AWS_SDK_LOAD_CONFIG=1 +or exported keys), AWS_REGION (default us-east-1), LOG_LEVEL (default warn), ORG_IMPORT_USE_APEX=true. +Without --apply nothing is written to EasyCLA, ACS, Salesforce or member-service; every run still writes the +report files and run.log under --out-dir, copies run.log to CloudWatch Logs and e-mails the full decision +record to SSM cla-org-import-report-emails- (or --email-to). +` + +type combinedRepo struct { + users.UserRepository + company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository +} + +type env struct { + deps orgimport.Deps + cfg config.Config + sess *session.Session +} + +type reportFlags struct { + emailTo string + noEmail bool + noAWSLog bool + logGroup string +} + +const ( + cmdAudit = "audit" + cmdIngest = "ingest" + trueString = "true" + unknownValue = "unknown" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdin)) +} + +func run(args []string, stdin io.Reader) int { + if len(args) < 1 { + fmt.Fprint(os.Stderr, usageText) + return 2 + } + if os.Getenv("LOG_LEVEL") == "" { + _ = os.Setenv("LOG_LEVEL", "warn") + } + cmd := args[0] + fs := flag.NewFlagSet("org_import "+cmd, flag.ContinueOnError) + fs.Usage = func() { fmt.Fprint(os.Stderr, usageText); fs.PrintDefaults() } + outDir := fs.String("out-dir", "./org-import-out", "directory for CSV reports and run.log (gitignored)") + apply := fs.Bool("apply", false, "perform writes (default: dry run)") + yes := fs.Bool("yes", false, "skip the confirmation prompt with --apply") + tranche := fs.Int("tranche", 0, "process at most N groups (0 = all)") + ids := fs.String("ids", "", "comma-separated company_external_id list (old or new ids)") + mapping := fs.String("mapping", "", "Salesforce mapping CSV old_id,new_id,action,approved") + decisions := fs.String("decisions", "", "duplicate-review decisions CSV decision,old_ids,target_sfid,reviewer,note (lfx-self-serve #3085)") + sharedDomains := fs.String("shared-domains", "", "shared-domain list, one domain per line (default: built-in list)") + state := fs.String("state", "", "append-only JSONL state file for rewrite tranches") + routes := fs.String("routes", "register,rewrite", "routes to process: register,rewrite") + skipWait := fs.Bool("skip-wait", false, "check new ids in org-service once instead of waiting up to 40 minutes") + useApex := fs.Bool("use-apex", os.Getenv("ORG_IMPORT_USE_APEX") == trueString, "resolve new ids through the Salesforce Apex endpoint (needs cla-salesforce-apex-* SSM params)") + waitMax := fs.Duration("wait-max", 40*time.Minute, "maximum org-service propagation wait") + var rf reportFlags + fs.StringVar(&rf.emailTo, "email-to", "", "report recipients (comma-separated); default SSM cla-org-import-report-emails-") + fs.BoolVar(&rf.noEmail, "no-email", false, "do not e-mail the report") + fs.BoolVar(&rf.noAWSLog, "no-aws-log", false, "do not copy run.log to CloudWatch Logs") + fs.StringVar(&rf.logGroup, "aws-log-group", "", "CloudWatch Logs group (default /easycla/org-import/)") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + if cmd != cmdAudit && cmd != cmdIngest { + fmt.Fprint(os.Stderr, usageText) + return 2 + } + stage := os.Getenv("STAGE") + if stage == "" { + fmt.Fprintln(os.Stderr, "STAGE is not set") + return 2 + } + if os.Getenv("AWS_REGION") == "" { + _ = os.Setenv("AWS_REGION", "us-east-1") + } + if rf.logGroup == "" { + rf.logGroup = "/easycla/org-import/" + stage + } + + opts := orgimport.Options{ + Stage: stage, Apply: *apply, Tranche: *tranche, Mapping: *mapping, Decisions: *decisions, SharedDomains: *sharedDomains, + State: *state, SkipWait: *skipWait, UseApex: *useApex, OutDir: *outDir, WaitMax: *waitMax, + } + if *ids != "" { + opts.IDs = strings.Split(*ids, ",") + } + for _, r := range strings.Split(*routes, ",") { + switch strings.TrimSpace(r) { + case "register": + opts.Routes = append(opts.Routes, orgimport.RouteRegister) + case "rewrite": + opts.Routes = append(opts.Routes, orgimport.RouteRewrite) + case "": + default: + fmt.Fprintf(os.Stderr, "unknown route %q (register|rewrite)\n", r) + return 2 + } + } + if !*apply && *yes { + fmt.Fprintln(os.Stderr, "note: --yes has no effect without --apply") + } + + if err := os.MkdirAll(*outDir, 0o750); err != nil { + fmt.Fprintf(os.Stderr, "cannot create %s: %v\n", *outDir, err) + return 2 + } + logPath := filepath.Join(*outDir, "run.log") + logFile, err := os.Create(filepath.Clean(logPath)) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot create %s: %v\n", logPath, err) + return 2 + } + out := io.MultiWriter(os.Stdout, logFile) + errOut := io.MultiWriter(os.Stderr, logFile) + log.GetLogger().SetOutput(io.MultiWriter(os.Stderr, logFile)) + + info := orgimport.RunInfo{ + Stage: stage, Command: cmd, Apply: *apply, Args: args, Start: time.Now().UTC(), Runner: runnerName(), + Repository: os.Getenv("GITHUB_REPOSITORY"), Revision: buildRevision(), OutDir: *outDir, + Workflow: orgimport.WorkflowInputs{Routes: *routes, Tranche: fmt.Sprint(*tranche), IDs: *ids, Mapping: *mapping, Decisions: *decisions, SharedDomains: *sharedDomains}, + } + if runID := os.Getenv("GITHUB_RUN_ID"); runID != "" { + info.RunURL = fmt.Sprintf("%s/%s/actions/runs/%s", os.Getenv("GITHUB_SERVER_URL"), os.Getenv("GITHUB_REPOSITORY"), runID) + info.Artifact = fmt.Sprintf("org-import-out-%s-%s", stage, runID) + } + fmt.Fprintf(out, "org-import %s %s stage=%s start=%s runner=%s revision=%s args=%s\n", cmd, info.Mode(), stage, info.Start.Format(time.RFC3339), info.Runner, info.Revision, strings.Join(args, " ")) + + ctx := context.Background() + var plan *orgimport.Plan + var audit *orgimport.AuditResult + var e env + code := func() int { + var err error + e, err = wire(ctx, stage, opts.UseApex) + if err != nil { + fmt.Fprintf(errOut, "setup failed: %v\n", err) + info.Err = err.Error() + return 2 + } + e.deps.Out = out + if cmd == cmdAudit { + if audit, err = orgimport.Audit(ctx, e.deps, opts); err != nil { + fmt.Fprintf(errOut, "audit failed: %v\n", err) + info.Err = err.Error() + return 1 + } + info.Summary = fmt.Sprintf("%d company rows audited", len(audit.Rows)) + fmt.Fprintf(out, "reports written to %s (audit.csv, unresolvable.csv, possible_duplicates.csv, run.log)\n", *outDir) + return 0 + } + if plan, err = orgimport.BuildPlan(ctx, e.deps, opts); err != nil { + fmt.Fprintf(errOut, "planning failed: %v\n", err) + info.Err = err.Error() + return 1 + } + plan.Print(out) + if *apply && !*yes && !confirm(stage, out, stdin) { + fmt.Fprintf(out, "aborted: nothing was written\n") + info.Err = "aborted at the confirmation prompt" + return 2 + } + summary, err := orgimport.Execute(ctx, e.deps, opts, plan) + fmt.Fprintf(out, "%s\n", summary.String()) + info.Summary = summary.String() + if err != nil { + fmt.Fprintf(errOut, "ingest finished with errors: %v\n", err) + info.Err = err.Error() + return 1 + } + fmt.Fprintf(out, "reports written to %s (plan.csv, manual_actions.csv, targets.csv, to_salesforce.csv, run.log)\n", *outDir) + return 0 + }() + + info.End = time.Now().UTC() + report(ctx, e, info, plan, audit, rf, logPath, out, errOut) + _ = logFile.Close() + return code +} + +// report ships run.log to CloudWatch Logs and e-mails the decision record; failures never change the exit code. +func report(ctx context.Context, e env, info orgimport.RunInfo, plan *orgimport.Plan, audit *orgimport.AuditResult, rf reportFlags, logPath string, out, errOut io.Writer) { + runLog, err := os.ReadFile(filepath.Clean(logPath)) + if err != nil { + fmt.Fprintf(errOut, "report: cannot read %s: %v\n", logPath, err) + } + if e.sess == nil { + fmt.Fprintf(errOut, "report: AWS session unavailable, skipping CloudWatch Logs and e-mail\n") + return + } + if !rf.noAWSLog { + info.LogGroup = rf.logGroup + info.LogStream = strings.NewReplacer(":", "-", "*", "-").Replace(fmt.Sprintf("%s-%s-%s-%s", info.Start.Format("2006-01-02T15-04-05Z"), info.Command, info.Mode(), info.Runner)) + shipper := &orgimport.LogShipper{Client: cloudwatchlogs.New(e.sess), Group: info.LogGroup, Stream: info.LogStream} + n, shipErr := shipper.Ship(ctx, runLog) + if shipErr != nil { + fmt.Fprintf(errOut, "report: CloudWatch Logs failed (%s/%s): %v\n", info.LogGroup, info.LogStream, shipErr) + info.LogGroup, info.LogStream = "", "" + } else { + fmt.Fprintf(out, "run.log copied to CloudWatch Logs %s stream %s (%d events)\n", info.LogGroup, info.LogStream, n) + } + } + if rf.noEmail { + fmt.Fprintf(out, "report e-mail disabled (--no-email)\n") + return + } + recipients := orgimport.ParseRecipients(rf.emailTo) + if len(recipients) == 0 { + value, found, ssmErr := readSSM(ctx, ssm.New(e.sess), fmt.Sprintf("cla-org-import-report-emails-%s", info.Stage), false) + if ssmErr != nil { + fmt.Fprintf(errOut, "report: %v\n", ssmErr) + } + if !found { + fmt.Fprintf(out, "report e-mail skipped: SSM cla-org-import-report-emails-%s is not set and --email-to is empty\n", info.Stage) + return + } + recipients = orgimport.ParseRecipients(value) + } + if len(recipients) == 0 || e.cfg.SenderEmailAddress == "" { + fmt.Fprintf(out, "report e-mail skipped: recipients=%d sender=%q\n", len(recipients), e.cfg.SenderEmailAddress) + return + } + rep := orgimport.BuildReport(info, plan, audit, runLog) + raw, err := rep.MIME(e.cfg.SenderEmailAddress, recipients) + if err != nil { + fmt.Fprintf(errOut, "report: building e-mail failed: %v\n", err) + return + } + id, err := orgimport.Mailer{Client: ses.New(e.sess)}.Send(ctx, e.cfg.SenderEmailAddress, recipients, raw) + if err != nil { + fmt.Fprintf(errOut, "report: SES send failed: %v\n", err) + return + } + fmt.Fprintf(out, "report e-mailed to %s (%d bytes, %d attachments, SES message id %s)\n", strings.Join(recipients, ","), len(raw), len(rep.Attachments), id) +} + +func runnerName() string { + if id := os.Getenv("GITHUB_RUN_ID"); id != "" { + return "gha-" + id + } + user := os.Getenv("USER") + if user == "" { + user = unknownValue + } + host, err := os.Hostname() + if err != nil || host == "" { + host = "localhost" + } + return user + "@" + host +} + +// set by the Makefile (-X main.commit=… -X main.branch=…); a package build stamps VCS info instead +var ( + commit string + branch string +) + +func buildRevision() string { + if commit != "" { + if branch != "" { + return commit + " (" + branch + ")" + } + return commit + } + if bi, ok := debug.ReadBuildInfo(); ok { + rev, modified := "", "" + for _, s := range bi.Settings { + switch s.Key { + case "vcs.revision": + rev = s.Value + case "vcs.modified": + if s.Value == trueString { + modified = "-dirty" + } + } + } + if rev != "" { + return rev + modified + } + } + if sha := os.Getenv("GITHUB_SHA"); sha != "" { + return sha + } + return unknownValue +} + +func confirm(stage string, out io.Writer, stdin io.Reader) bool { + fmt.Fprintf(out, "APPLY mode: the plan above will be executed against %s. Type the stage name to continue: ", stage) + line, err := bufio.NewReader(stdin).ReadString('\n') + if err != nil && line == "" { + return false + } + return strings.TrimSpace(line) == stage +} + +func wire(ctx context.Context, stage string, useApex bool) (env, error) { + awsSession := session.Must(session.NewSession(&aws.Config{})) + e := env{sess: awsSession} + cfg, err := config.LoadConfig("", awsSession, stage) + if err != nil { + return e, fmt.Errorf("loading SSM config: %w", err) + } + e.cfg = cfg + token.Init(cfg.Auth0Platform.ClientID, cfg.Auth0Platform.ClientSecret, cfg.Auth0Platform.URL, cfg.Auth0Platform.Audience) + + usersRepo := users.NewRepository(awsSession, stage) + companyRepo := company.NewRepository(awsSession, stage) + projectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) + repositoriesRepo := repositories.NewRepository(awsSession, stage) + gerritRepo := gerrits.NewRepository(awsSession, stage) + projectRepo := repository.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) + eventsRepo := events.NewRepository(awsSession, stage) + githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage) + approvalRepo := approvals.NewRepository(stage, awsSession, fmt.Sprintf("cla-%s-approvals", stage)) + eventsService := events.NewService(eventsRepo, combinedRepo{usersRepo, companyRepo, projectRepo, projectClaGroupRepo}) + signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo, gerrits.NewService(gerritRepo), approvalRepo) + + organization_service.InitClient(cfg.APIGatewayURL, eventsService) + acs_service.InitClient(cfg.APIGatewayURL, cfg.AcsAPIKey) + + e.deps = orgimport.Deps{ + Companies: companyRepo, + Signatures: signaturesRepo, + ECLAs: orgimport.DynamoECLACounter{DB: dynamodb.New(awsSession), Table: fmt.Sprintf("cla-%s-signatures", stage)}, + Events: events.NewRekeyRepository(awsSession, stage), + Orgs: orgimport.OrgServiceAdapter{Client: organization_service.GetClient()}, + ACS: acs_service.GetClient(), + } + + members, err := member_service.NewClient(member_service.Config{ + BaseURL: cfg.MemberService.BaseURL, + Audience: cfg.MemberService.Audience, + OAuthTokenURL: cfg.Auth0Platform.URL, + ClientID: cfg.Auth0Platform.ClientID, + ClientSecret: cfg.Auth0Platform.ClientSecret, + }) + switch { + case err == nil: + e.deps.Members = members + case errors.Is(err, member_service.ErrNotConfigured): + log.Warnf("member-service not configured (cla-member-service-base-url-%s / cla-member-service-auth0-audience-%s): liveness falls back to org-service, register steps will fail", stage, stage) + default: + return e, err + } + + if useApex { + ssmClient := ssm.New(awsSession) + baseURL, err := requireSSM(ctx, ssmClient, fmt.Sprintf("cla-salesforce-apex-base-url-%s", stage), false) + if err != nil { + return e, err + } + apexToken, err := requireSSM(ctx, ssmClient, fmt.Sprintf("cla-salesforce-apex-token-%s", stage), true) + if err != nil { + return e, err + } + apex, err := orgimport.NewApexClient(baseURL, apexToken) + if err != nil { + return e, err + } + e.deps.Apex = apex + } + return e, nil +} + +func requireSSM(ctx context.Context, client *ssm.SSM, key string, decrypt bool) (string, error) { + value, found, err := readSSM(ctx, client, key, decrypt) + if err != nil { + return "", err + } + if !found { + return "", fmt.Errorf("%w: %s", orgimport.ErrApexUnavailable, key) + } + return value, nil +} + +func readSSM(ctx context.Context, client *ssm.SSM, key string, decrypt bool) (string, bool, error) { + out, err := client.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(key), WithDecryption: aws.Bool(decrypt)}) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ssm.ErrCodeParameterNotFound { + return "", false, nil + } + return "", false, fmt.Errorf("reading SSM %s: %w", key, err) + } + return strings.TrimSpace(aws.StringValue(out.Parameter.Value)), true, nil +} diff --git a/cla-backend-go/cmd/org_import/main_test.go b/cla-backend-go/cmd/org_import/main_test.go new file mode 100644 index 000000000..5947122be --- /dev/null +++ b/cla-backend-go/cmd/org_import/main_test.go @@ -0,0 +1,45 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRunUsageErrors(t *testing.T) { + t.Setenv("STAGE", "") + assert.Equal(t, 2, run(nil, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"bogus"}, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"ingest", "--nope"}, strings.NewReader(""))) + assert.Equal(t, 2, run([]string{"ingest"}, strings.NewReader("")), "STAGE is required") + t.Setenv("STAGE", "dev") + assert.Equal(t, 2, run([]string{"ingest", "--routes", "delete"}, strings.NewReader(""))) +} + +func TestConfirm(t *testing.T) { + var out bytes.Buffer + assert.True(t, confirm("dev", &out, strings.NewReader("dev\n"))) + assert.Contains(t, out.String(), "Type the stage name") + assert.False(t, confirm("dev", &out, strings.NewReader("prod\n"))) + assert.False(t, confirm("dev", &out, strings.NewReader(""))) + assert.True(t, confirm("prod", &out, strings.NewReader("prod")), "last line without newline") +} + +func TestRunnerAndRevision(t *testing.T) { + t.Setenv("GITHUB_RUN_ID", "42") + assert.Equal(t, "gha-42", runnerName()) + t.Setenv("GITHUB_RUN_ID", "") + t.Setenv("USER", "lukasz") + assert.True(t, strings.HasPrefix(runnerName(), "lukasz@")) + assert.NotEmpty(t, buildRevision()) + commit, branch = "abc123", "unicron-x" + defer func() { commit, branch = "", "" }() + assert.Equal(t, "abc123 (unicron-x)", buildRevision(), "Makefile -X values win over VCS stamping") + branch = "" + assert.Equal(t, "abc123", buildRevision()) +} diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index c272b7410..7ba361ffa 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -508,7 +508,7 @@ func server(localMode bool) http.Handler { version.Configure(api, Version, Commit, Branch, BuildDate) v2Version.Configure(v2API, Version, Commit, Branch, BuildDate) events.Configure(api, eventsService) - v2Events.Configure(v2API, eventsService, v1CompanyRepo, v1ProjectClaGroupRepo, v1ProjectService) + v2Events.Configure(v2API, eventsService, v1CompanyService, v1ProjectClaGroupRepo, v1ProjectService) v2Metrics.Configure(v2API, v2MetricsService, v1CompanyRepo) github_organizations.Configure(api, githubOrganizationsService, eventsService) v2GithubOrganizations.Configure(v2API, v2GithubOrganizationsService, eventsService) diff --git a/cla-backend-go/company/handlers.go b/cla-backend-go/company/handlers.go index e90018996..9422a3d6d 100644 --- a/cla-backend-go/company/handlers.go +++ b/cla-backend-go/company/handlers.go @@ -23,7 +23,6 @@ import ( "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/company" log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/linuxfoundation/easycla/cla-backend-go/user" - orgService "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" "github.com/go-openapi/runtime/middleware" ) @@ -73,21 +72,10 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv api.CompanyGetCompanyByExternalIDHandler = company.GetCompanyByExternalIDHandlerFunc(func(params company.GetCompanyByExternalIDParams) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint - // Check for Salesforce org - orgClient := orgService.GetClient() - org, getErr := orgClient.GetOrganization(ctx, params.CompanySFID) - - if getErr != nil { - msg := fmt.Sprintf("Failed to get salesforce org for ID: %s ", params.CompanySFID) - log.Warn(msg) - return company.NewGetCompanyByExternalIDBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Code: "400", - Message: msg, - }) - } - companyModel, err := service.GetCompanyByExternalID(ctx, params.CompanySFID) + // Persisted row when it exists, otherwise a non-persisted virtual company backed by the Salesforce org (#2751) + companyModel, err := service.ResolveCompany(ctx, params.CompanySFID) if err != nil { - msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to get associated salesforce Organization: %s using SFID: %s, error: %v", org.Name, params.CompanySFID, err) + msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to get associated salesforce Organization using SFID: %s, error: %v", params.CompanySFID, err) log.Warnf("%s", msg) return company.NewGetCompanyByExternalIDBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", diff --git a/cla-backend-go/company/mocks/mock_repo.go b/cla-backend-go/company/mocks/mock_repo.go index c3d59cd2a..695e12cb2 100644 --- a/cla-backend-go/company/mocks/mock_repo.go +++ b/cla-backend-go/company/mocks/mock_repo.go @@ -1,5 +1,6 @@ // Copyright The Linux Foundation and each contributor to CommunityBridge. // SPDX-License-Identifier: MIT +// // Code generated by MockGen. DO NOT EDIT. // Source: company/repository.go @@ -69,6 +70,21 @@ func (mr *MockIRepositoryMockRecorder) ApproveCompanyAccessRequest(ctx, companyI return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveCompanyAccessRequest", reflect.TypeOf((*MockIRepository)(nil).ApproveCompanyAccessRequest), ctx, companyInviteID) } +// ClearCompanySanctionStatusIfSSS mocks base method. +func (m *MockIRepository) ClearCompanySanctionStatusIfSSS(ctx context.Context, companyID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClearCompanySanctionStatusIfSSS", ctx, companyID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ClearCompanySanctionStatusIfSSS indicates an expected call of ClearCompanySanctionStatusIfSSS. +func (mr *MockIRepositoryMockRecorder) ClearCompanySanctionStatusIfSSS(ctx, companyID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearCompanySanctionStatusIfSSS", reflect.TypeOf((*MockIRepository)(nil).ClearCompanySanctionStatusIfSSS), ctx, companyID) +} + // CreateCompany mocks base method. func (m *MockIRepository) CreateCompany(ctx context.Context, in *models.Company) (*models.Company, error) { m.ctrl.T.Helper() @@ -112,6 +128,22 @@ func (mr *MockIRepositoryMockRecorder) DeleteCompanyBySFID(ctx, companySFID inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCompanyBySFID", reflect.TypeOf((*MockIRepository)(nil).DeleteCompanyBySFID), ctx, companySFID) } +// EnsureCompanyForExternalID mocks base method. +func (m *MockIRepository) EnsureCompanyForExternalID(ctx context.Context, externalID, companyName, signingEntityName string) (*models.Company, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureCompanyForExternalID", ctx, externalID, companyName, signingEntityName) + ret0, _ := ret[0].(*models.Company) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// EnsureCompanyForExternalID indicates an expected call of EnsureCompanyForExternalID. +func (mr *MockIRepositoryMockRecorder) EnsureCompanyForExternalID(ctx, externalID, companyName, signingEntityName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureCompanyForExternalID", reflect.TypeOf((*MockIRepository)(nil).EnsureCompanyForExternalID), ctx, externalID, companyName, signingEntityName) +} + // GetCompanies mocks base method. func (m *MockIRepository) GetCompanies(ctx context.Context) (*models.Companies, error) { m.ctrl.T.Helper() @@ -262,6 +294,21 @@ func (mr *MockIRepositoryMockRecorder) GetCompanyInviteRequests(ctx, companyID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompanyInviteRequests", reflect.TypeOf((*MockIRepository)(nil).GetCompanyInviteRequests), ctx, companyID, status) } +// GetCompanyRecord mocks base method. +func (m *MockIRepository) GetCompanyRecord(ctx context.Context, companyID string) (*company.DBModel, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCompanyRecord", ctx, companyID) + ret0, _ := ret[0].(*company.DBModel) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCompanyRecord indicates an expected call of GetCompanyRecord. +func (mr *MockIRepositoryMockRecorder) GetCompanyRecord(ctx, companyID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompanyRecord", reflect.TypeOf((*MockIRepository)(nil).GetCompanyRecord), ctx, companyID) +} + // GetCompanyUserInviteRequests mocks base method. func (m *MockIRepository) GetCompanyUserInviteRequests(ctx context.Context, companyID, userID string) (*company.Invite, error) { m.ctrl.T.Helper() @@ -350,31 +397,30 @@ func (mr *MockIRepositoryMockRecorder) UpdateCompanyAccessList(ctx, companyID, c return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanyAccessList", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanyAccessList), ctx, companyID, companyACL) } -// UpdateCompanySanctionStatus mocks base method. -func (m *MockIRepository) UpdateCompanySanctionStatus(ctx context.Context, companyID string, sanctioned bool, origin string) error { +// UpdateCompanyExternalID mocks base method. +func (m *MockIRepository) UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateCompanySanctionStatus", ctx, companyID, sanctioned, origin) + ret := m.ctrl.Call(m, "UpdateCompanyExternalID", ctx, companyID, oldExternalID, newExternalID) ret0, _ := ret[0].(error) return ret0 } -// UpdateCompanySanctionStatus indicates an expected call of UpdateCompanySanctionStatus. -func (mr *MockIRepositoryMockRecorder) UpdateCompanySanctionStatus(ctx, companyID, sanctioned, origin interface{}) *gomock.Call { +// UpdateCompanyExternalID indicates an expected call of UpdateCompanyExternalID. +func (mr *MockIRepositoryMockRecorder) UpdateCompanyExternalID(ctx, companyID, oldExternalID, newExternalID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanySanctionStatus", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanySanctionStatus), ctx, companyID, sanctioned, origin) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanyExternalID", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanyExternalID), ctx, companyID, oldExternalID, newExternalID) } -// ClearCompanySanctionStatusIfSSS mocks base method. -func (m *MockIRepository) ClearCompanySanctionStatusIfSSS(ctx context.Context, companyID string) (bool, error) { +// UpdateCompanySanctionStatus mocks base method. +func (m *MockIRepository) UpdateCompanySanctionStatus(ctx context.Context, companyID string, sanctioned bool, origin string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClearCompanySanctionStatusIfSSS", ctx, companyID) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "UpdateCompanySanctionStatus", ctx, companyID, sanctioned, origin) + ret0, _ := ret[0].(error) + return ret0 } -// ClearCompanySanctionStatusIfSSS indicates an expected call of ClearCompanySanctionStatusIfSSS. -func (mr *MockIRepositoryMockRecorder) ClearCompanySanctionStatusIfSSS(ctx, companyID interface{}) *gomock.Call { +// UpdateCompanySanctionStatus indicates an expected call of UpdateCompanySanctionStatus. +func (mr *MockIRepositoryMockRecorder) UpdateCompanySanctionStatus(ctx, companyID, sanctioned, origin interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearCompanySanctionStatusIfSSS", reflect.TypeOf((*MockIRepository)(nil).ClearCompanySanctionStatusIfSSS), ctx, companyID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCompanySanctionStatus", reflect.TypeOf((*MockIRepository)(nil).UpdateCompanySanctionStatus), ctx, companyID, sanctioned, origin) } diff --git a/cla-backend-go/company/mocks/mock_service.go b/cla-backend-go/company/mocks/mock_service.go index d103990b0..ba764b9ea 100644 --- a/cla-backend-go/company/mocks/mock_service.go +++ b/cla-backend-go/company/mocks/mock_service.go @@ -1,5 +1,6 @@ // Copyright The Linux Foundation and each contributor to CommunityBridge. // SPDX-License-Identifier: MIT +// // Code generated by MockGen. DO NOT EDIT. // Source: company/service.go @@ -83,21 +84,6 @@ func (mr *MockIServiceMockRecorder) ApproveCompanyAccessRequest(ctx, companyInvi return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveCompanyAccessRequest", reflect.TypeOf((*MockIService)(nil).ApproveCompanyAccessRequest), ctx, companyInviteID) } -// CreateOrgFromExternalID mocks base method. -func (m *MockIService) CreateOrgFromExternalID(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateOrgFromExternalID", ctx, signingEntityName, companySFID) - ret0, _ := ret[0].(*models.Company) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateOrgFromExternalID indicates an expected call of CreateOrgFromExternalID. -func (mr *MockIServiceMockRecorder) CreateOrgFromExternalID(ctx, signingEntityName, companySFID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrgFromExternalID", reflect.TypeOf((*MockIService)(nil).CreateOrgFromExternalID), ctx, signingEntityName, companySFID) -} - // GetCompanies mocks base method. func (m *MockIService) GetCompanies(ctx context.Context) (*models.Companies, error) { m.ctrl.T.Helper() @@ -263,6 +249,21 @@ func (mr *MockIServiceMockRecorder) RejectCompanyAccessRequest(ctx, companyInvit return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RejectCompanyAccessRequest", reflect.TypeOf((*MockIService)(nil).RejectCompanyAccessRequest), ctx, companyInviteID) } +// ResolveCompany mocks base method. +func (m *MockIService) ResolveCompany(ctx context.Context, companyIDOrSFID string) (*models.Company, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveCompany", ctx, companyIDOrSFID) + ret0, _ := ret[0].(*models.Company) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveCompany indicates an expected call of ResolveCompany. +func (mr *MockIServiceMockRecorder) ResolveCompany(ctx, companyIDOrSFID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveCompany", reflect.TypeOf((*MockIService)(nil).ResolveCompany), ctx, companyIDOrSFID) +} + // SearchCompanyByName mocks base method. func (m *MockIService) SearchCompanyByName(ctx context.Context, companyName, nextKey string) (*models.Companies, error) { m.ctrl.T.Helper() @@ -292,55 +293,3 @@ func (mr *MockIServiceMockRecorder) SearchOrganizationByName(ctx, orgName, websi mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchOrganizationByName", reflect.TypeOf((*MockIService)(nil).SearchOrganizationByName), ctx, orgName, websiteName, includeSigningEntityName, filter) } - -// getPreferredNameAndEmail mocks base method. -func (m *MockIService) getPreferredNameAndEmail(ctx context.Context, lfid string) (string, string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "getPreferredNameAndEmail", ctx, lfid) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// getPreferredNameAndEmail indicates an expected call of getPreferredNameAndEmail. -func (mr *MockIServiceMockRecorder) getPreferredNameAndEmail(ctx, lfid interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getPreferredNameAndEmail", reflect.TypeOf((*MockIService)(nil).getPreferredNameAndEmail), ctx, lfid) -} - -// sendRequestAccessEmail mocks base method. -func (m *MockIService) sendRequestAccessEmail(ctx context.Context, companyModel *models.Company, requesterName, requesterEmail, recipientName, recipientAddress string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "sendRequestAccessEmail", ctx, companyModel, requesterName, requesterEmail, recipientName, recipientAddress) -} - -// sendRequestAccessEmail indicates an expected call of sendRequestAccessEmail. -func (mr *MockIServiceMockRecorder) sendRequestAccessEmail(ctx, companyModel, requesterName, requesterEmail, recipientName, recipientAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "sendRequestAccessEmail", reflect.TypeOf((*MockIService)(nil).sendRequestAccessEmail), ctx, companyModel, requesterName, requesterEmail, recipientName, recipientAddress) -} - -// sendRequestApprovedEmailToRecipient mocks base method. -func (m *MockIService) sendRequestApprovedEmailToRecipient(ctx context.Context, companyModel *models.Company, recipientName, recipientAddress string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "sendRequestApprovedEmailToRecipient", ctx, companyModel, recipientName, recipientAddress) -} - -// sendRequestApprovedEmailToRecipient indicates an expected call of sendRequestApprovedEmailToRecipient. -func (mr *MockIServiceMockRecorder) sendRequestApprovedEmailToRecipient(ctx, companyModel, recipientName, recipientAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "sendRequestApprovedEmailToRecipient", reflect.TypeOf((*MockIService)(nil).sendRequestApprovedEmailToRecipient), ctx, companyModel, recipientName, recipientAddress) -} - -// sendRequestRejectedEmailToRecipient mocks base method. -func (m *MockIService) sendRequestRejectedEmailToRecipient(ctx context.Context, companyModel *models.Company, recipientName, recipientAddress string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "sendRequestRejectedEmailToRecipient", ctx, companyModel, recipientName, recipientAddress) -} - -// sendRequestRejectedEmailToRecipient indicates an expected call of sendRequestRejectedEmailToRecipient. -func (mr *MockIServiceMockRecorder) sendRequestRejectedEmailToRecipient(ctx, companyModel, recipientName, recipientAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "sendRequestRejectedEmailToRecipient", reflect.TypeOf((*MockIService)(nil).sendRequestRejectedEmailToRecipient), ctx, companyModel, recipientName, recipientAddress) -} diff --git a/cla-backend-go/company/models.go b/cla-backend-go/company/models.go index 73f58a57c..7d3fab470 100644 --- a/cla-backend-go/company/models.go +++ b/cla-backend-go/company/models.go @@ -29,6 +29,9 @@ type DBModel struct { SanctionOrigin string `dynamodbav:"sanction_origin" json:"sanction_origin,omitempty"` SanctionedDate string `dynamodbav:"sanctioned_date" json:"sanctioned_date,omitempty"` Version string `dynamodbav:"version" json:"version"` + + // PreviousCompanyExternalID is set only by the org import tool when it rewrites company_external_id. + PreviousCompanyExternalID string `dynamodbav:"previous_company_external_id,omitempty" json:"previous_company_external_id,omitempty"` } // Invite data model diff --git a/cla-backend-go/company/repository.go b/cla-backend-go/company/repository.go index bfcb8313f..3c7f10e14 100644 --- a/cla-backend-go/company/repository.go +++ b/cla-backend-go/company/repository.go @@ -57,6 +57,9 @@ type IRepository interface { //nolint UpdateCompanySanctionStatus(ctx context.Context, companyID string, sanctioned bool, origin string) error ClearCompanySanctionStatusIfSSS(ctx context.Context, companyID string) (bool, error) IsCCLAEnabledForCompany(ctx context.Context, companyID string) (bool, error) + EnsureCompanyForExternalID(ctx context.Context, externalID, companyName, signingEntityName string) (*models.Company, bool, error) + GetCompanyRecord(ctx context.Context, companyID string) (*DBModel, error) + UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error } type repository struct { diff --git a/cla-backend-go/company/repository_external_id.go b/cla-backend-go/company/repository_external_id.go new file mode 100644 index 000000000..bb48fd58e --- /dev/null +++ b/cla-backend-go/company/repository_external_id.go @@ -0,0 +1,241 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package company + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/gofrs/uuid" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" +) + +// ErrExternalIDConditionFailed is returned by UpdateCompanyExternalID when the row no longer carries the expected external ID. +var ErrExternalIDConditionFailed = errors.New("company external id changed concurrently") + +// ErrEnsureCompanyConflict is returned by EnsureCompanyForExternalID when the deterministic company id is already taken by a row of another organization. +var ErrEnsureCompanyConflict = errors.New("company id already used by a different organization") + +// ErrEmptyExternalID is returned when an external ID is required but empty. +var ErrEmptyExternalID = errors.New("company external id is empty") + +const ensureCompanyNote = "created at CCLA signing" + +// canonicalSigningEntity returns the identity component of a signing entity: empty for the +// parent record (no name, or the name equals the company name), otherwise the normalized name. +func canonicalSigningEntity(companyName, signingEntityName string) string { + entity := strings.ToLower(strings.TrimSpace(signingEntityName)) + if entity == "" || entity == strings.ToLower(strings.TrimSpace(companyName)) { + return "" + } + return entity +} + +// deterministicCompanyID derives a stable, UUIDv4-shaped company id from the external ID and the +// canonical signing entity so that concurrent first creations for the same organization collide on +// the primary key instead of producing two rows. +func deterministicCompanyID(externalID, canonicalEntity string) string { + sum := sha256.Sum256([]byte("easycla-company|" + externalID + "|" + canonicalEntity)) + var id uuid.UUID + copy(id[:], sum[:16]) + id[6] = (id[6] & 0x0f) | 0x40 + id[8] = (id[8] & 0x3f) | 0x80 + return id.String() +} + +// pickCompanyForEntity returns the row matching the canonical signing entity, or nil. +func pickCompanyForEntity(rows []*models.Company, canonicalEntity string) *models.Company { + for _, row := range rows { + if row != nil && canonicalSigningEntity(row.CompanyName, row.SigningEntityName) == canonicalEntity { + return row + } + } + return nil +} + +// EnsureCompanyForExternalID returns the company row for (externalID, signing entity), creating it +// when it does not exist. Existing rows always win and keep their ids; a new row gets the +// deterministic id and a conditional put, and on a conditional failure the winning row is read back +// and returned when it belongs to the same organization. The returned flag reports a creation. +func (repo repository) EnsureCompanyForExternalID(ctx context.Context, externalID, companyName, signingEntityName string) (*models.Company, bool, error) { + f := logrus.Fields{ + "functionName": "company.repository.EnsureCompanyForExternalID", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companySFID": externalID, + "companyName": companyName, + "signingEntityName": signingEntityName, + } + externalID = strings.TrimSpace(externalID) + if externalID == "" { + return nil, false, ErrEmptyExternalID + } + companyName = strings.TrimSpace(companyName) + if companyName == "" { + return nil, false, errors.New("company name is empty") + } + canonical := canonicalSigningEntity(companyName, signingEntityName) + + var existing *models.Company + if canonical == "" { + rows, err := repo.GetCompaniesByExternalID(ctx, externalID, false) + if err != nil { + if _, notFound := err.(*utils.CompanyNotFound); !notFound { + return nil, false, err + } + } else if len(rows) > 0 { + existing = rows[0] + } + } else { + rows, err := repo.GetCompaniesByExternalID(ctx, externalID, true) + if err != nil { + if _, notFound := err.(*utils.CompanyNotFound); !notFound { + return nil, false, err + } + } else { + existing = pickCompanyForEntity(rows, canonical) + } + } + if existing != nil { + log.WithFields(f).Debugf("reusing existing company %s", existing.CompanyID) + return existing, false, nil + } + + _, now := utils.CurrentTime() + record := &DBModel{ + CompanyID: deterministicCompanyID(externalID, canonical), + CompanyName: companyName, + SigningEntityName: companyName, + CompanyExternalID: externalID, + Created: now, + Updated: now, + Note: ensureCompanyNote, + Version: "v1", + } + if canonical != "" { + record.SigningEntityName = strings.TrimSpace(signingEntityName) + } + f["companyID"] = record.CompanyID + + av, err := dynamodbattribute.MarshalMap(record) + if err != nil { + return nil, false, err + } + _, err = repo.dynamoDBClient.PutItem(&dynamodb.PutItemInput{ + Item: av, + TableName: aws.String(repo.companyTableName), + ConditionExpression: aws.String("attribute_not_exists(company_id)"), + }) + if err == nil { + log.WithFields(f).Info("company created") + created, convErr := record.toModel() + return created, true, convErr + } + if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != dynamodb.ErrCodeConditionalCheckFailedException { + log.WithFields(f).WithError(err).Warn("problem creating company") + return nil, false, err + } + + winner, err := repo.getCompanyRecord(ctx, record.CompanyID, true) + if err != nil { + return nil, false, err + } + if winner.CompanyExternalID != externalID || canonicalSigningEntity(winner.CompanyName, winner.SigningEntityName) != canonical { + log.WithFields(f).Warnf("company id collision with external id %s / entity %q", winner.CompanyExternalID, winner.SigningEntityName) + return nil, false, ErrEnsureCompanyConflict + } + log.WithFields(f).Debug("company created concurrently - returning the winning row") + model, err := winner.toModel() + return model, false, err +} + +// GetCompanyRecord returns the raw company row (including previous_company_external_id) by id. +func (repo repository) GetCompanyRecord(ctx context.Context, companyID string) (*DBModel, error) { + return repo.getCompanyRecord(ctx, companyID, false) +} + +func (repo repository) getCompanyRecord(ctx context.Context, companyID string, consistent bool) (*DBModel, error) { + f := logrus.Fields{ + "functionName": "company.repository.getCompanyRecord", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyID": companyID, + } + if companyID == "" { + return nil, &utils.CompanyNotFound{Message: "company_id cannot be empty", CompanyID: companyID} + } + out, err := repo.dynamoDBClient.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(repo.companyTableName), + Key: map[string]*dynamodb.AttributeValue{"company_id": {S: aws.String(companyID)}}, + ConsistentRead: aws.Bool(consistent), + }) + if err != nil { + log.WithFields(f).WithError(err).Warn("error fetching company record") + return nil, err + } + if len(out.Item) == 0 { + return nil, &utils.CompanyNotFound{Message: "no company matching company record", CompanyID: companyID} + } + record := &DBModel{} + if err = dynamodbattribute.UnmarshalMap(out.Item, record); err != nil { + return nil, err + } + return record, nil +} + +// UpdateCompanyExternalID rewrites company_external_id from oldExternalID to newExternalID for one +// row, remembering the old value in previous_company_external_id. The write is conditional on the +// row still carrying oldExternalID (an empty oldExternalID means the row must have no external id +// yet, and no previous value is recorded); otherwise ErrExternalIDConditionFailed is returned. +func (repo repository) UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error { + f := logrus.Fields{ + "functionName": "company.repository.UpdateCompanyExternalID", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyID": companyID, + "oldExternalID": oldExternalID, + "newExternalID": newExternalID, + } + if strings.TrimSpace(companyID) == "" || strings.TrimSpace(newExternalID) == "" { + return fmt.Errorf("company id and new external id are required") + } + update, condition := "SET #E = :new, #P = :old, #M = :m", "#E = :old" + if strings.TrimSpace(oldExternalID) == "" { + oldExternalID = "" + update, condition = "SET #E = :new, #M = :m", "attribute_not_exists(#E) OR #E = :old" + } + _, now := utils.CurrentTime() + _, err := repo.dynamoDBClient.UpdateItem(&dynamodb.UpdateItemInput{ + TableName: aws.String(repo.companyTableName), + Key: map[string]*dynamodb.AttributeValue{"company_id": {S: aws.String(companyID)}}, + UpdateExpression: aws.String(update), + ConditionExpression: aws.String(condition), + ExpressionAttributeNames: map[string]*string{ + "#E": aws.String("company_external_id"), + "#P": aws.String("previous_company_external_id"), + "#M": aws.String("date_modified"), + }, + ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ + ":new": {S: aws.String(newExternalID)}, + ":old": {S: aws.String(oldExternalID)}, + ":m": {S: aws.String(now)}, + }, + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == dynamodb.ErrCodeConditionalCheckFailedException { + return ErrExternalIDConditionFailed + } + log.WithFields(f).WithError(err).Warn("error updating company external id") + return err + } + log.WithFields(f).Info("company external id rewritten") + return nil +} diff --git a/cla-backend-go/company/repository_external_id_test.go b/cla-backend-go/company/repository_external_id_test.go new file mode 100644 index 000000000..f64d6d14c --- /dev/null +++ b/cla-backend-go/company/repository_external_id_test.go @@ -0,0 +1,362 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package company + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeCompaniesTable is a minimal in-memory DynamoDB endpoint for the companies table: GetItem, +// conditional PutItem (attribute_not_exists), Query on external-company-index and the conditional +// UpdateItem used by UpdateCompanyExternalID. +type fakeCompaniesTable struct { + mu sync.Mutex + items map[string]map[string]interface{} + hiddenFromIndex map[string]bool // simulates GSI propagation lag + puts int + conditionFailures int + lastCondition string + failPuts bool +} + +type fakeAttr struct { + S *string + BOOL *bool +} + +var fakePlaceholderPair = regexp.MustCompile(`(#[0-9A-Za-z_]+) = (:[0-9A-Za-z_]+)`) + +func (f *fakeCompaniesTable) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + f.mu.Lock() + defer f.mu.Unlock() + switch r.Header.Get("X-Amz-Target") { + case "DynamoDB_20120810.GetItem": + var req struct { + Key map[string]fakeAttr + } + fakeCompanyDecode(w, body, &req) + resp := map[string]interface{}{} + if item, ok := f.items[*req.Key["company_id"].S]; ok { + resp["Item"] = item + } + fakeCompanyJSON(w, resp) + case "DynamoDB_20120810.PutItem": + var req struct { + Item map[string]interface{} + ConditionExpression string + } + fakeCompanyDecode(w, body, &req) + id := fakeCompanyString(req.Item, "company_id") + if f.failPuts { + fakeCompanyError(w, "InternalServerError", "injected") + return + } + if _, exists := f.items[id]; exists && strings.Contains(req.ConditionExpression, "attribute_not_exists(company_id)") { + f.conditionFailures++ + fakeCompanyError(w, "ConditionalCheckFailedException", "exists") + return + } + f.puts++ + f.items[id] = req.Item + fakeCompanyJSON(w, map[string]interface{}{}) + case "DynamoDB_20120810.Query": + var req struct { + IndexName string + KeyConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeAttr + } + fakeCompanyDecode(w, body, &req) + m := fakePlaceholderPair.FindStringSubmatch(req.KeyConditionExpression) + attr, value := req.ExpressionAttributeNames[m[1]], *req.ExpressionAttributeValues[m[2]].S + matched := []map[string]interface{}{} + for id, item := range f.items { + if !f.hiddenFromIndex[id] && fakeCompanyString(item, attr) == value { + matched = append(matched, item) + } + } + fakeCompanyJSON(w, map[string]interface{}{"Items": matched, "Count": len(matched), "ScannedCount": len(matched)}) + case "DynamoDB_20120810.UpdateItem": + var req struct { + Key map[string]fakeAttr + UpdateExpression string + ConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeAttr + } + fakeCompanyDecode(w, body, &req) + item, ok := f.items[*req.Key["company_id"].S] + if !ok { + item = map[string]interface{}{"company_id": map[string]interface{}{"S": *req.Key["company_id"].S}} + } + f.lastCondition = req.ConditionExpression + if req.ConditionExpression != "" { + m := fakePlaceholderPair.FindStringSubmatch(req.ConditionExpression) + if fakeCompanyString(item, req.ExpressionAttributeNames[m[1]]) != *req.ExpressionAttributeValues[m[2]].S { + f.conditionFailures++ + fakeCompanyError(w, "ConditionalCheckFailedException", "condition failed") + return + } + } + for _, pair := range fakePlaceholderPair.FindAllStringSubmatch(strings.TrimPrefix(req.UpdateExpression, "SET "), -1) { + item[req.ExpressionAttributeNames[pair[1]]] = map[string]interface{}{"S": *req.ExpressionAttributeValues[pair[2]].S} + } + f.items[*req.Key["company_id"].S] = item + fakeCompanyJSON(w, map[string]interface{}{}) + default: + http.Error(w, "unsupported operation "+r.Header.Get("X-Amz-Target"), http.StatusBadRequest) + } +} + +func fakeCompanyString(item map[string]interface{}, name string) string { + if attr, ok := item[name].(map[string]interface{}); ok { + if s, ok := attr["S"].(string); ok { + return s + } + } + return "" +} + +func fakeCompanyJSON(w http.ResponseWriter, payload interface{}) { + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func fakeCompanyDecode(w http.ResponseWriter, body []byte, v interface{}) { + if err := json.Unmarshal(body, v); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } +} + +func fakeCompanyError(w http.ResponseWriter, code, message string) { + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + w.WriteHeader(http.StatusBadRequest) + fakeCompanyJSON(w, map[string]string{"__type": "com.amazonaws.dynamodb.v20120810#" + code, "message": message}) +} + +func fakeCompanyItem(id, name, entity, externalID string) map[string]interface{} { + return map[string]interface{}{ + "company_id": map[string]interface{}{"S": id}, + "company_name": map[string]interface{}{"S": name}, + "signing_entity_name": map[string]interface{}{"S": entity}, + "company_external_id": map[string]interface{}{"S": externalID}, + "date_created": map[string]interface{}{"S": "2024-01-01T00:00:00Z"}, + "date_modified": map[string]interface{}{"S": "2024-01-01T00:00:00Z"}, + } +} + +func newCompanyRepo(t *testing.T, table *fakeCompaniesTable) (repository, *fakeCompaniesTable) { + if table.items == nil { + table.items = map[string]map[string]interface{}{} + } + server := httptest.NewServer(table) + t.Cleanup(server.Close) + awsSession, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(server.URL), + Credentials: credentials.NewStaticCredentials("test", "test", ""), + DisableSSL: aws.Bool(true), + MaxRetries: aws.Int(0), + }) + require.NoError(t, err) + return repository{stage: "test", dynamoDBClient: dynamodb.New(awsSession), companyTableName: "cla-test-companies"}, table +} + +func TestCanonicalSigningEntity(t *testing.T) { + assert.Equal(t, "", canonicalSigningEntity("Acme Inc", "")) + assert.Equal(t, "", canonicalSigningEntity("Acme Inc", " acme inc ")) + assert.Equal(t, "acme gmbh", canonicalSigningEntity("Acme Inc", " Acme GmbH")) +} + +func TestDeterministicCompanyID(t *testing.T) { + id := deterministicCompanyID("0014100000Te1TUAAZ", "") + parsed, err := uuid.FromString(id) + require.NoError(t, err) + assert.Equal(t, byte(4), parsed.Version()) + assert.Equal(t, uuid.VariantRFC4122, parsed.Variant()) + assert.Regexp(t, `^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[89ab][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$`, id) + assert.Equal(t, id, deterministicCompanyID("0014100000Te1TUAAZ", ""), "stable") + assert.NotEqual(t, id, deterministicCompanyID("0014100000Te1TUAAZ", "acme gmbh"), "entity is part of the identity") + assert.NotEqual(t, id, deterministicCompanyID("0014100000Te1TUAAY", ""), "external id is part of the identity") +} + +func TestEnsureCompanyForExternalID(t *testing.T) { + const sfid = "0014100000Te1TUAAZ" + + t.Run("empty external id is rejected without a write", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{}) + _, created, err := repo.EnsureCompanyForExternalID(context.Background(), " ", "Acme", "") + assert.ErrorIs(t, err, ErrEmptyExternalID) + assert.False(t, created) + assert.Equal(t, 0, table.puts) + }) + + t.Run("existing parent row is reused with its original id", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "legacy-id": fakeCompanyItem("legacy-id", "Acme Inc", "Acme Inc", sfid), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, "legacy-id", comp.CompanyID) + assert.Equal(t, 0, table.puts) + }) + + t.Run("existing named signing entity row is reused", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "parent": fakeCompanyItem("parent", "Acme Inc", "Acme Inc", sfid), + "child": fakeCompanyItem("child", "Acme Inc", "Acme GmbH", sfid), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "acme gmbh ") + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, "child", comp.CompanyID) + assert.Equal(t, 0, table.puts) + }) + + t.Run("a same-name row with another external id is never reused", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "other": fakeCompanyItem("other", "Acme Inc", "Acme Inc", "0014100000Other00"), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.NoError(t, err) + assert.True(t, created) + assert.Equal(t, deterministicCompanyID(sfid, ""), comp.CompanyID) + assert.Equal(t, sfid, comp.CompanyExternalID) + assert.Equal(t, "Acme Inc", comp.SigningEntityName) + assert.Equal(t, ensureCompanyNote, comp.Note) + assert.Equal(t, 1, table.puts) + assert.Len(t, table.items, 2) + }) + + t.Run("named entity row is created with the requested entity name", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "parent": fakeCompanyItem("parent", "Acme Inc", "Acme Inc", sfid), + }}) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "Acme GmbH") + require.NoError(t, err) + assert.True(t, created) + assert.Equal(t, deterministicCompanyID(sfid, "acme gmbh"), comp.CompanyID) + assert.Equal(t, "Acme GmbH", comp.SigningEntityName) + }) + + t.Run("concurrent creation converges on the winning row", func(t *testing.T) { + // the winner's row exists on the primary key but the GSI has not caught up yet + winnerID := deterministicCompanyID(sfid, "") + repo, table := newCompanyRepo(t, &fakeCompaniesTable{ + items: map[string]map[string]interface{}{winnerID: fakeCompanyItem(winnerID, "Acme Inc", "Acme Inc", sfid)}, + hiddenFromIndex: map[string]bool{winnerID: true}, + }) + comp, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.NoError(t, err) + assert.False(t, created) + assert.Equal(t, winnerID, comp.CompanyID) + assert.Equal(t, 1, table.conditionFailures) + assert.Equal(t, 0, table.puts) + }) + + t.Run("a foreign row on the deterministic key is a conflict", func(t *testing.T) { + winnerID := deterministicCompanyID(sfid, "") + table := &fakeCompaniesTable{items: map[string]map[string]interface{}{ + winnerID: fakeCompanyItem(winnerID, "Someone Else", "Someone Else", "0014100000Other00"), + }} + repo, _ := newCompanyRepo(t, table) + _, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + assert.ErrorIs(t, err, ErrEnsureCompanyConflict) + assert.False(t, created) + assert.Equal(t, 1, table.conditionFailures) + assert.Equal(t, 0, table.puts) + }) + + t.Run("a write failure is returned", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{failPuts: true}) + _, created, err := repo.EnsureCompanyForExternalID(context.Background(), sfid, "Acme Inc", "") + require.Error(t, err) + assert.False(t, created) + }) +} + +func TestUpdateCompanyExternalID(t *testing.T) { + t.Run("rewrites and remembers the previous id", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "lf-old"), + }}) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "lf-old", "0014100000New0000")) + record, err := repo.GetCompanyRecord(context.Background(), "c1") + require.NoError(t, err) + assert.Equal(t, "0014100000New0000", record.CompanyExternalID) + assert.Equal(t, "lf-old", record.PreviousCompanyExternalID) + assert.NotEqual(t, "2024-01-01T00:00:00Z", record.Updated) + assert.Equal(t, 0, table.conditionFailures) + }) + + t.Run("a row that no longer carries the old id fails the condition", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "0014100000New0000"), + }}) + err := repo.UpdateCompanyExternalID(context.Background(), "c1", "lf-old", "0014100000New0000") + assert.ErrorIs(t, err, ErrExternalIDConditionFailed) + assert.Equal(t, 1, table.conditionFailures) + }) + + t.Run("a blank old id fills a row without an external id and records no previous value", func(t *testing.T) { + missing := fakeCompanyItem("c1", "Blank Inc", "Blank Inc", "") + delete(missing, "company_external_id") + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": missing, + "c2": fakeCompanyItem("c2", "Blank Two", "Blank Two", ""), + }}) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "", "0014100000New0000")) + require.NoError(t, repo.UpdateCompanyExternalID(context.Background(), "c2", " ", "0014100000New0001")) + for id, want := range map[string]string{"c1": "0014100000New0000", "c2": "0014100000New0001"} { + record, err := repo.GetCompanyRecord(context.Background(), id) + require.NoError(t, err) + assert.Equal(t, want, record.CompanyExternalID) + assert.Equal(t, "", record.PreviousCompanyExternalID) + } + assert.Equal(t, 0, table.conditionFailures) + assert.Contains(t, table.lastCondition, "attribute_not_exists(#E) OR #E = :old") + }) + + t.Run("a blank old id never overwrites an existing external id", func(t *testing.T) { + repo, table := newCompanyRepo(t, &fakeCompaniesTable{items: map[string]map[string]interface{}{ + "c1": fakeCompanyItem("c1", "Acme Inc", "Acme Inc", "lf-old"), + }}) + err := repo.UpdateCompanyExternalID(context.Background(), "c1", "", "0014100000New0000") + assert.ErrorIs(t, err, ErrExternalIDConditionFailed) + assert.Equal(t, 1, table.conditionFailures) + record, err := repo.GetCompanyRecord(context.Background(), "c1") + require.NoError(t, err) + assert.Equal(t, "lf-old", record.CompanyExternalID) + }) + + t.Run("blank company or new id is rejected", func(t *testing.T) { + repo, _ := newCompanyRepo(t, &fakeCompaniesTable{}) + assert.Error(t, repo.UpdateCompanyExternalID(context.Background(), "", "old", "new")) + assert.Error(t, repo.UpdateCompanyExternalID(context.Background(), "c1", "old", "")) + }) +} diff --git a/cla-backend-go/company/resolve_company_test.go b/cla-backend-go/company/resolve_company_test.go new file mode 100644 index 000000000..826ece0ba --- /dev/null +++ b/cla-backend-go/company/resolve_company_test.go @@ -0,0 +1,238 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package company_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/linuxfoundation/easycla/cla-backend-go/company" + mock_company "github.com/linuxfoundation/easycla/cla-backend-go/company/mocks" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + organizationService "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + resolvePlatformHost = "platform.resolve.invalid" + resolveAuthHost = "auth.resolve.invalid" + resolveSFID = "0014100000Te0yqQAB" + resolveCompanyID = "9b8e7d66-40a5-4cde-9f00-3e1d1a2b3c4d" +) + +// resolveHTTP answers the token endpoint and the organization-service routes the tests need; +// anything else fails the test (no network) +type resolveHTTP struct { + t *testing.T + orgs map[string]string + searchBody string + calls []string +} + +func (h *resolveHTTP) RoundTrip(r *http.Request) (*http.Response, error) { + h.calls = append(h.calls, r.Method+" "+r.URL.Path) + response := &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Request: r} + switch { + case r.URL.Host == resolveAuthHost && r.URL.Path == "/oauth/token": + response.Body = io.NopCloser(strings.NewReader(`{"access_token":"unit-test-token","token_type":"Bearer","expires_in":3600}`)) + case r.URL.Host == resolvePlatformHost && r.URL.Path == "/organization-service/v1/orgs/search": + response.Body = io.NopCloser(strings.NewReader(h.searchBody)) + case r.URL.Host == resolvePlatformHost && strings.HasPrefix(r.URL.Path, "/organization-service/v1/orgs/"): + body, ok := h.orgs[strings.TrimPrefix(r.URL.Path, "/organization-service/v1/orgs/")] + if !ok { + response.StatusCode = http.StatusNotFound + body = `{"Message":"not found"}` + } + response.Body = io.NopCloser(strings.NewReader(body)) + default: + h.t.Errorf("unexpected HTTP request (no network allowed): %s %s", r.Method, r.URL) + return nil, fmt.Errorf("unexpected HTTP request: %s %s", r.Method, r.URL) + } + return response, nil +} + +func setupResolveHTTP(t *testing.T, orgs map[string]string, searchBody string) *resolveHTTP { + t.Helper() + transport := &resolveHTTP{t: t, orgs: orgs, searchBody: searchBody} + oldTransport, oldClient := http.DefaultTransport, http.DefaultClient + http.DefaultTransport = transport + http.DefaultClient = &http.Client{Transport: transport} + t.Cleanup(func() { + http.DefaultTransport, http.DefaultClient = oldTransport, oldClient + }) + token.Init("test-client", "test-secret", "https://"+resolveAuthHost+"/oauth/token", "test-audience") + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := token.GetToken(); err == nil { + break + } + require.True(t, time.Now().Before(deadline), "mock token initialization did not complete") + time.Sleep(10 * time.Millisecond) + } + organizationService.InitClient("https://"+resolvePlatformHost, nil) + return transport +} + +func newResolveService(repo company.IRepository) company.IService { + return company.NewService(repo, "https://corporate.invalid", nil, nil) +} + +func TestResolveCompanyOrder(t *testing.T) { + setupResolveHTTP(t, map[string]string{ + resolveSFID: `{"ID":"` + resolveSFID + `","Name":"Acme Corp","SigningEntityName":["Acme Labs"]}`, + }, "") + notFoundByID := &utils.CompanyNotFound{CompanyID: resolveSFID} + notFoundBySFID := &utils.CompanyNotFound{CompanySFID: resolveSFID} + dynamoErr := errors.New("dynamodb unavailable") + persisted := &models.Company{CompanyID: resolveCompanyID, CompanyExternalID: resolveSFID, CompanyName: "Acme", IsSanctioned: true} + + t.Run("internal id wins", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveCompanyID).Return(persisted, nil) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveCompanyID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + }) + t.Run("external id row wins over the organization", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveSFID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(persisted, nil) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveSFID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + assert.True(t, got.IsSanctioned) + }) + t.Run("virtual company when no row exists", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveSFID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, notFoundBySFID) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveSFID) + + require.NoError(t, err) + assert.Equal(t, &models.Company{CompanyID: resolveSFID, CompanyExternalID: resolveSFID, CompanyName: "Acme Corp", SigningEntityName: "Acme Corp"}, got) + }) + t.Run("unknown organization is not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), "0014100000Unknown0").Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), "0014100000Unknown0").Return(nil, notFoundBySFID) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), "0014100000Unknown0") + + var notFound *utils.CompanyNotFound + require.ErrorAs(t, err, ¬Found) + assert.Nil(t, got) + }) + t.Run("repository errors propagate", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompany(gomock.Any(), resolveSFID).Return(nil, notFoundByID) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, dynamoErr) + + got, err := newResolveService(repo).ResolveCompany(context.Background(), resolveSFID) + + require.ErrorIs(t, err, dynamoErr) + assert.Nil(t, got) + }) +} + +func TestVirtualCompany(t *testing.T) { + got := company.VirtualCompany(resolveSFID, "Acme Corp") + assert.Equal(t, resolveSFID, got.CompanyID) + assert.Equal(t, resolveSFID, got.CompanyExternalID) + assert.Equal(t, "Acme Corp", got.CompanyName) + assert.Equal(t, "Acme Corp", got.SigningEntityName) + assert.False(t, got.IsSanctioned) +} + +func TestGetCompanyByExternalIDDoesNotCreate(t *testing.T) { + t.Run("no row", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + notFound := &utils.CompanyNotFound{CompanySFID: resolveSFID} + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, notFound) + + got, err := newResolveService(repo).GetCompanyByExternalID(context.Background(), resolveSFID) + + require.ErrorIs(t, err, notFound) + assert.Nil(t, got) + }) + t.Run("persisted row", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + persisted := &models.Company{CompanyID: resolveCompanyID, CompanyExternalID: resolveSFID, CompanyName: "Acme"} + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(persisted, nil) + + got, err := newResolveService(repo).GetCompanyByExternalID(context.Background(), resolveSFID) + + require.NoError(t, err) + assert.Same(t, persisted, got) + }) +} + +func TestSearchOrganizationByNameDoesNotCreate(t *testing.T) { + transport := setupResolveHTTP(t, nil, `{"Data":[{"ID":"`+resolveSFID+`","Name":"Acme Corp","Link":"https://acme.invalid","SigningEntityName":["Acme Labs"]}],"Metadata":{"TotalSize":1,"Offset":0,"PageSize":1000}}`) + for _, includeSigningEntityName := range []bool{false, true} { + t.Run(fmt.Sprintf("includeSigningEntityName=%t", includeSigningEntityName), func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(nil, &utils.CompanyNotFound{CompanySFID: resolveSFID}) + + result, err := newResolveService(repo).SearchOrganizationByName(context.Background(), "Acme", "", includeSigningEntityName, "") + + require.NoError(t, err) + require.Len(t, result.List, 1) + assert.Equal(t, resolveSFID, result.List[0].OrganizationID) + assert.False(t, *result.List[0].CclaEnabled) + if includeSigningEntityName { + assert.Equal(t, []string{"Acme Labs"}, result.List[0].SigningEntityNames) + } + }) + } + t.Run("persisted company reports its CCLA state", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + repo := mock_company.NewMockIRepository(ctrl) + repo.EXPECT().GetCompanyByExternalID(gomock.Any(), resolveSFID).Return(&models.Company{CompanyID: resolveCompanyID, CompanyExternalID: resolveSFID, CompanyName: "Acme"}, nil) + repo.EXPECT().IsCCLAEnabledForCompany(gomock.Any(), resolveCompanyID).Return(true, nil) + + result, err := newResolveService(repo).SearchOrganizationByName(context.Background(), "Acme", "", false, "") + + require.NoError(t, err) + require.Len(t, result.List, 1) + assert.Equal(t, resolveSFID, result.List[0].OrganizationID) + assert.True(t, *result.List[0].CclaEnabled) + }) + for _, call := range transport.calls { + if strings.Contains(call, "/organization-service/") { + assert.True(t, strings.HasPrefix(call, "GET "), "the search must only read from the organization service: %s", call) + } + } +} diff --git a/cla-backend-go/company/service.go b/cla-backend-go/company/service.go index ab8116356..6b690e8a2 100644 --- a/cla-backend-go/company/service.go +++ b/cla-backend-go/company/service.go @@ -40,11 +40,10 @@ const ( // IService interface defining the functions for the company service type IService interface { // nolint - CreateOrgFromExternalID(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) - GetCompanies(ctx context.Context) (*models.Companies, error) GetCompany(ctx context.Context, companyID string) (*models.Company, error) GetCompanyByExternalID(ctx context.Context, companySFID string) (*models.Company, error) + ResolveCompany(ctx context.Context, companyIDOrSFID string) (*models.Company, error) GetCompaniesByExternalID(ctx context.Context, companySFID string, includeChildCompanies bool) ([]*models.Company, error) GetCompanyBySigningEntityName(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) SearchCompanyByName(ctx context.Context, companyName string, nextKey string) (*models.Companies, error) @@ -641,19 +640,62 @@ func (s service) GetCompanyByExternalID(ctx context.Context, companySFID string) } log.WithFields(f).Debug("Searching company by external ID...") comp, err := s.repo.GetCompanyByExternalID(ctx, companySFID) + if err != nil { + return nil, err + } + log.WithFields(f).Debugf("Loaded and returning company: %+v...", comp) + return comp, nil +} + +// ResolveCompany returns the persisted company for an internal ID or a Salesforce ID; when no row +// exists it returns a non-persisted virtual company (CompanyID = CompanyExternalID = SFID) built +// from the organization service. Only "not found" outcomes fall through; other errors propagate. +func (s service) ResolveCompany(ctx context.Context, companyIDOrSFID string) (*models.Company, error) { + f := logrus.Fields{ + "functionName": "company.service.ResolveCompany", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyIDOrSFID": companyIDOrSFID, + } + comp, err := s.repo.GetCompany(ctx, companyIDOrSFID) if err == nil { - log.WithFields(f).Debugf("Loaded and returning company: %+v...", comp) return comp, nil } - - if _, ok := err.(*utils.CompanyNotFound); ok { - comp, err = s.CreateOrgFromExternalID(ctx, "", companySFID) - if err != nil { - return comp, err - } + if _, ok := err.(*utils.CompanyNotFound); !ok { + return nil, err + } + comp, err = s.repo.GetCompanyByExternalID(ctx, companyIDOrSFID) + if err == nil { return comp, nil } - return nil, err + if _, ok := err.(*utils.CompanyNotFound); !ok { + return nil, err + } + orgClient := organization_service.GetClient() + if orgClient == nil { + return nil, err + } + org, orgErr := orgClient.GetOrganization(ctx, companyIDOrSFID) + if orgErr != nil { + if _, ok := orgErr.(*organizations.GetOrgNotFound); ok { + log.WithFields(f).Debug("no company row and no organization - not found") + return nil, &utils.CompanyNotFound{Message: "no company or organization matching the id", CompanyID: companyIDOrSFID} + } + log.WithFields(f).WithError(orgErr).Warn("problem loading organization") + return nil, orgErr + } + log.WithFields(f).Debugf("no company row - returning virtual company for organization %s", org.Name) + return VirtualCompany(org.ID, org.Name), nil +} + +// VirtualCompany is the non-persisted view of an organization that has no EasyCLA row yet. +func VirtualCompany(companySFID, companyName string) *models.Company { + return &models.Company{ + CompanyID: companySFID, + CompanyExternalID: companySFID, + CompanyName: companyName, + SigningEntityName: companyName, + IsSanctioned: false, + } } func (s service) GetCompaniesByExternalID(ctx context.Context, companySFID string, includeChildCompanies bool) ([]*models.Company, error) { @@ -744,13 +786,6 @@ func (s service) SearchOrganizationByName(ctx context.Context, orgName string, w var signingEntityNames []string if len(org.SigningEntityName) > 0 { signingEntityNames = utils.TrimSpaceFromItems(org.SigningEntityName) - for _, signingEntityName := range signingEntityNames { - // Auto-create the internal record, if needed - _, err = s.CreateOrgFromExternalID(ctx, signingEntityName, org.ID) - if err != nil { - log.WithFields(f).WithError(err).Warnf("Unable to create organization from external ID: %s using signing entity name: %s", org.ID, signingEntityName) - } - } resultsChannel <- &models.Org{ OrganizationID: org.ID, OrganizationName: org.Name, @@ -798,144 +833,3 @@ func (s service) SearchOrganizationByName(ctx context.Context, orgName string, w return result, nil } - -// CreateOrgFromExternalID creates a new EasyCLA company from the external SF Organization ID -func (s service) CreateOrgFromExternalID(ctx context.Context, signingEntityName, companySFID string) (*models.Company, error) { - f := logrus.Fields{ - "functionName": "company.service.CreateOrgFromExternalID", - utils.XREQUESTID: ctx.Value(utils.XREQUESTID), - "companySFID": companySFID, - "signingEntityName": signingEntityName, - } - - var companyModel *models.Company - var lookupErr error - - // Lookup the company in our database...does it exist? - companyModel, lookupErr = s.GetCompanyBySigningEntityName(ctx, signingEntityName, companySFID) - if lookupErr != nil { - log.WithFields(f).WithError(lookupErr).Debug("problem locating internal company record by signing entity name and SFID - must not exist yet") - } - - // Already exists - no need to create in our own database - if companyModel != nil { - return companyModel, nil - } - - osc := organization_service.GetClient() - log.WithFields(f).Debugf("Searching organization by company SFID in the organization service...") - org, err := osc.GetOrganization(ctx, companySFID) - if err != nil { - log.WithFields(f).WithError(err).Warn("getting organization details failed") - return nil, err - } - - // Add some fields to the logger - f["companyName"] = org.Name - f["companyStatus"] = org.Status - - // Query the platform user service to locate the company admin - log.WithFields(f).Debugf("getting company-admin information...") - companyAdmin, err := getCompanyAdmin(ctx, companySFID) - if err != nil { - log.WithFields(f).WithError(err).Warnf("unable to load company admin information for company: %s", companySFID) - } - - var claUser *models.User - if companyAdmin != nil { - f["company-admin"] = companyAdmin - log.WithFields(f).Debugf("loaded company admin: %+v", companyAdmin) - - log.WithFields(f).Debugf("getting user information from cla") - claUser, err = s.userService.GetUserByLFUserName(companyAdmin.LfUsername) - if err != nil { - log.WithFields(f).WithError(err).Warnf("problem loading user by username: %s", companyAdmin.LfUsername) - return nil, err - } - - if claUser == nil { - // create cla-user - log.WithFields(f).Debugf("cla user not found. creating cla user.") - claUser, err = s.userService.CreateUser(companyAdmin, nil) - if err != nil { - log.WithFields(f).WithError(err).Warn("creating cla user failed") - return nil, err - } - } - } else { - log.WithFields(f).Debug("unable to load company admin from companySFID - admin not found") - } - - additionalNote := "" - if signingEntityName == "" { - additionalNote = fmt.Sprintf("signing entity name not set - using organization name: %s", org.Name) - log.WithFields(f).Debugf("%s", additionalNote) - signingEntityName = org.Name - } - - _, now := utils.CurrentTime() - newComp := &models.Company{ - CompanyExternalID: org.ID, - CompanyName: org.Name, - SigningEntityName: signingEntityName, - IsSanctioned: false, - Note: fmt.Sprintf("%s - Created based on SF Organization Service record - %s", now, additionalNote), - } - if companyAdmin != nil { - newComp.CompanyACL = []string{companyAdmin.LfUsername} - } - if claUser != nil { - newComp.CompanyManagerID = claUser.UserID - } - - f["company"] = newComp - log.WithFields(f).Debugf("creating cla company record") - // create company - comp, err := s.repo.CreateCompany(ctx, newComp) - if err != nil { - log.WithFields(f).WithError(err).Warnf("creating cla company failed") - return nil, err - } - - log.WithFields(f).Debugf("Created company %s with Signing Entity Name: %s with ID: %s", - comp.CompanyName, signingEntityName, comp.CompanyID) - return comp, nil -} - -// getCompanyAdmin is helper function which queries org-service to get first company-admin -func getCompanyAdmin(ctx context.Context, companySFID string) (*models.User, error) { - f := logrus.Fields{ - "functionName": "company.service.getCompanyAdmin", - utils.XREQUESTID: ctx.Value(utils.XREQUESTID), - "companySFID": companySFID, - } - osc := organization_service.GetClient() - result, err := osc.ListOrgUserAdminScopes(ctx, companySFID, nil) - if err != nil { - if _, ok := err.(*organizations.ListOrgUsrAdminScopesNotFound); !ok { - log.WithFields(f).Warnf("getting company-admin failed. error = %s", err.Error()) - return nil, err - } - } - if result != nil { - for _, usc := range result.Userroles { - for _, rs := range usc.RoleScopes { - if rs.RoleName == "company-admin" { - companyAdmin := &models.User{ - LfEmail: strfmt.Email(usc.Contact.EmailAddress), - LfUsername: usc.Contact.Username, - UserExternalID: usc.Contact.ID, - Username: usc.Contact.Name, - } - log.WithFields(f).WithField("company-admin", companyAdmin).Debug("company-admin found") - return companyAdmin, nil - } - } - } - } - log.WithFields(f).Warnf("no company-admin found") - return nil, &utils.CompanyAdminNotFound{ - CompanySFID: companySFID, - Err: nil, - } -} diff --git a/cla-backend-go/config/config.go b/cla-backend-go/config/config.go index f0df045e0..1797e3c2e 100644 --- a/cla-backend-go/config/config.go +++ b/cla-backend-go/config/config.go @@ -104,6 +104,18 @@ type Config struct { // SelfServe holds the LFX Self Serve trusted-caller configuration SelfServe SelfServe `json:"self_serve"` + + // MemberService holds the LFX v2 member-service client configuration (org import tool only) + MemberService MemberService `json:"member_service"` +} + +// MemberService holds the LFX v2 member-service configuration used by the org import tool to +// register organizations as B2B orgs. It reuses the shared Auth0Platform M2M credentials; only the +// base URL (cla-member-service-base-url-{stage}) and audience (cla-member-service-auth0-audience-{stage}) +// are configured here, both optional: when empty the register route is unavailable. +type MemberService struct { + BaseURL string `json:"base_url"` + Audience string `json:"audience"` } // SelfServe holds the LFX Self Serve trusted-caller configuration: the Auth0 azp (client ID) diff --git a/cla-backend-go/config/ssm.go b/cla-backend-go/config/ssm.go index c087bbad3..bf26d7ddc 100644 --- a/cla-backend-go/config/ssm.go +++ b/cla-backend-go/config/ssm.go @@ -279,6 +279,9 @@ func loadSSMConfig(awsSession *session.Session, stage string) Config { //nolint loadOptionalSelfServeConfig(ssmClient, stage, &config, f) + config.MemberService.BaseURL = getOptionalSSMString(ssmClient, fmt.Sprintf("cla-member-service-base-url-%s", stage), f) + config.MemberService.Audience = getOptionalSSMString(ssmClient, fmt.Sprintf("cla-member-service-auth0-audience-%s", stage), f) + return config } @@ -350,9 +353,9 @@ func getOptionalSSMString(ssmClient *ssm.SSM, key string, f logrus.Fields) strin }) if err != nil { if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ssm.ErrCodeParameterNotFound { - log.WithFields(f).Debugf("optional SSM key %s not provisioned - sanctions screening disabled until it is set", key) + log.WithFields(f).Debugf("optional SSM key %s not provisioned - the feature depending on it stays disabled until it is set", key) } else { - log.WithFields(f).WithError(err).Warnf("unable to read optional SSM key %s - sanctions screening disabled", key) + log.WithFields(f).WithError(err).Warnf("unable to read optional SSM key %s - the feature depending on it stays disabled", key) } return "" } diff --git a/cla-backend-go/events/repository_rekey.go b/cla-backend-go/events/repository_rekey.go new file mode 100644 index 000000000..917ea109e --- /dev/null +++ b/cla-backend-go/events/repository_rekey.go @@ -0,0 +1,187 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package events + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/expression" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" +) + +// ErrEventNotFound is returned by RekeyEventCompanySFID when the event does not exist. +var ErrEventNotFound = errors.New("event not found") + +// ErrEventCompanySFIDMismatch is returned when an event carries a company SFID that is neither the old nor the new one. +var ErrEventCompanySFIDMismatch = errors.New("event carries an unexpected company sfid") + +// ErrEventRekeyConflict is returned when the event changed between the read and the conditional write; the call can be retried. +var ErrEventRekeyConflict = errors.New("event changed concurrently") + +// company SFID prefixed composite index keys maintained by AddDataToEvent +var companySFIDCompositeAttributes = []string{"company_sfid_foundation_sfid", "company_sfid_project_id", "company_sfid_cla_group_id"} + +// RekeyRepository is the org-import view of the events table: locate the events of a company SFID and move them to a new SFID. +type RekeyRepository interface { + ListEventIDsByCompanySFID(ctx context.Context, companySFID string) ([]string, error) + ListEventIDsByCompanySFIDCLAGroup(ctx context.Context, companySFID, claGroupID string) ([]string, error) + RekeyEventCompanySFID(ctx context.Context, eventID, oldSFID, newSFID string) (bool, error) +} + +// NewRekeyRepository creates the org-import events repository +func NewRekeyRepository(awsSession *session.Session, stage string) RekeyRepository { + return &repository{ + stage: stage, + dynamoDBClient: dynamodb.New(awsSession), + eventsTable: fmt.Sprintf("cla-%s-events", stage), + } +} + +// ListEventIDsByCompanySFID returns the ids of all events with event_company_sfid = companySFID +func (repo *repository) ListEventIDsByCompanySFID(ctx context.Context, companySFID string) ([]string, error) { + if strings.TrimSpace(companySFID) == "" { + return nil, errors.New("company sfid is required") + } + return repo.listEventIDs(ctx, EventCompanySFIDEventDataLowerIndex, expression.Key("event_company_sfid").Equal(expression.Value(companySFID))) +} + +// ListEventIDsByCompanySFIDCLAGroup returns the ids of all events with company_sfid_cla_group_id = companySFID#claGroupID +func (repo *repository) ListEventIDsByCompanySFIDCLAGroup(ctx context.Context, companySFID, claGroupID string) ([]string, error) { + if strings.TrimSpace(companySFID) == "" || strings.TrimSpace(claGroupID) == "" { + return nil, errors.New("company sfid and cla group id are required") + } + return repo.listEventIDs(ctx, CompanySFIDClaGroupIDEpochIndex, expression.Key("company_sfid_cla_group_id").Equal(expression.Value(companySFID+"#"+claGroupID))) +} + +func (repo *repository) listEventIDs(ctx context.Context, indexName string, keyCondition expression.KeyConditionBuilder) ([]string, error) { + f := logrus.Fields{ + "functionName": "v1.events.repository.listEventIDs", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "indexName": indexName, + } + expr, err := expression.NewBuilder().WithKeyCondition(keyCondition).WithProjection(expression.NamesList(expression.Name("event_id"))).Build() + if err != nil { + return nil, err + } + input := &dynamodb.QueryInput{ + TableName: aws.String(repo.eventsTable), + IndexName: aws.String(indexName), + KeyConditionExpression: expr.KeyCondition(), + ProjectionExpression: expr.Projection(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + } + var ids []string + for { + out, queryErr := repo.dynamoDBClient.Query(input) + if queryErr != nil { + log.WithFields(f).WithError(queryErr).Warn("error listing events") + return nil, queryErr + } + for _, item := range out.Items { + if id := item["event_id"]; id != nil && id.S != nil { + ids = append(ids, *id.S) + } + } + if len(out.LastEvaluatedKey) == 0 { + return ids, nil + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } +} + +// RekeyEventCompanySFID moves one event from oldSFID to newSFID: event_company_sfid and the +// company_sfid_* composite keys that still start with oldSFID are rewritten with a conditional +// update. An event already carrying newSFID is left alone (false, nil); any other company SFID is +// an ErrEventCompanySFIDMismatch. +func (repo *repository) RekeyEventCompanySFID(ctx context.Context, eventID, oldSFID, newSFID string) (bool, error) { + f := logrus.Fields{ + "functionName": "v1.events.repository.RekeyEventCompanySFID", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "eventID": eventID, + "oldSFID": oldSFID, + "newSFID": newSFID, + } + if strings.TrimSpace(eventID) == "" || strings.TrimSpace(oldSFID) == "" || strings.TrimSpace(newSFID) == "" || oldSFID == newSFID { + return false, errors.New("event id and distinct old/new company sfids are required") + } + key := map[string]*dynamodb.AttributeValue{"event_id": {S: aws.String(eventID)}} + out, err := repo.dynamoDBClient.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(repo.eventsTable), + Key: key, + ConsistentRead: aws.Bool(true), + }) + if err != nil { + log.WithFields(f).WithError(err).Warn("error reading event") + return false, err + } + if len(out.Item) == 0 { + return false, ErrEventNotFound + } + + names := map[string]*string{} + values := map[string]*dynamodb.AttributeValue{} + var sets, conditions []string + rewrite := func(attribute, current, updated string) { + placeholder := fmt.Sprintf("#a%d", len(sets)) + names[placeholder] = aws.String(attribute) + values[":new"+placeholder[1:]] = &dynamodb.AttributeValue{S: aws.String(updated)} + values[":cur"+placeholder[1:]] = &dynamodb.AttributeValue{S: aws.String(current)} + sets = append(sets, fmt.Sprintf("%s = :new%s", placeholder, placeholder[1:])) + conditions = append(conditions, fmt.Sprintf("%s = :cur%s", placeholder, placeholder[1:])) + } + if current := stringAttribute(out.Item, "event_company_sfid"); current != "" && current != newSFID { + if current != oldSFID { + return false, ErrEventCompanySFIDMismatch + } + rewrite("event_company_sfid", current, newSFID) + } + for _, attribute := range companySFIDCompositeAttributes { + current := stringAttribute(out.Item, attribute) + if current == "" || strings.HasPrefix(current, newSFID+"#") { + continue + } + if !strings.HasPrefix(current, oldSFID+"#") { + return false, ErrEventCompanySFIDMismatch + } + rewrite(attribute, current, newSFID+strings.TrimPrefix(current, oldSFID)) + } + if len(sets) == 0 { + return false, nil + } + + _, err = repo.dynamoDBClient.UpdateItem(&dynamodb.UpdateItemInput{ + TableName: aws.String(repo.eventsTable), + Key: key, + UpdateExpression: aws.String("SET " + strings.Join(sets, ", ")), + ConditionExpression: aws.String(strings.Join(conditions, " AND ")), + ExpressionAttributeNames: names, + ExpressionAttributeValues: values, + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == dynamodb.ErrCodeConditionalCheckFailedException { + return false, ErrEventRekeyConflict + } + log.WithFields(f).WithError(err).Warn("error rekeying event") + return false, err + } + log.WithFields(f).Debugf("event rekeyed (%d attributes)", len(sets)) + return true, nil +} + +func stringAttribute(item map[string]*dynamodb.AttributeValue, name string) string { + if attr := item[name]; attr != nil && attr.S != nil { + return *attr.S + } + return "" +} diff --git a/cla-backend-go/events/repository_rekey_test.go b/cla-backend-go/events/repository_rekey_test.go new file mode 100644 index 000000000..e66728ae3 --- /dev/null +++ b/cla-backend-go/events/repository_rekey_test.go @@ -0,0 +1,313 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package events + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeEventsTable is a minimal in-memory DynamoDB endpoint for the events table: GetItem, Query on +// a single-attribute key (paginated pageSize items at a time) and conditional UpdateItem. +type fakeEventsTable struct { + mu sync.Mutex + items map[string]map[string]string + pageSize int + queries int + updates int + conditionFailures int + beforeUpdate func(items map[string]map[string]string) +} + +type fakeEventAttr struct { + S *string +} + +var fakeEventPair = regexp.MustCompile(`(#[0-9A-Za-z_]+) = (:[0-9A-Za-z_]+)`) + +func (f *fakeEventsTable) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + f.mu.Lock() + defer f.mu.Unlock() + switch r.Header.Get("X-Amz-Target") { + case "DynamoDB_20120810.GetItem": + var req struct{ Key map[string]fakeEventAttr } + fakeEventDecode(w, body, &req) + resp := map[string]interface{}{} + if item, ok := f.items[*req.Key["event_id"].S]; ok { + resp["Item"] = fakeEventWire(item) + } + fakeEventJSON(w, resp) + case "DynamoDB_20120810.Query": + f.queries++ + var req struct { + KeyConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeEventAttr + ExclusiveStartKey map[string]fakeEventAttr + } + fakeEventDecode(w, body, &req) + m := fakeEventPair.FindStringSubmatch(req.KeyConditionExpression) + attr, value := req.ExpressionAttributeNames[m[1]], *req.ExpressionAttributeValues[m[2]].S + var ids []string + for id, item := range f.items { + if item[attr] == value { + ids = append(ids, id) + } + } + sort.Strings(ids) + if start, ok := req.ExclusiveStartKey["event_id"]; ok { + for i, id := range ids { + if id == *start.S { + ids = ids[i+1:] + break + } + } + } + resp := map[string]interface{}{} + if f.pageSize > 0 && len(ids) > f.pageSize { + ids = ids[:f.pageSize] + resp["LastEvaluatedKey"] = map[string]interface{}{"event_id": map[string]string{"S": ids[len(ids)-1]}} + } + page := []map[string]interface{}{} + for _, id := range ids { + page = append(page, map[string]interface{}{"event_id": map[string]string{"S": id}}) + } + resp["Items"], resp["Count"] = page, len(page) + fakeEventJSON(w, resp) + case "DynamoDB_20120810.UpdateItem": + var req struct { + Key map[string]fakeEventAttr + UpdateExpression string + ConditionExpression string + ExpressionAttributeNames map[string]string + ExpressionAttributeValues map[string]fakeEventAttr + } + fakeEventDecode(w, body, &req) + if f.beforeUpdate != nil { + f.beforeUpdate(f.items) + } + item := f.items[*req.Key["event_id"].S] + for _, pair := range fakeEventPair.FindAllStringSubmatch(req.ConditionExpression, -1) { + if item[req.ExpressionAttributeNames[pair[1]]] != *req.ExpressionAttributeValues[pair[2]].S { + f.conditionFailures++ + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + w.WriteHeader(http.StatusBadRequest) + fakeEventJSON(w, map[string]string{"__type": "com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException", "message": "condition failed"}) + return + } + } + for _, pair := range fakeEventPair.FindAllStringSubmatch(strings.TrimPrefix(req.UpdateExpression, "SET "), -1) { + item[req.ExpressionAttributeNames[pair[1]]] = *req.ExpressionAttributeValues[pair[2]].S + } + f.updates++ + fakeEventJSON(w, map[string]interface{}{}) + default: + http.Error(w, "unsupported operation "+r.Header.Get("X-Amz-Target"), http.StatusBadRequest) + } +} + +func fakeEventWire(item map[string]string) map[string]interface{} { + wire := map[string]interface{}{} + for k, v := range item { + wire[k] = map[string]string{"S": v} + } + return wire +} + +func fakeEventJSON(w http.ResponseWriter, payload interface{}) { + w.Header().Set("Content-Type", "application/x-amz-json-1.0") + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func fakeEventDecode(w http.ResponseWriter, body []byte, v interface{}) { + if err := json.Unmarshal(body, v); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } +} + +func newRekeyRepo(t *testing.T, table *fakeEventsTable) RekeyRepository { + server := httptest.NewServer(table) + t.Cleanup(server.Close) + awsSession, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(server.URL), + Credentials: credentials.NewStaticCredentials("test", "test", ""), + DisableSSL: aws.Bool(true), + MaxRetries: aws.Int(0), + }) + require.NoError(t, err) + return NewRekeyRepository(awsSession, "test") +} + +func fakeEvent(id, sfid, foundation, project, claGroup string) map[string]string { + item := map[string]string{"event_id": id, "event_type": "TestEvent", "event_data_lower": "data " + id} + if sfid != "" { + item["event_company_sfid"] = sfid + } + if foundation != "" { + item["company_sfid_foundation_sfid"] = sfid + "#" + foundation + } + if project != "" { + item["company_sfid_project_id"] = sfid + "#" + project + } + if claGroup != "" { + item["company_sfid_cla_group_id"] = sfid + "#" + claGroup + } + return item +} + +func TestListEventIDsByCompanySFID(t *testing.T) { + table := &fakeEventsTable{pageSize: 2, items: map[string]map[string]string{ + "e1": fakeEvent("e1", "lf-old", "f1", "p1", "g1"), + "e2": fakeEvent("e2", "lf-old", "", "", "g1"), + "e3": fakeEvent("e3", "lf-old", "f1", "", ""), + "e4": fakeEvent("e4", "0014100000Other00", "f1", "p1", "g1"), + "e5": fakeEvent("e5", "", "", "", ""), + }} + repo := newRekeyRepo(t, table) + + ids, err := repo.ListEventIDsByCompanySFID(context.Background(), "lf-old") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"e1", "e2", "e3"}, ids) + assert.Equal(t, 2, table.queries, "pagination follows LastEvaluatedKey") + + ids, err = repo.ListEventIDsByCompanySFIDCLAGroup(context.Background(), "lf-old", "g1") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"e1", "e2"}, ids) + + ids, err = repo.ListEventIDsByCompanySFID(context.Background(), "unknown") + require.NoError(t, err) + assert.Empty(t, ids) + + _, err = repo.ListEventIDsByCompanySFID(context.Background(), " ") + assert.Error(t, err) + _, err = repo.ListEventIDsByCompanySFIDCLAGroup(context.Background(), "lf-old", "") + assert.Error(t, err) +} + +func TestRekeyEventCompanySFID(t *testing.T) { + const oldSFID, newSFID = "lf-old", "0014100000New0000" + + t.Run("rewrites the sfid and every composite key, keeps the rest", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e1": fakeEvent("e1", oldSFID, "f1", "p1", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, map[string]string{ + "event_id": "e1", + "event_type": "TestEvent", + "event_data_lower": "data e1", + "event_company_sfid": newSFID, + "company_sfid_foundation_sfid": newSFID + "#f1", + "company_sfid_project_id": newSFID + "#p1", + "company_sfid_cla_group_id": newSFID + "#g1", + }, table.items["e1"]) + assert.Equal(t, 1, table.updates) + }) + + t.Run("only the attributes present are rewritten", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e2": fakeEvent("e2", oldSFID, "", "", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e2", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newSFID, table.items["e2"]["event_company_sfid"]) + assert.Equal(t, newSFID+"#g1", table.items["e2"]["company_sfid_cla_group_id"]) + _, hasFoundation := table.items["e2"]["company_sfid_foundation_sfid"] + assert.False(t, hasFoundation) + }) + + t.Run("is idempotent", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e1": fakeEvent("e1", newSFID, "f1", "p1", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, 0, table.updates) + }) + + t.Run("finishes a half-rekeyed event", func(t *testing.T) { + item := fakeEvent("e1", newSFID, "", "", "") + item["company_sfid_cla_group_id"] = oldSFID + "#g1" + table := &fakeEventsTable{items: map[string]map[string]string{"e1": item}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newSFID+"#g1", table.items["e1"]["company_sfid_cla_group_id"]) + }) + + t.Run("refuses an event of another company", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e4": fakeEvent("e4", "0014100000Other00", "f1", "p1", "g1")}} + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e4", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventCompanySFIDMismatch) + assert.False(t, changed) + assert.Equal(t, 0, table.updates) + }) + + t.Run("refuses a composite key of another company", func(t *testing.T) { + item := fakeEvent("e6", oldSFID, "", "", "") + item["company_sfid_project_id"] = "0014100000Other00#p1" + table := &fakeEventsTable{items: map[string]map[string]string{"e6": item}} + repo := newRekeyRepo(t, table) + _, err := repo.RekeyEventCompanySFID(context.Background(), "e6", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventCompanySFIDMismatch) + assert.Equal(t, 0, table.updates) + }) + + t.Run("missing event", func(t *testing.T) { + repo := newRekeyRepo(t, &fakeEventsTable{items: map[string]map[string]string{}}) + _, err := repo.RekeyEventCompanySFID(context.Background(), "nope", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventNotFound) + }) + + t.Run("concurrent change between read and write is a retryable conflict", func(t *testing.T) { + table := &fakeEventsTable{items: map[string]map[string]string{"e1": fakeEvent("e1", oldSFID, "f1", "", "")}} + table.beforeUpdate = func(items map[string]map[string]string) { + items["e1"]["company_sfid_foundation_sfid"] = oldSFID + "#f2" + } + repo := newRekeyRepo(t, table) + changed, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + assert.ErrorIs(t, err, ErrEventRekeyConflict) + assert.False(t, changed) + assert.Equal(t, 1, table.conditionFailures) + + table.beforeUpdate = nil + changed, err = repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, newSFID) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newSFID+"#f2", table.items["e1"]["company_sfid_foundation_sfid"]) + }) + + t.Run("invalid arguments", func(t *testing.T) { + repo := newRekeyRepo(t, &fakeEventsTable{items: map[string]map[string]string{}}) + _, err := repo.RekeyEventCompanySFID(context.Background(), "e1", oldSFID, oldSFID) + assert.Error(t, err) + _, err = repo.RekeyEventCompanySFID(context.Background(), "", oldSFID, newSFID) + assert.Error(t, err) + }) +} diff --git a/cla-backend-go/orgimport/adapters.go b/cla-backend-go/orgimport/adapters.go new file mode 100644 index 000000000..dc65492e6 --- /dev/null +++ b/cla-backend-go/orgimport/adapters.go @@ -0,0 +1,86 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" + + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service/client/organizations" +) + +// OrgServiceAdapter maps the shared org-service client onto OrgService. +type OrgServiceAdapter struct { + Client *organization_service.Client +} + +// GetOrganization returns ErrOrgNotFound on 404. +func (a OrgServiceAdapter) GetOrganization(ctx context.Context, orgID string) (*Org, error) { + org, err := a.Client.GetOrganization(ctx, orgID) + if err != nil { + var notFound *organizations.GetOrgNotFound + if errors.As(err, ¬Found) { + return nil, ErrOrgNotFound + } + return nil, err + } + if org == nil { + return nil, ErrOrgNotFound + } + return &Org{ID: org.ID, Name: org.Name, Website: org.Link, SigningEntityNames: org.SigningEntityName}, nil +} + +// CreateUserRoleScope grants a role scope by LFID username; an existing grant is not an error. +func (a OrgServiceAdapter) CreateUserRoleScope(ctx context.Context, username, organizationID, objectType, objectID, roleID string) error { + return a.Client.CreateOrgUserRoleScopeByUsername(ctx, username, organizationID, objectType, objectID, roleID) +} + +// DeleteUserRoleScope removes one grant (ACS grant id); a missing grant is success. +func (a OrgServiceAdapter) DeleteUserRoleScope(ctx context.Context, organizationID, roleID, grantID, username string) error { + email := "" + err := a.Client.DeleteOrgUserRoleOrgScopeProjectOrg(ctx, organizationID, roleID, grantID, &username, &email) + if err != nil { + var notFound *organizations.DeleteOrgUsrRoleScopesNotFound + if errors.As(err, ¬Found) { + return nil + } + return err + } + return nil +} + +// DynamoECLACounter counts rows of the signature-user-ccla-company-index partition of a company. +type DynamoECLACounter struct { + DB dynamodbiface.DynamoDBAPI + Table string +} + +// CountECLAs sums Count over all pages. +func (c DynamoECLACounter) CountECLAs(ctx context.Context, companyID string) (int, error) { + input := &dynamodb.QueryInput{ + TableName: aws.String(c.Table), + IndexName: aws.String("signature-user-ccla-company-index"), + KeyConditionExpression: aws.String("signature_user_ccla_company_id = :id"), + ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{":id": {S: aws.String(companyID)}}, + Select: aws.String(dynamodb.SelectCount), + } + total := 0 + for { + out, err := c.DB.QueryWithContext(ctx, input) + if err != nil { + return 0, fmt.Errorf("counting ECLAs of %s: %w", companyID, err) + } + total += int(aws.Int64Value(out.Count)) + if len(out.LastEvaluatedKey) == 0 { + return total, nil + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } +} diff --git a/cla-backend-go/orgimport/apex.go b/cla-backend-go/orgimport/apex.go new file mode 100644 index 000000000..57702d32f --- /dev/null +++ b/cla-backend-go/orgimport/apex.go @@ -0,0 +1,92 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// ApexRequest is the design §4 find-or-create payload. +type ApexRequest struct { + Name string `json:"name"` + Website string `json:"website,omitempty"` + Source string `json:"source"` + ExternalKey string `json:"externalKey"` + CCLASignedDate string `json:"cclaSignedDate,omitempty"` + DryRun bool `json:"dryRun"` +} + +// ApexResult is the Apex response. +type ApexResult struct { + ID string `json:"id"` + Action string `json:"action"` +} + +// ApexClient calls the Salesforce Apex REST endpoint (off unless --use-apex and both SSM params exist). +type ApexClient struct { + BaseURL string + Token string + HTTPClient *http.Client +} + +// NewApexClient returns ErrApexUnavailable when the endpoint is not configured. +func NewApexClient(baseURL, token string) (*ApexClient, error) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" || strings.TrimSpace(token) == "" { + return nil, ErrApexUnavailable + } + return &ApexClient{BaseURL: baseURL, Token: strings.TrimSpace(token), HTTPClient: &http.Client{Timeout: 60 * time.Second}}, nil +} + +// FindOrCreate posts the request; the action is validated, the id must be a Salesforce account id unless ambiguous. +func (c *ApexClient) FindOrCreate(ctx context.Context, req ApexRequest) (*ApexResult, error) { + if req.Source == "" { + req.Source = "EasyCLA" + } + body, err := json.Marshal(req) + if err != nil { + return nil, err + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/services/apexrest/lfx/account", bytes.NewReader(body)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Authorization", "Bearer "+c.Token) + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "application/json") + resp, err := c.HTTPClient.Do(httpReq) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + _ = resp.Body.Close() + if readErr != nil { + return nil, readErr + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, fmt.Errorf("apex: %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + var res ApexResult + if err = json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("apex: decoding response: %w", err) + } + res.Action = strings.ToLower(strings.TrimSpace(res.Action)) + switch res.Action { + case ActionMatched, ActionCreated: + if !IsSFID(res.ID) { + return nil, fmt.Errorf("apex: action %s returned non-account id %q", res.Action, res.ID) + } + case ActionAmbiguous: + default: + return nil, fmt.Errorf("apex: unknown action %q", res.Action) + } + return &res, nil +} diff --git a/cla-backend-go/orgimport/audit.go b/cla-backend-go/orgimport/audit.go new file mode 100644 index 000000000..2a48a1b9c --- /dev/null +++ b/cla-backend-go/orgimport/audit.go @@ -0,0 +1,323 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "encoding/csv" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" +) + +// Tiers mirror utils/audit_company_reachability.sh so counts can be cross-checked. +const ( + TierMissing = "MISSING_SFID" + TierInvalid = "INVALID_SFID_FORMAT" + TierOK = "SFID_OK" + TierDangling = "SFID_DANGLING_OR_DELETED" + TierUnknown = "UNKNOWN" +) + +// AuditRow is one line of audit.csv. +type AuditRow struct { + Row *Row + Shape IDShape + ECLACount int + OrgStatus string + Website string + Duplicate bool + Route Route + ManualReason string + Tier string +} + +// AuditResult is the audit output. +type AuditResult struct { + Rows []*AuditRow + Groups []*Group + Tiers map[string]int + Routes map[Route]int + Duplicates [][]*Row +} + +// Audit classifies every company row (reads only) and writes audit.csv, unresolvable.csv and +// possible_duplicates.csv into opts.OutDir. +func Audit(ctx context.Context, deps Deps, opts Options) (*AuditResult, error) { + deps = withDefaults(deps) + inv, err := LoadInventory(ctx, deps) + if err != nil { + return nil, err + } + groups := inv.EligibleGroups() + for _, g := range groups { + classify(ctx, deps, g, nil, nil, Options{}) + } + groupByRow := map[string]*Group{} + for _, g := range groups { + for _, r := range g.Rows { + groupByRow[r.CompanyID] = g + } + } + + orgs := lookupOrgs(ctx, deps, inv) + res := &AuditResult{Tiers: map[string]int{}, Routes: map[Route]int{}} + byName := map[string][]*Row{} + for _, row := range inv.Rows { + ar := &AuditRow{Row: row, Shape: ShapeOf(row.ExternalID)} + if o := orgs[row.ExternalID]; o != nil { + ar.OrgStatus = o.status + if o.org != nil { + ar.Website = o.org.Website + } + } + ar.Tier = tier(row.ExternalID, ar.OrgStatus) + if g := groupByRow[row.CompanyID]; g != nil { + ar.Duplicate, ar.Route, ar.ManualReason = g.Duplicate, g.Route, g.ManualReason + if g.Err != nil { + ar.ManualReason = "error: " + g.Err.Error() + } + } + res.Tiers[ar.Tier]++ + if ar.Route != "" { + res.Routes[ar.Route]++ + } + res.Rows = append(res.Rows, ar) + byName[canonicalEntity(row.CompanyName)] = append(byName[canonicalEntity(row.CompanyName)], row) + } + res.Groups = groups + + for _, g := range groups { + if g.Duplicate { + res.Duplicates = append(res.Duplicates, g.Rows) + } + } + names := make([]string, 0, len(byName)) + for n := range byName { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + rows := byName[n] + if n == "" || len(rows) < 2 || !distinctExternalIDs(rows) { + continue + } + res.Duplicates = append(res.Duplicates, rows) + } + + if deps.ECLAs != nil { + for _, ar := range res.Rows { + if ar.Row.ActiveCCLA && (ar.Route == RouteManual || ar.Duplicate || ar.unresolvable()) { + if n, cErr := deps.ECLAs.CountECLAs(ctx, ar.Row.CompanyID); cErr == nil { + ar.ECLACount = n + } else { + ar.ECLACount = -1 + } + } + } + } + + if opts.OutDir != "" { + if err = writeAuditReports(opts.OutDir, res); err != nil { + return nil, err + } + } + fmt.Fprintf(deps.Out, "audit stage=%s companies=%d eligible_groups=%d", opts.Stage, len(res.Rows), len(groups)) + for _, t := range []string{TierMissing, TierInvalid, TierOK, TierDangling, TierUnknown} { + fmt.Fprintf(deps.Out, " %s=%d", t, res.Tiers[t]) + } + fmt.Fprintf(deps.Out, " register=%d rewrite=%d manual=%d duplicates=%d\n", res.Routes[RouteRegister], res.Routes[RouteRewrite], res.Routes[RouteManual], len(res.Duplicates)) + return res, nil +} + +func (ar *AuditRow) unresolvable() bool { + return ar.Shape == ShapeEmpty || ar.Shape == ShapeOther || (ar.Shape == ShapeSFID && ar.OrgStatus == "404") +} + +func distinctExternalIDs(rows []*Row) bool { + seen := map[string]bool{} + for _, r := range rows { + seen[r.ExternalID] = true + } + return len(seen) > 1 +} + +func tier(externalID, orgStatus string) string { + if externalID == "" { + return TierMissing + } + if len(externalID) != 15 && len(externalID) != 18 || !alnum(externalID) { + return TierInvalid + } + switch orgStatus { + case "200": + return TierOK + case "404": + return TierDangling + default: + return TierUnknown + } +} + +func alnum(s string) bool { + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +type orgLookup struct { + org *Org + status string +} + +// lookupOrgs resolves every distinct 15/18-char external id in org-service (4 workers). +func lookupOrgs(ctx context.Context, deps Deps, inv *Inventory) map[string]*orgLookup { + ids := make([]string, 0, len(inv.ByExternal)) + for id := range inv.ByExternal { + if (len(id) == 15 || len(id) == 18) && alnum(id) { + ids = append(ids, id) + } + } + sort.Strings(ids) + out := make(map[string]*orgLookup, len(ids)) + var mu sync.Mutex + var wg sync.WaitGroup + work := make(chan string) + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for id := range work { + l := &orgLookup{} + org, err := deps.Orgs.GetOrganization(ctx, id) + switch { + case err == nil: + l.org, l.status = org, "200" + case errors.Is(err, ErrOrgNotFound): + l.status = "404" + default: + l.status = orgStatusErr + } + mu.Lock() + out[id] = l + mu.Unlock() + } + }() + } + for _, id := range ids { + work <- id + } + close(work) + wg.Wait() + return out +} + +func writeAuditReports(dir string, res *AuditResult) error { + if err := os.MkdirAll(dir, 0o750); err != nil { + return err + } + rows := [][]string{{"company_id", "company_name", "signing_entity_name", "company_external_id", "id_shape", "active_ccla", "ccla_count", "ecla_count", "org_service", "website", "duplicate_sfid_group", "route", "manual_reason", "tier"}} + var unresolvable [][]string + unresolvable = append(unresolvable, []string{"company_id", "company_name", "signing_entity_name", "company_external_id", "id_shape", "ccla_count", "ecla_count", "org_service", "manual_reason"}) + for _, ar := range res.Rows { + r := ar.Row + rows = append(rows, []string{r.CompanyID, r.CompanyName, r.SigningEntityName, r.ExternalID, string(ar.Shape), strconv.FormatBool(r.ActiveCCLA), strconv.Itoa(r.CCLACount), strconv.Itoa(ar.ECLACount), ar.OrgStatus, ar.Website, strconv.FormatBool(ar.Duplicate), string(ar.Route), ar.ManualReason, ar.Tier}) + if r.ActiveCCLA && ar.unresolvable() { + unresolvable = append(unresolvable, []string{r.CompanyID, r.CompanyName, r.SigningEntityName, r.ExternalID, string(ar.Shape), strconv.Itoa(r.CCLACount), strconv.Itoa(ar.ECLACount), ar.OrgStatus, ar.ManualReason}) + } + } + dups := [][]string{{"group", "company_id", "company_name", "signing_entity_name", "company_external_id", "active_ccla", "ccla_count"}} + for i, set := range res.Duplicates { + for _, r := range set { + dups = append(dups, []string{strconv.Itoa(i + 1), r.CompanyID, r.CompanyName, r.SigningEntityName, r.ExternalID, strconv.FormatBool(r.ActiveCCLA), strconv.Itoa(r.CCLACount)}) + } + } + for name, data := range map[string][][]string{"audit.csv": rows, "unresolvable.csv": unresolvable, "possible_duplicates.csv": dups} { + if err := writeCSV(filepath.Join(dir, name), data); err != nil { + return err + } + } + return nil +} + +func writeIngestReports(dir string, plan *Plan) error { + if err := os.MkdirAll(dir, 0o750); err != nil { + return err + } + rows := [][]string{{"key", "old_id", "id_shape", "route", "manual_reason", "live", "org_service", "website", "domain", "shared_domain", "new_id", "action", "decision", "reviewer", "company_ids", "company_names", "error"}} + toSF := [][]string{{"old_id", "name", "website", "ccla_signed_date", "domain", "shared_domain"}} + manual := [][]string{{"key", "old_id", "route", "reason", "suggested_action", "live", "org_service", "website", "domain", "shared_domain", "new_id", "action", "company_ids", "company_names", "error"}} + for _, g := range plan.Groups { + names := strings.Join(g.Names(), ";") + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + domain, shared := plan.domainOf(g) + decision, reviewer := "", "" + if g.Decision != nil { + decision, reviewer = g.Decision.Kind, g.Decision.Reviewer + } + rows = append(rows, []string{g.Key, g.OldID, string(g.Shape), string(g.Route), g.ManualReason, g.Live, g.OrgStatus, g.Website(), domain, shared, g.NewID, g.Action, decision, reviewer, strings.Join(g.CompanyIDs(), ";"), names, errText}) + if g.ManualReason == ReasonNoMapping || g.ManualReason == ReasonDeadAccount { + req := apexRequest(g, true) + toSF = append(toSF, []string{g.OldID, req.Name, req.Website, req.CCLASignedDate, domain, shared}) + } + } + for _, a := range plan.ManualActions() { + g := a.Group + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + domain, shared := plan.domainOf(g) + manual = append(manual, []string{g.Key, g.OldID, string(g.Route), a.Reason, a.Suggested, g.Live, g.OrgStatus, g.Website(), domain, shared, g.NewID, g.Action, strings.Join(g.CompanyIDs(), ";"), strings.Join(g.Names(), ";"), errText}) + } + targets := [][]string{{"target_sfid", "groups", "old_ids", "company_ids", "company_names", "existing_rows", "decision", "reviewer", "status"}} + for _, t := range plan.Targets { + var ids, names []string + for _, g := range t.Groups { + ids = append(ids, g.CompanyIDs()...) + names = append(names, g.Names()...) + } + decision, reviewer := "", "" + if t.Decision != nil { + decision, reviewer = t.Decision.Kind, t.Decision.Reviewer + } + targets = append(targets, []string{t.SFID, strconv.Itoa(len(t.Groups)), strings.Join(t.OldIDs(), ";"), strings.Join(ids, ";"), strings.Join(names, ";"), strconv.Itoa(len(t.Existing)), decision, reviewer, t.Status}) + } + for name, data := range map[string][][]string{"plan.csv": rows, "to_salesforce.csv": toSF, "manual_actions.csv": manual, "targets.csv": targets} { + if err := writeCSV(filepath.Join(dir, name), data); err != nil { + return err + } + } + return nil +} + +// domainOf returns the group's website domain and "true"/"false" for the shared-domain flag. +func (p *Plan) domainOf(g *Group) (string, string) { + domain, reason := p.shared.Shared(g.Website()) + return domain, strconv.FormatBool(reason == ReasonSharedDomain) +} + +func writeCSV(path string, rows [][]string) error { + f, err := os.Create(filepath.Clean(path)) + if err != nil { + return err + } + w := csv.NewWriter(f) + if err = w.WriteAll(rows); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/cla-backend-go/orgimport/awsreport.go b/cla-backend-go/orgimport/awsreport.go new file mode 100644 index 000000000..3686b2bb2 --- /dev/null +++ b/cla-backend-go/orgimport/awsreport.go @@ -0,0 +1,131 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ses" + "github.com/aws/aws-sdk-go/service/ses/sesiface" +) + +const ( + // CloudWatch Logs limits: 256 KB per event (26 bytes overhead), 1 MB and 10,000 events per batch. + cwMaxEventBytes = 256*1024 - 26 + cwMaxBatchBytes = 1024*1024 - 26*10000 + cwMaxBatchCount = 10000 + cwEventOverhead = 26 +) + +// LogShipper copies the run log into a CloudWatch Logs stream (group and stream are created on demand). +type LogShipper struct { + Client cloudwatchlogsiface.CloudWatchLogsAPI + Group string + Stream string + Now func() time.Time +} + +// Ship writes data line by line as log events and returns the number of events written. +func (s *LogShipper) Ship(ctx context.Context, data []byte) (int, error) { + if s.Client == nil { + return 0, ErrNotConfigured + } + now := time.Now + if s.Now != nil { + now = s.Now + } + if _, err := s.Client.CreateLogGroupWithContext(ctx, &cloudwatchlogs.CreateLogGroupInput{LogGroupName: aws.String(s.Group)}); err != nil && !isAlreadyExists(err) { + return 0, fmt.Errorf("create log group %s: %w", s.Group, err) + } + if _, err := s.Client.CreateLogStreamWithContext(ctx, &cloudwatchlogs.CreateLogStreamInput{LogGroupName: aws.String(s.Group), LogStreamName: aws.String(s.Stream)}); err != nil && !isAlreadyExists(err) { + return 0, fmt.Errorf("create log stream %s: %w", s.Stream, err) + } + ts := now().UnixMilli() + var events []*cloudwatchlogs.InputLogEvent + for _, line := range splitEvents(data) { + events = append(events, &cloudwatchlogs.InputLogEvent{Message: aws.String(line), Timestamp: aws.Int64(ts)}) + } + var token *string + sent := 0 + for len(events) > 0 { + n, size := 0, 0 + for n < len(events) && n < cwMaxBatchCount { + size += len(*events[n].Message) + cwEventOverhead + if size > cwMaxBatchBytes && n > 0 { + break + } + n++ + } + out, err := s.Client.PutLogEventsWithContext(ctx, &cloudwatchlogs.PutLogEventsInput{ + LogGroupName: aws.String(s.Group), LogStreamName: aws.String(s.Stream), LogEvents: events[:n], SequenceToken: token, + }) + if err != nil { + return sent, fmt.Errorf("put log events: %w", err) + } + if out != nil { + token = out.NextSequenceToken + } + sent += n + events = events[n:] + } + return sent, nil +} + +// splitEvents splits the log into lines, chunking lines longer than the event limit; empty lines are kept +// as a single space so line numbering matches run.log. +func splitEvents(data []byte) []string { + var out []string + for _, line := range bytes.Split(bytes.TrimRight(data, "\n"), []byte("\n")) { + if len(line) == 0 { + out = append(out, " ") + continue + } + for len(line) > cwMaxEventBytes { + out = append(out, string(line[:cwMaxEventBytes])) + line = line[cwMaxEventBytes:] + } + out = append(out, string(line)) + } + return out +} + +func isAlreadyExists(err error) bool { + var aerr awserr.Error + return errors.As(err, &aerr) && aerr.Code() == cloudwatchlogs.ErrCodeResourceAlreadyExistsException +} + +// Mailer sends the report through SES SendRawEmail. +type Mailer struct { + Client sesiface.SESAPI +} + +// Send delivers a raw MIME message and returns the SES message id. +func (m Mailer) Send(ctx context.Context, from string, to []string, raw []byte) (string, error) { + if m.Client == nil { + return "", ErrNotConfigured + } + if len(to) == 0 { + return "", errors.New("no recipients") + } + if len(raw) > MaxRawEmailBytes { + return "", fmt.Errorf("message is %d bytes, SES limit is %d", len(raw), MaxRawEmailBytes) + } + out, err := m.Client.SendRawEmailWithContext(ctx, &ses.SendRawEmailInput{ + Source: aws.String(from), + Destinations: aws.StringSlice(to), + RawMessage: &ses.RawMessage{Data: raw}, + }) + if err != nil { + return "", err + } + return aws.StringValue(out.MessageId), nil +} diff --git a/cla-backend-go/orgimport/decisions.go b/cla-backend-go/orgimport/decisions.go new file mode 100644 index 000000000..5ab6ab30c --- /dev/null +++ b/cla-backend-go/orgimport/decisions.go @@ -0,0 +1,324 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bufio" + "encoding/csv" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" +) + +// Decision kinds recorded by the duplicate review (lfx-self-serve #3085). +const ( + DecisionCollapse = "collapse" + DecisionDistinct = "distinct" +) + +// Decision is one reviewed group: collapse (all old ids land on TargetSFID) or distinct (separate Accounts). +type Decision struct { + Kind string + OldIDs []string + Target string + Reviewer string + Note string +} + +// Decisions indexes the review file by old id. +type Decisions struct { + ByOldID map[string]*Decision + List []*Decision +} + +// LoadDecisions reads and validates the decisions CSV (header required). +func LoadDecisions(path string) (*Decisions, error) { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + return ParseDecisions(f) +} + +// ParseDecisions parses `decision,old_ids,target_sfid,reviewer,note` rows; old_ids are separated by `;` or spaces. +func ParseDecisions(r io.Reader) (*Decisions, error) { + cr := csv.NewReader(r) + cr.TrimLeadingSpace = true + cr.FieldsPerRecord = -1 + records, err := cr.ReadAll() + if err != nil { + return nil, fmt.Errorf("decisions: %w", err) + } + if len(records) == 0 { + return nil, fmt.Errorf("decisions: empty file") + } + col := map[string]int{} + for i, h := range records[0] { + col[strings.ToLower(strings.TrimSpace(h))] = i + } + for _, name := range []string{"decision", "old_ids", "target_sfid", "reviewer"} { + if _, ok := col[name]; !ok { + return nil, fmt.Errorf("decisions: missing column %q (need decision,old_ids,target_sfid,reviewer[,note])", name) + } + } + field := func(rec []string, name string) string { + i, ok := col[name] + if !ok || i >= len(rec) { + return "" + } + return strings.TrimSpace(rec[i]) + } + d := &Decisions{ByOldID: map[string]*Decision{}} + for n, rec := range records[1:] { + line := n + 2 + if len(rec) == 0 || (len(rec) == 1 && strings.TrimSpace(rec[0]) == "") { + continue + } + dec := &Decision{Kind: strings.ToLower(field(rec, "decision")), Target: field(rec, "target_sfid"), Reviewer: field(rec, "reviewer"), Note: field(rec, "note")} + dec.OldIDs = append(dec.OldIDs, strings.FieldsFunc(field(rec, "old_ids"), func(r rune) bool { return r == ';' || r == ' ' || r == '\t' })...) + if dec.Reviewer == "" { + return nil, fmt.Errorf("decisions line %d: reviewer is required", line) + } + if len(dec.OldIDs) == 0 { + return nil, fmt.Errorf("decisions line %d: old_ids is empty", line) + } + switch dec.Kind { + case DecisionCollapse: + if !IsSFID(dec.Target) { + return nil, fmt.Errorf("decisions line %d: collapse needs a Salesforce account id in target_sfid", line) + } + case DecisionDistinct: + if dec.Target != "" { + return nil, fmt.Errorf("decisions line %d: distinct must not set target_sfid", line) + } + if len(dec.OldIDs) < 2 { + return nil, fmt.Errorf("decisions line %d: distinct needs at least two old ids", line) + } + default: + return nil, fmt.Errorf("decisions line %d: decision must be collapse|distinct", line) + } + for _, id := range dec.OldIDs { + if prev, dup := d.ByOldID[id]; dup { + return nil, fmt.Errorf("decisions line %d: %s already appears in a %s decision", line, id, prev.Kind) + } + d.ByOldID[id] = dec + } + d.List = append(d.List, dec) + } + return d, nil +} + +// Collapse reports whether a recorded collapse decision lets oldID land on target. +func (d *Decisions) Collapse(oldID, target string) *Decision { + if d == nil { + return nil + } + dec := d.ByOldID[oldID] + if dec != nil && dec.Kind == DecisionCollapse && dec.Target == target { + return dec + } + return nil +} + +// Distinct reports whether a recorded distinct decision separates oldID from any id in others. +func (d *Decisions) Distinct(oldID string, others []string) *Decision { + if d == nil { + return nil + } + dec := d.ByOldID[oldID] + if dec == nil || dec.Kind != DecisionDistinct { + return nil + } + for _, o := range others { + if o != oldID && d.ByOldID[o] == dec { + return dec + } + } + return nil +} + +// Target groups the plan by destination Account (dry-run report of design §5 step 3). +type Target struct { + SFID string + Groups []*Group + Existing []*Row + Mapped []string + Decision *Decision + Status string +} + +// OldIDs lists the old ids landing on the target: tranche groups, rows already carrying it and +// mapping rows outside the tranche. +func (t *Target) OldIDs() []string { + seen := map[string]bool{} + var ids []string + add := func(id string) { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + for _, g := range t.Groups { + add(g.Key) + } + for _, r := range t.Existing { + add(r.ExternalID) + } + for _, id := range t.Mapped { + add(id) + } + return ids +} + +// applyDecisions is the approval-aware collision pass: a rewrite group may share its destination with +// other groups, other mapping rows or rows already carrying the id only under a recorded collapse decision. +func applyDecisions(groups []*Group, inv *Inventory, mapping *Mapping, decisions *Decisions) []*Target { + byTarget := map[string]*Target{} + for _, g := range groups { + if g.Route != RouteRewrite || g.NewID == "" || g.Err != nil { + continue + } + t := byTarget[g.NewID] + if t == nil { + t = &Target{SFID: g.NewID} + t.Existing = append(t.Existing, inv.ByExternal[g.NewID]...) + byTarget[g.NewID] = t + } + t.Groups = append(t.Groups, g) + } + targets := make([]*Target, 0, len(byTarget)) + for _, t := range byTarget { + targets = append(targets, t) + } + sort.Slice(targets, func(i, j int) bool { return targets[i].SFID < targets[j].SFID }) + for _, t := range targets { + t.Status = statusOK + if mapping != nil { + seen := t.OldIDs() + for oldID, row := range mapping.Rows { + if row.NewID == t.SFID && !containsString(seen, oldID) { + t.Mapped = append(t.Mapped, oldID) + } + } + sort.Strings(t.Mapped) + } + ids := t.OldIDs() + if len(ids) < 2 { + continue + } + for _, g := range t.Groups { + if g.Replayed { + continue + } + if dec := decisions.Distinct(g.Key, ids); dec != nil { + g.Route, g.ManualReason = RouteManual, ReasonDistinctConflict + t.Decision, t.Status = dec, ReasonDistinctConflict + continue + } + dec := decisions.Collapse(g.Key, t.SFID) + if dec == nil { + g.Route, g.ManualReason = RouteManual, "target_collision" + if t.Status == statusOK { + t.Status = "needs_decision" + } + continue + } + g.Decision, t.Decision = dec, dec + } + } + return targets +} + +func containsString(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// SharedDomains is the list of domains that never match an Account automatically (design §4). +type SharedDomains map[string]bool + +// DefaultSharedDomains is the built-in list; the #3085 review file replaces it (--shared-domains). +var DefaultSharedDomains = []string{ + "github.com", "nowebsite.com", + "gmail.com", "googlemail.com", "yahoo.com", "hotmail.com", "outlook.com", "live.com", "icloud.com", "protonmail.com", "qq.com", "163.com", +} + +// LoadSharedDomains reads one domain per line (`#` comments); an empty path yields the default list. +func LoadSharedDomains(path string) (SharedDomains, error) { + set := SharedDomains{} + if path == "" { + for _, d := range DefaultSharedDomains { + set[d] = true + } + return set, nil + } + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if i := strings.IndexByte(line, '#'); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + continue + } + set[normalizeDomain(line)] = true + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("shared domains: %w", err) + } + return set, nil +} + +// Domain extracts the registrable host of a website value (scheme, path and www. stripped). +func Domain(website string) string { + website = strings.TrimSpace(strings.ToLower(website)) + if website == "" { + return "" + } + if !strings.Contains(website, "://") { + website = "http://" + website + } + u, err := url.Parse(website) + if err != nil || u.Hostname() == "" { + return normalizeDomain(website) + } + return normalizeDomain(u.Hostname()) +} + +func normalizeDomain(host string) string { + host = strings.TrimSpace(strings.ToLower(host)) + host = strings.TrimPrefix(host, "http://") + host = strings.TrimPrefix(host, "https://") + if i := strings.IndexAny(host, "/:?"); i >= 0 { + host = host[:i] + } + host = strings.TrimSuffix(host, ".") + return strings.TrimPrefix(host, "www.") +} + +// Shared reports whether website has no domain or a listed shared domain; the reason is the manual reason. +func (s SharedDomains) Shared(website string) (string, string) { + d := Domain(website) + switch { + case d == "": + return "", "missing_website" + case s[d]: + return d, ReasonSharedDomain + } + return d, "" +} diff --git a/cla-backend-go/orgimport/decisions_test.go b/cla-backend-go/orgimport/decisions_test.go new file mode 100644 index 000000000..bb4a0648a --- /dev/null +++ b/cla-backend-go/orgimport/decisions_test.go @@ -0,0 +1,369 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeDecisions(t *testing.T, dir string, rows ...string) string { + t.Helper() + path := filepath.Join(dir, "decisions.csv") + require.NoError(t, os.WriteFile(path, []byte("decision,old_ids,target_sfid,reviewer,note\n"+strings.Join(rows, "\n")+"\n"), 0o600)) + return path +} + +func TestParseDecisions(t *testing.T) { + d, err := ParseDecisions(strings.NewReader("decision,old_ids,target_sfid,reviewer,note\n" + + "collapse,lf-a; lf-b," + targetSFID + ",michal,same legal entity\n" + + "DISTINCT,lf-c;lf-d,,michal,\n")) + require.NoError(t, err) + require.Len(t, d.List, 2) + assert.Equal(t, []string{"lf-a", "lf-b"}, d.List[0].OldIDs) + assert.NotNil(t, d.Collapse("lf-a", targetSFID)) + assert.Nil(t, d.Collapse("lf-a", targetSFID2), "collapse is bound to its target") + assert.Nil(t, d.Collapse("lf-c", targetSFID)) + assert.NotNil(t, d.Distinct("lf-c", []string{"lf-d", "lf-x"})) + assert.Nil(t, d.Distinct("lf-c", []string{"lf-a"}), "distinct only separates ids of the same decision") + assert.Nil(t, d.Distinct("lf-a", []string{"lf-b"})) + assert.Nil(t, (*Decisions)(nil).Collapse("lf-a", targetSFID)) + assert.Nil(t, (*Decisions)(nil).Distinct("lf-a", nil)) + spaced, err := ParseDecisions(strings.NewReader("decision,old_ids,target_sfid,reviewer\ncollapse,lf-a lf-b\t" + targetSFID + "," + targetSFID + ",michal\n")) + require.NoError(t, err) + assert.Equal(t, []string{"lf-a", "lf-b", targetSFID}, spaced.List[0].OldIDs, "old ids may be separated by spaces (workflow input rows use '|')") + + for _, bad := range []string{ + "", + "decision,old_ids,target_sfid\ncollapse,lf-a," + targetSFID, + "decision,old_ids,target_sfid,reviewer\ncollapse,lf-a," + targetSFID + ",", + "decision,old_ids,target_sfid,reviewer\ncollapse,lf-a,not-an-sfid,michal", + "decision,old_ids,target_sfid,reviewer\ncollapse,," + targetSFID + ",michal", + "decision,old_ids,target_sfid,reviewer\ndistinct,lf-a,,michal", + "decision,old_ids,target_sfid,reviewer\ndistinct,lf-a;lf-b," + targetSFID + ",michal", + "decision,old_ids,target_sfid,reviewer\nmerge,lf-a;lf-b," + targetSFID + ",michal", + "decision,old_ids,target_sfid,reviewer\ncollapse,lf-a," + targetSFID + ",michal\ndistinct,lf-a;lf-b,,michal", + } { + _, err = ParseDecisions(strings.NewReader(bad)) + assert.Error(t, err, bad) + } + _, err = LoadDecisions(filepath.Join(t.TempDir(), "missing.csv")) + assert.Error(t, err) +} + +func TestSharedDomains(t *testing.T) { + def, err := LoadSharedDomains("") + require.NoError(t, err) + assert.True(t, def["github.com"]) + assert.True(t, def["nowebsite.com"]) + assert.Equal(t, "legacy.example", Domain("https://www.legacy.example/about?x=1")) + assert.Equal(t, "legacy.example", Domain("Legacy.Example")) + assert.Equal(t, "github.com", Domain("github.com/some-org")) + assert.Equal(t, "", Domain(" ")) + d, reason := def.Shared("https://github.com/acme") + assert.Equal(t, "github.com", d) + assert.Equal(t, "shared_domain", reason) + _, reason = def.Shared("") + assert.Equal(t, "missing_website", reason) + d, reason = def.Shared("https://acme.example") + assert.Equal(t, "acme.example", d) + assert.Empty(t, reason) + + path := filepath.Join(t.TempDir(), "shared.txt") + require.NoError(t, os.WriteFile(path, []byte("# review list\nWWW.Example.ORG # comment\nhttps://foo.test/\n\n"), 0o600)) + custom, err := LoadSharedDomains(path) + require.NoError(t, err) + assert.Equal(t, SharedDomains{"example.org": true, "foo.test": true}, custom) + assert.False(t, custom["github.com"], "a review list replaces the built-in list") + _, err = LoadSharedDomains(filepath.Join(t.TempDir(), "missing")) + assert.Error(t, err) +} + +func collisionFixture(t *testing.T) (*fixture, string) { + t.Helper() + fx := newFixture() + fx.company("c-a", "Acme Inc", "", "lf-a", "cg-1") + fx.company("c-b", "Acme GmbH", "", "lf-b", "cg-1") + fx.company("c-c", "Acme Labs", "", "lf-c", "cg-2") + fx.platform.sfAccounts[targetSFID] = true + fx.platform.orgs[targetSFID] = &Org{ID: targetSFID, Name: "Acme"} + fx.platform.orgs["lf-a"] = &Org{ID: "lf-a", Name: "Acme Inc", Website: "https://acme.example"} + fx.platform.orgs["lf-b"] = &Org{ID: "lf-b", Name: "Acme GmbH", Website: "https://acme.example/de"} + fx.platform.orgs["lf-c"] = &Org{ID: "lf-c", Name: "Acme Labs", Website: "https://labs.acme.example"} + dir := t.TempDir() + return fx, dir +} + +func TestCollisionNeedsDecision(t *testing.T) { + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + opts := Options{Stage: "dev", Mapping: mapping, OutDir: filepath.Join(dir, "out")} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteManual, g.Route, id) + assert.Equal(t, "target_collision", g.ManualReason, id) + assert.Equal(t, targetSFID, g.NewID, id, "the resolved target stays on the group for the report") + } + c := groupByKey(plan.Groups, "lf-c") + assert.Equal(t, RouteRewrite, c.Route) + assert.Equal(t, targetSFID2, c.NewID) + require.Len(t, plan.Targets, 2) + tgt := targetByID(plan.Targets, targetSFID) + assert.Equal(t, "needs_decision", tgt.Status) + assert.Equal(t, []string{"lf-a", "lf-b"}, tgt.OldIDs()) + assert.Equal(t, "ok", targetByID(plan.Targets, targetSFID2).Status) + assert.Empty(t, plan.Rewrite[0].ManualReason) + require.Len(t, plan.Rewrite, 1) + + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 2, sum.Manual) + targets := readCSV(t, filepath.Join(dir, "out", "targets.csv")) + require.Len(t, targets, 3) + assert.Equal(t, []string{targetSFID, "2", "lf-a;lf-b", "c-a;c-b", "Acme Inc;Acme GmbH", "0", "", "", "needs_decision"}, targets[1]) + assert.Equal(t, targetSFID2, targets[2][0]) + assert.Equal(t, "ok", targets[2][8]) +} + +func TestCollisionCollapseDecision(t *testing.T) { + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + decisions := writeDecisions(t, dir, "collapse,lf-a;lf-b,"+targetSFID+",michal,one legal entity") + opts := Options{Stage: "dev", Mapping: mapping, Decisions: decisions, OutDir: filepath.Join(dir, "out")} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + require.Len(t, plan.Rewrite, 3) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteRewrite, g.Route, id) + assert.Empty(t, g.ManualReason, id) + assert.Equal(t, targetSFID, g.NewID, id) + require.NotNil(t, g.Decision, id) + assert.Equal(t, "michal", g.Decision.Reviewer) + } + tgt := targetByID(plan.Targets, targetSFID) + assert.Equal(t, "ok", tgt.Status) + assert.Equal(t, DecisionCollapse, tgt.Decision.Kind) + + // a collapse recorded for another target does not approve this one + other := writeDecisions(t, dir, "collapse,lf-a;lf-b,"+targetSFID2+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: other}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-b").ManualReason) + + // a collapse covering only one of the ids approves only that id + partial := writeDecisions(t, dir, "collapse,lf-a,"+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: partial}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-b").ManualReason) + assert.Equal(t, "needs_decision", targetByID(plan.Targets, targetSFID).Status) + + // dry run of the approved collapse writes the decision into the plan report + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + assert.Equal(t, 1, sum.Manual) + rows := readCSV(t, filepath.Join(dir, "out", "plan.csv")) + byKey := map[string][]string{} + for _, r := range rows[1:] { + byKey[r[0]] = r + } + assert.Equal(t, "collapse", byKey["lf-a"][12]) + assert.Equal(t, "michal", byKey["lf-a"][13]) + assert.Equal(t, "acme.example", byKey["lf-a"][8]) + assert.Equal(t, "false", byKey["lf-a"][9]) + var buf bytes.Buffer + plan.Print(&buf) + assert.Contains(t, buf.String(), targetSFID+" status=needs_decision old_ids=lf-a;lf-b") +} + +func TestCollisionDistinctDecision(t *testing.T) { + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true") + decisions := writeDecisions(t, dir, "distinct,lf-a;lf-b,,michal,separate entities") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + for _, id := range []string{"lf-a", "lf-b"} { + g := groupByKey(plan.Groups, id) + assert.Equal(t, RouteManual, g.Route, id) + assert.Equal(t, "distinct_conflict", g.ManualReason, id) + } + assert.Equal(t, "distinct_conflict", targetByID(plan.Targets, targetSFID).Status) + assert.Contains(t, Suggest("distinct_conflict"), "distinct decision") + + // distinct ids landing on different accounts are fine + mapping = writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID2+",created,true") + fx.platform.orgs[targetSFID2] = &Org{ID: targetSFID2} + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-b").Route) +} + +func TestCollisionWithExistingRowsAndMappingCoTargets(t *testing.T) { + fx, dir := collisionFixture(t) + fx.company("c-existing", "Acme Holdings", "", targetSFID) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + a := groupByKey(plan.Groups, "lf-a") + assert.Equal(t, "target_collision", a.ManualReason, "rows already carrying the target need a collapse decision") + tgt := targetByID(plan.Targets, targetSFID) + assert.Equal(t, []string{"lf-a", targetSFID}, tgt.OldIDs()) + require.Len(t, tgt.Existing, 1) + + decisions := writeDecisions(t, dir, "collapse,lf-a;"+targetSFID+","+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) + + // a mapping row outside the tranche landing on the same target is a co-target too + fx, dir = collisionFixture(t) + mapping = writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-z,"+targetSFID+",created,true") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, "target_collision", groupByKey(plan.Groups, "lf-a").ManualReason) + assert.Equal(t, []string{"lf-a", "lf-z"}, targetByID(plan.Targets, targetSFID).OldIDs()) + decisions = writeDecisions(t, dir, "collapse,lf-a;lf-z,"+targetSFID+",michal,") + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, Decisions: decisions}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, "lf-a").Route) +} + +func TestReplayedGroupSkipsDecisionPass(t *testing.T) { + fx, mapping, statePath := rewriteFixture(t) + fx.company("c-existing", "Legacy Holdings", "", targetSFID) + require.NoError(t, os.WriteFile(statePath, []byte(`{"ts":"2026-09-29T11:00:00Z","old_id":"`+lfID+`","new_id":"`+targetSFID+`","company_ids":["c-parent","c-sub"],"step":"rows","status":"ok"}`+"\n"), 0o600)) + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev", Mapping: mapping, State: statePath}) + require.NoError(t, err) + g := groupByKey(plan.Groups, lfID) + require.NotNil(t, g) + assert.True(t, g.Replayed) + assert.Equal(t, RouteRewrite, g.Route, "an unfinished rewrite resumes regardless of collisions") + assert.Empty(t, g.ManualReason) +} + +const legacyGitHubSite = "https://github.com/legacy-ltd" + +func TestApexSharedDomainGate(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":%q,"action":%q}`, targetSFID, ActionCreated) + })) + defer srv.Close() + apex, err := NewApexClient(srv.URL, "apex-token") + require.NoError(t, err) + + fx, mapping, _ := rewriteFixture(t) + fx.platform.orgs[lfID].Website = legacyGitHubSite + deps := fx.deps() + deps.Apex = apex + dir := t.TempDir() + opts := Options{Stage: "dev", UseApex: true, OutDir: dir} + plan, err := BuildPlan(context.Background(), deps, opts) + require.NoError(t, err) + g := groupByKey(plan.Groups, lfID) + assert.Equal(t, RouteManual, g.Route) + assert.Equal(t, "shared_domain", g.ManualReason) + assert.Equal(t, 0, calls, "shared domains never reach Apex") + _, err = Execute(context.Background(), deps, opts, plan) + require.NoError(t, err) + rows := readCSV(t, filepath.Join(dir, "plan.csv")) + var lfRow []string + for _, r := range rows[1:] { + if r[0] == lfID { + lfRow = r + } + } + require.NotNil(t, lfRow) + assert.Equal(t, legacyGitHubSite, lfRow[7]) + assert.Equal(t, "github.com", lfRow[8]) + assert.Equal(t, "true", lfRow[9]) + manual := readCSV(t, filepath.Join(dir, "manual_actions.csv")) + require.Len(t, manual, 2) + assert.Equal(t, "shared_domain", manual[1][3]) + assert.Contains(t, manual[1][4], "never matched automatically") + + fx.platform.orgs[lfID].Website = "" + plan, err = BuildPlan(context.Background(), deps, Options{UseApex: true}) + require.NoError(t, err) + assert.Equal(t, "missing_website", groupByKey(plan.Groups, lfID).ManualReason) + assert.Equal(t, 0, calls) + + // an approved mapping row is the operator's resolution and bypasses the gate and Apex + plan, err = BuildPlan(context.Background(), deps, Options{UseApex: true, Mapping: mapping}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, lfID).Route) + assert.Equal(t, targetSFID, groupByKey(plan.Groups, lfID).NewID) + assert.Equal(t, 0, calls) + + // a review list replaces the built-in list + fx.platform.orgs[lfID].Website = legacyGitHubSite + shared := filepath.Join(dir, "shared.txt") + require.NoError(t, os.WriteFile(shared, []byte("nowebsite.com\n"), 0o600)) + plan, err = BuildPlan(context.Background(), deps, Options{UseApex: true, SharedDomains: shared}) + require.NoError(t, err) + assert.Equal(t, RouteRewrite, groupByKey(plan.Groups, lfID).Route) + assert.Equal(t, 1, calls) + _, err = BuildPlan(context.Background(), deps, Options{UseApex: true, SharedDomains: filepath.Join(dir, "missing")}) + assert.Error(t, err) + _, err = BuildPlan(context.Background(), deps, Options{Decisions: filepath.Join(dir, "missing")}) + assert.Error(t, err) +} + +func TestManualActionsAndSuggestions(t *testing.T) { + fx := newFixture() + fx.company("c-empty", "Empty Inc", "", "", "cg-1") + fx.company("c-lf", "Legacy Ltd", "", lfID, "cg-1") + fx.company("c-dead", "Dead Corp", "", deadSFID, "cg-1") + plan, err := BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev"}) + require.NoError(t, err) + actions := plan.ManualActions() + require.Len(t, actions, 3) + reasons := map[string]string{} + for _, a := range actions { + reasons[a.Group.Key] = a.Reason + assert.NotEmpty(t, a.Suggested) + assert.Contains(t, a.String(), a.Reason) + } + assert.Equal(t, map[string]string{"c-empty": "empty_external_id", lfID: "no_mapping", deadSFID: "no_mapping"}, reasons) + assert.Contains(t, Suggest("empty_external_id"), "m3-org-cleanup.md") + assert.Contains(t, Suggest("error: boom"), "re-run") + assert.Contains(t, SuggestError(errors.New("liveness check failed for 001: member-service: authorization failed (403): Client \"x\" is not authorized to access resource server \"y\". You need to create a \"client-grant\" associated to this API.")), "client grant") + assert.Contains(t, SuggestError(errors.New("member-service: authorization failed (403): forbidden")), "Heimdall") + assert.Contains(t, SuggestError(ErrNotConfigured), "cla-member-service-base-url") + assert.Contains(t, SuggestError(errors.New("boom")), "re-run") + assert.Contains(t, Suggest("something_new"), "run.log") + + // a failed group is listed with its error + fx.platform.failGetB2B = true + plan, err = BuildPlan(context.Background(), fx.deps(), Options{Stage: "dev"}) + require.NoError(t, err) + var failed *ManualAction + for _, a := range plan.ManualActions() { + if a.Group.Key == deadSFID { + cp := a + failed = &cp + } + } + require.NotNil(t, failed) + assert.Equal(t, "error", failed.Reason) + assert.Contains(t, failed.String(), "liveness check failed") +} diff --git a/cla-backend-go/orgimport/eligible.go b/cla-backend-go/orgimport/eligible.go new file mode 100644 index 000000000..6354775bf --- /dev/null +++ b/cla-backend-go/orgimport/eligible.go @@ -0,0 +1,183 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" +) + +const sfidPrefix = "001" + +// ShapeOf classifies an external id: 15/18-char alphanumeric starting with 001 → SFID, lf… → lf. +func ShapeOf(id string) IDShape { + id = strings.TrimSpace(id) + switch { + case id == "": + return ShapeEmpty + case IsSFID(id): + return ShapeSFID + case strings.HasPrefix(id, "lf"): + return ShapeLF + default: + return ShapeOther + } +} + +// IsSFID reports whether id looks like a Salesforce Account ID. +func IsSFID(id string) bool { + if len(id) != 15 && len(id) != 18 || !strings.HasPrefix(id, sfidPrefix) { + return false + } + for _, c := range id { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +func canonicalEntity(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +// Inventory is the companies table joined with the active-CCLA set. +type Inventory struct { + Rows []*Row + ByID map[string]*Row + ByExternal map[string][]*Row +} + +// LoadInventory scans companies and active (signed+approved) CCLAs. +func LoadInventory(ctx context.Context, deps Deps) (*Inventory, error) { + cclas, err := deps.Signatures.GetCCLASignatures(ctx, aws.Bool(true), aws.Bool(true)) + if err != nil { + return nil, fmt.Errorf("listing active CCLAs: %w", err) + } + active := map[string][]string{} + signedOn := map[string]string{} + for _, s := range cclas { + if s == nil || s.SignatureReferenceID == "" { + continue + } + active[s.SignatureReferenceID] = append(active[s.SignatureReferenceID], s.SignatureProjectID) + if d := firstNonEmpty(s.SignedOn, s.DateCreated); d != "" && (signedOn[s.SignatureReferenceID] == "" || d < signedOn[s.SignatureReferenceID]) { + signedOn[s.SignatureReferenceID] = d + } + } + companies, err := deps.Companies.GetCompanies(ctx) + if err != nil { + return nil, fmt.Errorf("scanning companies: %w", err) + } + inv := &Inventory{ByID: map[string]*Row{}, ByExternal: map[string][]*Row{}} + for i := range companies.Companies { + c := companies.Companies[i] + row := &Row{ + CompanyID: c.CompanyID, + CompanyName: c.CompanyName, + SigningEntityName: c.SigningEntityName, + ExternalID: strings.TrimSpace(c.CompanyExternalID), + } + if groups, ok := active[c.CompanyID]; ok { + row.ActiveCCLA = true + row.CCLACount = len(groups) + row.CLAGroupIDs = distinct(groups) + row.CCLASignedOn = signedOn[c.CompanyID] + } + inv.Rows = append(inv.Rows, row) + inv.ByID[row.CompanyID] = row + if row.ExternalID != "" { + inv.ByExternal[row.ExternalID] = append(inv.ByExternal[row.ExternalID], row) + } + } + sort.Slice(inv.Rows, func(i, j int) bool { return inv.Rows[i].CompanyID < inv.Rows[j].CompanyID }) + return inv, nil +} + +// EligibleGroups returns one group per external id that has at least one active-CCLA row (all +// sibling rows included); rows with an empty or malformed external id form single-row groups keyed +// by company id (a blank or garbage value never identifies one organization). +func (inv *Inventory) EligibleGroups() []*Group { + var groups []*Group + seen := map[string]bool{} + for _, row := range inv.Rows { + if !row.ActiveCCLA { + continue + } + if shape := ShapeOf(row.ExternalID); shape == ShapeEmpty || shape == ShapeOther { + groups = append(groups, &Group{OldID: row.ExternalID, Key: row.CompanyID, Shape: shape, Rows: []*Row{row}}) + continue + } + if seen[row.ExternalID] { + continue + } + seen[row.ExternalID] = true + rows := append([]*Row(nil), inv.ByExternal[row.ExternalID]...) + sort.Slice(rows, func(i, j int) bool { return rows[i].CompanyID < rows[j].CompanyID }) + g := &Group{OldID: row.ExternalID, Key: row.ExternalID, Shape: ShapeOf(row.ExternalID), Rows: rows} + g.Duplicate = hasDuplicateEntity(rows) + groups = append(groups, g) + } + sort.Slice(groups, func(i, j int) bool { return groups[i].Key < groups[j].Key }) + return groups +} + +// hasDuplicateEntity is true when two rows of one external id share the same (or empty) entity name. +func hasDuplicateEntity(rows []*Row) bool { + seen := map[string]bool{} + for _, r := range rows { + k := canonicalEntity(r.SigningEntityName) + if seen[k] { + return true + } + seen[k] = true + } + return false +} + +// ExternalIDCollision reports whether a row outside the group already carries newID. +func (inv *Inventory) ExternalIDCollision(g *Group, newID string) bool { + for _, r := range inv.ByExternal[newID] { + if !containsRow(g.Rows, r.CompanyID) { + return true + } + } + return false +} + +func containsRow(rows []*Row, companyID string) bool { + for _, r := range rows { + if r.CompanyID == companyID { + return true + } + } + return false +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func distinct(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + } + sort.Strings(out) + return out +} diff --git a/cla-backend-go/orgimport/manual.go b/cla-backend-go/orgimport/manual.go new file mode 100644 index 000000000..cd04ac733 --- /dev/null +++ b/cla-backend-go/orgimport/manual.go @@ -0,0 +1,96 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "fmt" + "strings" +) + +// ManualAction is one row of manual_actions.csv: a group the tool will not process on its own. +type ManualAction struct { + Group *Group + Reason string + Suggested string +} + +// suggestions maps a manual/pending reason to what the operator should do (README §7, m3-org-cleanup.md). +var suggestions = map[string]string{ + "empty_external_id": "Row has no company_external_id: find or create the Account with sales ops (m3-org-cleanup.md, #2749), then add a mapping row company_id,new_id,matched,true and re-run.", + "invalid_id_shape": "company_external_id is neither a Salesforce id nor lf…: find the Account with sales ops (m3-org-cleanup.md, #2749), then add a mapping row company_id,new_id,matched,true and re-run.", + "no_mapping": "Rewrite candidate without a target: add a mapping row old_id,new_id,action,approved (data in to_salesforce.csv) or run with --use-apex.", + "dead_account": "Salesforce account no longer exists: treat as rewrite — add a mapping row (data in to_salesforce.csv) or run with --use-apex.", + "mapping_ambiguous": "Several Accounts share the domain: pick the right Account with sales ops and set action=matched,approved=true in the mapping.", + "mapping_not_approved": "Matched Account is not approved: confirm it is the same legal entity, then set approved=true.", + "apex_match_needs_approval": "Apex matched an existing Account: approve it with a mapping row old_id,new_id,matched,true (or reject with a different target).", + "mapping_same_id": "new_id equals old_id: fix the mapping row.", + "target_collision": "Several old ids land on this Account (or it already has EasyCLA rows): record a collapse (one Account) or distinct (separate Accounts) decision with a reviewer in the decisions file (#3085), then re-run.", + "distinct_conflict": "A distinct decision separates these ids but they resolve to the same Account: fix the mapping/Apex target or the decision.", + "missing_website": "Org-service has no website for the organization: automatic domain match is impossible — add a mapping row after sales ops name the Account.", + "shared_domain": "Website domain is shared (github.com, nowebsite.com, mail provider): never matched automatically — add a mapping row after sales ops name the Account.", + "apex_error": "Apex call failed: re-run; if it persists check cla-salesforce-apex-* and the endpoint.", +} + +// Suggest returns the operator guidance for a reason. +func Suggest(reason string) string { + if s, ok := suggestions[reason]; ok { + return s + } + if strings.HasPrefix(reason, LiveError) { + return "Transient error: re-run; if it persists inspect run.log." + } + return "Review run.log for this group." +} + +// SuggestError maps a group error to an action; access errors get the ops item instead of "re-run". +func SuggestError(err error) string { + msg := err.Error() + switch { + case strings.Contains(msg, "client-grant"), strings.Contains(msg, "not authorized to access resource server"): + return "EasyCLA's Auth0 M2M client has no client grant for the member-service audience (SSM cla-member-service-auth0-audience-): ops must add it (README §2), then re-run." + case strings.Contains(msg, "(403)"), strings.Contains(msg, "(401)"): + return "member-service refused the call: the client needs auditor (GET /b2b_orgs) and global_org_admin (POST /b2b_orgs) in the member-service Heimdall ruleset (README §2), then re-run." + case strings.Contains(msg, ErrNotConfigured.Error()): + return "SSM cla-member-service-base-url- / cla-member-service-auth0-audience- are missing for this stage (README §2)." + } + return Suggest(LiveError) +} + +// ManualActions lists every group that needs a human: manual, pending and failed groups. +func (p *Plan) ManualActions() []ManualAction { + var out []ManualAction + for _, g := range p.Groups { + var reason, suggested string + switch { + case g.Err != nil: + reason = g.ManualReason + if reason == "" { + reason = LiveError + } + suggested = SuggestError(g.Err) + case g.Route == RouteManual, g.Pending(): + reason = g.ManualReason + suggested = Suggest(reason) + default: + continue + } + out = append(out, ManualAction{Group: g, Reason: reason, Suggested: suggested}) + } + return out +} + +func (a ManualAction) String() string { + g := a.Group + s := fmt.Sprintf("%s [%s] %s", g.Key, a.Reason, a.Suggested) + if g.NewID != "" { + s += " target=" + g.NewID + } + if d := g.Domain(); d != "" { + s += " domain=" + d + } + if g.Err != nil { + s += " error=" + g.Err.Error() + } + return s + " | rows=" + strings.Join(g.CompanyIDs(), ";") +} diff --git a/cla-backend-go/orgimport/mapping.go b/cla-backend-go/orgimport/mapping.go new file mode 100644 index 000000000..790547579 --- /dev/null +++ b/cla-backend-go/orgimport/mapping.go @@ -0,0 +1,127 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const trueString = "true" + +// Mapping actions produced by Salesforce (mapping file or Apex). +const ( + ActionMatched = "matched" + ActionCreated = "created" + ActionAmbiguous = "ambiguous" +) + +// MappingRow is one old_id,new_id,action,approved line. +type MappingRow struct { + OldID string + NewID string + Action string + Approved bool +} + +// Mapping is the validated Salesforce mapping file; Targets counts old ids per new id (feeds the +// approval-aware collision pass). +type Mapping struct { + Rows map[string]MappingRow + Targets map[string]int +} + +// LoadMapping reads and validates a mapping CSV (header required). +func LoadMapping(path string) (*Mapping, error) { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + return ParseMapping(f) +} + +// ParseMapping parses a mapping CSV. +func ParseMapping(r io.Reader) (*Mapping, error) { + cr := csv.NewReader(r) + cr.TrimLeadingSpace = true + records, err := cr.ReadAll() + if err != nil { + return nil, fmt.Errorf("mapping: %w", err) + } + if len(records) == 0 { + return nil, fmt.Errorf("mapping: empty file") + } + header := records[0] + col := map[string]int{} + for i, h := range header { + col[strings.ToLower(strings.TrimSpace(h))] = i + } + for _, name := range []string{"old_id", "new_id", "action", "approved"} { + if _, ok := col[name]; !ok { + return nil, fmt.Errorf("mapping: missing column %q (need old_id,new_id,action,approved)", name) + } + } + m := &Mapping{Rows: map[string]MappingRow{}, Targets: map[string]int{}} + for n, rec := range records[1:] { + line := n + 2 + if len(rec) < len(header) { + return nil, fmt.Errorf("mapping line %d: expected %d columns", line, len(header)) + } + row := MappingRow{ + OldID: strings.TrimSpace(rec[col["old_id"]]), + NewID: strings.TrimSpace(rec[col["new_id"]]), + Action: strings.ToLower(strings.TrimSpace(rec[col["action"]])), + } + switch strings.ToLower(strings.TrimSpace(rec[col["approved"]])) { + case trueString, "yes", "1": + row.Approved = true + case "false", "no", "0", "": + default: + return nil, fmt.Errorf("mapping line %d: approved must be true|false", line) + } + if row.OldID == "" { + return nil, fmt.Errorf("mapping line %d: empty old_id", line) + } + if _, dup := m.Rows[row.OldID]; dup { + return nil, fmt.Errorf("mapping line %d: duplicate old_id %s", line, row.OldID) + } + switch row.Action { + case ActionMatched, ActionCreated: + if !IsSFID(row.NewID) { + return nil, fmt.Errorf("mapping line %d: new_id %q is not a Salesforce account id", line, row.NewID) + } + m.Targets[row.NewID]++ + case ActionAmbiguous: + default: + return nil, fmt.Errorf("mapping line %d: action must be matched|created|ambiguous", line) + } + m.Rows[row.OldID] = row + } + return m, nil +} + +// Resolve returns the new id for oldID or the manual reason why it cannot be used. +func (m *Mapping) Resolve(oldID string) (newID, action, reason string) { + if m == nil { + return "", "", ReasonNoMapping + } + row, ok := m.Rows[oldID] + if !ok { + return "", "", ReasonNoMapping + } + switch { + case row.Action == ActionAmbiguous: + return "", row.Action, ReasonMappingAmbiguous + case row.Action == ActionMatched && !row.Approved: + return "", row.Action, "mapping_not_approved" + case row.NewID == oldID: + return "", row.Action, ReasonMappingSameID + } + return row.NewID, row.Action, "" +} diff --git a/cla-backend-go/orgimport/orgimport.go b/cla-backend-go/orgimport/orgimport.go new file mode 100644 index 000000000..fe9ae2fff --- /dev/null +++ b/cla-backend-go/orgimport/orgimport.go @@ -0,0 +1,276 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +// Package orgimport implements the M3 organization import/sync tool (lfx-self-serve #2750): +// registration of EasyCLA companies with an active CCLA as B2B orgs and the rewrite of dead or +// legacy (lf…) external IDs to live Salesforce account IDs. It never creates, merges or deletes +// EasyCLA rows. +package orgimport + +import ( + "context" + "errors" + "io" + "strconv" + "strings" + "time" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + member_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/member-service" +) + +// Route is the import route of a company group. +type Route string + +// Routes. +const ( + RouteRegister Route = "register" + RouteRewrite Route = "rewrite" + RouteManual Route = "manual" +) + +// IDShape classifies a company_external_id. +type IDShape string + +// ID shapes. +const ( + ShapeSFID IDShape = "001" + ShapeLF IDShape = "lf" + ShapeEmpty IDShape = "empty" + ShapeOther IDShape = "other" +) + +// Liveness values (Group.Live), run modes and the manual/pending reasons shared by several files. +const ( + LiveLive = "live" + LiveDead = "dead" + LiveError = "error" + + ModeApply = "apply" + ModeDryRun = "dry-run" + + ReasonNoMapping = "no_mapping" + ReasonDeadAccount = "dead_account" + ReasonMappingAmbiguous = "mapping_ambiguous" + ReasonMappingSameID = "mapping_same_id" + ReasonSharedDomain = "shared_domain" + ReasonDistinctConflict = "distinct_conflict" + + orgStatusErr = "err" +) + +// Step names recorded in the state file. +const ( + StepResolve = "resolve" + StepWait = "wait" + StepGrants = "copy_grants" + StepRows = "rewrite_rows" + StepEvents = "rekey_events" + StepCleanup = "delete_old_grants" + StepRegister = "register" + StepDone = "done" +) + +// Sentinel errors. +var ( + ErrOrgNotFound = errors.New("organization not found") + ErrNotConfigured = errors.New("member-service is not configured (cla-member-service-* SSM parameters missing)") + ErrApexUnavailable = errors.New("apex endpoint is not configured (cla-salesforce-apex-* SSM parameters missing)") +) + +// CompanyStore is the subset of company.IRepository the tool uses. +type CompanyStore interface { + GetCompanies(ctx context.Context) (*models.Companies, error) + GetCompanyRecord(ctx context.Context, companyID string) (*company.DBModel, error) + UpdateCompanyExternalID(ctx context.Context, companyID, oldExternalID, newExternalID string) error +} + +// CCLALister lists CCLA signature rows (signatures.SignatureRepository satisfies it). +type CCLALister interface { + GetCCLASignatures(ctx context.Context, signed, approved *bool) ([]*signatures.ItemSignature, error) +} + +// ECLACounter counts employee acknowledgements referencing a company (audit only). +type ECLACounter interface { + CountECLAs(ctx context.Context, companyID string) (int, error) +} + +// EventRekeyer re-keys company-scoped events (events.RekeyRepository satisfies it). +type EventRekeyer interface { + ListEventIDsByCompanySFID(ctx context.Context, companySFID string) ([]string, error) + ListEventIDsByCompanySFIDCLAGroup(ctx context.Context, companySFID, claGroupID string) ([]string, error) + RekeyEventCompanySFID(ctx context.Context, eventID, oldSFID, newSFID string) (bool, error) +} + +// Org is the org-service view of an organization. +type Org struct { + ID string + Name string + Website string + SigningEntityNames []string +} + +// OrgService is the org-service surface the tool uses; GetOrganization returns ErrOrgNotFound on 404. +type OrgService interface { + GetOrganization(ctx context.Context, orgID string) (*Org, error) + CreateUserRoleScope(ctx context.Context, username, organizationID, objectType, objectID, roleID string) error + DeleteUserRoleScope(ctx context.Context, organizationID, roleID, grantID, username string) error +} + +// ACSService lists every grant scoped to an organization. +type ACSService interface { + ListOrgGrants(ctx context.Context, orgSFID string) ([]acs_service.OrgGrant, error) +} + +// MemberService registers B2B orgs; nil when the SSM parameters are absent. +type MemberService interface { + GetB2BOrg(ctx context.Context, uid string) (*member_service.B2BOrg, error) + RegisterB2BOrg(ctx context.Context, sfid string) (*member_service.B2BOrg, error) +} + +// ApexService is the Salesforce find-or-create contract (design §4), used only with --use-apex. +type ApexService interface { + FindOrCreate(ctx context.Context, req ApexRequest) (*ApexResult, error) +} + +// Deps are the external dependencies; Members, Events, ECLAs and Apex may be nil. +type Deps struct { + Companies CompanyStore + Signatures CCLALister + ECLAs ECLACounter + Events EventRekeyer + Orgs OrgService + ACS ACSService + Members MemberService + Apex ApexService + Out io.Writer + Now func() time.Time + Sleep func(context.Context, time.Duration) error +} + +// Options are the ingest/audit flags. +type Options struct { + Stage string + Apply bool + Tranche int + IDs []string + Mapping string + Decisions string + SharedDomains string + State string + Routes []Route + SkipWait bool + UseApex bool + OutDir string + WaitMax time.Duration + WaitPoll time.Duration +} + +// Row is one companies-table row. +type Row struct { + CompanyID string + CompanyName string + SigningEntityName string + ExternalID string + ActiveCCLA bool + CCLACount int + CLAGroupIDs []string + CCLASignedOn string +} + +// Group is the import unit: every row sharing one company_external_id. +type Group struct { + OldID string + Key string + Shape IDShape + Rows []*Row + Route Route + ManualReason string + Duplicate bool + NewID string + Action string + Org *Org + OrgStatus string + Live string + Decision *Decision + Replayed bool + Err error +} + +// Summary is the one-line run result. +type Summary struct { + Stage string + Mode string + Eligible int + Registered int + Rewritten int + Pending int + Manual int + Failed int +} + +func (s Summary) String() string { + return "stage=" + s.Stage + " mode=" + s.Mode + + " eligible=" + strconv.Itoa(s.Eligible) + " registered=" + strconv.Itoa(s.Registered) + + " rewritten=" + strconv.Itoa(s.Rewritten) + " pending=" + strconv.Itoa(s.Pending) + " manual=" + strconv.Itoa(s.Manual) + " failed=" + strconv.Itoa(s.Failed) +} + +// Pending is a rewrite candidate without a resolved new id (mapping row missing or account found dead). +func (g *Group) Pending() bool { + return g.Route == RouteRewrite && g.NewID == "" +} + +// RowTargeted is a single-row group whose company_external_id is empty or malformed: it is selected +// by its company id (Key) and nothing in ACS, org-service or the events table is keyed by the old value. +func (g *Group) RowTargeted() bool { + return g.Key != g.OldID +} + +// Website is the org-service website of the group's organization ("" when unknown). +func (g *Group) Website() string { + if g.Org == nil { + return "" + } + return strings.TrimSpace(g.Org.Website) +} + +// Domain is the normalized website domain ("" when unknown). +func (g *Group) Domain() string { + return Domain(g.Website()) +} + +// Names returns the signing entity (or company) name of each row. +func (g *Group) Names() []string { + names := make([]string, 0, len(g.Rows)) + for _, r := range g.Rows { + names = append(names, firstNonEmpty(r.SigningEntityName, r.CompanyName)) + } + return names +} + +// CompanyIDs returns the internal ids of the group's rows. +func (g *Group) CompanyIDs() []string { + ids := make([]string, 0, len(g.Rows)) + for _, r := range g.Rows { + ids = append(ids, r.CompanyID) + } + return ids +} + +// CLAGroupIDs returns the distinct CLA group ids of the group's active CCLAs. +func (g *Group) CLAGroupIDs() []string { + seen := map[string]bool{} + var out []string + for _, r := range g.Rows { + for _, id := range r.CLAGroupIDs { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + } + return out +} diff --git a/cla-backend-go/orgimport/report.go b/cla-backend-go/orgimport/report.go new file mode 100644 index 000000000..d8e27e7f7 --- /dev/null +++ b/cla-backend-go/orgimport/report.go @@ -0,0 +1,502 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "fmt" + "html" + "io" + "mime" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +const ( + // MaxRawEmailBytes is the SES SendRawEmail limit (v1 API). + MaxRawEmailBytes = 10 * 1024 * 1024 + // maxZipAttachment keeps the whole message under the SES limit with base64 overhead. + maxZipAttachment = 6 * 1024 * 1024 + // maxInlinePlanRows caps the inline plan table; plan.csv is attached in full. + maxInlinePlanRows = 2000 + // maxLogTailBytes caps the inline run.log tail; run.log is attached in full. + maxLogTailBytes = 96 * 1024 +) + +// RunInfo describes one CLI run for the report e-mail and the AWS log header. +type RunInfo struct { + Stage string + Command string + Apply bool + Args []string + Start time.Time + End time.Time + Runner string + RunURL string + Repository string + Artifact string + Revision string + OutDir string + LogGroup string + LogStream string + Summary string + Err string + Workflow WorkflowInputs +} + +// WorkflowInputs mirrors the org-import-sweep.yml dispatch inputs for the "how to apply" command. +type WorkflowInputs struct { + Routes string + Tranche string + IDs string + Mapping string + Decisions string + SharedDomains string +} + +// Mode is dry-run or apply. +func (i RunInfo) Mode() string { + if i.Apply { + return ModeApply + } + return ModeDryRun +} + +// Failed reports whether the run ended with an error. +func (i RunInfo) Failed() bool { return i.Err != "" } + +// Attachment is one e-mail attachment. +type Attachment struct { + Name string + ContentType string + Data []byte +} + +// Report is the complete decision record of a run: subject, HTML and text bodies, attachments. +type Report struct { + Subject string + HTML string + Text string + Attachments []Attachment +} + +// BuildReport renders the run as a self-contained decision record. plan or audit may be nil. +func BuildReport(info RunInfo, plan *Plan, audit *AuditResult, runLog []byte) Report { + status := "OK" + if info.Failed() { + status = "FAILED" + } + summary := info.Summary + if summary == "" { + summary = "no summary" + } + r := Report{Subject: fmt.Sprintf("[EasyCLA org-import][%s] %s %s: %s (%s)", info.Stage, info.Command, info.Mode(), summary, status)} + + var h, t strings.Builder + h.WriteString("") + fmt.Fprintf(&h, "

EasyCLA org-import: %s %s on %s — %s

", esc(info.Command), esc(info.Mode()), esc(info.Stage), status) + fmt.Fprintf(&t, "EasyCLA org-import: %s %s on %s - %s\n\n", info.Command, info.Mode(), info.Stage, status) + + header := [][2]string{ + {"Stage", info.Stage}, {"Command", info.Command}, {"Mode", info.Mode()}, {"Result", status}, + {"Summary", summary}, + {"Started (UTC)", info.Start.UTC().Format(time.RFC3339)}, {"Finished (UTC)", info.End.UTC().Format(time.RFC3339)}, + {"Duration", info.End.Sub(info.Start).Round(time.Second).String()}, + {"Runner", info.Runner}, {"Arguments", strings.Join(info.Args, " ")}, + {"Build revision", info.Revision}, {"Output directory", info.OutDir}, + } + if info.RunURL != "" { + header = append(header, [2]string{"GitHub Actions run", info.RunURL}) + } + if info.Artifact != "" { + header = append(header, [2]string{"Actions artifact", info.Artifact}) + } + if info.LogGroup != "" { + header = append(header, [2]string{"CloudWatch Logs", info.LogGroup + " / " + info.LogStream}) + } + if info.Err != "" { + header = append(header, [2]string{"Error", info.Err}) + } + h.WriteString("") + for _, kv := range header { + v := esc(kv[1]) + if kv[0] == "GitHub Actions run" { + v = fmt.Sprintf("%s", v, v) + } + fmt.Fprintf(&h, "", esc(kv[0]), v) + fmt.Fprintf(&t, "%-20s %s\n", kv[0]+":", kv[1]) + } + h.WriteString("
%s%s
") + t.WriteString("\n") + + if plan != nil { + writePlanSections(&h, &t, info, plan) + } + if audit != nil { + writeAuditSections(&h, &t, audit) + } + + tail := logTail(runLog) + fmt.Fprintf(&h, "

run.log%s

%s
", tailNote(runLog, tail), esc(string(tail))) + fmt.Fprintf(&t, "== run.log%s ==\n%s\n", tailNote(runLog, tail), string(tail)) + + r.Attachments = attachments(info.OutDir, runLog) + if len(r.Attachments) > 0 { + h.WriteString("

Attachments

    ") + t.WriteString("\n== Attachments ==\n") + for _, a := range r.Attachments { + fmt.Fprintf(&h, "
  • %s (%d bytes)
    • ", esc(a.Name), len(a.Data)) + fmt.Fprintf(&t, "- %s (%d bytes)\n", a.Name, len(a.Data)) + } + h.WriteString("
") + } + h.WriteString("") + r.HTML, r.Text = h.String(), t.String() + return r +} + +func writePlanSections(h, t *strings.Builder, info RunInfo, plan *Plan) { + actions := plan.ManualActions() + fmt.Fprintf(h, "

Manual actions (%d)

", len(actions)) + fmt.Fprintf(t, "== Manual actions (%d) ==\n", len(actions)) + if len(actions) == 0 { + h.WriteString("

None: every eligible group is registered, rewritten or ready.

") + t.WriteString("None.\n") + } else { + h.WriteString("

Each row needs a human before the tool can act on it; the suggested action says how.

") + writeTable(h, t, []string{"key", "old id", "route", "reason", "suggested action", "live", "org-service", "website", "domain", "new id", "action", "company ids", "company names", "error"}, manualRows(plan, actions)) + } + + if len(plan.Targets) > 0 { + fmt.Fprintf(h, "

Targets (%d)

Rewrite destinations grouped by Salesforce Account. needs_decision requires a collapse or distinct decision in the decisions file (lfx-self-serve #3085).

", len(plan.Targets)) + fmt.Fprintf(t, "\n== Targets (%d) ==\n", len(plan.Targets)) + var rows [][]string + for _, tg := range plan.Targets { + decision, reviewer := "", "" + if tg.Decision != nil { + decision, reviewer = tg.Decision.Kind, tg.Decision.Reviewer + } + var ids, names []string + for _, g := range tg.Groups { + ids = append(ids, g.CompanyIDs()...) + names = append(names, g.Names()...) + } + rows = append(rows, []string{tg.SFID, strconv.Itoa(len(tg.Groups)), strings.Join(tg.OldIDs(), "; "), strings.Join(ids, "; "), strings.Join(names, "; "), strconv.Itoa(len(tg.Existing)), decision, reviewer, tg.Status}) + } + writeTable(h, t, []string{"target", "groups", "old ids", "company ids", "company names", "existing rows", "decision", "reviewer", "status"}, rows) + } + + fmt.Fprintf(h, "

Plan (%d groups: %d register, %d rewrite)

", len(plan.Groups), len(plan.Register), len(plan.Rewrite)) + fmt.Fprintf(t, "\n== Plan (%d groups: %d register, %d rewrite) ==\n", len(plan.Groups), len(plan.Register), len(plan.Rewrite)) + var rows [][]string + for i, g := range plan.Groups { + if i == maxInlinePlanRows { + fmt.Fprintf(h, "

Only the first %d groups are shown inline; plan.csv (attached) has all %d.

", maxInlinePlanRows, len(plan.Groups)) + fmt.Fprintf(t, "(only the first %d groups shown; plan.csv has all %d)\n", maxInlinePlanRows, len(plan.Groups)) + break + } + domain, shared := plan.domainOf(g) + decision := "" + if g.Decision != nil { + decision = g.Decision.Kind + " by " + g.Decision.Reviewer + } + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + rows = append(rows, []string{g.Key, string(g.Shape), string(g.Route), g.ManualReason, g.Live, g.OrgStatus, g.Website(), domain, shared, g.NewID, g.Action, decision, strings.Join(g.CompanyIDs(), "; "), strings.Join(g.Names(), "; "), errText}) + } + writeTable(h, t, []string{"key", "shape", "route", "reason", "live", "org-service", "website", "domain", "shared", "new id", "action", "decision", "company ids", "company names", "error"}, rows) + + if !info.Apply { + h.WriteString("

How to apply this plan

After reviewing the record above, re-run the same command in apply mode. Locally:

") + t.WriteString("\n== How to apply this plan ==\nLocally:\n") + for _, c := range ApplyCommands(info) { + fmt.Fprintf(h, "
%s
", esc(c)) + fmt.Fprintf(t, " %s\n", c) + } + } +} + +func manualRows(plan *Plan, actions []ManualAction) [][]string { + var rows [][]string + for _, a := range actions { + g := a.Group + errText := "" + if g.Err != nil { + errText = g.Err.Error() + } + domain, _ := plan.domainOf(g) + rows = append(rows, []string{g.Key, g.OldID, string(g.Route), a.Reason, a.Suggested, g.Live, g.OrgStatus, g.Website(), domain, g.NewID, g.Action, strings.Join(g.CompanyIDs(), "; "), strings.Join(g.Names(), "; "), errText}) + } + return rows +} + +func writeAuditSections(h, t *strings.Builder, audit *AuditResult) { + fmt.Fprintf(h, "

Audit (%d company rows)

", len(audit.Rows)) + fmt.Fprintf(t, "\n== Audit (%d company rows) ==\n", len(audit.Rows)) + var rows [][]string + for _, k := range sortedKeys(audit.Tiers) { + rows = append(rows, []string{"tier", k, strconv.Itoa(audit.Tiers[k])}) + } + routes := make([]string, 0, len(audit.Routes)) + for r := range audit.Routes { + routes = append(routes, string(r)) + } + sort.Strings(routes) + for _, r := range routes { + rows = append(rows, []string{"route", r, strconv.Itoa(audit.Routes[Route(r)])}) + } + rows = append(rows, []string{"possible duplicate groups", "", strconv.Itoa(len(audit.Duplicates))}) + writeTable(h, t, []string{"kind", "value", "count"}, rows) + h.WriteString("

audit.csv, unresolvable.csv and possible_duplicates.csv are attached (zip).

") +} + +// ApplyCommands returns the exact local and GitHub Actions commands that re-run this plan in apply mode. +func ApplyCommands(info RunInfo) []string { + var args []string + for _, a := range info.Args { + switch a { + case "--apply", "-apply", "--yes", "-yes", "--no-email", "-no-email", "--no-aws-log", "-no-aws-log": + continue + } + args = append(args, shellQuote(a)) + } + local := fmt.Sprintf("STAGE=%s ./bin/org-import %s --apply --yes", info.Stage, strings.Join(args, " ")) + repo := info.Repository + if repo == "" { + repo = "linuxfoundation/easycla" + } + w := info.Workflow + gh := fmt.Sprintf("gh workflow run org-import-sweep.yml -R %s -f stage=%s -f mode=apply", repo, info.Stage) + if w.Routes != "" { + gh += " -f routes=" + shellQuote(w.Routes) + } + if w.Tranche != "" && w.Tranche != "0" { + gh += " -f tranche=" + shellQuote(w.Tranche) + } + if w.IDs != "" { + gh += " -f ids=" + shellQuote(w.IDs) + } + for _, f := range [][2]string{{"mapping", w.Mapping}, {"decisions", w.Decisions}, {"shared_domains", w.SharedDomains}} { + if f[1] != "" { + gh += fmt.Sprintf(" -f %s=\"$(tr '\\n' '|' < %s)\"", f[0], shellQuote(f[1])) + } + } + return []string{local, gh} +} + +func shellQuote(s string) string { + if s == "" { + return "''" + } + if strings.ContainsAny(s, " \t\n'\"$`\\!*?[]{}()<>|&;#~") { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" + } + return s +} + +func writeTable(h, t *strings.Builder, head []string, rows [][]string) { + h.WriteString("") + for _, c := range head { + fmt.Fprintf(h, "", esc(c)) + } + h.WriteString("") + t.WriteString(strings.Join(head, " | ") + "\n") + for _, row := range rows { + h.WriteString("") + for _, c := range row { + fmt.Fprintf(h, "", esc(c)) + } + h.WriteString("") + t.WriteString(strings.Join(row, " | ") + "\n") + } + h.WriteString("
%s
%s
") +} + +func esc(s string) string { return html.EscapeString(s) } + +func sortedKeys(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func logTail(runLog []byte) []byte { + if len(runLog) <= maxLogTailBytes { + return runLog + } + tail := runLog[len(runLog)-maxLogTailBytes:] + if i := bytes.IndexByte(tail, '\n'); i >= 0 { + tail = tail[i+1:] + } + return tail +} + +func tailNote(runLog, tail []byte) string { + if len(tail) == len(runLog) { + return "" + } + return fmt.Sprintf(" (last %d of %d bytes; full log attached)", len(tail), len(runLog)) +} + +// attachments returns manual_actions.csv and plan.csv (or audit.csv) inline, the run log, and a zip +// of the whole output directory when it fits. +func attachments(outDir string, runLog []byte) []Attachment { + var out []Attachment + for _, name := range []string{"manual_actions.csv", "plan.csv", "targets.csv", "to_salesforce.csv", "audit.csv", "unresolvable.csv", "possible_duplicates.csv"} { + data, err := os.ReadFile(filepath.Clean(filepath.Join(outDir, name))) + if err == nil && len(data) > 0 { + out = append(out, Attachment{Name: name, ContentType: "text/csv", Data: data}) + } + } + if len(runLog) > 0 { + out = append(out, Attachment{Name: "run.log", ContentType: "text/plain", Data: runLog}) + } + if zipped, err := ZipDir(outDir); err == nil && len(zipped) > 0 && len(zipped) <= maxZipAttachment { + out = append(out, Attachment{Name: filepath.Base(outDir) + ".zip", ContentType: "application/zip", Data: zipped}) + } + return out +} + +// ZipDir zips the regular files directly under dir (reports, state, run.log). +func ZipDir(dir string) ([]byte, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, e := range entries { + if !e.Type().IsRegular() { + continue + } + data, readErr := os.ReadFile(filepath.Clean(filepath.Join(dir, e.Name()))) + if readErr != nil { + return nil, readErr + } + w, createErr := zw.Create(e.Name()) + if createErr != nil { + return nil, createErr + } + if _, err = w.Write(data); err != nil { + return nil, err + } + } + if err = zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MIME renders the report as a raw RFC 5322 message (multipart/mixed with a multipart/alternative body). +// Attachments are dropped largest-first until the message fits MaxRawEmailBytes. +func (r Report) MIME(from string, to []string) ([]byte, error) { + atts := append([]Attachment(nil), r.Attachments...) + for { + msg, err := r.mime(from, to, atts) + if err != nil { + return nil, err + } + if len(msg) <= MaxRawEmailBytes || len(atts) == 0 { + return msg, nil + } + largest := 0 + for i, a := range atts { + if len(a.Data) > len(atts[largest].Data) { + largest = i + } + } + atts = append(atts[:largest], atts[largest+1:]...) + } +} + +func (r Report) mime(from string, to []string, atts []Attachment) ([]byte, error) { + var buf bytes.Buffer + mixed := multipart.NewWriter(&buf) + fmt.Fprintf(&buf, "From: %s\r\n", from) + fmt.Fprintf(&buf, "To: %s\r\n", strings.Join(to, ", ")) + fmt.Fprintf(&buf, "Subject: %s\r\n", mime.QEncoding.Encode("utf-8", r.Subject)) + fmt.Fprintf(&buf, "Date: %s\r\n", time.Now().UTC().Format(time.RFC1123Z)) + buf.WriteString("MIME-Version: 1.0\r\n") + fmt.Fprintf(&buf, "Content-Type: multipart/mixed; boundary=%q\r\n\r\n", mixed.Boundary()) + + var alt bytes.Buffer + altW := multipart.NewWriter(&alt) + for _, part := range []struct{ ctype, body string }{{"text/plain; charset=utf-8", r.Text}, {"text/html; charset=utf-8", r.HTML}} { + hdr := textproto.MIMEHeader{"Content-Type": {part.ctype}, "Content-Transfer-Encoding": {"base64"}} + w, err := altW.CreatePart(hdr) + if err != nil { + return nil, err + } + if err = writeBase64(w, []byte(part.body)); err != nil { + return nil, err + } + } + if err := altW.Close(); err != nil { + return nil, err + } + bodyPart, err := mixed.CreatePart(textproto.MIMEHeader{"Content-Type": {fmt.Sprintf("multipart/alternative; boundary=%q", altW.Boundary())}}) + if err != nil { + return nil, err + } + if _, err = bodyPart.Write(alt.Bytes()); err != nil { + return nil, err + } + for _, a := range atts { + hdr := textproto.MIMEHeader{ + "Content-Type": {fmt.Sprintf("%s; name=%q", a.ContentType, a.Name)}, + "Content-Disposition": {fmt.Sprintf("attachment; filename=%q", a.Name)}, + "Content-Transfer-Encoding": {"base64"}, + } + w, partErr := mixed.CreatePart(hdr) + if partErr != nil { + return nil, partErr + } + if partErr = writeBase64(w, a.Data); partErr != nil { + return nil, partErr + } + } + if err = mixed.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func writeBase64(w io.Writer, data []byte) error { + enc := base64.StdEncoding.EncodeToString(data) + for len(enc) > 0 { + n := 76 + if n > len(enc) { + n = len(enc) + } + if _, err := io.WriteString(w, enc[:n]+"\r\n"); err != nil { + return err + } + enc = enc[n:] + } + return nil +} + +// ParseRecipients splits a comma/semicolon/whitespace separated address list. +func ParseRecipients(s string) []string { + var out []string + for _, a := range strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\t' }) { + if a = strings.TrimSpace(a); a != "" { + out = append(out, a) + } + } + return out +} diff --git a/cla-backend-go/orgimport/report_test.go b/cla-backend-go/orgimport/report_test.go new file mode 100644 index 000000000..034fa1303 --- /dev/null +++ b/cla-backend-go/orgimport/report_test.go @@ -0,0 +1,311 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package orgimport + +import ( + "archive/zip" + "bytes" + "context" + "encoding/base64" + "errors" + "io" + "mime" + "mime/multipart" + "net/mail" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ses" + "github.com/aws/aws-sdk-go/service/ses/sesiface" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func reportFixture(t *testing.T) (RunInfo, *Plan, string) { + t.Helper() + fx, dir := collisionFixture(t) + mapping := writeMapping(t, dir, "lf-a,"+targetSFID+",matched,true", "lf-b,"+targetSFID+",created,true", "lf-c,"+targetSFID2+",created,true") + outDir := filepath.Join(dir, "out") + opts := Options{Stage: "dev", Mapping: mapping, OutDir: outDir, Routes: []Route{RouteRewrite}} + plan, err := BuildPlan(context.Background(), fx.deps(), opts) + require.NoError(t, err) + sum, err := Execute(context.Background(), fx.deps(), opts, plan) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(outDir, "run.log"), []byte("line 1\nline 2\n"), 0o600)) + start := time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC) + info := RunInfo{ + Stage: "dev", Command: "ingest", Args: []string{"ingest", "--mapping", mapping, "--routes", "rewrite", "--out-dir", outDir, "--no-aws-log"}, + Start: start, End: start.Add(90 * time.Second), Runner: "lukasz@dockaws", Repository: "linuxfoundation/easycla", + RunURL: "https://github.com/linuxfoundation/easycla/actions/runs/42", Artifact: "org-import-out-dev-42", Revision: "abc123-dirty", + OutDir: outDir, LogGroup: "/easycla/org-import/dev", LogStream: "s1", Summary: sum.String(), + Workflow: WorkflowInputs{Routes: "rewrite", Tranche: "0", Mapping: mapping}, + } + return info, plan, outDir +} + +func TestBuildReport(t *testing.T) { + info, plan, outDir := reportFixture(t) + rep := BuildReport(info, plan, nil, []byte("line 1\nline 2\n")) + assert.Equal(t, "[EasyCLA org-import][dev] ingest dry-run: "+info.Summary+" (OK)", rep.Subject) + for _, want := range []string{ + "Manual actions (2)", "target_collision", "decisions file", "Targets (2)", "needs_decision", "Plan (3 groups: 0 register, 1 rewrite)", + "How to apply this plan", "STAGE=dev ./bin/org-import ingest --mapping", "--apply --yes", "gh workflow run org-import-sweep.yml -R linuxfoundation/easycla -f stage=dev -f mode=apply -f routes=rewrite", + "-f mapping="$(tr '\\n' '|' < ", "https://github.com/linuxfoundation/easycla/actions/runs/42", "org-import-out-dev-42", "abc123-dirty", "/easycla/org-import/dev / s1", + "1m30s", "line 2", "Attachments", "manual_actions.csv", "plan.csv", "targets.csv", "run.log", "out.zip", "acme.example", + } { + assert.Contains(t, rep.HTML, want, want) + } + assert.NotContains(t, rep.HTML, "--no-aws-log --apply", "report-only flags are dropped from the apply command") + for _, want := range []string{"== Manual actions (2) ==", "== Targets (2) ==", "== Plan", "== How to apply this plan ==", "== run.log ==", "line 1", `-f mapping="$(tr '\n' '|' < `} { + assert.Contains(t, rep.Text, want, want) + } + names := []string{} + for _, a := range rep.Attachments { + names = append(names, a.Name) + } + assert.Equal(t, []string{"manual_actions.csv", "plan.csv", "targets.csv", "to_salesforce.csv", "run.log", "out.zip"}, names) + zr, err := zip.NewReader(bytes.NewReader(rep.Attachments[5].Data), int64(len(rep.Attachments[5].Data))) + require.NoError(t, err) + var zipped []string + for _, f := range zr.File { + zipped = append(zipped, f.Name) + } + assert.ElementsMatch(t, []string{"plan.csv", "manual_actions.csv", "targets.csv", "to_salesforce.csv", "run.log"}, zipped) + _, err = ZipDir(filepath.Join(outDir, "missing")) + assert.Error(t, err) + + // apply run: no "how to apply", error → FAILED subject and Error row + info.Apply, info.Err, info.Summary = true, "2 group(s) failed", "" + rep = BuildReport(info, plan, nil, nil) + assert.Equal(t, "[EasyCLA org-import][dev] ingest apply: no summary (FAILED)", rep.Subject) + assert.NotContains(t, rep.HTML, "How to apply") + assert.Contains(t, rep.HTML, "Error2 group(s) failed") +} + +func TestBuildReportEscapesAndTruncates(t *testing.T) { + info, plan, _ := reportFixture(t) + info.Args = append(info.Args, "