Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 246 additions & 0 deletions .github/scripts/e2e-dispatch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
#!/usr/bin/env bash
# Ported from .jfrog-pipelines/pipelines.yml (JFrog Pipelines is EOL).
#
# Runnable both from the "E2E Dispatch" GitHub Actions workflow and from a
# local shell. To run locally, export the env vars below (same names the

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To run locally
why do we need to run locally? Isnt it from CI 100% of the times?

# workflow injects from secrets/inputs) and run this script directly from a
# checkout of jfrog/jfrog-cli.
#
# Required:
# JENKINS_URL, JENKINS_USER, JENKINS_TOKEN (jenkins_entplus_rt integration)
# MASTER_KEY, JFROG_ADMIN_USERNAME,
# JFROG_ADMIN_PASSWORD, JFROG_ADMIN_TOKEN (jfrog_cli_tests integration)
# GITHUB_DISPATCH_TOKEN or JFROG_CLI_GH_TOKEN (github_dispatch / jfrog_cli_gh integration;
# fine-grained PAT or GitHub App token —
# github.jfrog.info rejects classic PATs)
#
# Optional (defaults shown):
# RT_VERSION= SKIP_ENV_SETUP=false ARTIFACTORY_URL=
# GITHUB_API_URL=https://github.jfrog.info/api/v3
# GITHUB_WORKFLOWS_REPO=JFROG/jfrog-cli-workflows GITHUB_WORKFLOWS_REF=master
# JFROG_CLI_GITHUB_REPO=jfrog/jfrog-cli JFROG_CLI_GITHUB_REF=<current HEAD sha>
# GHE_ACTIONS_RUNNER=artifactory-dind-amd-scale-set
# LOGS_TO_KIBANA=true DEPLOYMENT_SIZING=common
# MAX_RUN_RETRIES=2 MAX_WAIT_SECONDS=14400
#
# NOT independently verified against a real Jenkins instance: the provision/

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can create a test job verify to address any hidden scenarios

# teardown steps call the generic Jenkins remote-build API (crumb + queue
# polling) because JFrog Pipelines' native "Jenkins step type" doesn't expose
# how it itself talks to Jenkins — there was nothing literal to port here.

set -eo pipefail

fail() { echo "ERROR: $1" >&2; exit 1; }

SKIP_ENV_SETUP="${SKIP_ENV_SETUP:-false}"
GITHUB_API_URL="${GITHUB_API_URL:-https://github.jfrog.info/api/v3}"
GITHUB_WORKFLOWS_REPO="${GITHUB_WORKFLOWS_REPO:-JFROG/jfrog-cli-workflows}"
GITHUB_WORKFLOWS_REF="${GITHUB_WORKFLOWS_REF:-master}"
JFROG_CLI_GITHUB_REPO="${JFROG_CLI_GITHUB_REPO:-jfrog/jfrog-cli}"
JFROG_CLI_GITHUB_REF="${JFROG_CLI_GITHUB_REF:-$(git rev-parse HEAD 2>/dev/null || echo "")}"
GHE_ACTIONS_RUNNER="${GHE_ACTIONS_RUNNER:-artifactory-dind-amd-scale-set}"
LOGS_TO_KIBANA="${LOGS_TO_KIBANA:-true}"
DEPLOYMENT_SIZING="${DEPLOYMENT_SIZING:-common}"
MAX_RUN_RETRIES="${MAX_RUN_RETRIES:-2}"
MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-14400}"

GITHUB_TOKEN_RESOLVED="${GITHUB_DISPATCH_TOKEN:-${JFROG_CLI_GH_TOKEN:-}}"
[[ -n "${GITHUB_TOKEN_RESOLVED}" ]] || fail "Set GITHUB_DISPATCH_TOKEN (or JFROG_CLI_GH_TOKEN as a fallback)"
[[ -n "${JFROG_CLI_GITHUB_REF}" ]] || fail "Set JFROG_CLI_GITHUB_REF (no git checkout found to default from)"

echo "Starting CLI e2e dispatch"
echo "Using RT_VERSION=${RT_VERSION:-<none>}"

suffix=$(date +%s)
server_name="cli${suffix}"
prefix="${server_name%${suffix}}"
if [[ "${SKIP_ENV_SETUP}" == "true" ]]; then
[[ -n "${ARTIFACTORY_URL:-}" ]] || fail "SKIP_ENV_SETUP=true requires ARTIFACTORY_URL"
art_url="${ARTIFACTORY_URL}"
else
art_url="https://${server_name}.jfrogdev.org"
fi
echo "server_name=${server_name} art_url=${art_url}"

teardown() {
[[ "${SKIP_ENV_SETUP}" == "true" ]] && return 0
echo "Tearing down ephemeral Artifactory (${server_name})..."
local job_path="job/tools/job/platform/job/environment_operate"
local crumb_json crumb_field crumb
crumb_json=$(curl -sS -u "${JENKINS_USER}:${JENKINS_TOKEN}" "${JENKINS_URL}/crumbIssuer/api/json" || echo '{}')
crumb_field=$(echo "$crumb_json" | jq -r '.crumbRequestField // empty')
crumb=$(echo "$crumb_json" | jq -r '.crumb // empty')
curl -sS -o /dev/null -u "${JENKINS_USER}:${JENKINS_TOKEN}" \
${crumb:+-H "${crumb_field}: ${crumb}"} \
--data-urlencode "SERVER_NAME=${server_name}" \
--data-urlencode "ACTION=delete" \
"${JENKINS_URL}/${job_path}/buildWithParameters" || echo "WARNING: teardown request failed, check Jenkins manually"
}
[[ "${SKIP_ENV_SETUP}" == "true" ]] || trap teardown EXIT

# --- Jenkins step: tools/platform/environment_setup_gen2 ---
# VERIFY: confirm this job path resolves the same way against your Jenkins
# instance as it did through JFrog Pipelines' native Jenkins integration.
if [[ "${SKIP_ENV_SETUP}" != "true" ]]; then
echo "Provisioning ephemeral Artifactory..."
job_path="job/tools/job/platform/job/environment_setup_gen2"
crumb_json=$(curl -sS -u "${JENKINS_USER}:${JENKINS_TOKEN}" "${JENKINS_URL}/crumbIssuer/api/json" || echo '{}')

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use an existing github action or write reusable github action to reuse it across the workflows instead of simple script which can trigger jenkins job that way you can maintain and scale triggering jenkins jobs

crumb_field=$(echo "$crumb_json" | jq -r '.crumbRequestField // empty')
crumb=$(echo "$crumb_json" | jq -r '.crumb // empty')

queue_location=$(curl -sS -D - -o /dev/null -u "${JENKINS_USER}:${JENKINS_TOKEN}" \
${crumb:+-H "${crumb_field}: ${crumb}"} \
--data-urlencode "SERVER_NAME=${server_name}" \
--data-urlencode "DEPLOYMENT_TYPE=onprem" \
--data-urlencode "ACCOUNT_TYPE=enterprise_plus" \
--data-urlencode "GROUP=ARTIFACTORY" \
--data-urlencode "EXPIRY=2d" \
--data-urlencode "LOGS_TO_KIBANA=${LOGS_TO_KIBANA}" \
--data-urlencode "DEPLOYMENT_SIZING=${DEPLOYMENT_SIZING}" \
--data-urlencode "EXTRA_PARAMS=conf_artifactory_unified_version=${RT_VERSION:-} master_key=${MASTER_KEY}" \
"${JENKINS_URL}/${job_path}/buildWithParameters" \
| grep -i '^Location:' | tr -d '\r' | awk '{print $2}')
[[ -n "${queue_location}" ]] || fail "Jenkins did not return a queue item location"

build_url=""
for _ in $(seq 1 30); do
build_url=$(curl -sS -u "${JENKINS_USER}:${JENKINS_TOKEN}" "${queue_location}api/json" | jq -r '.executable.url // empty')
[[ -n "${build_url}" ]] && break
sleep 10
done
[[ -n "${build_url}" ]] || fail "environment_setup_gen2 never left the queue"
echo "Building at: ${build_url}"

while true; do
build_json=$(curl -sS -u "${JENKINS_USER}:${JENKINS_TOKEN}" "${build_url}api/json")
[[ "$(echo "$build_json" | jq -r '.building')" == "false" ]] && break
sleep 15
done
result=$(echo "$build_json" | jq -r '.result')
echo "environment_setup_gen2 result: ${result}"
[[ "${result}" == "SUCCESS" ]] || fail "environment provisioning failed"
fi

# --- Mint an OAuth token against the (now ready) environment ---
echo "Installing JFrog CLI 2.54.0"
curl -fL https://getcli.jfrog.io/v2 | sh -s 2.54.0

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not latest version?

export PATH=$PATH:$HOME/.jfrog
jfrog plugin install access@v7.66.0

support_output=$(jfrog access support-token --url="${art_url}/access" --join-key="${MASTER_KEY}" 2>&1)
support_token=$(echo "${support_output}" | grep -o 'JF_ACCESS_ADMIN_TOKEN=.*' | cut -d'=' -f2-)
[[ -n "${support_token}" ]] || fail "support token empty: ${support_output}"

oauth_response=$(curl -s -w "\nHTTP_CODE:%{http_code}" --location "${art_url}/access/api/v1/oauth/token" \
--header 'Content-Type: application/x-www-form-urlencoded' \
--header "Authorization: Bearer ${support_token}" \
--data-urlencode "username=${JFROG_ADMIN_USERNAME}" \
--data-urlencode 'scope=applied-permissions/admin' \
--data-urlencode 'expires_in=36000' \
--data-urlencode 'grant_type=client_credentials' \
--data-urlencode 'audience=*@*')
oauth_http_code=$(echo "${oauth_response}" | grep "HTTP_CODE:" | cut -d':' -f2)
oauth_body=$(echo "${oauth_response}" | grep -v "HTTP_CODE:")
[[ "${oauth_http_code}" == "200" ]] || fail "oauth HTTP ${oauth_http_code}: ${oauth_body}"

oauth_token=$(echo "${oauth_body}" | jq -r .access_token)
[[ -n "${oauth_token}" && "${oauth_token}" != "null" ]] || fail "could not parse access_token"
echo "OAuth token acquired for ${art_url}"

# --- Dispatch and monitor the GitHub Actions test workflows ---
workflow_files="artifactoryTests.yml goTests.yml npmTests.yml pnpmTests.yml pythonTests.yml mavenTests.yml gradleTests.yml nugetTests.yml conanTests.yml helmTests.yml lifecycleTests.yml accessTests.yml pluginsTests.yml dockerTests.yml podmanTests.yml distributionTests.yml"

repo_code=$(curl -sS -o /tmp/gh_repo.json -w "%{http_code}" \
-H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN_RESOLVED}" \
-H "X-GitHub-Api-Version: 2022-11-28" "${GITHUB_API_URL}/repos/${GITHUB_WORKFLOWS_REPO}")
[[ "${repo_code}" == "200" ]] || fail "Cannot access ${GITHUB_WORKFLOWS_REPO} (HTTP ${repo_code}): $(cat /tmp/gh_repo.json)"

curl -sS -o /tmp/gh_workflows.json \
-H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN_RESOLVED}" \
-H "X-GitHub-Api-Version: 2022-11-28" "${GITHUB_API_URL}/repos/${GITHUB_WORKFLOWS_REPO}/actions/workflows"

common_inputs=$(jq -n \
--arg jfrog_cli_repository "${JFROG_CLI_GITHUB_REPO}" --arg jfrog_cli_ref "${JFROG_CLI_GITHUB_REF}" \
--arg jfrog_url "${art_url}" --arg jfrog_admin_token "${oauth_token}" --arg runner "${GHE_ACTIONS_RUNNER}" \
'{jfrog_cli_repository:$jfrog_cli_repository, jfrog_cli_ref:$jfrog_cli_ref, jfrog_url:$jfrog_url, jfrog_admin_token:$jfrog_admin_token, runner:$runner}')

dispatched=""
for wf_file in ${workflow_files}; do
workflow_id=$(jq -r --arg f "${wf_file}" '.workflows[] | select(.path | endswith($f)) | .id' /tmp/gh_workflows.json | head -1)
[[ -z "${workflow_id}" || "${workflow_id}" == "null" ]] && { echo "WARNING: ${wf_file} not indexed, skipping"; continue; }

case "${wf_file}" in
distributionTests.yml) inputs=$(echo "${common_inputs}" | jq --arg jfrog_user "${JFROG_ADMIN_USERNAME:-admin}" '. + {jfrog_user:$jfrog_user}') ;;
*) inputs="${common_inputs}" ;;
esac
dispatch_body=$(jq -n --arg ref "${GITHUB_WORKFLOWS_REF}" --argjson inputs "${inputs}" '{ref:$ref, inputs:$inputs}')

http_code=$(curl -sS -o /tmp/gh_dispatch_resp.txt -w "%{http_code}" -X POST \
-H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN_RESOLVED}" \
-H "X-GitHub-Api-Version: 2022-11-28" -d "${dispatch_body}" \
"${GITHUB_API_URL}/repos/${GITHUB_WORKFLOWS_REPO}/actions/workflows/${workflow_id}/dispatches")
[[ "${http_code}" == "204" ]] || fail "Dispatch failed for ${wf_file} (HTTP ${http_code}): $(cat /tmp/gh_dispatch_resp.txt)"
echo "Dispatched ${wf_file} (id=${workflow_id})"
dispatched="${dispatched} ${wf_file}:${workflow_id}"
done
[[ -n "${dispatched}" ]] || fail "No workflows were dispatched"

echo "Waiting 30s for runs to appear..."
sleep 30
run_entries=""
for entry in ${dispatched}; do
wf_file="${entry%%:*}"; wf_id="${entry##*:}"; run_id=""
for attempt in $(seq 1 20); do
run_id=$(curl -sS -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN_RESOLVED}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"${GITHUB_API_URL}/repos/${GITHUB_WORKFLOWS_REPO}/actions/workflows/${wf_id}/runs?per_page=1&event=workflow_dispatch" \
| jq -r '.workflow_runs[0].id // empty')
[[ -n "${run_id}" && "${run_id}" != "null" ]] && break
sleep 10
done
[[ -n "${run_id}" && "${run_id}" != "null" ]] || fail "Could not resolve run id for ${wf_file}"
run_entries="${run_entries} ${wf_file}:${run_id}"
done

retries_left=""
for entry in ${run_entries}; do wf_file="${entry%%:*}"; retries_left="${retries_left} ${wf_file}=${MAX_RUN_RETRIES}"; done
get_retries_left() { echo "${retries_left}" | tr ' ' '\n' | awk -F= -v n="$1" '$1==n{print $2; exit}'; }
set_retries_left() { local n="$1" c="$2" r=""; for kv in ${retries_left}; do case "$kv" in "$n="*) r="$r $n=$c";; *) r="$r $kv";; esac; done; retries_left="$r"; }

elapsed=0; interval=60
while [[ ${elapsed} -lt ${MAX_WAIT_SECONDS} ]]; do
all_done=true; any_failed=false; summary=""
for entry in ${run_entries}; do
wf_file="${entry%%:*}"; run_id="${entry##*:}"
run_json=$(curl -sS -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN_RESOLVED}" \
-H "X-GitHub-Api-Version: 2022-11-28" "${GITHUB_API_URL}/repos/${GITHUB_WORKFLOWS_REPO}/actions/runs/${run_id}")
status=$(echo "$run_json" | jq -r '.status // empty')
conclusion=$(echo "$run_json" | jq -r '.conclusion // empty')
if [[ "${status}" != "completed" ]]; then
all_done=false; summary="${summary}${wf_file}:${status};"
elif [[ "${conclusion}" != "success" ]]; then
rl=$(get_retries_left "${wf_file}"); rl="${rl:-0}"
if [[ "${rl}" -gt 0 ]]; then
rr_code=$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN_RESOLVED}" \
-H "X-GitHub-Api-Version: 2022-11-28" "${GITHUB_API_URL}/repos/${GITHUB_WORKFLOWS_REPO}/actions/runs/${run_id}/rerun-failed-jobs")
if [[ "${rr_code}" == "201" ]]; then
set_retries_left "${wf_file}" "$((rl-1))"; all_done=false; summary="${summary}${wf_file}:retrying;"
else
any_failed=true; summary="${summary}${wf_file}:${conclusion};"
fi
else
any_failed=true; summary="${summary}${wf_file}:${conclusion};"
fi
else
summary="${summary}${wf_file}:success;"
fi
done
if [[ "${all_done}" == "true" ]]; then
[[ "${any_failed}" == "true" ]] && fail "One or more workflow runs failed. ${summary}"
echo "All workflow runs completed successfully. ${summary}"
exit 0
fi
sleep "${interval}"; elapsed=$((elapsed + interval))
done
fail "Timed out after ${MAX_WAIT_SECONDS}s. ${summary}"
67 changes: 67 additions & 0 deletions .github/workflows/e2e-dispatch.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Ported from .jfrog-pipelines/pipelines.yml (JFrog Pipelines is EOL)
#
# The actual logic lives in .github/scripts/e2e-dispatch.sh, which is also
# runnable locally — export the same env vars this workflow sets and run it
# directly from a checkout of jfrog/jfrog-cli.
name: E2E Dispatch
on:
workflow_dispatch:
inputs:
rt_version:
description: "Artifactory version for the ephemeral environment"
required: false
skip_env_setup:
description: "Skip ephemeral environment provisioning (use an existing artifactory_url)"
required: false
default: "false"
artifactory_url:
description: "Artifactory URL to use for tests when skip_env_setup is true"
required: false
github_workflows_ref:
description: "Branch or tag in the workflows repo to dispatch from"
required: false
default: "master"
jfrog_cli_github_ref:
description: "Git ref of jfrog/jfrog-cli to pass to the dispatched workflows (defaults to this run's commit)"
required: false
max_run_retries:
description: "Per-workflow rerun-failed-jobs budget"
required: false
default: "2"
max_wait_seconds:
description: "Total seconds to wait for all dispatched runs to complete"
required: false
default: "14400"

jobs:
e2e-dispatch:
runs-on: ubuntu-latest
env:
RT_VERSION: ${{ github.event.inputs.rt_version }}
SKIP_ENV_SETUP: ${{ github.event.inputs.skip_env_setup }}
ARTIFACTORY_URL: ${{ github.event.inputs.artifactory_url }}
GITHUB_API_URL: "https://github.jfrog.info/api/v3"
GITHUB_WORKFLOWS_REPO: "JFROG/jfrog-cli-workflows"
GITHUB_WORKFLOWS_REF: ${{ github.event.inputs.github_workflows_ref }}
JFROG_CLI_GITHUB_REPO: jfrog/jfrog-cli
JFROG_CLI_GITHUB_REF: ${{ github.event.inputs.jfrog_cli_github_ref || github.sha }}
GHE_ACTIONS_RUNNER: "artifactory-dind-amd-scale-set"
LOGS_TO_KIBANA: "true"
DEPLOYMENT_SIZING: "common"
MAX_RUN_RETRIES: ${{ github.event.inputs.max_run_retries }}
MAX_WAIT_SECONDS: ${{ github.event.inputs.max_wait_seconds }}
MASTER_KEY: ${{ secrets.JFROG_CLI_TESTS_MASTER_KEY }}
JFROG_ADMIN_USERNAME: ${{ secrets.JFROG_CLI_TESTS_USERNAME }}
JFROG_ADMIN_PASSWORD: ${{ secrets.JFROG_CLI_TESTS_PASSWORD }}
JFROG_ADMIN_TOKEN: ${{ secrets.JFROG_CLI_TESTS_TOKEN }}
GITHUB_DISPATCH_TOKEN: ${{ secrets.GITHUB_DISPATCH_TOKEN }}
JFROG_CLI_GH_TOKEN: ${{ secrets.JFROG_CLI_GH_TOKEN }}
JENKINS_URL: ${{ secrets.JENKINS_ENTPLUS_RT_URL }}
JENKINS_USER: ${{ secrets.JENKINS_ENTPLUS_RT_USER }}
JENKINS_TOKEN: ${{ secrets.JENKINS_ENTPLUS_RT_TOKEN }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Run e2e dispatch
run: .github/scripts/e2e-dispatch.sh

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of using the script, cant we invoke using github actions itself and check their status or using gh cli.

Loading