From 53a5b83e0a126c320c752704a88776244ff542ed Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Fri, 31 Jul 2026 18:57:10 -0400 Subject: [PATCH 01/10] prodx: add requirements file for dependabot --- ci/build | 2 +- requirements.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/ci/build b/ci/build index 0abf44ef..6214ac05 100755 --- a/ci/build +++ b/ci/build @@ -9,7 +9,7 @@ git config --global user.email "concourseteam+concourse-github-bot@gmail.com" git config --global user.name "Concourse Bot" pushd docs - pip install mkdocs-material mkdocs-redirects mkdocs-glightbox nodeenv + pip install -r requirements.txt nodeenv env --node=22.21.1 source env/bin/activate diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..422cda56 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +mkdocs-glightbox==0.5.2 +mkdocs-material==9.7.7 +mkdocs-redirects==1.2.3 +nodeenv==1.10.0 From 6882423cdac28303d258f78d34adfcc292aea2e7 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Fri, 31 Jul 2026 19:12:17 -0400 Subject: [PATCH 02/10] docx: add credhub shared path documentation (concourse#9615) --- docs/docs/operation/creds/credhub.md | 32 +++++++++++++++++++++++++--- docs/libs/examples | 2 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/docs/operation/creds/credhub.md b/docs/docs/operation/creds/credhub.md index 717d2cd2..195ae0e7 100644 --- a/docs/docs/operation/creds/credhub.md +++ b/docs/docs/operation/creds/credhub.md @@ -24,8 +24,6 @@ When resolving a parameter such as `((foo_param))`, it will look in the followin * `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` * `/concourse/TEAM_NAME/foo_param` -The leading `/concourse` can be changed by specifying `--credhub-path-prefix`. - CredHub credentials actually have different types, which may contain multiple values. For example, the `user` type specifies both `username` and `password.` You can specify the field to grab via `.` syntax, e.g. `((foo_param.username))`. @@ -34,4 +32,32 @@ If the action is being run in the context of a pipeline (e.g. a `check` or a ste first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be scoped widely if they're common across many pipelines. -If an action is being run in a one-off build, the ATC will only look in the team path. \ No newline at end of file +When executing a one-off task, there is no pipeline: so in this case, only the team path `/concourse/TEAM_NAME/foo` is +searched. + +There are several ways to customize the lookup logic: + +1. Add a "shared path", for secrets common to all teams. +2. Change the path prefix from `/concourse` to something else. + +Each of these can be controlled by Concourse command line flags, or environment variables. + +### Configuring a shared path + +A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, +foregoing the default team/pipeline namespacing. Use with care! + +```properties +CONCOURSE_CREDHUB_SHARED_PATH=some-shared-path +``` + +This path must exist under the configured path prefix. The above configuration would correspond +to `/concourse/some-shared-path` with the default `/concourse` prefix. + +### Changing the path prefix + +The leading `/concourse` can be changed by specifying the following: + +```properties +CONCOURSE_CREDHUB_PATH_PREFIX=/some-other-prefix +``` \ No newline at end of file diff --git a/docs/libs/examples b/docs/libs/examples index 254c2f62..66703307 160000 --- a/docs/libs/examples +++ b/docs/libs/examples @@ -1 +1 @@ -Subproject commit 254c2f62b8e6abc7af810e4f6e6bbe221c33d9be +Subproject commit 667033071b4f78f2a525fe156f6ddaf1d41102ea From 516cbe9c764d4d44e5e595a324be3296bdd1f84f Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 12:25:58 -0400 Subject: [PATCH 03/10] docx: set new cred manager doc standard with credhub --- docs/docs/operation/creds/credhub.md | 54 +++++++++++++++++++++------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/docs/docs/operation/creds/credhub.md b/docs/docs/operation/creds/credhub.md index 195ae0e7..db372ec8 100644 --- a/docs/docs/operation/creds/credhub.md +++ b/docs/docs/operation/creds/credhub.md @@ -2,19 +2,47 @@ title: The CredHub credential manager --- +Concourse can be configured to pull credentials from a [CredHub](https://github.com/cloudfoundry/credhub) instance. + ## Configuration -The ATC is statically configured with a CredHub server URL with TLS and client config. +To enable this credential manager, configure the following environment variables on the [ +`web` node](../../install/running-web.md): + +```properties +CONCOURSE_CREDHUB_URL=https://credhub-server:9000 +CONCOURSE_CREDHUB_CLIENT_ID=db02de05-fa39-4855-059b-67221c5c2f63 +CONCOURSE_CREDHUB_CLIENT_SECRET=6a174c20-f6de-a53c-74d2-6018fcceff64 +``` + +### TLS Configuration + +If your CredHub instance is signed with TLS by a local Certificate Authority, you can use the following environment +variable: + +```properties +CONCOURSE_CREDHUB_CA_CERT=/etc/ca.crt +``` + +??? danger "Skip SSL Verification" + + CredHub can also be configured to skip SSL Verification. This property should not be used within production. -For example, to point the ATC at an internal CredHub server with TLS signed by a local CA, using client id and secret, -you may configure: + ```properties + CONCOURSE_CREDHUB_INSECURE_SKIP_VERIFY=true + ``` -```shell -concourse web ... \ - --credhub-url https://10.2.0.3:9000 \ - --credhub-ca-cert /etc/my-ca.cert \ - --credhub-client-id =db02de05-fa39-4855-059b-67221c5c2f63 \ - --credhub-client-secret 6a174c20-f6de-a53c-74d2-6018fcceff64 +### mTLS Configuration + +CredHub can also be configured to authenticate +using [Mutual TLS (mTLS)](https://github.com/cloudfoundry/credhub/blob/main/docs/mutual-tls.md) instead of traditional +password or token-based methods. This can be accomplished using the following environment variables: + +```properties +CONCOURSE_CREDHUB_URL=https://credhub-server:9000 +CONCOURSE_CREDHUB_CA_CERT=/etc/ca.crt +CONCOURSE_CREDHUB_CLIENT_CERT=/etc/client.crt +CONCOURSE_CREDHUB_CLIENT_KEY=/etc/client.pem ``` ## Credential Lookup Rules @@ -25,8 +53,8 @@ When resolving a parameter such as `((foo_param))`, it will look in the followin * `/concourse/TEAM_NAME/foo_param` CredHub credentials actually have different types, which may contain multiple values. For example, the `user` type -specifies both `username` and `password.` You can specify the field to grab via `.` syntax, -e.g. `((foo_param.username))`. +specifies both `username` and `password.` You can specify the field to grab via `.` syntax, e.g. +`((foo_param.username))`. If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be @@ -51,8 +79,8 @@ foregoing the default team/pipeline namespacing. Use with care! CONCOURSE_CREDHUB_SHARED_PATH=some-shared-path ``` -This path must exist under the configured path prefix. The above configuration would correspond -to `/concourse/some-shared-path` with the default `/concourse` prefix. +This path must exist under the configured path prefix. The above configuration would correspond to +`/concourse/some-shared-path` with the default `/concourse` prefix. ### Changing the path prefix From b624eec082c9a06044aea9578b598fafe3bcd50a Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 14:53:54 -0400 Subject: [PATCH 04/10] docx: align aws secrets manager docs --- docs/docs/operation/creds/aws-secrets.md | 372 +++++++++++++++-------- 1 file changed, 242 insertions(+), 130 deletions(-) diff --git a/docs/docs/operation/creds/aws-secrets.md b/docs/docs/operation/creds/aws-secrets.md index c886b628..abc3157d 100644 --- a/docs/docs/operation/creds/aws-secrets.md +++ b/docs/docs/operation/creds/aws-secrets.md @@ -2,183 +2,295 @@ title: The AWS Secrets Manager credential manager --- +Concourse can be configured to pull credentials from [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). + ## Configuration -In order to integrate with AWS Secrets Manager for credential management, the web node must be configured with: +To enable this credential manager, configure the following environment variables on the [ +`web` node](../../install/running-web.md): -* an access key and secret key, or a session token -* the AWS region that your parameters are stored within. +```properties +CONCOURSE_AWS_SECRETSMANAGER_REGION=us-east-1 +CONCOURSE_AWS_SECRETSMANAGER_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE +CONCOURSE_AWS_SECRETSMANAGER_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +``` -If no access key, secret key, or session token is provided, Concourse will attempt to use environment variables or the -instance credentials assigned to the instance. +### Session Token Configuration -The web node's configuration specifies the following: +AWS Secrets Manager can also be configured to use a Session Token for short lived credentials by using the following +environment variables: -**`aws-secretsmanager-access-key`**: string +```properties +CONCOURSE_AWS_SECRETSMANAGER_REGION=us-east-1 +CONCOURSE_AWS_SECRETSMANAGER_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE +CONCOURSE_AWS_SECRETSMANAGER_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +CONCOURSE_AWS_SECRETSMANAGER_SESSION_TOKEN=AQoDYXdzEJr... +``` -: A valid AWS access key. +### Instance Profile Configuration -: Environment variable `CONCOURSE_AWS_SECRETSMANAGER_ACCESS_KEY`. +AWS Secrets Manager can also be configured to use an +[IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) assigned on the [ +`web` node](../../install/running-web.md). When using an IAM Role, credentials are fetched automatically from the EC2 +metadata service and only the region needs to be configured: -**`aws-secretsmanager-secret-key`**: string +```properties +CONCOURSE_AWS_SECRETSMANAGER_REGION=us-east-1 +``` -: The secret key that corresponds to the access key defined above. +### IAM Permissions -: Environment variable `CONCOURSE_AWS_SECRETSMANAGER_SECRET_KEY`. +The following is an example of an IAM policy that can be used to grant permissions to an IAM user or instance role. -**`aws-secretsmanager-session-token`**: string +!!! note -: A valid AWS session token. + The `Resource` section can contain a wildcard to a secret or be restricted to an individual secret. -: Environment variable `CONCOURSE_AWS_SECRETSMANAGER_SESSION_TOKEN`. +In order for the health check to work properly (see [Scaling](#scaling)), Concourse needs to have access to the +`__concourse-health-check` secret. -**`aws-secretsmanager-region`**: string +=== "JSON" -: The AWS region that requests to Secrets Manager will be sent to. + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAccessToSecretManagerParameters", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:ListSecrets" + ], + + "Resource": "*" + }, + { + "Sid": "AllowAccessGetSecret", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ], + + "Resource": [ + "arn:aws:secretsmanager:*:*:secret:/concourse/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + ] + } + ``` + +=== "Terraform / OpenTofu" + + ```hcl + data "aws_iam_policy_document" "secrets_lookup" { + statement { + sid = "AllowAccessToSecretManagerParameters" + + effect = "Allow" + + actions = [ + "secretsmanager:ListSecrets" + ] + + resources = [ + "*", + ] + } + + statement { + sid = "AllowAccessGetSecret" + + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ] + + resources = [ + "arn:aws:secretsmanager:*:*:secret:/concourse/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + } + ``` -: Environment variable `CONCOURSE_AWS_SECRETSMANAGER_REGION`. +If you wish to restrict concourse to only have access to secrets for a specific pipeline, you can replace +`"arn:aws:secretsmanager:*:*:secret:/concourse/*"` in the example above with: -**`aws-secretsmanager-pipeline-secret-template`**: string +=== "JSON" -: The base path used when attempting to locate a pipeline-level secret. + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAccessToSecretManagerParameters", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:ListSecrets" + ], + + "Resource": "*" + }, + { + "Sid": "AllowAccessGetSecret", + + "Effect": "Allow", + + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ], + + "Resource": [ + "arn:aws:secretsmanager:*:*:secret:/concourse/TEAM_NAME/*", + "arn:aws:secretsmanager:*:*:secret:/concourse/TEAM_NAME/PIPELINE_NAME/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + ] + } + ``` -: Environment variable `CONCOURSE_AWS_SECRETSMANAGER_PIPELINE_SECRET_TEMPLATE`. +=== "Terraform / OpenTofu" -: !!! example + ```hcl + variable "team_name" { + type = string + default = "my_team" + } + + variable "pipeline_name" { + type = string + default = "my_pipeline" + } + + data "aws_iam_policy_document" "secrets_lookup" { + statement { + sid = "AllowAccessToSecretManagerParameters" + + effect = "Allow" + + actions = [ + "secretsmanager:ListSecrets" + ] + + resources = [ + "*", + ] + } + + statement { + sid = "AllowAccessGetSecret" + + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ] + + resources = [ + "arn:aws:secretsmanager:*:*:secret:/concourse/${var.team_name}/*", + "arn:aws:secretsmanager:*:*:secret:/concourse/${var.team_name}/${var.pipeline_name}/*", + "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" + ] + } + } + ``` - Default: `/concourse/{{.Team}}/{{.Pipeline}}/{{.Secret}}` +where `TEAM_NAME` and `PIPELINE_NAME` are replaced with the team and name of the pipeline in question. -**`aws-secretsmanager-team-secret-template`**: string +For more information on how to use IAM roles to restrict access to Secrets Manager, review +the [official documentation](https://docs.aws.amazon.com/secretsmanager/latest/userguide/auth-and-access_identity-based-policies.html). -: The base path used when attempting to locate a team-level secret. +### Scaling -: Environment variable `CONCOURSE_AWS_SECRETSMANAGER_TEAM_SECRET_TEMPLATE`. +If your cluster has a large workload, in particular if there are many resources, Concourse can generate a lot of traffic +to AWS and subsequently get rate-limited. -: !!! example +As long as Concourse has permission to get the value of the `__concourse-health-check` secret, you should be able to +measure an error rate by polling the `/api/v1/info/creds` endpoint when authenticated as +a [Concourse Admin](../../auth-and-teams/user-roles.md#concourse-admin). - Default: `/concourse/{{.Team}}/{{.Secret}}` +Depending on your workflow for updating secrets and your reliability requirements it may be +worth [Caching credentials](caching.md) and/or [Retrying failed fetches](retrying-failed.md) to mitigate +rate-limit-related errors. -For example, to launch the ATC and enable Secrets Manager, you may configure: +## Credential Lookup Rules -```shell -concourse web ... \ - --aws-secretsmanager-region us-east-1 \ - --aws-secretsmanager-access-key AKIAIOSFODNN7EXAMPLE \ - --aws-secretsmanager-secret-key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +When resolving a parameter such as `((foo_param))`, it will look in the following paths, in order: -# or use env variables -CONCOURSE_AWS_SECRETSMANAGER_REGION="us-east-1" \ -CONCOURSE_AWS_SECRETSMANAGER_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" \ -CONCOURSE_AWS_SECRETSMANAGER_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ -concourse web ... -``` +* `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` +* `/concourse/TEAM_NAME/foo_param` -A more secure method is to configure -an [IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) for your EC2 ATC -instance so that credentials are fetched automatically from the EC2 metadata service. +CredHub credentials actually have different types, which may contain multiple values. For example, the `user` type +specifies both `username` and `password.` You can specify the field to grab via `.` syntax, e.g. +`((foo_param.username))`. -## Saving credentials in AWS +If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will +first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be +scoped widely if they're common across many pipelines. -It seems to be best to use the 'other type of secret' option and the 'plaintext' entry (otherwise your secrets will be -interpolated as JSON) for best results. Make sure your secret locations match the lookup templates exactly; include the -leading `/`, for example. +When executing a one-off task, there is no pipeline: so in this case, only the team path `/concourse/TEAM_NAME/foo` is +searched. -## IAM Permissions +There are several ways to customize the lookup logic: -The following is an example of an IAM policy that can be used to grant permissions to an IAM user or instance role. Note -that the `Resource` section can contain a wildcard to a secret or be restricted to an individual secret. In order for -the health check to work properly (see [Scaling](#scaling)), Concourse needs to have access to -the `__concourse-health-check` secret. +1. Add a "shared path", for secrets common to all teams. +2. Change the team- and pipeline-dependent path templates. -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowAccessToSecretManagerParameters", - "Effect": "Allow", - "Action": [ - "secretsmanager:ListSecrets" - ], - "Resource": "*" - }, - { - "Sid": "AllowAccessGetSecret", - "Effect": "Allow", - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret" - ], - "Resource": [ - "arn:aws:secretsmanager:*:*:secret:/concourse/*", - "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" - ] - } - ] -} -``` +Each of these can be controlled by Concourse command line flags, or environment variables. -If you wish to restrict concourse to only have access to secrets for a specific pipeline, you can -replace `"arn:aws:secretsmanager:*:*:secret:/concourse/*"` in the example above with: +### Configuring a shared path -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowAccessToSecretManagerParameters", - "Effect": "Allow", - "Action": [ - "secretsmanager:ListSecrets" - ], - "Resource": "*" - }, - { - "Sid": "AllowAccessGetSecret", - "Effect": "Allow", - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret" - ], - "Resource": [ - "arn:aws:secretsmanager:*:*:secret:/concourse/TEAM_NAME/*", - "arn:aws:secretsmanager:*:*:secret:/concourse/TEAM_NAME/PIPELINE_NAME/*", - "arn:aws:secretsmanager:*:*:secret:__concourse-health-check-??????" - ] - } - ] -} +A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, +foregoing the default team/pipeline namespacing. Use with care! + +```properties +CONCOURSE_AWS_SECRETSMANAGER_SHARED_SECRET_TEMPLATE=some-shared-path ``` -where `TEAM_NAME` and `PIPELINE_NAME` are replaced with the team and name of the pipeline in question. +This path must exist under the configured path prefix. The above configuration would correspond to +`/concourse/some-shared-path` with the default `/concourse` prefix. -For more information on how to use IAM roles to restrict access to Secrets Manager, review -the [official documentation](https://docs.aws.amazon.com/secretsmanager/latest/userguide/auth-and-access_identity-based-policies.html). +### Changing the path templates -## Credential Lookup Rules +You can choose your own list of templates, which will expand to team- or pipeline-specific paths. By default, the +templates used are: -When resolving a parameter such as `((foo_param))`, Concourse will look in the following paths, in order: +```properties +CONCOURSE_AWS_SECRETSMANAGER_TEAM_SECRET_TEMPLATE=/concourse/{{.Team}}/{{.Secret}} +CONCOURSE_AWS_SECRETSMANAGER_PIPELINE_SECRET_TEMPLATE=/concourse/{{.Team}}/{{.Pipeline}}/{{.Secret}} -* `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` -* `/concourse/TEAM_NAME/foo_param` - -The leading `/concourse` can be changed by specifying `--aws-secretsmanager-pipeline-secret-template` -or `--aws-secretsmanager-team-secret-template` variables. +``` -!!! note +When secrets are to be looked up, these are evaluated where `{{.Team}}` expands to the current team, `{{.Pipeline}}` to +the current pipeline (if any), and `{{.Secret}}` to the name of the secret. So if the settings are: - If Concourse does not have [permission](#iam-permissions) to access the pipeline-scoped paths, then credential - lookups will fail even for credentials which are stored at the team level. +```properties +CONCOURSE_AWS_SECRETSMANAGER_TEAM_SECRET_TEMPLATE=/{{.Team}}/concourse/{{.Secret}} +CONCOURSE_AWS_SECRETSMANAGER_PIPELINE_SECRET_TEMPLATE=/{{.Team}}/concourse/{{.Pipeline}}/{{.Secret}} +CONCOURSE_AWS_SECRETSMANAGER_SHARED_SECRET_TEMPLATE=/common/{{.Secret}} +``` -## Scaling +and `((password))` is used in team `myteam` and pipeline `mypipeline`, Concourse will look for the following, in order: -If your cluster has a large workload, in particular if there are many resources, Concourse can generate a lot of traffic -to AWS and subsequently get rate-limited. +1. `/myteam/concourse/mypipeline/password` +2. `/myteam/concourse/password` +3. `/common/password` -As long as Concourse has permission to get the value of the `__concourse-health-check` secret, you should be able to -measure an error rate by polling the `/api/v1/info/creds` endpoint when authenticated as -a [Concourse Admin](../../auth-and-teams/user-roles.md#concourse-admin). +## Saving credentials in AWS -Depending on your workflow for updating secrets and your reliability requirements it may be -worth [Caching credentials](caching.md) and/or [Retrying failed fetches](retrying-failed.md) to mitigate -rate-limit-related errors. \ No newline at end of file +It seems to be best to use the 'other type of secret' option and the 'plaintext' entry (otherwise your secrets will be +interpolated as JSON) for best results. Make sure your secret locations match the lookup templates exactly; include the +leading `/`, for example. \ No newline at end of file From 7c328d1e24293fa585550e7f6b2cee8f92f514c5 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 14:59:09 -0400 Subject: [PATCH 05/10] fix: remove credhub line --- docs/docs/operation/creds/aws-secrets.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/docs/operation/creds/aws-secrets.md b/docs/docs/operation/creds/aws-secrets.md index abc3157d..9ddb3fc8 100644 --- a/docs/docs/operation/creds/aws-secrets.md +++ b/docs/docs/operation/creds/aws-secrets.md @@ -233,10 +233,6 @@ When resolving a parameter such as `((foo_param))`, it will look in the followin * `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` * `/concourse/TEAM_NAME/foo_param` -CredHub credentials actually have different types, which may contain multiple values. For example, the `user` type -specifies both `username` and `password.` You can specify the field to grab via `.` syntax, e.g. -`((foo_param.username))`. - If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be scoped widely if they're common across many pipelines. From 3153b371cf5400db80b640bf3ca827fd505ca7dd Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 15:31:11 -0400 Subject: [PATCH 06/10] docx: align AWS Systems Manger to new format --- base.yml | 8 +- docs/docs/operation/creds/aws-ssm.md | 386 ++++++++++++++++++++------- docs/docs/operation/creds/index.md | 28 +- 3 files changed, 313 insertions(+), 109 deletions(-) diff --git a/base.yml b/base.yml index c88ed002..a4429e9a 100644 --- a/base.yml +++ b/base.yml @@ -224,13 +224,13 @@ nav: - docs/operation/encryption.md - Credential Management: - docs/operation/creds/index.md - - docs/operation/creds/vault.md - - docs/operation/creds/credhub.md - - docs/operation/creds/aws-ssm.md - docs/operation/creds/aws-secrets.md - - docs/operation/creds/kubernetes.md + - docs/operation/creds/aws-ssm.md - docs/operation/creds/conjur.md + - docs/operation/creds/credhub.md - docs/operation/creds/id-token.md + - docs/operation/creds/kubernetes.md + - docs/operation/creds/vault.md - docs/operation/creds/caching.md - docs/operation/creds/redacting.md - docs/operation/creds/retrying-failed.md diff --git a/docs/docs/operation/creds/aws-ssm.md b/docs/docs/operation/creds/aws-ssm.md index 62c2a355..c0dfe501 100644 --- a/docs/docs/operation/creds/aws-ssm.md +++ b/docs/docs/operation/creds/aws-ssm.md @@ -1,133 +1,337 @@ --- -title: The AWS SSM credential manager +title: The AWS Systems Manager credential manager --- +Concourse can be configured to pull credentials +from [AWS Systems Manager]([https://aws.amazon.com/secrets-manager/](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html)). + ## Configuration -The ATC is configured with an access key and secret key or session token and the AWS region that your parameters are -stored within. If no access key, secret key, or session token is provided, Concourse will attempt to use environment -variables or the instance credentials assigned to the instance. +To enable this credential manager, configure the following environment variables on the [ +`web` node](../../install/running-web.md): + +```properties +CONCOURSE_AWS_SSM_REGION=us-east-1 +CONCOURSE_AWS_SSM_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE +CONCOURSE_AWS_SSM_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +``` + +### Session Token Configuration + +AWS Systems Manager can also be configured to use a Session Token for short lived credentials by using the following +environment variables: -The ATC's configuration specifies the following: +```properties +CONCOURSE_AWS_SSM_REGION=us-east-1 +CONCOURSE_AWS_SSM_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE +CONCOURSE_AWS_SSM_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +CONCOURSE_AWS_SSM_SESSION_TOKEN=AQoDYXdzEJr... +``` -**`aws-ssm-access-key`**: string +### Instance Profile Configuration -: A valid AWS access key. +AWS Systems Manager can also be configured to use an +[IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) assigned on the [ +`web` node](../../install/running-web.md). When using an IAM Role, credentials are fetched automatically from the EC2 +metadata service and only the region needs to be configured: -: Environment variable `CONCOURSE_AWS_SSM_ACCESS_KEY`. +```properties +CONCOURSE_AWS_SSM_REGION=us-east-1 +``` -**`aws-ssm-secret-key`**: string +### IAM Permissions -: The secret key that corresponds to the access key defined above. +The following is an example of an IAM policy that can be used to grant permissions to an IAM user or instance role. -: Environment variable `CONCOURSE_AWS_SSM_SECRET_KEY`. +!!! note -**`aws-ssm-session-token`**: string + The `Resource` section can contain a wildcard to a secret or be restricted to an individual secret. -: A valid AWS session token. +=== "JSON" -: Environment variable `CONCOURSE_AWS_SSM_SESSION_TOKEN`. + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowListKeys", + + "Effect": "Allow", + + "Action": [ + "kms:ListAliases", + "kms:ListKeys" + ], + + "Resource": "*" + }, + { + "Sid": "AllowAccessToSsmParameters", + + "Effect": "Allow", + + "Action": [ + "ssm:GetParameter", + "ssm:GetParametersByPath" + ], + + "Resource": [ + "arn:aws:ssm:::parameter/concourse/*", + ] + }, + { + "Sid": "AllowAccessToDecryptSsmParameters", + + "Effect": "Allow", + + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + + "Resource": [ + "arn:aws:kms:::key/KMS_KEY_ID" + ] + }, + ] + } + ``` -**`aws-ssm-region`**: string +=== "Terraform / OpenTofu" -: The AWS region that requests to parameter store will be sent to. + ```hcl + variable "kms_key_id" { + type = string + } + + data "aws_iam_policy_document" "params_lookup" { + statement { + sid = "AllowListKeys" + + effect = "Allow" + + actions = [ + "kms:ListAliases", + "kms:ListKeys" + ] + + resources = [ + "*", + ] + } + + statement { + sid = "AllowAccessToSsmParameters" + + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParametersByPath" + ] + + resources = [ + "arn:aws:ssm:::parameter/concourse/*", + ] + } + + statement { + sid = "AllowAccessToDecryptSsmParameters" + + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey" + ] + + resources = [ + "arn:aws:kms:::key/${var.kms_key_id}", + ] + } + } + ``` -: Environment variable `CONCOURSE_AWS_SSM_REGION`. +If you wish to restrict Concourse to only have access to parameters for a specific pipeline, you can replace +`"arn:aws:ssm:::parameter/concourse/*"` in the example above with: -**`aws-ssm-pipeline-secret-template`**: string +=== "JSON" -: The base path used when attempting to locate a pipeline-level secret. + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowListKeys", + + "Effect": "Allow", + + "Action": [ + "kms:ListAliases", + "kms:ListKeys" + ], + + "Resource": "*" + }, + { + "Sid": "AllowAccessToSsmParameters", + + "Effect": "Allow", + + "Action": [ + "ssm:GetParameter", + "ssm:GetParametersByPath" + ], + + "Resource": [ + "arn:aws:ssm:::parameter/concourse/TEAM_NAME/*", + "arn:aws:ssm:::parameter/concourse/TEAM_NAME/PIPELINE_NAME/*" + ] + }, + { + "Sid": "AllowAccessToDecryptSsmParameters", + + "Effect": "Allow", + + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + + "Resource": [ + "arn:aws:kms:::key/KMS_KEY_ID" + ] + }, + ] + } + ``` -: Environment variable `CONCOURSE_AWS_SSM_PIPELINE_SECRET_TEMPLATE`. +=== "Terraform / OpenTofu" -: !!! example + ```hcl + variable "team_name" { + type = string + default = "my_team" + } + + variable "pipeline_name" { + type = string + default = "my_pipeline" + } + + variable "kms_key_id" { + type = string + } + + data "aws_iam_policy_document" "params_lookup" { + statement { + sid = "AllowListKeys" + + effect = "Allow" + + actions = [ + "kms:ListAliases", + "kms:ListKeys" + ] + + resources = [ + "*", + ] + } + + statement { + sid = "AllowAccessToSsmParameters" + + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParametersByPath" + ] + + resources = [ + "arn:aws:ssm:::parameter/concourse/${var.team_name}/*", + "arn:aws:ssm:::parameter/concourse/${var.team_name}/${var.pipeline_name}/*", + ] + } + + statement { + sid = "AllowAccessToDecryptSsmParameters" + + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey" + ] + + resources = [ + "arn:aws:kms:::key/${var.kms_key_id}", + ] + } + } + ``` - Default: `/concourse/{{.Team}}/{{.Pipeline}}/{{.Secret}}` +where `TEAM_NAME`, `PIPELINE_NAME`, and `KMS_KEY_ID` are replaced with the team, pipeline, and key id in question. -**`aws-ssm-team-secret-template`**: string +For more information on how to use IAM roles to restrict access to SSM parameters, review +the [official documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-access.html). -: The base path used when attempting to locate a team-level secret. +## Credential Lookup Rules -: Environment variable `CONCOURSE_AWS_SSM_TEAM_SECRET_TEMPLATE`. +When resolving a parameter such as `((foo_param))`, it will look in the following paths, in order: -: !!! example +* `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` +* `/concourse/TEAM_NAME/foo_param` - Default: `/concourse/{{.Team}}/{{.Secret}}` +If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will +first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be +scoped widely if they're common across many pipelines. -For example, to launch the ATC and enable the parameter store, you may configure: +When executing a one-off task, there is no pipeline: so in this case, only the team path `/concourse/TEAM_NAME/foo` is +searched. -```shell -concourse web ... \ - --aws-ssm-region us-east-1 \ - --aws-ssm-access-key AKIAIOSFODNN7EXAMPLE \ - --aws-ssm-secret-key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +There are several ways to customize the lookup logic: -# or use env variables -CONCOURSE_AWS_SSM_REGION="us-east-1" \ -CONCOURSE_AWS_SSM_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" \ -CONCOURSE_AWS_SSM_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ -concourse web ... -``` +1. Add a "shared path", for secrets common to all teams. +2. Change the team- and pipeline-dependent path templates. -A more secure method is to configure -an [IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) for your EC2 ATC -instance so that credentials are fetched automatically from the EC2 metadata service. +Each of these can be controlled by Concourse command line flags, or environment variables. -## IAM Permissions +### Configuring a shared path -The following is an example of an IAM policy that can be used to grant permissions to an IAM user or instance role. Note -that the `Resource` section can contain a wildcard to a parameter or be restricted to an individual parameter. +A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, +foregoing the default team/pipeline namespacing. Use with care! -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowAccessToSsmParameters", - "Effect": "Allow", - "Action": [ - "ssm:GetParameter", - "ssm:GetParametersByPath" - ], - "Resource": [ - "arn:aws:ssm:::parameter/concourse/*", - "arn:aws:ssm:::parameter/concourse/TEAM_NAME/*", - "arn:aws:ssm:::parameter/concourse/TEAM_NAME/PIPELINE_NAME/*" - ] - }, - { - "Sid": "AllowAccessToDecryptSsmParameters", - "Effect": "Allow", - "Action": [ - "kms:Decrypt", - "kms:DescribeKey" - ], - "Resource": "arn:aws:kms:::key/KMS_KEY_ID" - }, - { - "Sid": "AllowListKeys", - "Effect": "Allow", - "Action": [ - "kms:ListAliases", - "kms:ListKeys" - ], - "Resource": "*" - } - ] -} +```properties +CONCOURSE_AWS_SSM_SHARED_PATH=some-shared-path ``` -Note that the `TEAM_NAME`, `PIPELINE_NAME`, and `KMS_KEY_ID` text above should be replaced to fit your Concourse setup. +This path must exist under the configured path prefix. The above configuration would correspond to +`/concourse/some-shared-path` with the default `/concourse` prefix. -For more information on how to use IAM roles to restrict access to SSM parameters, review -the [official documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-access.html). +### Changing the path templates -## Credential Lookup Rules +You can choose your own list of templates, which will expand to team- or pipeline-specific paths. By default, the +templates used are: -When resolving a parameter such as `((foo_param))`, Concourse will look in the following paths, in order: +```properties +CONCOURSE_AWS_SSM_TEAM_SECRET_TEMPLATE=/concourse/{{.Team}}/{{.Secret}} +CONCOURSE_AWS_SSM_PIPELINE_SECRET_TEMPLATE=/concourse/{{.Team}}/{{.Pipeline}}/{{.Secret}} -* `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` -* `/concourse/TEAM_NAME/foo_param` +``` + +When secrets are to be looked up, these are evaluated where `{{.Team}}` expands to the current team, `{{.Pipeline}}` to +the current pipeline (if any), and `{{.Secret}}` to the name of the secret. So if the settings are: + +```properties +CONCOURSE_AWS_SSM_TEAM_SECRET_TEMPLATE=/{{.Team}}/concourse/{{.Secret}} +CONCOURSE_AWS_SSM_PIPELINE_SECRET_TEMPLATE=/{{.Team}}/concourse/{{.Pipeline}}/{{.Secret}} +CONCOURSE_AWS_SSM_SHARED_PATH=/common/{{.Secret}} +``` + +and `((password))` is used in team `myteam` and pipeline `mypipeline`, Concourse will look for the following, in order: -The leading `/concourse` can be changed by specifying `--aws-ssm-pipeline-secret-template` -or `--aws-ssm-team-secret-template` variables. \ No newline at end of file +1. `/myteam/concourse/mypipeline/password` +2. `/myteam/concourse/password` +3. `/common/password` diff --git a/docs/docs/operation/creds/index.md b/docs/docs/operation/creds/index.md index e4536763..1ef9e971 100644 --- a/docs/docs/operation/creds/index.md +++ b/docs/docs/operation/creds/index.md @@ -25,39 +25,39 @@ relevant section below for whichever backend you want to use.
-- :simple-vault: Vault +- :material-aws: AWS Secrets Manager --- - [:octicons-arrow-right-24: Configure](vault.md) + [:octicons-arrow-right-24: Configure](aws-secrets.md) -- :material-lock: CredHub +- :material-aws: AWS SSM --- - [:octicons-arrow-right-24: Configure](credhub.md) + [:octicons-arrow-right-24: Configure](aws-ssm.md) -- :material-aws: AWS SSM +- :material-lock: Conjur --- - [:octicons-arrow-right-24: Configure](aws-ssm.md) + [:octicons-arrow-right-24: Configure](conjur.md) -- :material-aws: AWS Secrets Manager +- :material-lock: CredHub --- - [:octicons-arrow-right-24: Configure](aws-secrets.md) + [:octicons-arrow-right-24: Configure](credhub.md) -- :material-kubernetes: Kubernetes +- :material-openid: IDToken --- - [:octicons-arrow-right-24: Configure](kubernetes.md) + [:octicons-arrow-right-24: Configure](id-token.md) -- :material-lock: Conjur +- :material-kubernetes: Kubernetes --- - [:octicons-arrow-right-24: Configure](conjur.md) + [:octicons-arrow-right-24: Configure](kubernetes.md) -- :material-openid: IDToken +- :simple-vault: Vault --- - [:octicons-arrow-right-24: Configure](id-token.md) + [:octicons-arrow-right-24: Configure](vault.md)
\ No newline at end of file From 2da3ac62bcd66e1fe6cddd7a8b5a62a7b961df46 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 16:37:19 -0400 Subject: [PATCH 07/10] docx: align CyberArk Conjur to new format --- docs/docs/operation/creds/conjur.md | 160 +++++++++++++--------------- 1 file changed, 74 insertions(+), 86 deletions(-) diff --git a/docs/docs/operation/creds/conjur.md b/docs/docs/operation/creds/conjur.md index 0fb5f74b..d119be26 100644 --- a/docs/docs/operation/creds/conjur.md +++ b/docs/docs/operation/creds/conjur.md @@ -2,99 +2,39 @@ title: The Conjur credential manager --- -## Configuration - Concourse can be configured to pull credentials from a [CyberArk Conjur](https://conjur.org/) instance. -The ATC is configured with a Conjur host username and api key or session token. If no host username, api key, or session -token is provided, Concourse will attempt to use environment variables. - -The ATC's configuration specifies the following: - -**`conjur-appliance-url`**: string - -: URL of the Conjur instance. - -: Environment variable `CONCOURSE_CONJUR_APPLIANCE_URL`. - -**`conjur-account`**: string - -: The Conjur account. - -: Environment variable `CONCOURSE_CONJUR_ACCOUNT`. - -**`conjur-authn-login`**: string - -: A valid Conjur host username. - -: Environment variable `CONCOURSE_CONJUR_AUTHN_LOGIN`. - -**`conjur-authn-api-key`**: string - -: The api key that corresponds to the Conjur host username. - -: Environment variable `CONCOURSE_CONJUR_AUTHN_API_KEY`. - -**`conjur-authn-token-file`**: string - -: Token file used if Conjur instance is running in k8s or iam. - -: Environment variable `CONCOURSE_CONJUR_AUTHN_TOKEN_FILE`. - -**`conjur-cert-file`**: string - -: Cert file used if conjur instance is using a self-signed cert. - -: Environment variable `CONCOURSE_CONJUR_CERT_FILE`. - -**`conjur-pipeline-secret-template`**: string - -: The base path used when attempting to locate a pipeline-level secret. - -: Environment variable `CONCOURSE_CONJUR_PIPELINE_SECRET_TEMPLATE`. - -: !!! example - - Default: `/concourse/{{.Team}}/{{.Secret}}` - -**`conjur-team-secret-template`**: string - -: The base path used when attempting to locate a team-level secret. - -: Environment variable `CONCOURSE_CONJUR_TEAM_SECRET_TEMPLATE`. - -: !!! example - - Default: `/concourse/{{.Team}}/{{.Secret}}` +## Configuration -**`conjur-secret-template`**: string +To enable this credential manager, configure the following environment variables on the [ +`web` node](../../install/running-web.md): -: The base path used when attempting to locate a vault or safe level secret. +```properties +CONCOURSE_CONJUR_APPLIANCE_URL=https://credhub-server:9000 +CONCOURSE_CONJUR_ACCOUNT=db02de05-fa39-4855-059b-67221c5c2f63 +CONCOURSE_CONJUR_AUTHN_LOGIN=6a174c20-f6de-a53c-74d2-6018fcceff64 +CONCOURSE_CONJUR_AUTHN_API_KEY=6a174c20-f6de-a53c-74d2-6018fcceff64 +``` -: Environment variable `CONCOURSE_CONJUR_SECRET_TEMPLATE`. +### Token File Configuration -: !!! example +Conjur can also be configured to use a token file instead of login and API keys by configuring the following environment +variable: - Default: `vaultName/{{.Secret}}` +```properties +CONCOURSE_CONJUR_AUTHN_TOKEN_FILE=/etc/token +``` -For example, to launch the ATC and enable Conjur, you may configure: +### TLS Configuration -```shell -concourse web ... \ - --conjur-appliance-url https://conjur-master.local \ - --conjur-account conjur \ - --conjur-authn-login host/concourse/dev \ - --conjur-authn-api-key 107eaqz167jkzm2q8wjv4mnyj0z12gfkws9wq9gzsjt29v2sn7yvy +If your Conjur instance is signed with TLS by a local Certificate Authority, you can use the following environment +variable: -# or use env variables -CONCOURSE_CONJUR_APPLIANCE_URL="https://conjur-master.local" \ -CONCOURSE_CONJUR_ACCOUNT="conjur" \ -CONCOURSE_CONJUR_AUTHN_LOGIN="host/concourse/dev" \ -CONCOURSE_CONJUR_AUTHN_API_KEY="107eaqz167jkzm2q8wjv4mnyj0z12gfkws9wq9gzsjt29v2sn7yvy" \ -concourse web ... +```properties +CONCOURSE_CONJUR_CERT_FILE=/etc/ca.crt ``` -## Conjur Permissions +### Permissions The following is an example Conjur policy that can be used to grant permissions to a Conjur host. In this example `host/concourse` will have permissions to read and update all the secrets within the `TEAM_NAME` @@ -123,13 +63,61 @@ the [official documentation](https://docs.conjur.org/Latest/en/Content/Operation ## Credential Lookup Rules -When resolving a parameter such as `((foo_param))`, Concourse will look in the following paths, in order: +When resolving a parameter such as `((foo_param))`, it will look in the following paths, in order: * `/concourse/TEAM_NAME/PIPELINE_NAME/foo_param` * `/concourse/TEAM_NAME/foo_param` -* `vaultName/foo_param` -The leading `/concourse` can be changed by specifying `--conjur-pipeline-secret-template` -or `--conjur-team-secret-template` variables. +If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will +first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be +scoped widely if they're common across many pipelines. + +When executing a one-off task, there is no pipeline: so in this case, only the team path `/concourse/TEAM_NAME/foo` is +searched. + +There are several ways to customize the lookup logic: + +1. Add a "shared path", for secrets common to all teams. +2. Change the team-, pipeline-, and absolute secret dependent path templates. + +Each of these can be controlled by Concourse command line flags, or environment variables. + +### Configuring a shared path + +A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, +foregoing the default team/pipeline namespacing. Use with care! + +```properties +CONCOURSE_CONJUR_SHARED_PATH=some-shared-path +``` + +This path must exist under the configured path prefix. The above configuration would correspond to +`/concourse/some-shared-path` with the default `/concourse` prefix. + +### Changing the path templates + +You can choose your own list of templates, which will expand to team-, pipeline-, and absolute secret specific paths. By +default, the templates used are: + +```properties +CONCOURSE_CONJUR_TEAM_SECRET_TEMPLATE="concourse/{{.Team}}/{{.Secret}} +CONCOURSE_CONJUR_PIPELINE_SECRET_TEMPLATE=/concourse/{{.Team}}/{{.Pipeline}}/{{.Secret}} +CONCOURSE_CONJUR_SECRET_TEMPLATE=vaultName/{{.Secret}} +``` + +When secrets are to be looked up, these are evaluated where `{{.Team}}` expands to the current team, `{{.Pipeline}}` to +the current pipeline (if any), and `{{.Secret}}` to the name of the secret. So if the settings are: + +```properties +CONCOURSE_CONJUR_TEAM_SECRET_TEMPLATE="{{.Team}}/concourse/{{.Secret}} +CONCOURSE_CONJUR_PIPELINE_SECRET_TEMPLATE=/{{.Team}}/concourse/{{.Pipeline}}/{{.Secret}} +CONCOURSE_CONJUR_SECRET_TEMPLATE=conjur/{{.Secret}} +CONCOURSE_CONJUR_SHARED_PATH=common +``` + +and `((password))` is used in team `myteam` and pipeline `mypipeline`, Concourse will look for the following, in order: -The leading `vaultName` can be changed by specifying `--conjur-secret-template` variable. \ No newline at end of file +1. `/myteam/concourse/mypipeline/password` +2. `/myteam/concourse/password` +3. `/conjur/password` +4. `/common/password` \ No newline at end of file From 2b42367adecf7f2ccf405ce114ffa0ce6ac39bc6 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 17:09:55 -0400 Subject: [PATCH 08/10] docx: align kubernetes.md to new format and include shared suffix --- docs/docs/operation/creds/kubernetes.md | 131 +++++++++++++----------- 1 file changed, 73 insertions(+), 58 deletions(-) diff --git a/docs/docs/operation/creds/kubernetes.md b/docs/docs/operation/creds/kubernetes.md index fecdfc7e..a2ab07cd 100644 --- a/docs/docs/operation/creds/kubernetes.md +++ b/docs/docs/operation/creds/kubernetes.md @@ -2,11 +2,12 @@ title: Kubernetes Credential Manager --- -Concourse can be configured to pull credentials -from [Kubernetes `secret` objects](https://kubernetes.io/docs/concepts/configuration/secret). +Concourse can be configured to pull credentials from [Kubernetes `secret` objects](https://kubernetes.io/docs/concepts/configuration/secret). -To configure it, either enable the in-cluster client by setting the following environment variable on -the [`web` node](../../install/running-web.md): +## Configuration + +To enable this credential manager, configure the following environment variable on the [ +`web` node](../../install/running-web.md): ```properties CONCOURSE_KUBERNETES_IN_CLUSTER=true @@ -18,47 +19,7 @@ or set the path to a `kubeconfig` file: CONCOURSE_KUBERNETES_CONFIG_PATH=~/.kube/config ``` -## Credential lookup rules - -When resolving a parameter such as `((foo))`, Concourse will look for it in the following order in the namespace -configured for that team: - -1. - ``` - Name: PIPELINE_NAME.foo - Namespace: concourse-TEAM_NAME - Type: Opaque - - Data - ==== - value: 32 bytes - ``` -1. - ``` - Name: foo - Namespace: concourse-TEAM_NAME - Type: Opaque - - Data - ==== - value: 32 bytes - ``` - -You can also have nested fields if the contents of the secret is JSON, which can -be accessed using `.` syntax (e.g. `((foo.bar))`). - -The prefix prepended to the namespace used by Concourse to search for secrets -(in the examples above, `concourse-`) can be changed by configuring the following -in the web node: - -```properties -CONCOURSE_KUBERNETES_NAMESPACE_PREFIX=some-other-prefix- -``` - -If an action is being run in a one-off build, Concourse will not include the -pipeline name in the secret that it looks for. - -## Configuring Kubernetes RBAC +### RBAC Permissions As the Web nodes need to retrieve secrets from namespaces that are not their own, they needs extra permissions to do so. @@ -77,16 +38,10 @@ For instance, if you have the following teams which you want to read secrets fro * team-a * team-b -Assuming the following [web node](../../install/running-web.md) configuration: - -```properties -CONCOURSE_KUBERNETES_NAMESPACE_PREFIX=myprefix- -``` - The web node must be able to get secrets from the following namespaces: -* `myprefix-team-a` -* `myprefix-team-b` +* `concourse-team-a` +* `concourse-team-b` To allow the web node to interpolate credentials for "team-a" and "team-b", we'd then need to create a few Kubernetes RBAC objects. @@ -146,7 +101,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: web-team-a - namespace: myprefix-team-a + namespace: concourse-team-a labels: app: web roleRef: @@ -166,7 +121,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: web-team-b - namespace: myprefix-team-b + namespace: concourse-team-b labels: app: web roleRef: @@ -203,8 +158,68 @@ spec: - name: web image: "concourse/concourse:latest" args: [ web ] - env: - - name: CONCOURSE_KUBERNETES_NAMESPACE_PREFIX - value: "myprefix-" # ... ``` + +## Credential lookup rules + +When resolving a parameter such as `((foo))`, Concourse will look for it in the following order in the namespace +configured for that team: + +1. + ``` + Name: PIPELINE_NAME.foo + Namespace: concourse-TEAM_NAME + Type: Opaque + + Data + ==== + value: 32 bytes + ``` +1. + ``` + Name: foo + Namespace: concourse-TEAM_NAME + Type: Opaque + + Data + ==== + value: 32 bytes + ``` + +You can also have nested fields if the contents of the secret is JSON, which can +be accessed using `.` syntax (e.g. `((foo.bar))`). + +If the action is being run in the context of a pipeline (e.g. a `check` or a step in a build of a job), the ATC will +first look in the pipeline path. If it's not found there, it will look in the team path. This allows credentials to be +scoped widely if they're common across many pipelines. + +When executing a one-off task, there is no pipeline: so in this case, only the namespace of `concourse-TEAM_NAME` is +searched. + +There are several ways to customize the lookup logic: + +1. Add a "shared namespace suffix", for secrets common to all teams. +2. Change the namespace prefix from `concourse-` to something else. + +Each of these can be controlled by Concourse command line flags, or environment variables. + +### Configuring a shared namespace suffix + +A "shared namespace suffix" can also be configured for credentials that you would like to share across all teams and +pipelines, foregoing the default team/pipeline namespacing. Use with care! + +```properties +CONCOURSE_KUBERNETES_NAMESPACE_SHARED_SUFFIX=shared-suffix +``` + +This namespace must exist when combined with the namespace prefix. The above configuration would correspond to +a namespace of `concourse-shared-suffix` with the default `concourse-` prefix. + +### Changing the namespace prefix + +The leading `concourse-` can be changed by specifying the following: + +```properties +CONCOURSE_KUBERNETES_NAMESPACE_PREFIX=some-other-prefix- +``` \ No newline at end of file From e58eb9bf8292974fb5948ea2d73d9758a7a68312 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 17:12:38 -0400 Subject: [PATCH 09/10] other: strip "the" from each cred manager --- docs/docs/operation/creds/aws-secrets.md | 2 +- docs/docs/operation/creds/aws-ssm.md | 2 +- docs/docs/operation/creds/conjur.md | 2 +- docs/docs/operation/creds/credhub.md | 2 +- docs/docs/operation/creds/id-token.md | 2 +- docs/docs/operation/creds/vault.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/operation/creds/aws-secrets.md b/docs/docs/operation/creds/aws-secrets.md index 9ddb3fc8..d928d234 100644 --- a/docs/docs/operation/creds/aws-secrets.md +++ b/docs/docs/operation/creds/aws-secrets.md @@ -1,5 +1,5 @@ --- -title: The AWS Secrets Manager credential manager +title: AWS Secrets Manager credential manager --- Concourse can be configured to pull credentials from [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). diff --git a/docs/docs/operation/creds/aws-ssm.md b/docs/docs/operation/creds/aws-ssm.md index c0dfe501..8fda376e 100644 --- a/docs/docs/operation/creds/aws-ssm.md +++ b/docs/docs/operation/creds/aws-ssm.md @@ -1,5 +1,5 @@ --- -title: The AWS Systems Manager credential manager +title: AWS Systems Manager credential manager --- Concourse can be configured to pull credentials diff --git a/docs/docs/operation/creds/conjur.md b/docs/docs/operation/creds/conjur.md index d119be26..044ea834 100644 --- a/docs/docs/operation/creds/conjur.md +++ b/docs/docs/operation/creds/conjur.md @@ -1,5 +1,5 @@ --- -title: The Conjur credential manager +title: Conjur credential manager --- Concourse can be configured to pull credentials from a [CyberArk Conjur](https://conjur.org/) instance. diff --git a/docs/docs/operation/creds/credhub.md b/docs/docs/operation/creds/credhub.md index db372ec8..03b4bb5b 100644 --- a/docs/docs/operation/creds/credhub.md +++ b/docs/docs/operation/creds/credhub.md @@ -1,5 +1,5 @@ --- -title: The CredHub credential manager +title: CredHub credential manager --- Concourse can be configured to pull credentials from a [CredHub](https://github.com/cloudfoundry/credhub) instance. diff --git a/docs/docs/operation/creds/id-token.md b/docs/docs/operation/creds/id-token.md index 8b706902..5d3be63e 100644 --- a/docs/docs/operation/creds/id-token.md +++ b/docs/docs/operation/creds/id-token.md @@ -1,5 +1,5 @@ --- -title: The IDToken credential manager +title: IDToken credential manager --- This idtoken credential manager is a bit special. It doesn't load any credentials from an external source but instead diff --git a/docs/docs/operation/creds/vault.md b/docs/docs/operation/creds/vault.md index 441e6690..55105fad 100644 --- a/docs/docs/operation/creds/vault.md +++ b/docs/docs/operation/creds/vault.md @@ -1,5 +1,5 @@ --- -title: The Vault credential manager +title: Vault credential manager --- Concourse can be configured to pull credentials from a [Vault](https://vaultproject.io/) instance. From baa2909d0dbe87b2455a12a61e62386c00d4a41f Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sun, 2 Aug 2026 17:42:43 -0400 Subject: [PATCH 10/10] docx: align Vault to new format and fix ssm link --- docs/docs/operation/creds/aws-ssm.md | 2 +- docs/docs/operation/creds/vault.md | 198 ++++++++++++++------------- 2 files changed, 102 insertions(+), 98 deletions(-) diff --git a/docs/docs/operation/creds/aws-ssm.md b/docs/docs/operation/creds/aws-ssm.md index 8fda376e..9a49b128 100644 --- a/docs/docs/operation/creds/aws-ssm.md +++ b/docs/docs/operation/creds/aws-ssm.md @@ -3,7 +3,7 @@ title: AWS Systems Manager credential manager --- Concourse can be configured to pull credentials -from [AWS Systems Manager]([https://aws.amazon.com/secrets-manager/](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html)). +from [AWS Systems Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html). ## Configuration diff --git a/docs/docs/operation/creds/vault.md b/docs/docs/operation/creds/vault.md index 55105fad..101b3d4f 100644 --- a/docs/docs/operation/creds/vault.md +++ b/docs/docs/operation/creds/vault.md @@ -2,112 +2,34 @@ title: Vault credential manager --- -Concourse can be configured to pull credentials from a [Vault](https://vaultproject.io/) instance. +Concourse can be configured to pull credentials from [Vault](https://vaultproject.io/). -To configure this, first configure the URL of your Vault server by setting the following env on -the [`web` node](../../install/running-web.md): +## Configuration + +To enable this credential manager, configure the following environment variables on the [ +`web` node](../../install/running-web.md): ```properties CONCOURSE_VAULT_URL=https://vault.example.com:8200 ``` -You may also need to configure the CA cert for Vault: +### TLS Configuration + +If your Vault instance is signed with TLS by a local Certificate Authority, you can use the following environment +variable: ```properties -CONCOURSE_VAULT_CA_CERT=path/to/ca.crt +CONCOURSE_VAULT_CA_CERT=/etc/ca.crt ``` You'll also need to configure how the `web` node authenticates with Vault - -see [Authenticating with Vault](#authenticating-with-vault) for +see [Authenticating with Vault](#configuring-authentication) for more details as that step is quite involved. -## Credential lookup rules - -Vault lets you organize secrets into hierarchies, which is useful for when they should be accessible for particular -pipelines or teams. When you have a parameter like `((foo))` in a pipeline definition, Concourse will (by default) look -for it in the following paths, in order: - -* `/concourse/TEAM_NAME/PIPELINE_NAME/foo` -* `/concourse/TEAM_NAME/foo` - -Vault credentials are actually key-value, so for `((foo))` Concourse will default to the field name value. You can -specify the field to grab via `.` syntax, e.g. `((foo.bar))`. - -If you have multiple, intermediate levels in your path, you can use the `/` separator to reach your intended field, -e.g. `((foo/bar/baz.qux))`. - -When executing a one-off task, there is no pipeline: so in this case, only the team path `/concourse/TEAM_NAME/foo` is -searched. - -There are several ways to customize the lookup logic: - -1. Add a "shared path", for secrets common to all teams. -2. Change the team- and pipeline-dependent path templates. -3. Change the path prefix from `/concourse` to something else. -4. Set a [Vault namespace](https://www.vaultproject.io/docs/enterprise/namespaces/) for isolation within a Vault - Enterprise installation. - -Each of these can be controlled by Concourse command line flags, or environment variables. - -### Configuring a shared path - -A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, -foregoing the default team/pipeline namespacing. Use with care! - -```properties -CONCOURSE_VAULT_SHARED_PATH=some-shared-path -``` - -This path must exist under the configured path prefix. The above configuration would correspond -to `/concourse/some-shared-path` with the default `/concourse` prefix. - -### Changing the path templates - -You can choose your own list of templates, which will expand to team- or pipeline-specific paths. These are subject to -the path prefix. By default, the templates used are: - -```properties -CONCOURSE_VAULT_LOOKUP_TEMPLATES=/{{.Team}}/{{.Pipeline}}/{{.Secret}},/{{.Team}}/{{.Secret}} -``` - -When secrets are to be looked up, these are evaluated subject to the configured path prefix, where `{{.Team}}` expands -to the current team, `{{.Pipeline}}` to the current pipeline (if any), and `{{.Secret}}` to the name of the secret. So -if the settings are: - -```properties -CONCOURSE_VAULT_PATH_PREFIX=/secrets -CONCOURSE_VAULT_LOOKUP_TEMPLATES=/{{.Team}}/concourse/{{.Pipeline}}/{{.Secret}},/{{.Team}}/concourse/{{.Secret}},/common/{{.Secret}} -``` - -and `((password))` is used in team `myteam` and pipeline `mypipeline`, Concourse will look for the following, in order: - -1. `/secrets/myteam/concourse/mypipeline/password` -2. `/secrets/myteam/concourse/password` -3. `/secrets/common/password` - -### Changing the path prefix - -The leading `/concourse` can be changed by specifying the following: - -```properties -CONCOURSE_VAULT_PATH_PREFIX=/some-other-prefix -``` - -### Using a Vault namespace - -If you are using Vault Enterprise, you can make secret lookups and authentication happen under a namespace. - -```properties -CONCOURSE_VAULT_NAMESPACE=chosen/namespace/path -``` - -This setting applies to all teams equally. - -## Configuring the secrets engine +### Configuring the secrets engine Concourse is currently limited to looking under a single path, meaning enabling only one secrets engine is -supported: `kv`, or `kv_v2`. This may change in the future - we're still collecting ideas -in [RFC #21](https://github.com/concourse/rfcs/pull/21). +supported: `kv`, or `kv_v2`. Using kv version 2 enables versioned secrets and the ability to restore previous versions or deleted secrets. Concourse will read the latest version of a secret at all times and if it is deleted it will appear as if the secret does not @@ -143,7 +65,7 @@ vault policy write concourse ./concourse-policy.hcl This configuration will allow Concourse to read all credentials under `/concourse`. This should match your configured path prefix. -### Enabling KV mount caching +#### Enabling KV mount caching When Concourse looks up secrets in Vault, it needs to determine which KV mount version (v1 or v2) contains the secret. By default, this information is fetched from Vault for each lookup. Enabling KV mount caching allows this information to be fetched once and reused for subsequent lookups, significantly reducing the number of Vault API calls. @@ -154,7 +76,7 @@ To enable KV mount caching, set the following environment variable on the `web` CONCOURSE_VAULT_ENABLE_KV_MOUNT_CACHE=true ``` -## Authenticating with Vault +### Configuring Authentication There are many ways to authenticate with a Vault server. The `web` node can be configured with either a token or an arbitrary auth backend and arbitrary auth params, so just about all of them should be configurable. @@ -162,7 +84,7 @@ arbitrary auth backend and arbitrary auth params, so just about all of them shou When the `web` node acquires a token, either by logging in with an auth backend or by being given one directly, it will continuously renew the token to ensure it doesn't expire. The renewal interval is half of the token's lease duration. -### Using a periodic token +#### Using a periodic token The simplest way to authenticate is by generating a periodic token: @@ -192,7 +114,7 @@ Periodic tokens are the quickest way to get started, but they have one fatal fla than the token's configured period, the token will expire and a new one will have to be created and configured. This can be avoided by using the [`approle` auth backend](#using-the-approle-auth-backend). -### Using the `userpass` auth backend +#### Using the `userpass` auth backend The [`userpass`](https://www.vaultproject.io/docs/auth/userpass.html) backend allows for _users_ (in this case, Concourse) to authenticate with a _user_ pre-configured in Vault. @@ -251,7 +173,7 @@ CONCOURSE_VAULT_AUTH_BACKEND="userpass" CONCOURSE_VAULT_AUTH_PARAM="username:concourse,password:<....>" ``` -### Using the `approle` auth backend +#### Using the `approle` auth backend The [`approle`](https://www.vaultproject.io/docs/auth/approle.html) backend allows for an _app_ (in this case, Concourse) to authenticate with a _role_ pre-configured in Vault. @@ -331,7 +253,7 @@ CONCOURSE_VAULT_AUTH_BACKEND="approle" CONCOURSE_VAULT_AUTH_PARAM="role_id:5f3420cd-3c66-2eff-8bcc-0e8e258a7d18,secret_id:f7ec2ac8-ad07-026a-3e1c-4c9781423155" ``` -### Using the `cert` auth backend +#### Using the `cert` auth backend The [`cert`](https://www.vaultproject.io/docs/auth/cert.html) auth method allows authentication using SSL/TLS client certificates. @@ -386,3 +308,85 @@ CONCOURSE_VAULT_CLIENT_KEY=vault-certs/concourse.key In this case no additional auth params are necessary, as the Vault's TLS auth backend will check the certificate against all roles if no name is specified. + +## Credential lookup rules + +Vault lets you organize secrets into hierarchies, which is useful for when they should be accessible for particular +pipelines or teams. When you have a parameter like `((foo))` in a pipeline definition, Concourse will (by default) look +for it in the following paths, in order: + +* `/concourse/TEAM_NAME/PIPELINE_NAME/foo` +* `/concourse/TEAM_NAME/foo` + +Vault credentials are actually key-value, so for `((foo))` Concourse will default to the field name value. You can +specify the field to grab via `.` syntax, e.g. `((foo.bar))`. + +If you have multiple, intermediate levels in your path, you can use the `/` separator to reach your intended field, +e.g. `((foo/bar/baz.qux))`. + +When executing a one-off task, there is no pipeline: so in this case, only the team path `/concourse/TEAM_NAME/foo` is +searched. + +There are several ways to customize the lookup logic: + +1. Add a "shared path", for secrets common to all teams. +2. Change the team- and pipeline-dependent path templates. +3. Change the path prefix from `/concourse` to something else. +4. Set a [Vault namespace](https://www.vaultproject.io/docs/enterprise/namespaces/) for isolation within a Vault + Enterprise installation. + +Each of these can be controlled by Concourse command line flags, or environment variables. + +### Configuring a shared path + +A "shared path" can also be configured for credentials that you would like to share across all teams and pipelines, +foregoing the default team/pipeline namespacing. Use with care! + +```properties +CONCOURSE_VAULT_SHARED_PATH=some-shared-path +``` + +This path must exist under the configured path prefix. The above configuration would correspond +to `/concourse/some-shared-path` with the default `/concourse` prefix. + +### Changing the path templates + +You can choose your own list of templates, which will expand to team- or pipeline-specific paths. These are subject to +the path prefix. By default, the templates used are: + +```properties +CONCOURSE_VAULT_LOOKUP_TEMPLATES=/{{.Team}}/{{.Pipeline}}/{{.Secret}},/{{.Team}}/{{.Secret}} +``` + +When secrets are to be looked up, these are evaluated subject to the configured path prefix, where `{{.Team}}` expands +to the current team, `{{.Pipeline}}` to the current pipeline (if any), and `{{.Secret}}` to the name of the secret. So +if the settings are: + +```properties +CONCOURSE_VAULT_PATH_PREFIX=/secrets +CONCOURSE_VAULT_LOOKUP_TEMPLATES=/{{.Team}}/concourse/{{.Pipeline}}/{{.Secret}},/{{.Team}}/concourse/{{.Secret}},/common/{{.Secret}} +``` + +and `((password))` is used in team `myteam` and pipeline `mypipeline`, Concourse will look for the following, in order: + +1. `/secrets/myteam/concourse/mypipeline/password` +2. `/secrets/myteam/concourse/password` +3. `/secrets/common/password` + +### Changing the path prefix + +The leading `/concourse` can be changed by specifying the following: + +```properties +CONCOURSE_VAULT_PATH_PREFIX=/some-other-prefix +``` + +### Using a Vault namespace + +If you are using Vault Enterprise, you can make secret lookups and authentication happen under a namespace. + +```properties +CONCOURSE_VAULT_NAMESPACE=chosen/namespace/path +``` + +This setting applies to all teams equally. \ No newline at end of file