From d4b674a2b7077d6497afcd03fbe292f32376415d Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 23 Jun 2026 18:03:02 -0800 Subject: [PATCH 01/15] PR-7509 method for pulling all metadata out of the cumulus RDS --- ctorm/cumulus-db-md-extract/README.md | 74 +++++++++++++++++++ .../export-granules-daily.sh | 47 ++++++++++++ .../export-granules-slice-daily.sql | 67 +++++++++++++++++ .../generate-slices-daily.sh | 31 ++++++++ 4 files changed, 219 insertions(+) create mode 100644 ctorm/cumulus-db-md-extract/README.md create mode 100644 ctorm/cumulus-db-md-extract/export-granules-daily.sh create mode 100644 ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql create mode 100644 ctorm/cumulus-db-md-extract/generate-slices-daily.sh diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md new file mode 100644 index 0000000..3b04473 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/README.md @@ -0,0 +1,74 @@ +# Cumulus DB MD Extractor + +The purpose of this is to get enough medata from the prod cumulus RDS db and into a Dynamo DB to be able to recreate +CNM-S messages. + +# How to use: + +## From the web console + +* Open a web console from Kion + * Open a tab with the secrets manager + * Find the secret with `db_login` in the name. This is where you find the host, user/pass, etc. + * Open a tab with the RDS + * Go to the query editor + * Open a CloudShell in anohter tab. + * Pick the VPC + * Pick the same subnet as the RDS + * Pick the default security group and the one(s) with `_rds_` in the name. + * TODO: see if we can do this with EC2 instead of cloudshell + +## From the shell + +### prepare the environment + +Create AWS credentials to upload to the CTORM bucket. + +```bash +mkdir .aws +vi .aws/credentials + +[ctorm] +aws_access_key_id = AKYEAH... +aws_secret_access_key = foofoofooofooo +``` + +Make some env vars: + +```bash + +read -s PGPASSWORD +export PGPASSWORD +export PGHOST='from secrets mgr' +export PGUSER='from secrets mgr' +export PGDATABASE='from secrets mgr' + +# this is the ctorm bucket and path info where the db dumps are stored +export CTORM_BUCKET='ctorm-dev-scratch' +export DUMP_SUBDIR='nisar' + +``` + +Get these files into the instance one way or another. + +* ctorm/cumulus-db-md-extract/export-granules-daily.sh +* ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql +* ctorm/cumulus-db-md-extract/generate-slices-daily.sh + +## Run the scripts + +```bash +bash generate-slices-daily.sh +``` + +This creates a `slices_daily.tsv` file. Basically a list of chunked work for export-granules-daily.sh to use. + +The following step will query the database and create gzipped ljson files and upload them to S3: + +```bash +bash export-granules-daily.sh +``` + +If running in cloudshell, you need to put some sort of input into the console every 10 minutes or so, or AWS will kill +the session. +Really should try to create a EC2 next time. diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh new file mode 100644 index 0000000..ec07cc7 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -0,0 +1,47 @@ +set -euo pipefail + +S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" + +while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do + if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then + echo "Skipping malformed slice row: collection=${COLLECTION:-} yyyymmdd=${YYYYMMDD:-}" >&2 + continue + fi + + YYYYMM="${YYYYMMDD:0:6}" + SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') + S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + + echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" + echo "Destination: ${S3_URI}" + + psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --set=ON_ERROR_STOP=1 \ + --set=collection="$COLLECTION" \ + --set=yyyymmdd="$YYYYMMDD" \ + --file=export-granules-slice-daily.sql \ + | awk -v label="${COLLECTION} ${YYYYMMDD}" ' + { + print; + count++; + if (count % 1000 == 0) { + printf("[%s] exported %d JSONL rows\n", label, count) > "/dev/stderr"; + fflush("/dev/stderr"); + } + } + END { + printf("[%s] export complete: %d JSONL rows\n", label, count) > "/dev/stderr"; + } + ' \ + | gzip -c \ + | aws --profile=ctorm s3 cp - "$S3_URI" + + echo "Uploaded ${S3_URI}" + +done < slices_daily.tsv diff --git a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql new file mode 100644 index 0000000..b23af06 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql @@ -0,0 +1,67 @@ +\pset tuples_only on +\pset format unaligned +\pset pager off + +WITH export_rows AS ( + SELECT + g.granule_id, + c.name AS collection, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMM') AS yyyymm, + to_char( + g.beginning_date_time AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) AS beginning_date_time, + jsonb_agg( + jsonb_build_object( + 'name', f.file_name, + 'type', f.type, + 'uri', 's3://' || f.bucket || '/' || f.key, + 'size', f.file_size, + 'checksum', f.checksum_value, + 'checksumType', 'md5' + ) + ORDER BY f.file_name + ) AS files + FROM public.granules g + JOIN public.collections c + ON c.cumulus_id = g.collection_cumulus_id + JOIN public.files f + ON f.granule_cumulus_id = g.cumulus_id + AND f.checksum_value IS NOT NULL + WHERE g.status = 'completed' + AND c.name = :'collection' + AND g.beginning_date_time >= to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') AT TIME ZONE 'UTC' + AND g.beginning_date_time < ( + to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') + interval '1 month' + ) AT TIME ZONE 'UTC' + GROUP BY + g.granule_id, + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMM'), + to_char( + g.beginning_date_time AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) +) +SELECT jsonb_strip_nulls( + jsonb_build_object( + 'pk', collection || '#' || yyyymm, + 'sk', beginning_date_time || '#' || granule_id, + + 'gsi1pk', granule_id, + 'gsi1sk', collection || '#' || yyyymm, + + 'collection', collection, + 'yyyymm', yyyymm, + 'beginning_date_time', beginning_date_time, + 'granule_id', granule_id, + 'load_test_count', 0, + + 'n', granule_id, + 'c', collection, + 'cv', '1', + 'f', files + ) +) +FROM export_rows +ORDER BY beginning_date_time, granule_id; \ No newline at end of file diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh new file mode 100644 index 0000000..60b9407 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -0,0 +1,31 @@ +psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --tuples-only \ + --no-align \ + --field-separator $'\t' \ + --set=ON_ERROR_STOP=1 \ + --command " + SELECT + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') AS yyyymmdd, + count(DISTINCT g.cumulus_id) AS granule_count, + count(f.cumulus_id) AS file_count + FROM public.granules g + JOIN public.collections c + ON c.cumulus_id = g.collection_cumulus_id + JOIN public.files f + ON f.granule_cumulus_id = g.cumulus_id + AND f.checksum_value IS NOT NULL + WHERE g.status = 'completed' + GROUP BY + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') + ORDER BY + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); + " > slices_daily.tsv From a32bd6c935465780d686e86086796dc25445152b Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 24 Jun 2026 11:53:25 -0800 Subject: [PATCH 02/15] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/README.md | 33 + .../terraform/.terraform.lock.hcl | 45 + .../terraform/codebuild-db-md-extract.tf | 296 ++++ .../terraform/nisar-prod.tfvars | 22 + .../terraform/opera-prod.tfvars | 22 + .../terraform/opera-uat.tfvars | 23 + .../terraform/terraform.tfstate | 360 +++++ .../terraform/terraform.tfvars | 1 + ctorm/infra/terraform/main.tf | 87 ++ ctorm/infra/terraform/outputs.tf | 4 + ctorm/infra/terraform/placeholder-lambda.zip | Bin 339 -> 344 bytes ctorm/infra/terraform/terraform.tfstate | 238 +++- .../infra/terraform/terraform.tfstate.backup | 1226 +++++++++++++++++ ctorm/infra/terraform/variables.tf | 5 + 14 files changed, 2344 insertions(+), 18 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl create mode 100644 ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf create mode 100644 ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars create mode 100644 ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars create mode 100644 ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform.tfstate create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform.tfvars create mode 100644 ctorm/infra/terraform/terraform.tfstate.backup diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index 3b04473..65c4d70 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -72,3 +72,36 @@ bash export-granules-daily.sh If running in cloudshell, you need to put some sort of input into the console every 10 minutes or so, or AWS will kill the session. Really should try to create a EC2 next time. + +## Using Codebuild + +### Terraform + +```bash +```bash +export AWS_PROFILE="cumulus-uat-6921" +export VARFILE=opera-uat.tfvars +terraform init + +terraform plan \ + -var-file="${VARFILE}" + +terraform apply \ + -var-file="${VARFILE}" + +terraform destroy + + +``` + +``` + +```bash +aws codebuild create-project \ + --name "OneTimeDataPull" \ + --source '{"type": "S3", "location": "ctorm-dev-scratch/codebuild/codebuild-noop.zip"}' \ + --environment '{"type": "LINUX_CONTAINER", "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", "computeType": "BUILD_GENERAL1_SMALL"}' \ + --artifacts '{"type": "NO_ARTIFACTS"}' \ + --service-role "arn:aws:iam::123456789012:role/service-role/YourCodeBuildRole" \ + --timeout-in-minutes 240 +``` diff --git a/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl b/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..8e5ba2e --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl @@ -0,0 +1,45 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.0" + hashes = [ + "h1:WB6H5ksIZiyq1lQlD/PWeh+tn4FLsbSjVnRW3+4xe2Y=", + "zh:0d14713fdc259fb377d0b899ad3c650a34194bd52194c863303ef22a65a580e2", + "zh:369b56040c7a8085d04e7e8ffac1e2b321a3170e502f788819bc34b868ec016f", + "zh:4d1a3b983ed6af5a52bfe12794674ae55cbadfa6021b37106ade68b433ad216a", + "zh:5c547549e26e083573c78a966ca68ce6d7df6bb8f3948f66a575f07da46b74ea", + "zh:6de093e62a975eb19a5e3017ce38e6e3cb639c17b79648d2000e0a8348f0e997", + "zh:7267936c2cdbc448efeb594d73e6b56a53d6a7ae14fe88cdd2a4133adc3302f0", + "zh:7482f023050ed426b4b45116e1761643bc33b1fd4ce4a6fab207ae2571f35940", + "zh:76bbd93b234e5a2927d98b511d86565700f549b570871a194c35f944b96cefb7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:c6afc4bc1f002bac9c173007dd4da05fde788cd14c2916089f958c33fedb0dfa", + "zh:d3ba40bd806a3a08e9237dece679193c99afb2085de6b45d7f5d1f673cfcd368", + "zh:e1ad7ded53ecd6f0e5b473a3b44eae2b2e885653a56050ab583d387332be02e4", + "zh:e93e78575ce82be6084cc153c24ba8f385dc8d6880888ee66e918460c870953d", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.52.0" + hashes = [ + "h1:lpXqosKH8yAahK3SA1P5Pdy1ziXJcY+blUidY0q9yGk=", + "zh:1ab1d78f2336fed42b4e13fa0077a0be9d86a7899897cda5b9f1a60051ec2e93", + "zh:1df11f5f252030803939a1c778931dbdcee1b1070b38b98d9a8cbfc26c2aeb8e", + "zh:20ec9af03c0c1f2f8582a8805d43a4cd1a0a65082308e00f025d87605715a88f", + "zh:2d5562ed0e7cb0892fb537c7989d25fdde1d0ac1f3a768ba15fa087985f2a0f5", + "zh:40d64f668961a172355c3d11d258e19172ffafb421c40d89266b129ed8f92a5d", + "zh:497792bccc33001247473bc32a148c067982cb3d245b8f3610afb920635d2235", + "zh:8011f9167082af74ed9257582a83d54e889388656597721b2691797f1dd6fb58", + "zh:8b10d1bbca51b0da1e1be0967ce75b039f2d5d86f2ca7339994a92d35cdf47a8", + "zh:9549647dbf7c913512c26fed6badd7bf24a29a36c2bd308536849303a85427da", + "zh:97c4b726cdbe48166f4b9e6a1230312a7933dc19dd139d941ca6aca86706eb33", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a484bc8166278b546bfe53698fa9e0a919dffe3bfda3c8bf8a0fc6811b5b9e66", + "zh:aa96e76bd9f93e5395a553fccec485554a74acae46b3834b1296374eba4f010f", + "zh:cf290ee3d0dfe91b596445f860440d9f18826f7395ff785f83f70d36cedadd1c", + "zh:d56b6a79207663673f66d9ed378d705c1bd1b4d117eeb5e2be3938cf1b75b7be", + "zh:febef068317f8e49bd438ccdf2f54345fc3bc4b80a2699d9ab3c449d7e521d8e", + ] +} diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf new file mode 100644 index 0000000..61c2f10 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -0,0 +1,296 @@ +# codebuild-db-md-extract.tf + +variable "name_prefix" { + type = string + default = "ctorm" +} + +variable "aws_region" { + type = string + default = "us-west-2" +} + +variable "db_md_extract_secret_arn" { + type = string + description = "ARN of the Secrets Manager secret containing host, username, password, and database." +} + +variable "db_md_extract_vpc_id" { + type = string + description = "VPC where CodeBuild should run to reach the RDS instance." +} + +variable "db_md_extract_subnet_ids" { + type = list(string) + description = "Private subnet IDs that can reach the RDS instance." +} + +variable "db_md_extract_security_group_ids" { + type = list(string) + description = "Security group IDs for CodeBuild. These must allow egress to the RDS port and be allowed by the RDS SG." +} + +variable "db_md_extract_dump_subdir" { + type = string + default = "nisar" +} + +variable "db_md_extract_target_bucket" { + type = string + default = "" + description = "Destination S3 bucket for exported dump files." +} + +variable "db_md_extract_target_role_arn" { + type = string + description = "Role ARN in the target account that CodeBuild can assume to write exported files." +} + +locals { + db_md_extract_project_name = "${var.name_prefix}-cumulus-db-md-extract" + db_md_extract_source_key = "codebuild/cumulus-db-md-extract.zip" +} + +data "archive_file" "cumulus_db_md_extract" { + type = "zip" + source_dir = "${path.module}/../../cumulus-db-md-extract" + output_path = "${path.module}/cumulus-db-md-extract.zip" +} + +resource "aws_s3_object" "cumulus_db_md_extract_source" { + bucket = aws_s3_bucket.scratch.bucket + key = local.db_md_extract_source_key + source = data.archive_file.cumulus_db_md_extract.output_path + source_hash = data.archive_file.cumulus_db_md_extract.output_base64sha256 +} + +data "aws_iam_policy_document" "codebuild_db_md_extract_assume_role" { + statement { + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "Service" + + identifiers = [ + "codebuild.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role" "codebuild_db_md_extract" { + name = "${local.db_md_extract_project_name}-role" + assume_role_policy = data.aws_iam_policy_document.codebuild_db_md_extract_assume_role.json +} + +resource "aws_cloudwatch_log_group" "codebuild_db_md_extract" { + name = "/aws/codebuild/${local.db_md_extract_project_name}" + retention_in_days = 14 +} + +data "aws_iam_policy_document" "codebuild_db_md_extract" { + statement { + sid = "WriteCodeBuildLogs" + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = [ + "${aws_cloudwatch_log_group.codebuild_db_md_extract.arn}:*", + ] + } + + statement { + sid = "ReadBuildSourceFromScratchBucket" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + ] + + resources = [ + "${aws_s3_bucket.scratch.arn}/${local.db_md_extract_source_key}", + ] + } + + statement { + sid = "ListScratchBucketForBuildSource" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.scratch.arn, + ] + + condition { + test = "StringLike" + variable = "s3:prefix" + + values = [ + "codebuild/*", + ] + } + } + + statement { + sid = "ReadDatabaseSecret" + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ] + + resources = [ + var.db_md_extract_secret_arn, + ] + } + + statement { + sid = "AssumeTargetAccountUploadRole" + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + resources = [ + var.db_md_extract_target_role_arn, + ] + } + + statement { + sid = "DescribeNetworkForVpcBuild" + effect = "Allow" + + actions = [ + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:CreateNetworkInterface", + ] + + resources = [ + "*", + ] + } + + statement { + sid = "CreateCodeBuildNetworkInterfacePermission" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterfacePermission", + ] + + resources = [ + "arn:aws:ec2:${var.aws_region}:*:network-interface/*", + ] + + condition { + test = "StringEquals" + variable = "ec2:AuthorizedService" + + values = [ + "codebuild.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role_policy" "codebuild_db_md_extract" { + name = "${local.db_md_extract_project_name}-policy" + role = aws_iam_role.codebuild_db_md_extract.id + policy = data.aws_iam_policy_document.codebuild_db_md_extract.json +} + +resource "aws_codebuild_project" "cumulus_db_md_extract" { + name = local.db_md_extract_project_name + description = "One-time Cumulus RDS metadata export to S3" + service_role = aws_iam_role.codebuild_db_md_extract.arn + build_timeout = 480 + + artifacts { + type = "NO_ARTIFACTS" + } + + source { + type = "S3" + location = "${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + buildspec = "buildspec.yaml" + } + + environment { + type = "LINUX_CONTAINER" + image = "aws/codebuild/amazonlinux2-x86_64-standard:5.0" + compute_type = "BUILD_GENERAL1_SMALL" + + environment_variable { + name = "AWS_DEFAULT_REGION" + value = var.aws_region + } + + environment_variable { + name = "CTORM_BUCKET" + value = var.db_md_extract_target_bucket + } + + environment_variable { + name = "DUMP_SUBDIR" + value = var.db_md_extract_dump_subdir + } + + environment_variable { + name = "DB_SECRET_ARN" + value = var.db_md_extract_secret_arn + } + + environment_variable { + name = "TARGET_ROLE_ARN" + value = var.db_md_extract_target_role_arn + } + } + + vpc_config { + vpc_id = var.db_md_extract_vpc_id + subnets = var.db_md_extract_subnet_ids + security_group_ids = var.db_md_extract_security_group_ids + } + + logs_config { + cloudwatch_logs { + group_name = aws_cloudwatch_log_group.codebuild_db_md_extract.name + status = "ENABLED" + } + } + + depends_on = [ + aws_iam_role_policy.codebuild_db_md_extract, + aws_s3_object.cumulus_db_md_extract_source, + ] +} + +output "cumulus_db_md_extract_codebuild_project_name" { + value = aws_codebuild_project.cumulus_db_md_extract.name +} + +output "cumulus_db_md_extract_codebuild_role_arn" { + value = aws_iam_role.codebuild_db_md_extract.arn +} + +output "cumulus_db_md_extract_source_s3_uri" { + value = "s3://${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" +} diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars new file mode 100644 index 0000000..de3cc0a --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -0,0 +1,22 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" + +db_md_extract_vpc_id = "vpc-..." + +db_md_extract_subnet_ids = [ + "subnet-...", + "subnet-...", +] + +db_md_extract_security_group_ids = [ + "sg-...", +] + +db_md_extract_source_bucket = "" + +db_md_extract_target_bucket = "ctorm-dev-scratch" + +db_md_extract_target_role_arn = "arn:aws:iam:::role/" + +db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars new file mode 100644 index 0000000..de3cc0a --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -0,0 +1,22 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" + +db_md_extract_vpc_id = "vpc-..." + +db_md_extract_subnet_ids = [ + "subnet-...", + "subnet-...", +] + +db_md_extract_security_group_ids = [ + "sg-...", +] + +db_md_extract_source_bucket = "" + +db_md_extract_target_bucket = "ctorm-dev-scratch" + +db_md_extract_target_role_arn = "arn:aws:iam:::role/" + +db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars new file mode 100644 index 0000000..b27da44 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars @@ -0,0 +1,23 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + +db_md_extract_vpc_id = "vpc-080322e163a0e7390" + +db_md_extract_subnet_ids = [ + "subnet-0ac04d1e36d0a50ea", +] + +db_md_extract_security_group_ids = [ + "sg-03beee748d2a43cc9", + "sg-0afdcb46b1a38db46", + "sg-06255aca3a87784a0", +] + +db_md_extract_source_bucket = "ctorm-dev-scratch" + +db_md_extract_target_bucket = "ctorm-dev-scratch" + +db_md_extract_target_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + +db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate new file mode 100644 index 0000000..a6e1dad --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate @@ -0,0 +1,360 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 6, + "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "baed98d70a081d5ddd95b21dade6d5b45112af30", + "output_base64sha256": "bS+Xh1vcj006QvtACc+o8twj7kCPntn13M4MAkySTvw=", + "output_base64sha512": "bB6iw4IJxB+G1ieprcxj96PdqyZbKT6ahc6BbPDrK5ONIk43JvXrz0xbCXud8vubsLC5o46yqXIhXQUqBtMYjw==", + "output_file_mode": null, + "output_md5": "c5beacdcad3c55176fedde20acb8616f", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "baed98d70a081d5ddd95b21dade6d5b45112af30", + "output_sha256": "6d2f97875bdc8f4d3a42fb4009cfa8f2dc23ee408f9ed9f5dcce0c024c924efc", + "output_sha512": "6c1ea2c38209c41f86d627a9adcc63f7a3ddab265b293e9a85ce816cf0eb2b938d224e3726f5ebcf4c5b097b9df2fb9bb0b0b9a38eb2a972215d052a06d3188f", + "output_size": 5074, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3670884144", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeNetworkInterfaces\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"ManageCodeBuildNetworkInterfaces\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Resource\":\"*\"},{\"Sid\":\"ManageCodeBuildNetworkInterfaces\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormUploadRole" + }, + { + "actions": [ + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "ManageCodeBuildNetworkInterfaces" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T19:45:21Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGEV76IAVBEI" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"ManageCodeBuildNetworkInterfaces\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars b/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars new file mode 100644 index 0000000..d078194 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars @@ -0,0 +1 @@ +aws_region = "us-west-2" diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index ebcc003..c84e99c 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -280,3 +280,90 @@ resource "aws_iam_role_policy" "scratch_bucket_access" { role = aws_iam_role.cnm-sender.id # assumes this role already exists policy = data.aws_iam_policy_document.allow_scratch_bucket_access.json } + + +data "aws_iam_policy_document" "cumulus_db_md_extract_upload_assume_role" { + statement { + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "AWS" + + identifiers = [ + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::082931748743:root", + "arn:aws:iam::510296831643:root" + ] + } + + condition { + test = "ArnEquals" + variable = "aws:PrincipalArn" + + values = [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + ] + } + } +} + +resource "aws_iam_role" "cumulus_db_md_extract_upload" { + name = "${var.name_prefix}-cumulus-db-md-extract-upload" + assume_role_policy = data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role.json +} + +data "aws_iam_policy_document" "cumulus_db_md_extract_upload" { + statement { + sid = "UploadExtractedGranules" + effect = "Allow" + + actions = [ + "s3:PutObject", + "s3:AbortMultipartUpload", + ] + + resources = [ + "${aws_s3_bucket.scratch.arn}/cumulus-granules/*", + ] + } + + statement { + sid = "ListDestinationPrefix" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.scratch.arn, + ] + + condition { + test = "StringLike" + variable = "s3:prefix" + + values = [ + "cumulus-granules/*", + ] + } + } +} + +resource "aws_iam_role_policy" "cumulus_db_md_extract_upload" { + name = "${var.name_prefix}-cumulus-db-md-extract-upload" + role = aws_iam_role.cumulus_db_md_extract_upload.id + policy = data.aws_iam_policy_document.cumulus_db_md_extract_upload.json +} diff --git a/ctorm/infra/terraform/outputs.tf b/ctorm/infra/terraform/outputs.tf index 11ca41e..6e734a2 100644 --- a/ctorm/infra/terraform/outputs.tf +++ b/ctorm/infra/terraform/outputs.tf @@ -29,3 +29,7 @@ output "prepare_ec2_role_arn" { output "scratch_bucket" { value = aws_s3_bucket.scratch.bucket } + +output "cumulus_db_md_extract_upload_role_arn" { + value = aws_iam_role.cumulus_db_md_extract_upload.arn +} diff --git a/ctorm/infra/terraform/placeholder-lambda.zip b/ctorm/infra/terraform/placeholder-lambda.zip index 767a6863d86904218024568cb08edb144365f8de..e71ce41ec254e01f994788da16c3a3105f7b80fd 100644 GIT binary patch delta 248 zcmcc2bc1PvL;a@P8^b1h`WkgB?Ax6azG0EYa(VgBXXoG4S4^w$YdjLp=h-1Be=xFM zW^Lsr6RRH9c@sbJ&S6TeIi0G@z%b1sSWmCabW?oL$9<2yPliq~jX9cc=j#2&U}m_R z+u@5RrJPnxI9JRQ;yZ(Tr?tu4ywH5PEtgKcaMnJ3wkM~bk2f-{zkY{v@olZkrqy2$ zTW(B0e)5b-;;H|6`!84DeRc0nx%E~4{rNvMyf?OW%v3)3?DYqwYzd{uTLWv@7#RNl t5AbH^*t=P9-ZcgWhD1gNh5&CyCJ_b^1`Z$u0l3^`1x8&q9-s;a1^|x)X7K<3 delta 243 zcmcb?beU;_L;b6=HCIc5CeL&gxF5SsJ0h^m&))9YZ2QZ5T26eJ#I*9g%%mV@yMx94 z&Btdv|ORE=n@940v>9OrR{^TAD>s*)h z-W)3(tb7dv43*NW%2!B5&%S0Gk=(nvJ?i;v$vgH{C0EY>=)WDBxP5D(?^g5ua~7X@ zq`I@y&8MdH_uRX;Z*6;7!?~4o!3{i{>3<2JZOd<>-3>-iR0&uy>s*Jj9>_8O^3;-(=Y?%N6 diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index bb737dd..9137406 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,9 +1,13 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 23, + "serial": 28, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { + "cumulus_db_md_extract_upload_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "type": "string" + }, "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" @@ -49,19 +53,19 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "f09bd3c52a0ccf18e91ad5fb208898ddd415b903", - "output_base64sha256": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", - "output_base64sha512": "YA5YdhMOE4f3g+N2rFuq2vpYa8KXnFSH+C7bkq9ixIvMxZqDxZBI/MR5E9vrLJ8hLk/mV0kMKS1wQ269QdyIFg==", + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", "output_file_mode": null, - "output_md5": "de3b86299704b85f7d5621f69bdb2914", + "output_md5": "94564c5dd9caf22baaf170b00693c497", "output_path": "./placeholder-lambda.zip", - "output_sha": "f09bd3c52a0ccf18e91ad5fb208898ddd415b903", - "output_sha256": "bbf01937538c639828cf7e2bd3fb95d4cd105dd13563565b0ced59b2ef90e291", - "output_sha512": "600e5876130e1387f783e376ac5baadafa586bc2979c5487f82edb92af62c48bccc59a83c59048fcc47913dbeb2c9f212e4fe657490c292d70436ebd41dc8816", - "output_size": 339, + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, "source": [ { - "content": "import json\nimport os\n\ndef handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", "filename": "index.py" } ], @@ -211,6 +215,133 @@ } ] }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "923578907", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:AbortMultipartUpload", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*" + ], + "sid": "UploadExtractedGranules" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "cumulus-granules/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch" + ], + "sid": "ListDestinationPrefix" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2147741451", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnEquals", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -565,7 +696,7 @@ "name_prefix": "", "path": "/", "role": "ctorm-dev-prepare-ec2-role", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AIPA4VW62R3RCASOXJGOK" }, @@ -621,6 +752,45 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T19:29:26Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cumulus-db-md-extract-upload", + "inline_policy": [ + { + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RJW53W2D5X" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -636,14 +806,19 @@ "description": "", "force_detach_policies": false, "id": "ctorm-dev-prepare-ec2-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-dev-prepare-ec2-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-dev-prepare-ec2-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROA4VW62R3RKO3MKUI7D" }, @@ -684,6 +859,32 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "role": "ctorm-dev-cumulus-db-md-extract-upload" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cumulus_db_md_extract_upload", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -750,7 +951,7 @@ "x86_64" ], "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", - "code_sha256": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", + "code_sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", "code_signing_config_arn": "", "dead_letter_config": [], "description": "", @@ -759,6 +960,7 @@ "variables": { "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "LOG_LEVEL": "INFO", "TABLE_NAME": "ctorm-dev-state" } } @@ -771,13 +973,13 @@ "file_system_config": [], "filename": "./placeholder-lambda.zip", "function_name": "ctorm-dev-cnm-sender", - "handler": "index.handler", + "handler": "load_tester.lambda_handler", "id": "ctorm-dev-cnm-sender", "image_config": [], "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", - "last_modified": "2026-06-10T20:02:16.662+0000", + "last_modified": "2026-06-24T19:29:31.000+0000", "layers": [], "logging_config": [ { @@ -804,8 +1006,8 @@ "signing_profile_version_arn": "", "skip_destroy": false, "snap_start": [], - "source_code_hash": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", - "source_code_size": 339, + "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "source_code_size": 344, "tags": {}, "tags_all": {}, "timeout": 120, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup new file mode 100644 index 0000000..0f77000 --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -0,0 +1,1226 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 27, + "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", + "outputs": { + "granules_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "type": "string" + }, + "granules_queue_arn": { + "value": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "type": "string" + }, + "granules_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "type": "string" + }, + "prepare_ec2_instance_profile_name": { + "value": "ctorm-dev-prepare-ec2-profile", + "type": "string" + }, + "prepare_ec2_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-dev-scratch", + "type": "string" + }, + "state_table_name": { + "value": "ctorm-dev-state", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-dev-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", + "output_file_mode": null, + "output_md5": "94564c5dd9caf22baaf170b00693c497", + "output_path": "./placeholder-lambda.zip", + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, + "source": [ + { + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "allow_scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "632361540", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:ListBucket\",\n \"s3:GetObject\",\n \"s3:DeleteObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::ctorm-dev-scratch/*\",\n \"arn:aws:s3:::ctorm-dev-scratch\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch", + "arn:aws:s3:::ctorm-dev-scratch/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2455266930", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "923578907", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:AbortMultipartUpload", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*" + ], + "sid": "UploadExtractedGranules" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "cumulus-granules/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch" + ], + "sid": "ListDestinationPrefix" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2147741451", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnEquals", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1999618104", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sqs:GetQueueAttributes", + "sqs:SendMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::asf-cumulus-dev-opera-products", + "arn:aws:s3:::asf-cumulus-prod-opera-products", + "arn:aws:s3:::asf-cumulus-test-opera-products", + "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", + "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", + "arn:aws:s3:::sds-n-cumulus-test-nisar-products" + ], + "sid": "" + }, + { + "actions": [ + "s3:GetObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::asf-cumulus-dev-opera-products/*", + "arn:aws:s3:::asf-cumulus-prod-opera-products/*", + "arn:aws:s3:::asf-cumulus-test-opera-products/*", + "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", + "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", + "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "prepare_ec2_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2851119427", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "ec2.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-dev-every-minute", + "is_enabled": false, + "name": "ctorm-dev-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "appsync_target": [], + "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [], + "event_bus_name": "default", + "force_destroy": false, + "http_target": [], + "id": "ctorm-dev-every-minute-terraform-20260610200222615700000001", + "input": "{\"source\":\"eventbridge\"}", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "", + "rule": "ctorm-dev-every-minute", + "run_command_targets": [], + "sagemaker_pipeline_target": [], + "sqs_target": [], + "target_id": "terraform-20260610200222615700000001" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender", + "id": "/aws/lambda/ctorm-dev-cnm-sender", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/lambda/ctorm-dev-cnm-sender", + "name_prefix": "", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "state", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", + "attribute": [ + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [], + "hash_key": "pk", + "id": "ctorm-dev-state", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-dev-state", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", + "create_date": "2026-06-19T01:00:34Z", + "id": "ctorm-dev-prepare-ec2-profile", + "name": "ctorm-dev-prepare-ec2-profile", + "name_prefix": "", + "path": "/", + "role": "ctorm-dev-prepare-ec2-role", + "tags": {}, + "tags_all": {}, + "unique_id": "AIPA4VW62R3RCASOXJGOK" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.prepare_ec2", + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cnm-sender-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-10T20:01:24Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cnm-sender-role", + "inline_policy": [ + { + "name": "ctorm-dev-cnm-sender-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + }, + { + "name": "ctorm-dev-scratch-access", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cnm-sender-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RPOQ35FULV" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T19:29:26Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cumulus-db-md-extract-upload", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROA4VW62R3RJW53W2D5X" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-19T01:00:33Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-prepare-ec2-role", + "inline_policy": [ + { + "name": "ctorm-dev-prepare-ec2-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-prepare-ec2-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RKO3MKUI7D" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", + "name": "ctorm-dev-cnm-sender-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "role": "ctorm-dev-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "role": "ctorm-dev-cumulus-db-md-extract-upload" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cumulus_db_md_extract_upload", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", + "name": "ctorm-dev-prepare-ec2-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", + "role": "ctorm-dev-prepare-ec2-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.prepare_ec2", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.prepare_ec2", + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cnm-sender-role:ctorm-dev-scratch-access", + "name": "ctorm-dev-scratch-access", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}", + "role": "ctorm-dev-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cnm-sender", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.allow_scratch_bucket_access", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", + "code_sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [ + { + "variables": { + "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", + "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "LOG_LEVEL": "INFO", + "TABLE_NAME": "ctorm-dev-state" + } + } + ], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "./placeholder-lambda.zip", + "function_name": "ctorm-dev-cnm-sender", + "handler": "load_tester.lambda_handler", + "id": "ctorm-dev-cnm-sender", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", + "kms_key_arn": "", + "last_modified": "2026-06-24T19:29:31.000+0000", + "layers": [], + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/ctorm-dev-cnm-sender", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::871271927522:role/ctorm-dev-cnm-sender-role", + "runtime": "python3.12", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "source_code_size": 344, + "tags": {}, + "tags_all": {}, + "timeout": 120, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_eventbridge", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "ctorm-dev-cnm-sender", + "function_url_auth_type": null, + "id": "AllowExecutionFromEventBridge", + "principal": "events.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "statement_id": "AllowExecutionFromEventBridge", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "scratch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch", + "bucket": "ctorm-dev-scratch", + "bucket_domain_name": "ctorm-dev-scratch.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "ctorm-dev-scratch.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [ + { + "id": "a7e8aeefe5d49ed5062fbc1a1d29f8b05e5150edf37ef1739efda49e5443a5ee", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-dev-scratch", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Name": "ctorm-dev-scratch" + }, + "tags_all": { + "Name": "ctorm-dev-scratch" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": {}, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": {}, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/terraform/variables.tf b/ctorm/infra/terraform/variables.tf index e661b9b..030971c 100644 --- a/ctorm/infra/terraform/variables.tf +++ b/ctorm/infra/terraform/variables.tf @@ -31,3 +31,8 @@ variable "prepare_source_bucket_names" { "sds-n-cumulus-test-nisar-products", ] } + +# variable "cumulus_db_md_extract_remote_codebuild_role_arn" { +# type = string +# description = "CodeBuild service role ARN from the remote/RDS account." +# } From c23bb8a28e7c767f759c08aabdbbe200012e4db4 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 24 Jun 2026 17:53:24 -0800 Subject: [PATCH 03/15] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/README.md | 47 +- ctorm/cumulus-db-md-extract/buildspec.yaml | 78 +++ .../export-granules-daily.sh | 44 +- .../export-granules-slice-daily.sql | 7 +- .../generate-slices-daily.sh | 66 +++ .../terraform/codebuild-db-md-extract.tf | 86 ++- .../terraform/nisar-prod.tfvars | 6 +- .../terraform/opera-prod.tfvars | 8 +- .../terraform/opera-uat.tfvars | 10 +- .../terraform/terraform.tfstate | 493 ++++++++++++++++-- ctorm/infra/terraform/main.tf | 15 +- ctorm/infra/terraform/terraform.tfstate | 22 +- 12 files changed, 753 insertions(+), 129 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/buildspec.yaml diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index 65c4d70..f3556bd 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -77,7 +77,6 @@ Really should try to create a EC2 next time. ### Terraform -```bash ```bash export AWS_PROFILE="cumulus-uat-6921" export VARFILE=opera-uat.tfvars @@ -94,14 +93,44 @@ terraform destroy ``` -``` +### Codebuild ```bash -aws codebuild create-project \ - --name "OneTimeDataPull" \ - --source '{"type": "S3", "location": "ctorm-dev-scratch/codebuild/codebuild-noop.zip"}' \ - --environment '{"type": "LINUX_CONTAINER", "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", "computeType": "BUILD_GENERAL1_SMALL"}' \ - --artifacts '{"type": "NO_ARTIFACTS"}' \ - --service-role "arn:aws:iam::123456789012:role/service-role/YourCodeBuildRole" \ - --timeout-in-minutes 240 +export AWS_PROFILE="cumulus-uat-6921" +export AWS_REGION="us-west-2" +export CODEBUILD_PROJECT="$(terraform output -raw cumulus_db_md_extract_codebuild_project_name)" + +# start build: +BUILD_ID="$( + aws codebuild start-build \ + --project-name "${CODEBUILD_PROJECT}" \ + --region "${AWS_REGION}" \ + --query 'build.id' \ + --output text +)" + +echo "${BUILD_ID}" + +# get status: +aws codebuild batch-get-builds \ + --ids "${BUILD_ID}" \ + --region "${AWS_REGION}" \ + --query 'builds[0].{status:buildStatus,phase:currentPhase,start:startTime,end:endTime,logs:logs.deepLink}' \ + --output table + +# get build logs: +aws codebuild batch-get-builds \ + --ids "${BUILD_ID}" \ + --region "${AWS_REGION}" \ + --query 'builds[0].logs.deepLink' \ + --output text + +LOG_GROUP="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.groupName' --output text)" +LOG_STREAM="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.streamName' --output text)" + +aws logs tail "${LOG_GROUP}" \ + --log-stream-names "${LOG_STREAM}" \ + --follow \ + --region "${AWS_REGION}" + ``` diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml new file mode 100644 index 0000000..e5da973 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -0,0 +1,78 @@ +version: 0.2 +env: + variables: + AWS_DEFAULT_REGION: "us-west-2" + secrets-manager: + PGHOST: "${DB_SECRET_ARN}:host" + PGUSER: "${DB_SECRET_ARN}:username" + PGPASSWORD: "${DB_SECRET_ARN}:password" + PGDATABASE: "${DB_SECRET_ARN}:database" + +phases: + install: + commands: + - echo "Inspecting build image..." + - cat /etc/os-release || true + - aws --version || true + - jq --version || true + - unzip -v | head -n 2 || true + - psql --version || true + - echo "Installing required packages if missing..." + - | + if ! command -v jq >/dev/null 2>&1; then + yum install -y jq + fi + - | + if ! command -v unzip >/dev/null 2>&1; then + yum install -y unzip + fi + - | + if ! command -v psql >/dev/null 2>&1; then + echo "psql not found; trying PostgreSQL client package installs..." + + yum install -y postgresql15 \ + || yum install -y postgresql14 \ + || yum install -y postgresql13 \ + || yum install -y postgresql12 \ + || yum install -y postgresql \ + || { + echo "Unable to install psql from enabled yum repositories." + echo "Available postgres-related packages:" + yum search postgresql || true + exit 1 + } + fi + - psql --version + - aws --version + + pre_build: + commands: + - ls -lah + - chmod +x generate-slices-daily.sh export-granules-daily.sh + - | + echo 'Before assuming role: ' + - aws sts get-caller-identity + - | + echo "Assuming ctorm upload role..." + - | + CREDS_JSON="$(aws sts assume-role \ + --role-arn "${CTORM_S3_ROLE_ARN}" \ + --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER}")" + + export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" + export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" + export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + - | + echo 'After assuming role: ' + - aws sts get-caller-identity + + build: + commands: + + - aws sts get-caller-identity + - ./generate-slices-daily.sh + - ./export-granules-daily.sh + post_build: + commands: + - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + - echo "Build complete." diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index ec07cc7..8e5a9e8 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -1,6 +1,20 @@ set -euo pipefail S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" +FAILED_SLICES_FILE="failed_slices_daily.tsv" + +rm -f "$FAILED_SLICES_FILE" + +echo "Statement timeout: " +psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --no-psqlrc \ + -c "SHOW statement_timeout;" + + while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then @@ -15,7 +29,7 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" echo "Destination: ${S3_URI}" - psql \ + if ! PGOPTIONS="-c statement_timeout=${PG_STATEMENT_TIMEOUT:-0}" psql \ --host="$PGHOST" \ --port="${PGPORT:-5432}" \ --username="$PGUSER" \ @@ -26,22 +40,22 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do --set=collection="$COLLECTION" \ --set=yyyymmdd="$YYYYMMDD" \ --file=export-granules-slice-daily.sql \ - | awk -v label="${COLLECTION} ${YYYYMMDD}" ' - { - print; - count++; - if (count % 1000 == 0) { - printf("[%s] exported %d JSONL rows\n", label, count) > "/dev/stderr"; - fflush("/dev/stderr"); - } - } - END { - printf("[%s] export complete: %d JSONL rows\n", label, count) > "/dev/stderr"; - } - ' \ | gzip -c \ - | aws --profile=ctorm s3 cp - "$S3_URI" + | aws s3 cp - "$S3_URI"; then + + echo "FAILED collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + continue + fi echo "Uploaded ${S3_URI}" done < slices_daily.tsv + +if [[ -s "$FAILED_SLICES_FILE" ]]; then + FAILED_SLICES_S3_URI="${S3_PREFIX}/failed_slices_daily.tsv" + aws s3 cp "$FAILED_SLICES_FILE" "$FAILED_SLICES_S3_URI" + echo "Some slices failed. Uploaded failure list to ${FAILED_SLICES_S3_URI}" >&2 +else + echo "All slices exported successfully." +fi diff --git a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql index b23af06..61a8f1f 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql +++ b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql @@ -30,9 +30,10 @@ WITH export_rows AS ( AND f.checksum_value IS NOT NULL WHERE g.status = 'completed' AND c.name = :'collection' - AND g.beginning_date_time >= to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') AT TIME ZONE 'UTC' + + AND g.beginning_date_time >= to_date(:'yyyymmdd', 'YYYYMMDD') AND g.beginning_date_time < ( - to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') + interval '1 month' + to_date(:'yyyymmdd', 'YYYYMMDD') + interval '1 day' ) AT TIME ZONE 'UTC' GROUP BY g.granule_id, @@ -64,4 +65,4 @@ SELECT jsonb_strip_nulls( ) ) FROM export_rows -ORDER BY beginning_date_time, granule_id; \ No newline at end of file +ORDER BY beginning_date_time, granule_id; diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh index 60b9407..3651d74 100644 --- a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -1,3 +1,9 @@ + +set -euo pipefail + + +S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" + psql \ --host="$PGHOST" \ --port="${PGPORT:-5432}" \ @@ -22,6 +28,65 @@ psql \ ON f.granule_cumulus_id = g.cumulus_id AND f.checksum_value IS NOT NULL WHERE g.status = 'completed' + AND c.name in ( + 'ANTPAT', + 'BFPQ', + 'CORNER_REFL', + 'DCOP', + 'DC_RADAR', + 'DSG_STATIC', + 'EA_L0B_L_CRSD', + 'EA_L0B_L_RRSD', + 'EA_L1_L_RIFG', + 'EA_L1_L_ROFF', + 'EA_L1_L_RSLC', + 'EA_L1_L_RUNW', + 'EA_L2_L_GCOV', + 'EA_L2_L_GOFF', + 'EA_L2_L_GSLC', + 'EA_L2_L_GUNW', + 'EA_L3_L_SME2', + 'FOE', + 'FRP', + 'FT_PARAM', + 'FT_WAVEFORM', + 'LRCLK_UTC', + 'L_CHAN_DATA', + 'MOE', + 'NISAR_DEM_TIF', + 'NISAR_DEM_VRT', + 'NISAR_L0A_RRST_BETA_V1', + 'NISAR_L0B_CRSD_BETA_V1', + 'NISAR_L0B_RRSD_BETA_V1', + 'NISAR_L1_RIFG_BETA_V1', + 'NISAR_L1_ROFF_BETA_V1', + 'NISAR_L1_RSLC_BETA_V1', + 'NISAR_L1_RUNW_BETA_V1', + 'NISAR_L2_GCOV_BETA_V1', + 'NISAR_L2_GOFF_BETA_V1', + 'NISAR_L2_GSLC_BETA_V1', + 'NISAR_L2_GUNW_BETA_V1', + 'NISAR_L3_SME2_BETA_V1', + 'NISAR_VWC', + 'NISAR_WATERMASK_TIF', + 'NISAR_WATERMASK_VRT', + 'NOE', + 'NRP', + 'OROST', + 'PA_L0B_L_RRSD', + 'POE', + 'PRP', + 'STUF', + 'TEC', + 'TFDB', + 'UR_L0B_L_RRSD', + + 'OPERA_L2_CSLC-S1_V1', + 'OPERA_L2_RTC-S1_V1', + 'OPERA_L3_DISP-S1_V1', + 'OPERA_L3_DIST-ALERT-S1_V1', + 'OPERA_L4_TROPO-ZENITH_V1' + ) GROUP BY c.name, to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') @@ -29,3 +94,4 @@ psql \ c.name, to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); " > slices_daily.tsv + aws s3 cp slices_daily.tsv "$S3_URI/" diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf index 61c2f10..c2acbcb 100644 --- a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -12,7 +12,7 @@ variable "aws_region" { variable "db_md_extract_secret_arn" { type = string - description = "ARN of the Secrets Manager secret containing host, username, password, and database." + description = "ARN of the Secrets Manager secret containing host, username, password, and database in the RDS account." } variable "db_md_extract_vpc_id" { @@ -35,15 +35,15 @@ variable "db_md_extract_dump_subdir" { default = "nisar" } -variable "db_md_extract_target_bucket" { +variable "db_md_extract_ctorm_bucket" { type = string - default = "" - description = "Destination S3 bucket for exported dump files." + default = "ctorm-dev-scratch" + description = "Destination CTORM S3 bucket for exported dump files." } -variable "db_md_extract_target_role_arn" { +variable "db_md_extract_ctorm_s3_role_arn" { type = string - description = "Role ARN in the target account that CodeBuild can assume to write exported files." + description = "Role ARN in the CTORM account that CodeBuild can assume to write exported files." } locals { @@ -51,14 +51,56 @@ locals { db_md_extract_source_key = "codebuild/cumulus-db-md-extract.zip" } +resource "aws_s3_bucket" "codebuild_source" { + bucket = "${local.db_md_extract_project_name}-source-${substr(data.aws_caller_identity.current.account_id, 8, 4)}" + force_destroy = true + +} + +resource "aws_s3_bucket_ownership_controls" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +data "aws_caller_identity" "current" {} + data "archive_file" "cumulus_db_md_extract" { type = "zip" - source_dir = "${path.module}/../../cumulus-db-md-extract" - output_path = "${path.module}/cumulus-db-md-extract.zip" + source_dir = "${path.module}/.." + output_path = "${path.module}/.terraform/cumulus-db-md-extract.zip" + + excludes = [ + "terraform", + "terraform/*", + ".terraform", + ".terraform/*", + ] } resource "aws_s3_object" "cumulus_db_md_extract_source" { - bucket = aws_s3_bucket.scratch.bucket + bucket = aws_s3_bucket.codebuild_source.bucket key = local.db_md_extract_source_key source = data.archive_file.cumulus_db_md_extract.output_path source_hash = data.archive_file.cumulus_db_md_extract.output_base64sha256 @@ -108,7 +150,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { } statement { - sid = "ReadBuildSourceFromScratchBucket" + sid = "ReadBuildSource" effect = "Allow" actions = [ @@ -117,12 +159,12 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { ] resources = [ - "${aws_s3_bucket.scratch.arn}/${local.db_md_extract_source_key}", + "${aws_s3_bucket.codebuild_source.arn}/${local.db_md_extract_source_key}", ] } statement { - sid = "ListScratchBucketForBuildSource" + sid = "ListBuildSourceBucket" effect = "Allow" actions = [ @@ -130,7 +172,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { ] resources = [ - aws_s3_bucket.scratch.arn, + aws_s3_bucket.codebuild_source.arn, ] condition { @@ -158,7 +200,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { } statement { - sid = "AssumeTargetAccountUploadRole" + sid = "AssumeCtormAccountUploadRole" effect = "Allow" actions = [ @@ -166,7 +208,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { ] resources = [ - var.db_md_extract_target_role_arn, + var.db_md_extract_ctorm_s3_role_arn, ] } @@ -175,6 +217,8 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { effect = "Allow" actions = [ + "ec2:DescribeDhcpOptions", + "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs", @@ -219,7 +263,7 @@ resource "aws_iam_role_policy" "codebuild_db_md_extract" { resource "aws_codebuild_project" "cumulus_db_md_extract" { name = local.db_md_extract_project_name - description = "One-time Cumulus RDS metadata export to S3" + description = "One-time Cumulus RDS metadata export to CTORM S3" service_role = aws_iam_role.codebuild_db_md_extract.arn build_timeout = 480 @@ -229,7 +273,7 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { source { type = "S3" - location = "${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + location = "${aws_s3_bucket.codebuild_source.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" buildspec = "buildspec.yaml" } @@ -245,7 +289,7 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { environment_variable { name = "CTORM_BUCKET" - value = var.db_md_extract_target_bucket + value = var.db_md_extract_ctorm_bucket } environment_variable { @@ -259,8 +303,8 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { } environment_variable { - name = "TARGET_ROLE_ARN" - value = var.db_md_extract_target_role_arn + name = "CTORM_S3_ROLE_ARN" + value = var.db_md_extract_ctorm_s3_role_arn } } @@ -292,5 +336,5 @@ output "cumulus_db_md_extract_codebuild_role_arn" { } output "cumulus_db_md_extract_source_s3_uri" { - value = "s3://${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + value = "s3://${aws_s3_bucket.codebuild_source.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" } diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars index de3cc0a..66ff300 100644 --- a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -13,10 +13,6 @@ db_md_extract_security_group_ids = [ "sg-...", ] -db_md_extract_source_bucket = "" - -db_md_extract_target_bucket = "ctorm-dev-scratch" - -db_md_extract_target_role_arn = "arn:aws:iam:::role/" +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index de3cc0a..588e880 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -13,10 +13,6 @@ db_md_extract_security_group_ids = [ "sg-...", ] -db_md_extract_source_bucket = "" +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." -db_md_extract_target_bucket = "ctorm-dev-scratch" - -db_md_extract_target_role_arn = "arn:aws:iam:::role/" - -db_md_extract_dump_subdir = "nisar" +db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars index b27da44..ac198b3 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars @@ -1,4 +1,4 @@ -name_prefix = "ctorm" +name_prefix = "ctorm-opera-uat" db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" @@ -14,10 +14,6 @@ db_md_extract_security_group_ids = [ "sg-06255aca3a87784a0", ] -db_md_extract_source_bucket = "ctorm-dev-scratch" +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" -db_md_extract_target_bucket = "ctorm-dev-scratch" - -db_md_extract_target_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" - -db_md_extract_dump_subdir = "opera" +db_md_extract_dump_subdir = "opera-uat" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate index a6e1dad..de16529 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate @@ -1,19 +1,19 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 6, + "serial": 51, "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { - "value": "ctorm-cumulus-db-md-extract", + "value": "ctorm-opera-uat-cumulus-db-md-extract", "type": "string" }, "cumulus_db_md_extract_codebuild_role_arn": { - "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "value": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", "type": "string" }, "cumulus_db_md_extract_source_s3_uri": { - "value": "s3://ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip", + "value": "s3://ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", "type": "string" } }, @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "baed98d70a081d5ddd95b21dade6d5b45112af30", - "output_base64sha256": "bS+Xh1vcj006QvtACc+o8twj7kCPntn13M4MAkySTvw=", - "output_base64sha512": "bB6iw4IJxB+G1ieprcxj96PdqyZbKT6ahc6BbPDrK5ONIk43JvXrz0xbCXud8vubsLC5o46yqXIhXQUqBtMYjw==", + "id": "c9cfb643537aef04775bf2b4289a4952f88fe75c", + "output_base64sha256": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", + "output_base64sha512": "dJ4dl6jPOMuVC7pkaqZ3IbGEwbzDfqI7gCJfVi4BUVV4jZH66Ks3YolX82N88kJFuwErzgcAk/DxCJGHoPvcRg==", "output_file_mode": null, - "output_md5": "c5beacdcad3c55176fedde20acb8616f", + "output_md5": "c5761ab3cd6148053f00bd7b09e6571c", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "baed98d70a081d5ddd95b21dade6d5b45112af30", - "output_sha256": "6d2f97875bdc8f4d3a42fb4009cfa8f2dc23ee408f9ed9f5dcce0c024c924efc", - "output_sha512": "6c1ea2c38209c41f86d627a9adcc63f7a3ddab265b293e9a85ce816cf0eb2b938d224e3726f5ebcf4c5b097b9df2fb9bb0b0b9a38eb2a972215d052a06d3188f", - "output_size": 5074, + "output_sha": "c9cfb643537aef04775bf2b4289a4952f88fe75c", + "output_sha256": "d5071c6d85da396d685a4606832345349755303b707a8ddea5f856655b86fb20", + "output_sha512": "749e1d97a8cf38cb950bba646aa67721b184c1bcc37ea23b80225f562e015155788d91fae8ab37628957f3637cf24245bb012bce070093f0f1089187a0fbdc46", + "output_size": 5457, "source": [], "source_content": null, "source_content_filename": null, @@ -55,6 +55,24 @@ } ] }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "097260566921", + "arn": "arn:aws:iam::097260566921:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "097260566921", + "user_id": "AIDARNJJOPGERLGUKQ553" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -64,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "3670884144", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeNetworkInterfaces\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"ManageCodeBuildNetworkInterfaces\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Resource\":\"*\"},{\"Sid\":\"ManageCodeBuildNetworkInterfaces\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "2729057441", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -85,7 +103,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" ], "sid": "WriteCodeBuildLogs" }, @@ -101,7 +119,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip" + "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip" ], "sid": "ReadBuildSource" }, @@ -124,7 +142,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:s3:::ctorm-dev-scratch" + "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921" ], "sid": "ListBuildSourceBucket" }, @@ -157,11 +175,15 @@ "resources": [ "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" ], - "sid": "AssumeCtormUploadRole" + "sid": "AssumeCtormAccountUploadRole" }, { "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs" @@ -177,22 +199,6 @@ ], "sid": "DescribeNetworkForVpcBuild" }, - { - "actions": [ - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "*" - ], - "sid": "ManageCodeBuildNetworkInterfaces" - }, { "actions": [ "ec2:CreateNetworkInterfacePermission" @@ -277,17 +283,17 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", "deletion_protection_enabled": false, - "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "id": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", "kms_key_id": "", "log_group_class": "STANDARD", - "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", "name_prefix": "", "region": "us-west-2", "retention_in_days": 14, "skip_destroy": false, - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -295,6 +301,163 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-opera-uat-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:097260566921:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "opera-uat" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-opera-uat-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-opera-uat-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-03beee748d2a43cc9", + "sg-06255aca3a87784a0", + "sg-0afdcb46b1a38db46" + ], + "subnets": [ + "subnet-0ac04d1e36d0a50ea" + ], + "vpc_id": "vpc-080322e163a0e7390" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -304,22 +467,27 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", - "create_date": "2026-06-24T19:45:21Z", + "create_date": "2026-06-24T23:12:29Z", "description": "", "force_detach_policies": false, - "id": "ctorm-cumulus-db-md-extract-role", - "inline_policy": [], + "id": "ctorm-opera-uat-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, - "name": "ctorm-cumulus-db-md-extract-role", + "name": "ctorm-opera-uat-cumulus-db-md-extract-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, - "unique_id": "AROARNJJOPGEV76IAVBEI" + "unique_id": "AROARNJJOPGE2BF3AA7RB" }, "sensitive_attributes": [], "private": "bnVsbA==", @@ -338,22 +506,247 @@ { "schema_version": 0, "attributes": { - "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", - "name": "ctorm-cumulus-db-md-extract-policy", + "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", + "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"ManageCodeBuildNetworkInterfaces\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", - "role": "ctorm-cumulus-db-md-extract-role" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-opera-uat-cumulus-db-md-extract-role" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_cloudwatch_log_group.codebuild_db_md_extract", "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", "data.aws_iam_policy_document.codebuild_db_md_extract", "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" ] } ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket_domain_name": "ctorm-opera-uat-cumulus-db-md-extract-source-6921.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-opera-uat-cumulus-db-md-extract-source-6921.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "expected_bucket_owner": "", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "c5761ab3cd6148053f00bd7b09e6571c", + "force_destroy": false, + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] } ], "check_results": null diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index c84e99c..f1e6ce5 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -304,16 +304,27 @@ data "aws_iam_policy_document" "cumulus_db_md_extract_upload_assume_role" { } condition { - test = "ArnEquals" + test = "ArnLike" variable = "aws:PrincipalArn" values = [ "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" ] } } diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 9137406..7426ae8 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 28, + "serial": 30, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -288,9 +288,9 @@ { "schema_version": 0, "attributes": { - "id": "2147741451", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "id": "4124935366", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -305,12 +305,12 @@ { "test": "ArnEquals", "values": [ - "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + "arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role" ], "variable": "aws:PrincipalArn" } @@ -762,7 +762,7 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", "create_date": "2026-06-24T19:29:26Z", "description": "", "force_detach_policies": false, From e77628914b2d65560ddac1edf6bbc17adb04d33f Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 24 Jun 2026 20:17:47 -0800 Subject: [PATCH 04/15] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/README.md | 10 +- .../terraform/opera-prod.tfvars | 13 +- .../terraform/terraform-opera-prod.tfstate | 752 +++++++++++++ ...rm.tfstate => terraform-opera-uat.tfstate} | 32 +- .../opera-prod/terraform.tfstate | 563 ++++++++++ ctorm/infra/README.md | 9 + ctorm/infra/terraform/main.tf | 71 +- ctorm/infra/terraform/outputs.tf | 12 +- ctorm/infra/terraform/terraform.tfstate | 299 +----- .../infra/terraform/terraform.tfstate.backup | 428 +------- .../ctorm-dev/terraform.tfstate | 294 ++++++ .../ctorm-prod/terraform.tfstate | 998 ++++++++++++++++++ 12 files changed, 2722 insertions(+), 759 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate rename ctorm/cumulus-db-md-extract/terraform/{terraform.tfstate => terraform-opera-uat.tfstate} (82%) create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate create mode 100644 ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate create mode 100644 ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index f3556bd..b8b3293 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -80,12 +80,20 @@ Really should try to create a EC2 next time. ```bash export AWS_PROFILE="cumulus-uat-6921" export VARFILE=opera-uat.tfvars +export STATEFILE=terraform-opera-uat.tfstate + +terraform workspace new opera-uat + +export TF_WORKSPACE=opera-uat + terraform init terraform plan \ + -state="${STATEFILE}" \ -var-file="${VARFILE}" terraform apply \ + -state="${STATEFILE}" \ -var-file="${VARFILE}" terraform destroy @@ -98,7 +106,7 @@ terraform destroy ```bash export AWS_PROFILE="cumulus-uat-6921" export AWS_REGION="us-west-2" -export CODEBUILD_PROJECT="$(terraform output -raw cumulus_db_md_extract_codebuild_project_name)" +export CODEBUILD_PROJECT="$(terraform output -state="${STATEFILE}" -raw cumulus_db_md_extract_codebuild_project_name)" # start build: BUILD_ID="$( diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index 588e880..b7cb678 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -1,18 +1,19 @@ name_prefix = "ctorm" -db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" -db_md_extract_vpc_id = "vpc-..." +db_md_extract_vpc_id = "vpc-0e36a2fca2e2a73d5" db_md_extract_subnet_ids = [ - "subnet-...", - "subnet-...", + "subnet-02dda0af17b4a0d19", ] db_md_extract_security_group_ids = [ - "sg-...", + "sg-0249655cac679df2c", + "sg-0ed18cd0fb6e79f58", + "sg-0ed18cd0fb6e79f58", ] -db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate new file mode 100644 index 0000000..691cf7d --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -0,0 +1,752 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 41, + "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", + "output_base64sha256": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", + "output_base64sha512": "J8ZtLQzzPeFUcNb4IW4DV3ar9M0/uFonIhPlfZDv6Z1cGcr+nSGpuJCcASX4HlLX6dJcILMBEtdwGbEuB/3rvQ==", + "output_file_mode": null, + "output_md5": "692e40b861d2f40dcc12eb78cbe57de4", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", + "output_sha256": "0a3e592d9256955ef7682a2b64dff82c1d25096c1ef8b6eaf2eb817fc1850723", + "output_sha512": "27c66d2d0cf33de15470d6f8216e035776abf4cd3fb85a272213e57d90efe99d5c19cafe9d21a9b8909c0125f81e52d7e9d25c20b30112d77019b12e07fdebbd", + "output_size": 5350, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "510296831643", + "arn": "arn:aws:iam::510296831643:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "510296831643", + "user_id": "AIDAXNUAPW2NZFPXV6JKY" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1104794953", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:510296831643:project/ctorm-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:510296831643:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "opera" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:510296831643:project/ctorm-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-0249655cac679df2c", + "sg-0ed18cd0fb6e79f58" + ], + "subnets": [ + "subnet-02dda0af17b4a0d19" + ], + "vpc_id": "vpc-0e36a2fca2e2a73d5" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T03:25:11Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROAXNUAPW2N44OBYZNKD" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643", + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-1643.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-1643.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "a055848272cdca6935f41c8d486234d66c39ff78b8f9ef89db0d949987311ce3", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "692e40b861d2f40dcc12eb78cbe57de4", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate similarity index 82% rename from ctorm/cumulus-db-md-extract/terraform/terraform.tfstate rename to ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate index de16529..71af6a5 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 51, + "serial": 62, "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "c9cfb643537aef04775bf2b4289a4952f88fe75c", - "output_base64sha256": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", - "output_base64sha512": "dJ4dl6jPOMuVC7pkaqZ3IbGEwbzDfqI7gCJfVi4BUVV4jZH66Ks3YolX82N88kJFuwErzgcAk/DxCJGHoPvcRg==", + "id": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", + "output_base64sha256": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", + "output_base64sha512": "PZDUjfN60Uim+bdem7zB2ilfEUKqP4pE0UADkxHIkR4YTYGcP+Txtuj+Rm7zSpLQ2fZk+rgOjhJKIWLy0lyWYQ==", "output_file_mode": null, - "output_md5": "c5761ab3cd6148053f00bd7b09e6571c", + "output_md5": "63b93d1d0d04eb5a9595f406ba9653cc", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "c9cfb643537aef04775bf2b4289a4952f88fe75c", - "output_sha256": "d5071c6d85da396d685a4606832345349755303b707a8ddea5f856655b86fb20", - "output_sha512": "749e1d97a8cf38cb950bba646aa67721b184c1bcc37ea23b80225f562e015155788d91fae8ab37628957f3637cf24245bb012bce070093f0f1089187a0fbdc46", - "output_size": 5457, + "output_sha": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", + "output_sha256": "b7ddad71c2668d0d4a2d31668752520c9e0b4e9dee0d8188eb3d3f9a89a79f4c", + "output_sha512": "3d90d48df37ad148a6f9b75e9bbcc1da295f1142aa3f8a44d140039311c8911e184d819c3fe4f1b6e8fe466ef34a92d0d9f664fab80e8e124a2162f2d25c9661", + "output_size": 5347, "source": [], "source_content": null, "source_content_filename": null, @@ -82,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "2729057441", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "1034869219", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -103,7 +103,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract" ], "sid": "WriteCodeBuildLogs" }, @@ -509,7 +509,7 @@ "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", "role": "ctorm-opera-uat-cumulus-db-md-extract-role" }, "sensitive_attributes": [], @@ -718,7 +718,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "c5761ab3cd6148053f00bd7b09e6571c", + "etag": "63b93d1d0d04eb5a9595f406ba9653cc", "force_destroy": false, "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -731,7 +731,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", + "source_hash": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate new file mode 100644 index 0000000..b7216a0 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate @@ -0,0 +1,563 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 10, + "lineage": "9578e429-f029-9119-8845-752daa480ee7", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "77650f5f87b9d429f0ae3ee4141a2d4408515477", + "output_base64sha256": "W1Cg6btOTGKbYpBaANU0oW3rd3EKlsQKsMP1/ARFPnY=", + "output_base64sha512": "TqEppCWs6YEod3UPDPcniu7MgiOD3UWx1JXpUkKFr476zHH0D6UVkqJOXuR8seBnRlq9HgydFHLTEcTYFps4Sw==", + "output_file_mode": null, + "output_md5": "34b9f1764b49aa11cd3c0f0cd3db5538", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "77650f5f87b9d429f0ae3ee4141a2d4408515477", + "output_sha256": "5b50a0e9bb4e4c629b62905a00d534a16deb77710a96c40ab0c3f5fc04453e76", + "output_sha512": "4ea129a425ace9812877750f0cf7278aeecc822383dd45b1d495e9524285af8efacc71f40fa51592a24e5ee47cb1e067465abd1e0c9d1472d311c4d8169b384b", + "output_size": 5323, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "097260566921", + "arn": "arn:aws:iam::097260566921:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "097260566921", + "user_id": "AIDARNJJOPGERLGUKQ553" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3830342824", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:\\u003cremote-account-id\\u003e:secret:\\u003csecret-name\\u003e\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::...\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:\\u003cremote-account-id\\u003e:secret:\\u003csecret-name\\u003e\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::...\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:\u003cremote-account-id\u003e:secret:\u003csecret-name\u003e" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::..." + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:48:50Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGEVXWJQGTNS" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921", + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-6921.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-6921.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": null, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "34b9f1764b49aa11cd3c0f0cd3db5538", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": null, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "W1Cg6btOTGKbYpBaANU0oW3rd3EKlsQKsMP1/ARFPnY=", + "storage_class": "STANDARD", + "tags": null, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 67aaa58..06d8a93 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -1,7 +1,16 @@ # CTORM infrastructure ```bash +export AWS_PROFILE="cumulus-sbx-7522" export VARFILE=dev.tfvars + +# create workspaces and select the one to use +terraform workspace new ctorm-dev + +terraform workspace new ctorm-prod + +export TF_WORKSPACE=ctorm-dev + terraform init terraform plan \ diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index f1e6ce5..2ba024e 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -17,8 +17,8 @@ resource "aws_sqs_queue" "granules" { }) } -resource "aws_dynamodb_table" "state" { - name = "${var.name_prefix}-state" +resource "aws_dynamodb_table" "granules" { + name = "${var.name_prefix}-granules" billing_mode = "PAY_PER_REQUEST" hash_key = "pk" @@ -101,7 +101,7 @@ data "aws_iam_policy_document" "cnm-sender" { ] resources = [ - aws_dynamodb_table.state.arn + aws_dynamodb_table.granules.arn ] } } @@ -153,7 +153,7 @@ resource "aws_lambda_function" "cnm-sender" { environment { variables = { GRANULES_QUEUE_URL = aws_sqs_queue.granules.url - TABLE_NAME = aws_dynamodb_table.state.name + TABLE_NAME = aws_dynamodb_table.granules.name CUMULUS_INGEST_QUEUE_URL = var.cumulus_ingest_queue_url LOG_LEVEL = "INFO" } @@ -188,69 +188,6 @@ resource "aws_lambda_permission" "allow_eventbridge" { source_arn = aws_cloudwatch_event_rule.every_minute.arn } - -data "aws_iam_policy_document" "prepare_ec2_assume_role" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } -} - -resource "aws_iam_role" "prepare_ec2" { - name = "${var.name_prefix}-prepare-ec2-role" - assume_role_policy = data.aws_iam_policy_document.prepare_ec2_assume_role.json -} - -data "aws_iam_policy_document" "prepare_ec2" { - statement { - actions = [ - "sqs:SendMessage", - "sqs:GetQueueAttributes" - ] - - resources = [ - aws_sqs_queue.granules.arn - ] - } - - statement { - actions = [ - "s3:ListBucket" - ] - - resources = [ - for bucket_name in var.prepare_source_bucket_names : "arn:aws:s3:::${bucket_name}" - ] - } - - statement { - actions = [ - "s3:GetObject" - ] - - resources = [ - for bucket_name in var.prepare_source_bucket_names : "arn:aws:s3:::${bucket_name}/*" - ] - } -} - -resource "aws_iam_role_policy" "prepare_ec2" { - name = "${var.name_prefix}-prepare-ec2-policy" - role = aws_iam_role.prepare_ec2.id - policy = data.aws_iam_policy_document.prepare_ec2.json -} - -resource "aws_iam_instance_profile" "prepare_ec2" { - name = "${var.name_prefix}-prepare-ec2-profile" - role = aws_iam_role.prepare_ec2.name -} - - - resource "aws_s3_bucket" "scratch" { bucket = "${var.name_prefix}-scratch" diff --git a/ctorm/infra/terraform/outputs.tf b/ctorm/infra/terraform/outputs.tf index 6e734a2..4dc5e83 100644 --- a/ctorm/infra/terraform/outputs.tf +++ b/ctorm/infra/terraform/outputs.tf @@ -10,22 +10,14 @@ output "granules_dlq_url" { value = aws_sqs_queue.granules_dlq.url } -output "state_table_name" { - value = aws_dynamodb_table.state.name +output "granule_table_name" { + value = aws_dynamodb_table.granules.name } output "worker_lambda_name" { value = aws_lambda_function.cnm-sender.function_name } -output "prepare_ec2_instance_profile_name" { - value = aws_iam_instance_profile.prepare_ec2.name -} - -output "prepare_ec2_role_arn" { - value = aws_iam_role.prepare_ec2.arn -} - output "scratch_bucket" { value = aws_s3_bucket.scratch.bucket } diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 7426ae8..c50a504 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,13 +1,17 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 30, + "serial": 45, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", "type": "string" }, + "granule_table_name": { + "value": "ctorm-dev-granules", + "type": "string" + }, "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" @@ -20,22 +24,10 @@ "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "type": "string" }, - "prepare_ec2_instance_profile_name": { - "value": "ctorm-dev-prepare-ec2-profile", - "type": "string" - }, - "prepare_ec2_role_arn": { - "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "type": "string" - }, "scratch_bucket": { "value": "ctorm-dev-scratch", "type": "string" }, - "state_table_name": { - "value": "ctorm-dev-state", - "type": "string" - }, "worker_lambda_name": { "value": "ctorm-dev-cnm-sender", "type": "string" @@ -132,9 +124,9 @@ { "schema_version": 0, "attributes": { - "id": "2455266930", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "id": "3548715077", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -204,7 +196,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules" ], "sid": "" } @@ -288,9 +280,9 @@ { "schema_version": 0, "attributes": { - "id": "4124935366", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}}}]}", + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -303,14 +295,20 @@ ], "condition": [ { - "test": "ArnEquals", + "test": "ArnLike", "values": [ - "arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role" + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" ], "variable": "aws:PrincipalArn" } @@ -387,132 +385,6 @@ } ] }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "1999618104", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sqs:GetQueueAttributes", - "sqs:SendMessage" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" - ], - "sid": "" - }, - { - "actions": [ - "s3:ListBucket" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products", - "arn:aws:s3:::asf-cumulus-prod-opera-products", - "arn:aws:s3:::asf-cumulus-test-opera-products", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products" - ], - "sid": "" - }, - { - "actions": [ - "s3:GetObject" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products/*", - "arn:aws:s3:::asf-cumulus-prod-opera-products/*", - "arn:aws:s3:::asf-cumulus-test-opera-products/*", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" - ], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2_assume_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "2851119427", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sts:AssumeRole" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [ - { - "identifiers": [ - "ec2.amazonaws.com" - ], - "type": "Service" - } - ], - "resources": [], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, { "mode": "managed", "type": "aws_cloudwatch_event_rule", @@ -578,7 +450,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", @@ -619,13 +491,13 @@ { "mode": "managed", "type": "aws_dynamodb_table", - "name": "state", + "name": "granules", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 1, "attributes": { - "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", "attribute": [ { "name": "pk", @@ -640,10 +512,10 @@ "deletion_protection_enabled": false, "global_secondary_index": [], "hash_key": "pk", - "id": "ctorm-dev-state", + "id": "ctorm-dev-granules", "import_table": [], "local_secondary_index": [], - "name": "ctorm-dev-state", + "name": "ctorm-dev-granules", "on_demand_throughput": [], "point_in_time_recovery": [ { @@ -664,7 +536,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -680,35 +552,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", - "create_date": "2026-06-19T01:00:34Z", - "id": "ctorm-dev-prepare-ec2-profile", - "name": "ctorm-dev-prepare-ec2-profile", - "name_prefix": "", - "path": "/", - "role": "ctorm-dev-prepare-ec2-role", - "tags": {}, - "tags_all": {}, - "unique_id": "AIPA4VW62R3RCASOXJGOK" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role", @@ -762,7 +605,7 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", "create_date": "2026-06-24T19:29:26Z", "description": "", "force_detach_policies": false, @@ -791,45 +634,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_role", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", - "create_date": "2026-06-19T01:00:33Z", - "description": "", - "force_detach_policies": false, - "id": "ctorm-dev-prepare-ec2-role", - "inline_policy": [ - { - "name": "ctorm-dev-prepare-ec2-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" - } - ], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "ctorm-dev-prepare-ec2-role", - "name_prefix": "", - "path": "/", - "permissions_boundary": "", - "tags": {}, - "tags_all": {}, - "unique_id": "AROA4VW62R3RKO3MKUI7D" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -842,14 +646,14 @@ "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", "name": "ctorm-dev-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", "role": "ctorm-dev-cnm-sender-role" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_sqs_queue.granules", "aws_sqs_queue.granules_dlq", @@ -885,33 +689,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_role_policy", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", - "name": "ctorm-dev-prepare-ec2-policy", - "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "role": "ctorm-dev-prepare-ec2-role" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "aws_sqs_queue.granules", - "aws_sqs_queue.granules_dlq", - "data.aws_iam_policy_document.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -961,7 +738,7 @@ "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "LOG_LEVEL": "INFO", - "TABLE_NAME": "ctorm-dev-state" + "TABLE_NAME": "ctorm-dev-granules" } } ], @@ -979,7 +756,7 @@ "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", - "last_modified": "2026-06-24T19:29:31.000+0000", + "last_modified": "2026-06-25T02:29:36.000+0000", "layers": [], "logging_config": [ { @@ -1024,7 +801,7 @@ "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_sqs_queue.granules", @@ -1063,7 +840,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 0f77000..270feb7 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,9 +1,17 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 27, + "serial": 40, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { + "cumulus_db_md_extract_upload_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "type": "string" + }, + "granule_table_name": { + "value": "ctorm-dev-granules", + "type": "string" + }, "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" @@ -16,22 +24,10 @@ "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "type": "string" }, - "prepare_ec2_instance_profile_name": { - "value": "ctorm-dev-prepare-ec2-profile", - "type": "string" - }, - "prepare_ec2_role_arn": { - "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "type": "string" - }, "scratch_bucket": { "value": "ctorm-dev-scratch", "type": "string" }, - "state_table_name": { - "value": "ctorm-dev-state", - "type": "string" - }, "worker_lambda_name": { "value": "ctorm-dev-cnm-sender", "type": "string" @@ -119,98 +115,6 @@ } ] }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "cnm-sender", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "2455266930", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" - ], - "sid": "" - }, - { - "actions": [ - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:ReceiveMessage" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" - ], - "sid": "" - }, - { - "actions": [ - "sqs:SendMessage", - "sqs:SendMessageBatch" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" - ], - "sid": "" - }, - { - "actions": [ - "dynamodb:GetItem", - "dynamodb:PutItem", - "dynamodb:Query", - "dynamodb:UpdateItem" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" - ], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, { "mode": "data", "type": "aws_iam_policy_document", @@ -284,9 +188,9 @@ { "schema_version": 0, "attributes": { - "id": "2147741451", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -299,14 +203,20 @@ ], "condition": [ { - "test": "ArnEquals", + "test": "ArnLike", "values": [ "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" ], "variable": "aws:PrincipalArn" } @@ -383,132 +293,6 @@ } ] }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "1999618104", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sqs:GetQueueAttributes", - "sqs:SendMessage" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" - ], - "sid": "" - }, - { - "actions": [ - "s3:ListBucket" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products", - "arn:aws:s3:::asf-cumulus-prod-opera-products", - "arn:aws:s3:::asf-cumulus-test-opera-products", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products" - ], - "sid": "" - }, - { - "actions": [ - "s3:GetObject" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products/*", - "arn:aws:s3:::asf-cumulus-prod-opera-products/*", - "arn:aws:s3:::asf-cumulus-test-opera-products/*", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" - ], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2_assume_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "2851119427", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sts:AssumeRole" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [ - { - "identifiers": [ - "ec2.amazonaws.com" - ], - "type": "Service" - } - ], - "resources": [], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, { "mode": "managed", "type": "aws_cloudwatch_event_rule", @@ -574,7 +358,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", @@ -612,99 +396,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_dynamodb_table", - "name": "state", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", - "attribute": [ - { - "name": "pk", - "type": "S" - }, - { - "name": "sk", - "type": "S" - } - ], - "billing_mode": "PAY_PER_REQUEST", - "deletion_protection_enabled": false, - "global_secondary_index": [], - "hash_key": "pk", - "id": "ctorm-dev-state", - "import_table": [], - "local_secondary_index": [], - "name": "ctorm-dev-state", - "on_demand_throughput": [], - "point_in_time_recovery": [ - { - "enabled": false, - "recovery_period_in_days": 0 - } - ], - "range_key": "sk", - "read_capacity": 0, - "replica": [], - "restore_date_time": null, - "restore_source_name": null, - "restore_source_table_arn": null, - "restore_to_latest_time": null, - "server_side_encryption": [], - "stream_arn": "", - "stream_enabled": false, - "stream_label": "", - "stream_view_type": "", - "table_class": "STANDARD", - "tags": {}, - "tags_all": {}, - "timeouts": null, - "ttl": [ - { - "attribute_name": "", - "enabled": false - } - ], - "write_capacity": 0 - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", - "create_date": "2026-06-19T01:00:34Z", - "id": "ctorm-dev-prepare-ec2-profile", - "name": "ctorm-dev-prepare-ec2-profile", - "name_prefix": "", - "path": "/", - "role": "ctorm-dev-prepare-ec2-role", - "tags": {}, - "tags_all": {}, - "unique_id": "AIPA4VW62R3RCASOXJGOK" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role", @@ -758,65 +449,31 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", "create_date": "2026-06-24T19:29:26Z", "description": "", "force_detach_policies": false, "id": "ctorm-dev-cumulus-db-md-extract-upload", - "inline_policy": [], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "ctorm-dev-cumulus-db-md-extract-upload", - "name_prefix": "", - "path": "/", - "permissions_boundary": "", - "tags": null, - "tags_all": {}, - "unique_id": "AROA4VW62R3RJW53W2D5X" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_iam_role", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", - "create_date": "2026-06-19T01:00:33Z", - "description": "", - "force_detach_policies": false, - "id": "ctorm-dev-prepare-ec2-role", "inline_policy": [ { - "name": "ctorm-dev-prepare-ec2-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], "max_session_duration": 3600, - "name": "ctorm-dev-prepare-ec2-role", + "name": "ctorm-dev-cumulus-db-md-extract-upload", "name_prefix": "", "path": "/", "permissions_boundary": "", "tags": {}, "tags_all": {}, - "unique_id": "AROA4VW62R3RKO3MKUI7D" + "unique_id": "AROA4VW62R3RJW53W2D5X" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ - "data.aws_iam_policy_document.prepare_ec2_assume_role" + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" ] } ] @@ -840,6 +497,7 @@ "private": "bnVsbA==", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_sqs_queue.granules", @@ -876,33 +534,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_role_policy", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", - "name": "ctorm-dev-prepare-ec2-policy", - "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "role": "ctorm-dev-prepare-ec2-role" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "aws_sqs_queue.granules", - "aws_sqs_queue.granules_dlq", - "data.aws_iam_policy_document.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -1015,6 +646,7 @@ "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", @@ -1054,7 +686,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate new file mode 100644 index 0000000..e4ddffc --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate @@ -0,0 +1,294 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 9, + "lineage": "848003f0-637c-6830-7392-8ca48da894a1", + "outputs": { + "granule_table_name": { + "value": "ctorm-dev-granules", + "type": "string" + }, + "granules_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "type": "string" + }, + "granules_queue_arn": { + "value": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "type": "string" + }, + "granules_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-dev-scratch", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-dev-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", + "output_file_mode": null, + "output_md5": "94564c5dd9caf22baaf170b00693c497", + "output_path": "./placeholder-lambda.zip", + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, + "source": [ + { + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnLike", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-dev-every-minute", + "is_enabled": false, + "name": "ctorm-dev-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate new file mode 100644 index 0000000..278e8a2 --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -0,0 +1,998 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 18, + "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", + "outputs": { + "cumulus_db_md_extract_upload_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload", + "type": "string" + }, + "granule_table_name": { + "value": "ctorm-granules", + "type": "string" + }, + "granules_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", + "type": "string" + }, + "granules_queue_arn": { + "value": "arn:aws:sqs:us-west-2:097260566921:ctorm-granules", + "type": "string" + }, + "granules_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-scratch", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", + "output_file_mode": null, + "output_md5": "94564c5dd9caf22baaf170b00693c497", + "output_path": "./placeholder-lambda.zip", + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, + "source": [ + { + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "allow_scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3026183601", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:ListBucket\",\n \"s3:GetObject\",\n \"s3:DeleteObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::ctorm-scratch/*\",\n \"arn:aws:s3:::ctorm-scratch\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Resource\":[\"arn:aws:s3:::ctorm-scratch/*\",\"arn:aws:s3:::ctorm-scratch\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-scratch", + "arn:aws:s3:::ctorm-scratch/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2687090556", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:097260566921:ctorm-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2685467495", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:AbortMultipartUpload", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-scratch/cumulus-granules/*" + ], + "sid": "UploadExtractedGranules" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "cumulus-granules/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-scratch" + ], + "sid": "ListDestinationPrefix" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnLike", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:097260566921:rule/ctorm-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-every-minute", + "is_enabled": false, + "name": "ctorm-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "appsync_target": [], + "arn": "arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [], + "event_bus_name": "default", + "force_destroy": false, + "http_target": [], + "id": "ctorm-every-minute-terraform-20260625024146730800000001", + "input": "{\"source\":\"eventbridge\"}", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "", + "rule": "ctorm-every-minute", + "run_command_targets": [], + "sagemaker_pipeline_target": [], + "sqs_target": [], + "target_id": "terraform-20260625024146730800000001" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender", + "id": "/aws/lambda/ctorm-cnm-sender", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/lambda/ctorm-cnm-sender", + "name_prefix": "", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules", + "attribute": [ + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [], + "hash_key": "pk", + "id": "ctorm-granules", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-granules", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": null, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cnm-sender-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:40:48Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cnm-sender-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cnm-sender-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGE3PVM46W7W" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:40:48Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-upload", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-upload", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGE4XUWPCFMM" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cnm-sender-role:ctorm-cnm-sender-policy", + "name": "ctorm-cnm-sender-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "role": "ctorm-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-upload:ctorm-cumulus-db-md-extract-upload", + "name": "ctorm-cumulus-db-md-extract-upload", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "role": "ctorm-cumulus-db-md-extract-upload" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cumulus_db_md_extract_upload", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cnm-sender-role:ctorm-scratch-access", + "name": "ctorm-scratch-access", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch/*\",\"arn:aws:s3:::ctorm-scratch\"]}]}", + "role": "ctorm-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cnm-sender", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.allow_scratch_bucket_access", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender", + "code_sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [ + { + "variables": { + "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", + "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "LOG_LEVEL": "INFO", + "TABLE_NAME": "ctorm-granules" + } + } + ], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "./placeholder-lambda.zip", + "function_name": "ctorm-cnm-sender", + "handler": "load_tester.lambda_handler", + "id": "ctorm-cnm-sender", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender/invocations", + "kms_key_arn": "", + "last_modified": "2026-06-25T02:41:40.841+0000", + "layers": null, + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/ctorm-cnm-sender", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::097260566921:role/ctorm-cnm-sender-role", + "runtime": "python3.12", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "source_code_size": 344, + "tags": null, + "tags_all": {}, + "timeout": 120, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_eventbridge", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "ctorm-cnm-sender", + "function_url_auth_type": null, + "id": "AllowExecutionFromEventBridge", + "principal": "events.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:events:us-west-2:097260566921:rule/ctorm-every-minute", + "statement_id": "AllowExecutionFromEventBridge", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "scratch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-scratch", + "bucket": "ctorm-scratch", + "bucket_domain_name": "ctorm-scratch.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "ctorm-scratch.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-scratch", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Name": "ctorm-scratch" + }, + "tags_all": { + "Name": "ctorm-scratch" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:097260566921:ctorm-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:097260566921:ctorm-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} From 0ee5897b61c64a09d7e9aed73dd8ab969855df06 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 11:24:58 -0800 Subject: [PATCH 05/15] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/buildspec.yaml | 7 ++++ .../terraform/nisar-prod.tfvars | 13 +++---- .../terraform/opera-prod.tfvars | 3 +- .../terraform/terraform-opera-prod.tfstate | 36 +++++++++---------- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index e5da973..665d8f7 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -65,6 +65,13 @@ phases: - | echo 'After assuming role: ' - aws sts get-caller-identity + - | + id=$(aws sts get-caller-identity) + if [ -z "$id" ]; then + echo "Error: `aws sts get-caller-identity` returned no output." >&2 + exit 1 + fi + build: commands: diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars index 66ff300..474dc28 100644 --- a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -1,18 +1,19 @@ name_prefix = "ctorm" -db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" -db_md_extract_vpc_id = "vpc-..." +db_md_extract_vpc_id = "vpc-05bb1a5fa3d38fddd" db_md_extract_subnet_ids = [ - "subnet-...", - "subnet-...", + "subnet-028b8ab53b5232fa5", ] db_md_extract_security_group_ids = [ - "sg-...", + "sg-01b8118dc0c9ea180", + "sg-0e56e1310879eaef3", + "sg-0a4f1b9d764ef7b38", ] -db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index b7cb678..7701459 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -14,6 +14,7 @@ db_md_extract_security_group_ids = [ "sg-0ed18cd0fb6e79f58", ] -db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." +# Look at the ctorm tf output for `cumulus_db_md_extract_upload_role_arn` +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 691cf7d..2c75342 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 41, + "serial": 55, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", - "output_base64sha256": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", - "output_base64sha512": "J8ZtLQzzPeFUcNb4IW4DV3ar9M0/uFonIhPlfZDv6Z1cGcr+nSGpuJCcASX4HlLX6dJcILMBEtdwGbEuB/3rvQ==", + "id": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", + "output_base64sha256": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", + "output_base64sha512": "bbAx+FqrN2kSRGI8VXAMr9dj+nGcUHwVSq0X7JE9pjAeuF11V4Xqscovy/5i7a3dXnIK9EQB1zZL7sAh3lY8og==", "output_file_mode": null, - "output_md5": "692e40b861d2f40dcc12eb78cbe57de4", + "output_md5": "f565d14b76dda95c7f2e971e583ce9d7", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", - "output_sha256": "0a3e592d9256955ef7682a2b64dff82c1d25096c1ef8b6eaf2eb817fc1850723", - "output_sha512": "27c66d2d0cf33de15470d6f8216e035776abf4cd3fb85a272213e57d90efe99d5c19cafe9d21a9b8909c0125f81e52d7e9d25c20b30112d77019b12e07fdebbd", - "output_size": 5350, + "output_sha": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", + "output_sha256": "78de28700a9ba06d35d8caa2edb812266d418165dcaecd30c128188f36b12fff", + "output_sha512": "6db031f85aab37691244623c55700cafd763fa719c507c154aad17ec913da6301eb85d755785eab1ca2fcbfe62edaddd5e720af44401d7364beec021de563ca2", + "output_size": 5410, "source": [], "source_content": null, "source_content_filename": null, @@ -82,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "1104794953", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "3636791408", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -173,7 +173,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" ], "sid": "AssumeCtormAccountUploadRole" }, @@ -370,7 +370,7 @@ { "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", - "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" } ], "fleet": [], @@ -475,7 +475,7 @@ "inline_policy": [ { "name": "ctorm-cumulus-db-md-extract-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" } ], "managed_policy_arns": [], @@ -508,7 +508,7 @@ "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", "name": "ctorm-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", "role": "ctorm-cumulus-db-md-extract-role" }, "sensitive_attributes": [], @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "692e40b861d2f40dcc12eb78cbe57de4", + "etag": "f565d14b76dda95c7f2e971e583ce9d7", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", + "source_hash": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From 1582c025878d86267893f6d9cec2cfaea90db356 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 13:13:31 -0800 Subject: [PATCH 06/15] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/buildspec.yaml | 11 +- .../terraform/nisar-prod.tfvars | 2 + .../terraform/opera-prod.tfvars | 2 + .../terraform/terraform-nisar-prod.tfstate | 753 ++++++++++++++++++ .../terraform/terraform-opera-prod.tfstate | 26 +- ctorm/cumulus-db-md-extract/test-s3-cp.sh | 7 + 6 files changed, 780 insertions(+), 21 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate create mode 100644 ctorm/cumulus-db-md-extract/test-s3-cp.sh diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index 665d8f7..33b0998 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -48,7 +48,7 @@ phases: pre_build: commands: - ls -lah - - chmod +x generate-slices-daily.sh export-granules-daily.sh + - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh - | echo 'Before assuming role: ' - aws sts get-caller-identity @@ -65,18 +65,13 @@ phases: - | echo 'After assuming role: ' - aws sts get-caller-identity - - | - id=$(aws sts get-caller-identity) - if [ -z "$id" ]; then - echo "Error: `aws sts get-caller-identity` returned no output." >&2 - exit 1 - fi build: commands: + # fail fast if we can't actually copy to S3: + - ./test-s3-cp.sh - - aws sts get-caller-identity - ./generate-slices-daily.sh - ./export-granules-daily.sh post_build: diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars index 474dc28..ac07320 100644 --- a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -17,3 +17,5 @@ db_md_extract_security_group_ids = [ db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "nisar" + +db_md_extract_ctorm_bucket = "ctorm-scratch" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index 7701459..353e7c9 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -18,3 +18,5 @@ db_md_extract_security_group_ids = [ db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "opera" + +db_md_extract_ctorm_bucket = "ctorm-scratch" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate new file mode 100644 index 0000000..71c1cb1 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate @@ -0,0 +1,753 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 17, + "lineage": "2aab81d2-395f-d5fa-becf-2772136eaa03", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", + "output_file_mode": null, + "output_md5": "8d91e1621f990906971ffa67900b6743", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", + "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", + "output_size": 5692, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "372059463218", + "arn": "arn:aws:iam::372059463218:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "372059463218", + "user_id": "AIDAVNIDY3YZHQAL4QVMF" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "202837701", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:372059463218:project/ctorm-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:372059463218:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "nisar" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:372059463218:project/ctorm-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-01b8118dc0c9ea180", + "sg-0a4f1b9d764ef7b38", + "sg-0e56e1310879eaef3" + ], + "subnets": [ + "subnet-028b8ab53b5232fa5" + ], + "vpc_id": "vpc-05bb1a5fa3d38fddd" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T19:31:14Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROAVNIDY3YZPQXYNWXSP" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218", + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-3218.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-3218.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "d6f242167244d0b7b820212432e41dbdcbde816cf4992ec7f7967eab0154eab5", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "8d91e1621f990906971ffa67900b6743", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 2c75342..238ecee 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 55, + "serial": 78, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", - "output_base64sha256": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", - "output_base64sha512": "bbAx+FqrN2kSRGI8VXAMr9dj+nGcUHwVSq0X7JE9pjAeuF11V4Xqscovy/5i7a3dXnIK9EQB1zZL7sAh3lY8og==", + "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", "output_file_mode": null, - "output_md5": "f565d14b76dda95c7f2e971e583ce9d7", + "output_md5": "8d91e1621f990906971ffa67900b6743", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", - "output_sha256": "78de28700a9ba06d35d8caa2edb812266d418165dcaecd30c128188f36b12fff", - "output_sha512": "6db031f85aab37691244623c55700cafd763fa719c507c154aad17ec913da6301eb85d755785eab1ca2fcbfe62edaddd5e720af44401d7364beec021de563ca2", - "output_size": 5410, + "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", + "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", + "output_size": 5692, "source": [], "source_content": null, "source_content_filename": null, @@ -355,7 +355,7 @@ { "name": "CTORM_BUCKET", "type": "PLAINTEXT", - "value": "ctorm-dev-scratch" + "value": "ctorm-scratch" }, { "name": "DUMP_SUBDIR", @@ -475,7 +475,7 @@ "inline_policy": [ { "name": "ctorm-cumulus-db-md-extract-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" } ], "managed_policy_arns": [], @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "f565d14b76dda95c7f2e971e583ce9d7", + "etag": "8d91e1621f990906971ffa67900b6743", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", + "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/test-s3-cp.sh b/ctorm/cumulus-db-md-extract/test-s3-cp.sh new file mode 100644 index 0000000..7b32da5 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/test-s3-cp.sh @@ -0,0 +1,7 @@ +set -euo pipefail + +S3_URI_COPYTEST="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" +echo "Failing fast if we can't write to ${S3_URI_COPYTEST}..." +touch ./test-s3-write.txt +aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write.txt" +rm ./test-s3-write.txt From 994e7df01313e595c6b1c52695f986fe5233e441 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 15:35:26 -0800 Subject: [PATCH 07/15] RP-7509 handle short assume role and long queries --- ctorm/cumulus-db-md-extract/buildspec.yaml | 2 +- .../export-granules-daily.sh | 30 ++++++++++--------- .../generate-slices-daily.sh | 10 ++++--- .../refresh-ctorm-upload-role.sh | 19 ++++++++++++ .../terraform/terraform-opera-prod.tfstate | 22 +++++++------- 5 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index 33b0998..a766071 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -48,7 +48,7 @@ phases: pre_build: commands: - ls -lah - - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh + - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh refresh-ctorm-upload-role.sh - | echo 'Before assuming role: ' - aws sts get-caller-identity diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index 8e5a9e8..b95b13f 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -5,17 +5,6 @@ FAILED_SLICES_FILE="failed_slices_daily.tsv" rm -f "$FAILED_SLICES_FILE" -echo "Statement timeout: " -psql \ - --host="$PGHOST" \ - --port="${PGPORT:-5432}" \ - --username="$PGUSER" \ - --dbname="$PGDATABASE" \ - --no-psqlrc \ - -c "SHOW statement_timeout;" - - - while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then echo "Skipping malformed slice row: collection=${COLLECTION:-} yyyymmdd=${YYYYMMDD:-}" >&2 @@ -25,9 +14,11 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do YYYYMM="${YYYYMMDD:0:6}" SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + TMP_OUTPUT="$(mktemp "${SAFE_COLLECTION}.${YYYYMMDD}.XXXXXX.jsonl.gz")" echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" echo "Destination: ${S3_URI}" + echo "Temporary output: ${TMP_OUTPUT}" if ! PGOPTIONS="-c statement_timeout=${PG_STATEMENT_TIMEOUT:-0}" psql \ --host="$PGHOST" \ @@ -40,19 +31,30 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do --set=collection="$COLLECTION" \ --set=yyyymmdd="$YYYYMMDD" \ --file=export-granules-slice-daily.sql \ - | gzip -c \ - | aws s3 cp - "$S3_URI"; then + | gzip -c > "$TMP_OUTPUT"; then + + echo "FAILED query/gzip collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + rm -f "$TMP_OUTPUT" + continue + fi + + source ./refresh-ctorm-upload-role.sh - echo "FAILED collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + if ! aws s3 cp "$TMP_OUTPUT" "$S3_URI"; then + echo "FAILED upload collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + rm -f "$TMP_OUTPUT" continue fi + rm -f "$TMP_OUTPUT" echo "Uploaded ${S3_URI}" done < slices_daily.tsv if [[ -s "$FAILED_SLICES_FILE" ]]; then + source ./refresh-ctorm-upload-role.sh FAILED_SLICES_S3_URI="${S3_PREFIX}/failed_slices_daily.tsv" aws s3 cp "$FAILED_SLICES_FILE" "$FAILED_SLICES_S3_URI" echo "Some slices failed. Uploaded failure list to ${FAILED_SLICES_S3_URI}" >&2 diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh index 3651d74..b506300 100644 --- a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -1,9 +1,6 @@ set -euo pipefail - -S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" - psql \ --host="$PGHOST" \ --port="${PGPORT:-5432}" \ @@ -94,4 +91,9 @@ psql \ c.name, to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); " > slices_daily.tsv - aws s3 cp slices_daily.tsv "$S3_URI/" + +source ./refresh-ctorm-upload-role.sh + +SLICES_S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily.tsv" +aws s3 cp slices_daily.tsv "${SLICES_S3_URI}" +echo "Uploaded ${SLICES_S3_URI}" diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh new file mode 100644 index 0000000..28ca391 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${CTORM_S3_ROLE_ARN:-}" ]]; then + echo "CTORM_S3_ROLE_ARN is required" >&2 + exit 1 +fi + +echo "Refreshing CTORM upload role credentials..." >&2 + +CREDS_JSON="$(aws sts assume-role \ + --role-arn "${CTORM_S3_ROLE_ARN}" \ + --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER:-manual}-$(date +%s)")" + +export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" +export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" +export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + +aws sts get-caller-identity >&2 diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 238ecee..7ac4399 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 78, + "serial": 83, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", - "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", + "id": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", + "output_base64sha256": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", + "output_base64sha512": "dyN57NtzXihAKHxJioCsORk+6seeqMPqmOjRjZ/MGtoqISkL/zQBVIy+bMmMzSWV9TWAQjgrVrNlyR/n32V7fA==", "output_file_mode": null, - "output_md5": "8d91e1621f990906971ffa67900b6743", + "output_md5": "b1956744784cbb7f29513ed64d125aa8", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", - "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", - "output_size": 5692, + "output_sha": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", + "output_sha256": "ec5523d82308e38fce8172f5035607b2fa19ff702d80ecfadd9ca22a83993773", + "output_sha512": "772379ecdb735e2840287c498a80ac39193eeac79ea8c3ea98e8d18d9fcc1ada2a21290bff3401548cbe6cc98ccd2595f5358042382b56b365c91fe7df657b7c", + "output_size": 6385, "source": [], "source_content": null, "source_content_filename": null, @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "8d91e1621f990906971ffa67900b6743", + "etag": "b1956744784cbb7f29513ed64d125aa8", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "source_hash": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From 814fd1274e9fdbd95a5f768fbc69a8b4a80a0d0e Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 17:22:49 -0800 Subject: [PATCH 08/15] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/buildspec.yaml | 11 ++-------- .../refresh-ctorm-upload-role.sh | 1 + .../terraform/terraform-opera-prod.tfstate | 22 +++++++++---------- ctorm/cumulus-db-md-extract/test-s3-cp.sh | 5 ++++- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index a766071..7702062 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -54,17 +54,10 @@ phases: - aws sts get-caller-identity - | echo "Assuming ctorm upload role..." - - | - CREDS_JSON="$(aws sts assume-role \ - --role-arn "${CTORM_S3_ROLE_ARN}" \ - --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER}")" - - export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" - export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" - export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + source ./refresh-ctorm-upload-role.sh - | echo 'After assuming role: ' - - aws sts get-caller-identity + aws sts get-caller-identity build: diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh index 28ca391..e28fc6a 100644 --- a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -8,6 +8,7 @@ fi echo "Refreshing CTORM upload role credentials..." >&2 +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CREDS_JSON="$(aws sts assume-role \ --role-arn "${CTORM_S3_ROLE_ARN}" \ --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER:-manual}-$(date +%s)")" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 7ac4399..50f232d 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 83, + "serial": 99, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", - "output_base64sha256": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", - "output_base64sha512": "dyN57NtzXihAKHxJioCsORk+6seeqMPqmOjRjZ/MGtoqISkL/zQBVIy+bMmMzSWV9TWAQjgrVrNlyR/n32V7fA==", + "id": "9fbb26adcb1cd2335401493f40f3f51313a97f96", + "output_base64sha256": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", + "output_base64sha512": "/0V83kNA5dBpxhDHHVYvvxhHTSdeerh1OAfI0c0IAXSyIp1NF2g0uFabWQ8AezYL/wARFo6uyGKw0qa8acR6iw==", "output_file_mode": null, - "output_md5": "b1956744784cbb7f29513ed64d125aa8", + "output_md5": "b3291b9c9adcab32d68819222e5d76f8", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", - "output_sha256": "ec5523d82308e38fce8172f5035607b2fa19ff702d80ecfadd9ca22a83993773", - "output_sha512": "772379ecdb735e2840287c498a80ac39193eeac79ea8c3ea98e8d18d9fcc1ada2a21290bff3401548cbe6cc98ccd2595f5358042382b56b365c91fe7df657b7c", - "output_size": 6385, + "output_sha": "9fbb26adcb1cd2335401493f40f3f51313a97f96", + "output_sha256": "6d3a2810ff5968aeae413f9189e0596f11fe57a1d5b6ac6afa871cce11c34bc0", + "output_sha512": "ff457cde4340e5d069c610c71d562fbf18474d275e7ab8753807c8d1cd080174b2229d4d176834b8569b590f007b360bff0011168eaec862b0d2a6bc69c47a8b", + "output_size": 6320, "source": [], "source_content": null, "source_content_filename": null, @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "b1956744784cbb7f29513ed64d125aa8", + "etag": "b3291b9c9adcab32d68819222e5d76f8", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", + "source_hash": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/test-s3-cp.sh b/ctorm/cumulus-db-md-extract/test-s3-cp.sh index 7b32da5..1f3c596 100644 --- a/ctorm/cumulus-db-md-extract/test-s3-cp.sh +++ b/ctorm/cumulus-db-md-extract/test-s3-cp.sh @@ -1,7 +1,10 @@ set -euo pipefail + + S3_URI_COPYTEST="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" echo "Failing fast if we can't write to ${S3_URI_COPYTEST}..." touch ./test-s3-write.txt -aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write.txt" +source ./refresh-ctorm-upload-role.sh +aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write-${CODEBUILD_BUILD_NUMBER}.txt" rm ./test-s3-write.txt From 3e19e8303495fe9fdcedcea126a93e222ee0aacc Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 26 Jun 2026 15:40:26 -0800 Subject: [PATCH 09/15] PR-7510 addition of Dynamo DB loader --- ctorm/README.md | 40 +- .../terraform/terraform-opera-prod.tfstate | 22 +- ctorm/granule-md-db-loader/README.md | 20 + ctorm/granule-md-db-loader/buildspec.yaml | 16 + .../granule_md_db_loader.py | 99 ++++ ctorm/infra/terraform/main.tf | 130 ++++- ctorm/infra/terraform/terraform.tfstate | 312 +++++++++++- .../infra/terraform/terraform.tfstate.backup | 457 +++++++++++++++++- .../ctorm-prod/terraform.tfstate | 38 +- 9 files changed, 1063 insertions(+), 71 deletions(-) create mode 100644 ctorm/granule-md-db-loader/README.md create mode 100644 ctorm/granule-md-db-loader/buildspec.yaml create mode 100644 ctorm/granule-md-db-loader/granule_md_db_loader.py diff --git a/ctorm/README.md b/ctorm/README.md index d44d3f6..eeb4e20 100644 --- a/ctorm/README.md +++ b/ctorm/README.md @@ -1,37 +1,15 @@ # CTORM + Cumulus Throughput... uh... ORM? Load tester tool for Cumulus. -Operates as two steps: - -* Prepare - * The prepare step goes through the UMMG directory of specified prod buckets and - extracts the metadata necessary to recreate CNM messages. It then creates SQS messages - containing lists of these metadata. -* CNM Sender - * The CNM Sender step reads the SQS messages and sends them to the Cumulus ingest queue. - -## Prepare - -### Cfg file - -See the `ctorm.cfg.example` file. - -#### source_buckets - -| key | value | -|--------------|-----------------------------------------------------------| -| bucketname | Name of bucket in which to find the UMMG files | -| keypair_name | Arbritrary name to suffix to the AWS keypair env var | -| share | Number of granules from this collection to add to the CNM | -| ummg_prefix | Path in bucketname where to find the desired UMMG files | - -Let's say you have the following source buckets set up: - -### Env vars - -You will need to have the following environment variables set: +Operates as x steps: -* `AWS_ACCESS_KEY_ID_[keypair_name]` where `keypair_name` matches an entry in the ctorm cfg toml file. -* `AWS_SECRET_ACCESS_KEY_[keypair_name]` where `keypair_name` matches an entry in the ctorm cfg toml file. +* cumulus-db-md-extract + * Operates in the prod account. + * Gets the metadata from the cumulus database and writes it to a bunch of .jsonl files in a S3 bucket in the + operational account. +* granule-md-db-loader + * Operates in the account in which the load test will operate. + * Reads the .jsonl files from the S3 bucket and loads them into the CTORM granules dynamo db. diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 50f232d..be02ac5 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 99, + "serial": 112, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "9fbb26adcb1cd2335401493f40f3f51313a97f96", - "output_base64sha256": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", - "output_base64sha512": "/0V83kNA5dBpxhDHHVYvvxhHTSdeerh1OAfI0c0IAXSyIp1NF2g0uFabWQ8AezYL/wARFo6uyGKw0qa8acR6iw==", + "id": "15e0432811ac45b77440e902f20b433130b97e70", + "output_base64sha256": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", + "output_base64sha512": "loaVRKTjqeDeRXYLTynxgJIcoeFMuaH5+Yx1GZPZymzfK4GSIarTAvusSWNs7DHEH+baqi6JJ8D5SjrMvSmT4Q==", "output_file_mode": null, - "output_md5": "b3291b9c9adcab32d68819222e5d76f8", + "output_md5": "cdd0aab259d06df74b4ae3ddbbd7615e", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "9fbb26adcb1cd2335401493f40f3f51313a97f96", - "output_sha256": "6d3a2810ff5968aeae413f9189e0596f11fe57a1d5b6ac6afa871cce11c34bc0", - "output_sha512": "ff457cde4340e5d069c610c71d562fbf18474d275e7ab8753807c8d1cd080174b2229d4d176834b8569b590f007b360bff0011168eaec862b0d2a6bc69c47a8b", - "output_size": 6320, + "output_sha": "15e0432811ac45b77440e902f20b433130b97e70", + "output_sha256": "f37531a4a276be2b5853d2383331e343604a81c2b63926c24f83c318e3960d9f", + "output_sha512": "96869544a4e3a9e0de45760b4f29f180921ca1e14cb9a1f9f98c751993d9ca6cdf2b819221aad302fbac49636cec31c41fe6daaa2e8927c0f94a3accbd2993e1", + "output_size": 6341, "source": [], "source_content": null, "source_content_filename": null, @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "b3291b9c9adcab32d68819222e5d76f8", + "etag": "cdd0aab259d06df74b4ae3ddbbd7615e", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", + "source_hash": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md new file mode 100644 index 0000000..8a75dcb --- /dev/null +++ b/ctorm/granule-md-db-loader/README.md @@ -0,0 +1,20 @@ +# Granule MD DB Loader + +## Deploying + +The infrastructure for this is provided by the [CTORM terraform](../infra/README.md) + +## Running + +```bash +# for dev: +export AWS_PROFILE="cumulus-sbx-7522" +export CODEBUILD_PROJECT="ctorm-dev-granule-md-db-loader" + +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--query 'build.id' \ +--output text + +``` diff --git a/ctorm/granule-md-db-loader/buildspec.yaml b/ctorm/granule-md-db-loader/buildspec.yaml new file mode 100644 index 0000000..d40dd5b --- /dev/null +++ b/ctorm/granule-md-db-loader/buildspec.yaml @@ -0,0 +1,16 @@ +version: 0.2 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - echo "Installing dependencies..." + - pip install boto3 + build: + commands: + - echo "Starting DynamoDB Import script..." + - python granule_md_db_loader.py + post_build: + commands: + - echo "Import finished." diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py new file mode 100644 index 0000000..b7ebc5b --- /dev/null +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -0,0 +1,99 @@ +import os +import sys +import gzip +import json +import logging +import datetime +from decimal import Decimal +import boto3 + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + + +def main(): + bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") + table_name = os.environ.get("TABLE_NAME") + prefix = os.environ.get("PREFIX", "cumulus-granules/") + + if not table_name: + logger.error("TABLE_NAME environment variable is required.") + sys.exit(1) + + s3 = boto3.client("s3") + dynamodb = boto3.resource("dynamodb") + table = dynamodb.Table(table_name) + + logger.info(f"Starting import from s3://{bucket_name}/{prefix} into DynamoDB table {table_name}") + + paginator = s3.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + total_files = 0 + total_records = 0 + + # Walk through the bucket objects + for page in pages: + if "Contents" not in page: + continue + + for obj in page["Contents"]: + key = obj["Key"] + if not key.endswith(".jsonl.gz"): + continue + + logger.info(f"Processing s3://{bucket_name}/{key}") + total_files += 1 + + # Download and decompress the file + local_path = "/tmp/temp.jsonl.gz" + try: + s3.download_file(bucket_name, key, local_path) + except Exception as e: + logger.error(f"Failed to download s3://{bucket_name}/{key}: {e}") + continue + + # Open, decompress and parse + records_in_file = 0 + try: + with gzip.open(local_path, "rt", encoding="utf-8") as f: + with table.batch_writer() as batch: + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + logger.error(f"Failed to parse JSON line: {e}") + continue + + # Add/modify fields if needed + # Example additional fields: + item["imported_at"] = datetime.datetime.utcnow().isoformat() + "Z" + + # Ensure partition and sort keys are present + if "pk" not in item or "sk" not in item: + logger.warning(f"Skipping record missing pk/sk: {item.get('granule_id')}") + continue + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 + + logger.info(f"Successfully imported {records_in_file} records from {key}") + except Exception as e: + logger.error(f"Error processing file {key}: {e}") + finally: + if os.path.exists(local_path): + os.remove(local_path) + + logger.info(f"Import complete. Processed {total_files} files, imported {total_records} total records.") + + +if __name__ == "__main__": + main() diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 2ba024e..ab1a595 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -33,6 +33,23 @@ resource "aws_dynamodb_table" "granules" { name = "sk" type = "S" } + + attribute { + name = "gsi1pk" + type = "S" + } + + attribute { + name = "gsi1sk" + type = "S" + } + + global_secondary_index { + name = "gsi1" + hash_key = "gsi1pk" + range_key = "gsi1sk" + projection_type = "ALL" + } } @@ -101,7 +118,8 @@ data "aws_iam_policy_document" "cnm-sender" { ] resources = [ - aws_dynamodb_table.granules.arn + aws_dynamodb_table.granules.arn, + "${aws_dynamodb_table.granules.arn}/index/*" ] } } @@ -315,3 +333,113 @@ resource "aws_iam_role_policy" "cumulus_db_md_extract_upload" { role = aws_iam_role.cumulus_db_md_extract_upload.id policy = data.aws_iam_policy_document.cumulus_db_md_extract_upload.json } + + +# Granule DynamoDB loader. +data "archive_file" "granule_md_db_loader" { + type = "zip" + source_dir = "${path.module}/../../granule-md-db-loader" + output_path = "${path.module}/.terraform/granule-md-db-loader.zip" +} + +resource "aws_s3_object" "granule_md_db_loader_source" { + bucket = aws_s3_bucket.scratch.bucket + key = "codebuild/granule-md-db-loader.zip" + source = data.archive_file.granule_md_db_loader.output_path + source_hash = data.archive_file.granule_md_db_loader.output_base64sha256 +} + +resource "aws_iam_role" "granule_md_db_loader_role" { + name = "${var.name_prefix}-codebuild-granule_md_db_loader-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "codebuild.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy" "codebuild_granule_md_db_loader_role_policy" { + name = "${var.name_prefix}-codebuild-granule-md-db-loader-role-policy" + role = aws_iam_role.granule_md_db_loader_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:*:*:*" + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:ListBucket" + ] + Resource = [ + aws_s3_bucket.scratch.arn, + "${aws_s3_bucket.scratch.arn}/*" + ] + }, + { + Effect = "Allow" + Action = [ + "dynamodb:PutItem", + "dynamodb:BatchWriteItem" + ] + Resource = [ + aws_dynamodb_table.granules.arn + ] + } + ] + }) +} + +resource "aws_codebuild_project" "granule_md_db_loader" { + name = "${var.name_prefix}-granule-md-db-loader" + description = "Imports granules jsonl.gz files from S3 to DynamoDB" + service_role = aws_iam_role.granule_md_db_loader_role.arn + build_timeout = 480 + + artifacts { + type = "NO_ARTIFACTS" + } + + source { + type = "S3" + location = "${aws_s3_bucket.scratch.bucket}/${aws_s3_object.granule_md_db_loader_source.key}" + } + + environment { + type = "LINUX_CONTAINER" + image = "aws/codebuild/amazonlinux2-x86_64-standard:5.0" + compute_type = "BUILD_GENERAL1_SMALL" + + environment_variable { + name = "CTORM_BUCKET" + value = aws_s3_bucket.scratch.bucket + } + + environment_variable { + name = "TABLE_NAME" + value = aws_dynamodb_table.granules.name + } + + environment_variable { + name = "PREFIX" + value = "cumulus-granules/" + } + } +} diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index c50a504..99bc4f2 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 45, + "serial": 55, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -34,6 +34,38 @@ } }, "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", + "output_base64sha256": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", + "output_base64sha512": "p79O0G8V4Rv5GWXKpy6XA/SPVqK8Ubrs3aXSg4WbWcOxqk9n0cIV3xDEI9QufnQ6LABj00SEA9y4SIGQ5h+NsA==", + "output_file_mode": null, + "output_md5": "efc9aa787b5b0e96b00fb4bab67e84d2", + "output_path": "./.terraform/granule-md-db-loader.zip", + "output_sha": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", + "output_sha256": "4ddb1329757729a39d4ed38faa7221a0aae698515e00a1c271733c9f7989ac4c", + "output_sha512": "a7bf4ed06f15e11bf91965caa72e9703f48f56a2bc51baecdda5d283859b59c3b1aa4f67d1c215df10c423d42e7e743a2c0063d3448403dcb8488190e61f8db0", + "output_size": 2135, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./../../granule-md-db-loader", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "archive_file", @@ -124,9 +156,9 @@ { "schema_version": 0, "attributes": { - "id": "3548715077", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", + "id": "2517841586", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": [\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -196,7 +228,8 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules" + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*" ], "sid": "" } @@ -488,6 +521,133 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "Imports granules jsonl.gz files from S3 to DynamoDB", + "encryption_key": "arn:aws:kms:us-west-2:871271927522:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [ + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "TABLE_NAME", + "type": "PLAINTEXT", + "value": "ctorm-dev-granules" + }, + { + "name": "PREFIX", + "type": "PLAINTEXT", + "value": "cumulus-granules/" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-dev-granule-md-db-loader", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch", + "aws_s3_object.granule_md_db_loader_source", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_dynamodb_table", @@ -499,6 +659,14 @@ "attributes": { "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", "attribute": [ + { + "name": "gsi1pk", + "type": "S" + }, + { + "name": "gsi1sk", + "type": "S" + }, { "name": "pk", "type": "S" @@ -510,7 +678,18 @@ ], "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, - "global_secondary_index": [], + "global_secondary_index": [ + { + "hash_key": "gsi1pk", + "name": "gsi1", + "non_key_attributes": [], + "on_demand_throughput": [], + "projection_type": "ALL", + "range_key": "gsi1sk", + "read_capacity": 0, + "write_capacity": 0 + } + ], "hash_key": "pk", "id": "ctorm-dev-granules", "import_table": [], @@ -536,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -570,7 +749,7 @@ "inline_policy": [ { "name": "ctorm-dev-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" }, { "name": "ctorm-dev-scratch-access", @@ -634,6 +813,42 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "granule_md_db_loader_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-26T21:54:18Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-codebuild-granule_md_db_loader-role", + "inline_policy": [ + { + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-codebuild-granule_md_db_loader-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RISLTCQYHZ" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -646,7 +861,7 @@ "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", "name": "ctorm-dev-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", "role": "ctorm-dev-cnm-sender-role" }, "sensitive_attributes": [], @@ -663,6 +878,31 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_granule_md_db_loader_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-codebuild-granule_md_db_loader-role:ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", + "role": "ctorm-dev-codebuild-granule_md_db_loader-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -928,6 +1168,60 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "granule_md_db_loader_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "bucket": "ctorm-dev-scratch", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "efc9aa787b5b0e96b00fb4bab67e84d2", + "force_destroy": false, + "id": "codebuild/granule-md-db-loader.zip", + "key": "codebuild/granule-md-db-loader.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "server_side_encryption": "AES256", + "source": "./.terraform/granule-md-db-loader.zip", + "source_hash": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.scratch", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 270feb7..378680f 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 40, + "serial": 53, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -34,6 +34,38 @@ } }, "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "95710df96fcf68aad0084e50ecd58c0f928f7862", + "output_base64sha256": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", + "output_base64sha512": "6bcsla7ow6YaZjJnx8kt7WausRTy/XS/3H9A2fuy56ueg+uXlNVN8jwaY45wS2CEbsZhPPop64S8Vw7JJ3Uk3A==", + "output_file_mode": null, + "output_md5": "19d483f8f2202fea76c210c12ce8ccb2", + "output_path": "./.terraform/granule-md-db-loader.zip", + "output_sha": "95710df96fcf68aad0084e50ecd58c0f928f7862", + "output_sha256": "d1f89ebdac2a779f3fcc3d32a31f518418d9868a77e7ed2c05e7ee864318bc78", + "output_sha512": "e9b72c95aee8c3a61a663267c7c92ded66aeb114f2fd74bfdc7f40d9fbb2e7ab9e83eb9794d54df23c1a638e704b60846ec6613cfa29eb84bc570ec9277524dc", + "output_size": 1719, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./../../granule-md-db-loader", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "archive_file", @@ -115,6 +147,99 @@ } ] }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2517841586", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": [\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -396,6 +521,216 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "Imports granules jsonl.gz files from S3 to DynamoDB", + "encryption_key": "arn:aws:kms:us-west-2:871271927522:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [ + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "TABLE_NAME", + "type": "PLAINTEXT", + "value": "ctorm-dev-granules" + }, + { + "name": "PREFIX", + "type": "PLAINTEXT", + "value": "cumulus-granules/" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-dev-granule-md-db-loader", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": null, + "tags_all": {}, + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch", + "aws_s3_object.granule_md_db_loader_source", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", + "attribute": [ + { + "name": "gsi1pk", + "type": "S" + }, + { + "name": "gsi1sk", + "type": "S" + }, + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [ + { + "hash_key": "gsi1pk", + "name": "gsi1", + "non_key_attributes": [], + "on_demand_throughput": [], + "projection_type": "ALL", + "range_key": "gsi1sk", + "read_capacity": 0, + "write_capacity": 0 + } + ], + "hash_key": "pk", + "id": "ctorm-dev-granules", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-dev-granules", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -414,7 +749,7 @@ "inline_policy": [ { "name": "ctorm-dev-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}" }, { "name": "ctorm-dev-scratch-access", @@ -478,6 +813,37 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "granule_md_db_loader_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-26T21:54:18Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-codebuild-granule_md_db_loader-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-codebuild-granule_md_db_loader-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROA4VW62R3RISLTCQYHZ" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -490,7 +856,7 @@ "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", "name": "ctorm-dev-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", "role": "ctorm-dev-cnm-sender-role" }, "sensitive_attributes": [], @@ -498,7 +864,6 @@ "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", "aws_dynamodb_table.granules", - "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_sqs_queue.granules", "aws_sqs_queue.granules_dlq", @@ -508,6 +873,31 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_granule_md_db_loader_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-codebuild-granule_md_db_loader-role:ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", + "role": "ctorm-dev-codebuild-granule_md_db_loader-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -583,7 +973,7 @@ "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "LOG_LEVEL": "INFO", - "TABLE_NAME": "ctorm-dev-state" + "TABLE_NAME": "ctorm-dev-granules" } } ], @@ -601,7 +991,7 @@ "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", - "last_modified": "2026-06-24T19:29:31.000+0000", + "last_modified": "2026-06-25T02:29:36.000+0000", "layers": [], "logging_config": [ { @@ -647,7 +1037,6 @@ "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", "aws_dynamodb_table.granules", - "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_sqs_queue.granules", @@ -774,6 +1163,60 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "granule_md_db_loader_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "bucket": "ctorm-dev-scratch", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "19d483f8f2202fea76c210c12ce8ccb2", + "force_destroy": false, + "id": "codebuild/granule-md-db-loader.zip", + "key": "codebuild/granule-md-db-loader.zip", + "kms_key_id": null, + "metadata": null, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "server_side_encryption": "AES256", + "source": "./.terraform/granule-md-db-loader.zip", + "source_hash": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", + "storage_class": "STANDARD", + "tags": null, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.scratch", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate index 278e8a2..8c6c0eb 100644 --- a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 18, + "serial": 25, "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -406,7 +406,7 @@ "role_arn": "", "schedule_expression": "rate(1 minute)", "state": "DISABLED", - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -480,7 +480,7 @@ "name_prefix": "", "retention_in_days": 14, "skip_destroy": false, - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -536,7 +536,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -567,14 +567,23 @@ "description": "", "force_detach_policies": false, "id": "ctorm-cnm-sender-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-cnm-sender-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}" + }, + { + "name": "ctorm-scratch-access", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch/*\",\"arn:aws:s3:::ctorm-scratch\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-cnm-sender-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROARNJJOPGE3PVM46W7W" }, @@ -601,14 +610,19 @@ "description": "", "force_detach_policies": false, "id": "ctorm-cumulus-db-md-extract-upload", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-upload", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-cumulus-db-md-extract-upload", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROARNJJOPGE4XUWPCFMM" }, @@ -743,7 +757,7 @@ "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender/invocations", "kms_key_arn": "", "last_modified": "2026-06-25T02:41:40.841+0000", - "layers": null, + "layers": [], "logging_config": [ { "application_log_level": "", @@ -771,7 +785,7 @@ "snap_start": [], "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", "source_code_size": 344, - "tags": null, + "tags": {}, "tags_all": {}, "timeout": 120, "timeouts": null, @@ -941,7 +955,7 @@ "redrive_allow_policy": "", "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granule-dlq\",\"maxReceiveCount\":5}", "sqs_managed_sse_enabled": true, - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", @@ -982,7 +996,7 @@ "redrive_allow_policy": "", "redrive_policy": "", "sqs_managed_sse_enabled": true, - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", From 93bf745fdee082364ba79285c225916962dac37f Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 26 Jun 2026 17:04:57 -0800 Subject: [PATCH 10/15] PR-7510 Added ability to use provisioned dynamodb for loading granules --- .../export-granules-daily.sh | 3 +- ctorm/granule-md-db-loader/README.md | 72 +++++++++++++++++++ .../granule_md_db_loader.py | 59 +++++++++++++-- ctorm/infra/README.md | 12 ++++ ctorm/infra/terraform/main.tf | 15 ++-- ctorm/infra/terraform/terraform.tfstate | 34 ++++----- .../infra/terraform/terraform.tfstate.backup | 41 ++++++----- ctorm/infra/terraform/variables.tf | 12 ++++ 8 files changed, 202 insertions(+), 46 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index b95b13f..c497664 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -11,9 +11,10 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do continue fi + YYYY="${YYYYMMDD:0:4)" YYYYMM="${YYYYMMDD:0:6}" SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') - S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYY}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" TMP_OUTPUT="$(mktemp "${SAFE_COLLECTION}.${YYYYMMDD}.XXXXXX.jsonl.gz")" echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md index 8a75dcb..5289584 100644 --- a/ctorm/granule-md-db-loader/README.md +++ b/ctorm/granule-md-db-loader/README.md @@ -4,17 +4,89 @@ The infrastructure for this is provided by the [CTORM terraform](../infra/README.md) +## Before running + +AWS throttles writes to dynamo DB if we insert repeatedly into the same partion, so when we can run +multiple collections at once, each going into their own partition. + +Additionally, running as PAY_PER_REQUEST will get us throttled no matter what. So for the loading of the granules, we +will switch the tf to use PROVISIONED. See the [CTORM infrastruture README](../infra/README.md) for the command to do +this. + ## Running ```bash # for dev: export AWS_PROFILE="cumulus-sbx-7522" export CODEBUILD_PROJECT="ctorm-dev-granule-md-db-loader" +export CTORM_BUCKET="ctorm-dev-scratch" + + +# get list of collections +aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/nisar/ |grep PRE +aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/opera/ |grep PRE + +# If you want do them all sequentially, not recommended and not going to work with all the data: +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--query 'build.id' \ +--output text + + +# Running a specific collection in parallel (by overriding the PREFIX): +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L2_CSLC-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L2_RTC-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DISP-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DIST-ALERT-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L0B_CRSD_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text aws codebuild start-build --profile="${AWS_PROFILE}" \ --project-name "${CODEBUILD_PROJECT}" \ --region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L0B_RRSD_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L2/,type=PLAINTEXT \ --query 'build.id' \ --output text ``` + +## After running + +Restore the main.tf to its original state with billing_mode set to PAY_PER_REQUEST, etc. diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index b7ebc5b..6ef48c6 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -1,17 +1,49 @@ -import os -import sys +import datetime import gzip import json import logging -import datetime +import os +import sys +import time from decimal import Decimal + import boto3 +from botocore.config import Config # Configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) +class PartitionRateLimiter: + """ + Limits the number of write requests to any single partition key (pk) + to prevent exceeding DynamoDB's physical limit of 1000 operations/sec per partition. + """ + + def __init__(self, max_rate_per_sec: int = 890): + self.max_rate = max_rate_per_sec + self.history = {} # pk -> list of timestamps + + def limit(self, pk): + now = time.time() + if pk not in self.history: + self.history[pk] = [] + + # Retain only timestamps from the last 1.0 second + self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] + + if len(self.history[pk]) >= self.max_rate: + # Calculate sleep duration to let the oldest request roll off the 1-second window + sleep_time = 1.0 - (now - self.history[pk][0]) + if sleep_time > 0: + time.sleep(sleep_time) + now = time.time() + self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] + + self.history[pk].append(now) + + def main(): bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") table_name = os.environ.get("TABLE_NAME") @@ -22,7 +54,15 @@ def main(): sys.exit(1) s3 = boto3.client("s3") - dynamodb = boto3.resource("dynamodb") + + # Configure boto3 with more aggressive retries to gracefully handle scale peaks + retry_config = Config( + retries={ + "max_attempts": 10, + "mode": "standard" + } + ) + dynamodb = boto3.resource("dynamodb", config=retry_config) table = dynamodb.Table(table_name) logger.info(f"Starting import from s3://{bucket_name}/{prefix} into DynamoDB table {table_name}") @@ -33,6 +73,9 @@ def main(): total_files = 0 total_records = 0 + # Initialize partition-level rate limiter set to a safe threshold (850 writes/sec per pk) + rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) + # Walk through the bucket objects for page in pages: if "Contents" not in page: @@ -72,14 +115,18 @@ def main(): continue # Add/modify fields if needed - # Example additional fields: item["imported_at"] = datetime.datetime.utcnow().isoformat() + "Z" # Ensure partition and sort keys are present - if "pk" not in item or "sk" not in item: + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: logger.warning(f"Skipping record missing pk/sk: {item.get('granule_id')}") continue + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + # Batch insert into DynamoDB batch.put_item(Item=item) records_in_file += 1 diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 06d8a93..5ae884c 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -19,6 +19,18 @@ terraform plan \ terraform apply \ -var-file="${VARFILE}" +# For temporary use while loading granule metadata: +terraform plan \ + -var-file="${VARFILE}" \ + -var="granule_table_billing_mode=PROVISIONED" \ + -var="granule_table_write_capacity=10000" + +terraform apply \ + -var-file="${VARFILE}" \ + -var="granule_table_billing_mode=PROVISIONED" \ + -var="granule_table_write_capacity=10000" + + terraform destroy diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index ab1a595..53ccb0d 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -18,11 +18,14 @@ resource "aws_sqs_queue" "granules" { } resource "aws_dynamodb_table" "granules" { - name = "${var.name_prefix}-granules" - billing_mode = "PAY_PER_REQUEST" + name = "${var.name_prefix}-granules" + # billing_mode = "PROVISIONED" # Temporarily switch to PROVISIONED for bulk load + billing_mode = var.granule_table_billing_mode - hash_key = "pk" - range_key = "sk" + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + hash_key = "pk" + range_key = "sk" attribute { name = "pk" @@ -49,10 +52,14 @@ resource "aws_dynamodb_table" "granules" { hash_key = "gsi1pk" range_key = "gsi1sk" projection_type = "ALL" + + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null } } + resource "aws_cloudwatch_log_group" "cnm-sender" { name = "/aws/lambda/${local.lambda_name}" retention_in_days = 14 diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 99bc4f2..f2714d5 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 55, + "serial": 63, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", - "output_base64sha256": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", - "output_base64sha512": "p79O0G8V4Rv5GWXKpy6XA/SPVqK8Ubrs3aXSg4WbWcOxqk9n0cIV3xDEI9QufnQ6LABj00SEA9y4SIGQ5h+NsA==", + "id": "15541977bfb6559cdcb3dce4cb03186ba760a70c", + "output_base64sha256": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", + "output_base64sha512": "5JDpSb4/VU/oaT7BC6qYuY+/ljGpjHWd5SY8B/8SFH98/DPmit3/MLGGHvjgZzhbsrOghg4nq8IQ168xlz/mpg==", "output_file_mode": null, - "output_md5": "efc9aa787b5b0e96b00fb4bab67e84d2", + "output_md5": "8198f06853d8821f035afa1477cc3d3c", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", - "output_sha256": "4ddb1329757729a39d4ed38faa7221a0aae698515e00a1c271733c9f7989ac4c", - "output_sha512": "a7bf4ed06f15e11bf91965caa72e9703f48f56a2bc51baecdda5d283859b59c3b1aa4f67d1c215df10c423d42e7e743a2c0063d3448403dcb8488190e61f8db0", - "output_size": 2135, + "output_sha": "15541977bfb6559cdcb3dce4cb03186ba760a70c", + "output_sha256": "ed4bcd8457abb06013f1d7949c3363de3d829ae4778ccbbf36f9d69e80807a28", + "output_sha512": "e490e949be3f554fe8693ec10baa98b98fbf9631a98c759de5263c07ff12147f7cfc33e68addff30b1861ef8e067385bb2b3a0860e27abc210d7af31973fe6a6", + "output_size": 3346, "source": [], "source_content": null, "source_content_filename": null, @@ -676,7 +676,7 @@ "type": "S" } ], - "billing_mode": "PAY_PER_REQUEST", + "billing_mode": "PROVISIONED", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -686,8 +686,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 0, - "write_capacity": 0 + "read_capacity": 5, + "write_capacity": 10000 } ], "hash_key": "pk", @@ -703,7 +703,7 @@ } ], "range_key": "sk", - "read_capacity": 0, + "read_capacity": 5, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -715,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -724,7 +724,7 @@ "enabled": false } ], - "write_capacity": 0 + "write_capacity": 10000 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -1194,7 +1194,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "efc9aa787b5b0e96b00fb4bab67e84d2", + "etag": "8198f06853d8821f035afa1477cc3d3c", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1206,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", + "source_hash": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 378680f..0bd8316 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 53, + "serial": 59, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "95710df96fcf68aad0084e50ecd58c0f928f7862", - "output_base64sha256": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", - "output_base64sha512": "6bcsla7ow6YaZjJnx8kt7WausRTy/XS/3H9A2fuy56ueg+uXlNVN8jwaY45wS2CEbsZhPPop64S8Vw7JJ3Uk3A==", + "id": "5def5225e70ce457c014d699b2bd3e2b93a9788e", + "output_base64sha256": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", + "output_base64sha512": "yt3czhsZeaAaC4zMOD5W5zjajGbY/Z09AXYEwpFDW2hmbfqTepKrLT5xoDg5zTEU4CnE2iQyza54OPqIOOZb+w==", "output_file_mode": null, - "output_md5": "19d483f8f2202fea76c210c12ce8ccb2", + "output_md5": "b6108bd6d78e6ebfc4b059f9b0e2f033", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "95710df96fcf68aad0084e50ecd58c0f928f7862", - "output_sha256": "d1f89ebdac2a779f3fcc3d32a31f518418d9868a77e7ed2c05e7ee864318bc78", - "output_sha512": "e9b72c95aee8c3a61a663267c7c92ded66aeb114f2fd74bfdc7f40d9fbb2e7ab9e83eb9794d54df23c1a638e704b60846ec6613cfa29eb84bc570ec9277524dc", - "output_size": 1719, + "output_sha": "5def5225e70ce457c014d699b2bd3e2b93a9788e", + "output_sha256": "da5f4df5ac6f5c647bd90006b2d8ac57ea305bbe74620e7efd6aa95a5244aace", + "output_sha512": "cadddcce1b1979a01a0b8ccc383e56e738da8c66d8fd9d3d017604c291435b68666dfa937a92ab2d3e71a03839cd3114e029c4da2432cdae7838fa8838e65bfb", + "output_size": 3128, "source": [], "source_content": null, "source_content_filename": null, @@ -632,7 +632,7 @@ } ], "source_version": "", - "tags": null, + "tags": {}, "tags_all": {}, "vpc_config": [] }, @@ -715,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -749,7 +749,7 @@ "inline_policy": [ { "name": "ctorm-dev-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" }, { "name": "ctorm-dev-scratch-access", @@ -828,14 +828,19 @@ "description": "", "force_detach_policies": false, "id": "ctorm-dev-codebuild-granule_md_db_loader-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-dev-codebuild-granule_md_db_loader-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROA4VW62R3RISLTCQYHZ" }, @@ -1189,21 +1194,21 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "19d483f8f2202fea76c210c12ce8ccb2", + "etag": "b6108bd6d78e6ebfc4b059f9b0e2f033", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", "kms_key_id": null, - "metadata": null, + "metadata": {}, "object_lock_legal_hold_status": "", "object_lock_mode": "", "object_lock_retain_until_date": "", "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", + "source_hash": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", "storage_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "version_id": "", "website_redirect": "" diff --git a/ctorm/infra/terraform/variables.tf b/ctorm/infra/terraform/variables.tf index 030971c..ef9903b 100644 --- a/ctorm/infra/terraform/variables.tf +++ b/ctorm/infra/terraform/variables.tf @@ -32,6 +32,18 @@ variable "prepare_source_bucket_names" { ] } +variable "granule_table_billing_mode" { + type = string + default = "PAY_PER_REQUEST" + description = "Must be either PROVISIONED or PAY_PER_REQUEST" +} + +variable "granule_table_write_capacity" { + type = number + default = 10000 + description = "When billing_mode is PROVISIONED, forces write_capacity/10 physical partition splits immediately. Set to 1000*number of parallel granule_md_db_loader runs." +} + # variable "cumulus_db_md_extract_remote_codebuild_role_arn" { # type = string # description = "CodeBuild service role ARN from the remote/RDS account." From 03dcbf6884930a8cd3b1a7d8717088e6f0c8c53a Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Sun, 28 Jun 2026 17:33:29 -0800 Subject: [PATCH 11/15] Fixing bugs relating to specifying a slices tsv file --- ctorm/cumulus-db-md-extract/README.md | 18 + .../export-granules-daily.sh | 2 +- .../generate-slices-daily.sh | 182 +++++----- .../generate-slices-daily.sql | 21 ++ .../refresh-ctorm-upload-role.sh | 2 +- .../terraform/codebuild-db-md-extract.tf | 10 + .../terraform/terraform-opera-prod.tfstate | 32 +- .../terraform/terraform-opera-uat.tfstate | 44 ++- ctorm/infra/terraform/main.tf | 3 +- ctorm/infra/terraform/terraform.tfstate | 43 +-- .../infra/terraform/terraform.tfstate.backup | 24 +- .../ctorm-prod/terraform.tfstate | 321 +++++++++++++++++- 12 files changed, 532 insertions(+), 170 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/generate-slices-daily.sql diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index b8b3293..1fe8459 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -108,17 +108,35 @@ export AWS_PROFILE="cumulus-uat-6921" export AWS_REGION="us-west-2" export CODEBUILD_PROJECT="$(terraform output -state="${STATEFILE}" -raw cumulus_db_md_extract_codebuild_project_name)" +# You can do one collection at a time, many in parallel if necessary: +export COLLECTION="OPERA_L2_RTC-S1_V1" + # start build: BUILD_ID="$( aws codebuild start-build \ --project-name "${CODEBUILD_PROJECT}" \ --region "${AWS_REGION}" \ + --environment-variables-override name=COLLECTION,value="${COLLECTION}",type=PLAINTEXT \ --query 'build.id' \ --output text )" echo "${BUILD_ID}" +# Or start build specifying a custom slices.tsv file from S3: +# BUILD_ID="$( + aws codebuild start-build \ + --project-name "${CODEBUILD_PROJECT}" \ + --region "${AWS_REGION}" \ + --environment-variables-override '[ + {"name": "SLICES_S3_URI", "value": "s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_RTC-S1_V1-20170910-20181203.tsv", "type": "PLAINTEXT"}, + {"name": "COLLECTION", "value": "OPERA_L2_RTC-S1_V1", "type": "PLAINTEXT"} + ]' \ + --query 'build.id' \ + --output text +# )" + + # get status: aws codebuild batch-get-builds \ --ids "${BUILD_ID}" \ diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index c497664..c28d176 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -11,7 +11,7 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do continue fi - YYYY="${YYYYMMDD:0:4)" + YYYY="${YYYYMMDD:0:4}" YYYYMM="${YYYYMMDD:0:6}" SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYY}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh index b506300..3599117 100644 --- a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -1,99 +1,95 @@ set -euo pipefail -psql \ - --host="$PGHOST" \ - --port="${PGPORT:-5432}" \ - --username="$PGUSER" \ - --dbname="$PGDATABASE" \ - --quiet \ - --no-psqlrc \ - --tuples-only \ - --no-align \ - --field-separator $'\t' \ - --set=ON_ERROR_STOP=1 \ - --command " - SELECT - c.name, - to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') AS yyyymmdd, - count(DISTINCT g.cumulus_id) AS granule_count, - count(f.cumulus_id) AS file_count - FROM public.granules g - JOIN public.collections c - ON c.cumulus_id = g.collection_cumulus_id - JOIN public.files f - ON f.granule_cumulus_id = g.cumulus_id - AND f.checksum_value IS NOT NULL - WHERE g.status = 'completed' - AND c.name in ( - 'ANTPAT', - 'BFPQ', - 'CORNER_REFL', - 'DCOP', - 'DC_RADAR', - 'DSG_STATIC', - 'EA_L0B_L_CRSD', - 'EA_L0B_L_RRSD', - 'EA_L1_L_RIFG', - 'EA_L1_L_ROFF', - 'EA_L1_L_RSLC', - 'EA_L1_L_RUNW', - 'EA_L2_L_GCOV', - 'EA_L2_L_GOFF', - 'EA_L2_L_GSLC', - 'EA_L2_L_GUNW', - 'EA_L3_L_SME2', - 'FOE', - 'FRP', - 'FT_PARAM', - 'FT_WAVEFORM', - 'LRCLK_UTC', - 'L_CHAN_DATA', - 'MOE', - 'NISAR_DEM_TIF', - 'NISAR_DEM_VRT', - 'NISAR_L0A_RRST_BETA_V1', - 'NISAR_L0B_CRSD_BETA_V1', - 'NISAR_L0B_RRSD_BETA_V1', - 'NISAR_L1_RIFG_BETA_V1', - 'NISAR_L1_ROFF_BETA_V1', - 'NISAR_L1_RSLC_BETA_V1', - 'NISAR_L1_RUNW_BETA_V1', - 'NISAR_L2_GCOV_BETA_V1', - 'NISAR_L2_GOFF_BETA_V1', - 'NISAR_L2_GSLC_BETA_V1', - 'NISAR_L2_GUNW_BETA_V1', - 'NISAR_L3_SME2_BETA_V1', - 'NISAR_VWC', - 'NISAR_WATERMASK_TIF', - 'NISAR_WATERMASK_VRT', - 'NOE', - 'NRP', - 'OROST', - 'PA_L0B_L_RRSD', - 'POE', - 'PRP', - 'STUF', - 'TEC', - 'TFDB', - 'UR_L0B_L_RRSD', - 'OPERA_L2_CSLC-S1_V1', - 'OPERA_L2_RTC-S1_V1', - 'OPERA_L3_DISP-S1_V1', - 'OPERA_L3_DIST-ALERT-S1_V1', - 'OPERA_L4_TROPO-ZENITH_V1' - ) - GROUP BY - c.name, - to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') - ORDER BY - c.name, - to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); - " > slices_daily.tsv +#ANTPAT', +#BFPQ', +#CORNER_REFL', +#DCOP', +#DC_RADAR', +#DSG_STATIC', +#EA_L0B_L_CRSD', +#EA_L0B_L_RRSD', +#EA_L1_L_RIFG', +#EA_L1_L_ROFF', +#EA_L1_L_RSLC', +#EA_L1_L_RUNW',-- +#EA_L2_L_GCOV', +#EA_L2_L_GOFF', +#EA_L2_L_GSLC', +#EA_L2_L_GUNW', +#EA_L3_L_SME2', +#FOE', +#FRP', +#FT_PARAM', +#FT_WAVEFORM', +#LRCLK_UTC', +#L_CHAN_DATA', +#MOE', +#NISAR_DEM_TIF', +#NISAR_DEM_VRT', +#NISAR_L0A_RRST_BETA_V1', +#NISAR_L0B_CRSD_BETA_V1', +#NISAR_L0B_RRSD_BETA_V1', +#NISAR_L1_RIFG_BETA_V1', +#NISAR_L1_ROFF_BETA_V1', +#NISAR_L1_RSLC_BETA_V1', +#NISAR_L1_RUNW_BETA_V1', +#NISAR_L2_GCOV_BETA_V1', +#NISAR_L2_GOFF_BETA_V1', +#NISAR_L2_GSLC_BETA_V1', +#NISAR_L2_GUNW_BETA_V1', +#NISAR_L3_SME2_BETA_V1', +#NISAR_VWC', +#NISAR_WATERMASK_TIF', +#NISAR_WATERMASK_VRT', +#NOE', +#NRP', +#OROST', +#PA_L0B_L_RRSD', +#POE', +#PRP', +#STUF', +#TEC', +#TFDB', +#UR_L0B_L_RRSD', +# Opera collections: +#COLLECTION="OPERA_L2_CSLC-S1_V1" +#COLLECTION="OPERA_L2_RTC-S1_V1" +#COLLECTION="OPERA_L3_DISP-S1_V1" +#COLLECTION="OPERA_L3_DIST-ALERT-S1_V1" +#COLLECTION="OPERA_L4_TROPO-ZENITH_V1" -source ./refresh-ctorm-upload-role.sh +echo "SLICES_S3_URI: ${SLICES_S3_URI}" +echo "COLLECTION: ${COLLECTION}" -SLICES_S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily.tsv" -aws s3 cp slices_daily.tsv "${SLICES_S3_URI}" -echo "Uploaded ${SLICES_S3_URI}" +if [[ -n "${SLICES_S3_URI:-}" ]]; then + echo "SLICES_S3_URI is specified: ${SLICES_S3_URI}" + echo "Downloading specified slices file instead of querying database..." + source ./refresh-ctorm-upload-role.sh + aws s3 cp "${SLICES_S3_URI}" slices_daily.tsv +else + # Generate slices TSV file + echo "SLICES_S3_URI not set. Going to run a big query now..." + psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --tuples-only \ + --no-align \ + --field-separator $'\t' \ + --set=ON_ERROR_STOP=1 \ + --set=collection="$COLLECTION" \ + --file=generate-slices-daily.sql \ + > slices_daily.tsv + + source ./refresh-ctorm-upload-role.sh + + # Upload slices TSV file to S3 for archive purposes. + SLICES_S3_UPLOAD_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily-${COLLECTION}-${CODEBUILD_BUILD_NUMBER}.tsv" + aws s3 cp slices_daily.tsv "${SLICES_S3_UPLOAD_URI}" + echo "Uploaded ${SLICES_S3_UPLOAD_URI}" +fi diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sql b/ctorm/cumulus-db-md-extract/generate-slices-daily.sql new file mode 100644 index 0000000..ae81935 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sql @@ -0,0 +1,21 @@ +SELECT + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') AS YYYYMMDD, + COUNT(DISTINCT G.CUMULUS_ID) AS GRANULE_COUNT, + COUNT(F.CUMULUS_ID) AS FILE_COUNT +FROM + PUBLIC.GRANULES G + JOIN PUBLIC.COLLECTIONS C + ON C.CUMULUS_ID = G.COLLECTION_CUMULUS_ID + JOIN PUBLIC.FILES F + ON F.GRANULE_CUMULUS_ID = G.CUMULUS_ID + AND F.CHECKSUM_VALUE IS NOT NULL +WHERE + G.STATUS = 'completed' + AND C.NAME = :'collection' +GROUP BY + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') +ORDER BY + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD'); diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh index e28fc6a..ced1a23 100644 --- a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -17,4 +17,4 @@ export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKey export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" -aws sts get-caller-identity >&2 +# aws sts get-caller-identity >&2 diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf index c2acbcb..aeb8efe 100644 --- a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -306,6 +306,16 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { name = "CTORM_S3_ROLE_ARN" value = var.db_md_extract_ctorm_s3_role_arn } + + environment_variable { + name = "COLLECTION" + value = "" + } + + environment_variable { + name = "SLICES_S3_URI" + value = "" + } } vpc_config { diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index be02ac5..84f099f 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 112, + "serial": 142, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "15e0432811ac45b77440e902f20b433130b97e70", - "output_base64sha256": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", - "output_base64sha512": "loaVRKTjqeDeRXYLTynxgJIcoeFMuaH5+Yx1GZPZymzfK4GSIarTAvusSWNs7DHEH+baqi6JJ8D5SjrMvSmT4Q==", + "id": "a96216dceaf9e6e503bf02981b53784e03ef13b8", + "output_base64sha256": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=", + "output_base64sha512": "RLp6HJBUgfaHuTiqD2qyQKwBJyHtvZ+/VDK/urPEE58Gf5Md2MDIJeCVoQIux45wacZU5HiEe/eoFtx28IitLw==", "output_file_mode": null, - "output_md5": "cdd0aab259d06df74b4ae3ddbbd7615e", + "output_md5": "5a4c52cf01416b48350304df429135a5", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "15e0432811ac45b77440e902f20b433130b97e70", - "output_sha256": "f37531a4a276be2b5853d2383331e343604a81c2b63926c24f83c318e3960d9f", - "output_sha512": "96869544a4e3a9e0de45760b4f29f180921ca1e14cb9a1f9f98c751993d9ca6cdf2b819221aad302fbac49636cec31c41fe6daaa2e8927c0f94a3accbd2993e1", - "output_size": 6341, + "output_sha": "a96216dceaf9e6e503bf02981b53784e03ef13b8", + "output_sha256": "8882f14372977b33e1aa89c08d14ad49217280b96c1c984d91c0f02f7042b750", + "output_sha512": "44ba7a1c905481f687b938aa0f6ab240ac012721edbd9fbf5432bfbab3c4139f067f931dd8c0c825e095a1022ec78e7069c654e478847bf7a816dc76f088ad2f", + "output_size": 7000, "source": [], "source_content": null, "source_content_filename": null, @@ -371,6 +371,16 @@ "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_URI", + "type": "PLAINTEXT", + "value": "" } ], "fleet": [], @@ -717,7 +727,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "cdd0aab259d06df74b4ae3ddbbd7615e", + "etag": "5a4c52cf01416b48350304df429135a5", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +740,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", + "source_hash": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate index 71af6a5..bdc7311 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 62, + "serial": 69, "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", - "output_base64sha256": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", - "output_base64sha512": "PZDUjfN60Uim+bdem7zB2ilfEUKqP4pE0UADkxHIkR4YTYGcP+Txtuj+Rm7zSpLQ2fZk+rgOjhJKIWLy0lyWYQ==", + "id": "2ea86e5ad0e886c9ead4d329ab2a5a1e5025bc8f", + "output_base64sha256": "PvweHBk8zXkKLm/Sf7cg9BQCZO4d5G7XeJiAdyjkEBs=", + "output_base64sha512": "PxlBXxsKqs9FBH1t/uUkUhmh7jm22Jluqzp7HD9CzqGMktLgIoGwqYQiZwHgo9zBcjFaTo1AIvEUmj2OTQUe5g==", "output_file_mode": null, - "output_md5": "63b93d1d0d04eb5a9595f406ba9653cc", + "output_md5": "72ff56f7afe5d2511991649ddca2a839", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", - "output_sha256": "b7ddad71c2668d0d4a2d31668752520c9e0b4e9dee0d8188eb3d3f9a89a79f4c", - "output_sha512": "3d90d48df37ad148a6f9b75e9bbcc1da295f1142aa3f8a44d140039311c8911e184d819c3fe4f1b6e8fe466ef34a92d0d9f664fab80e8e124a2162f2d25c9661", - "output_size": 5347, + "output_sha": "2ea86e5ad0e886c9ead4d329ab2a5a1e5025bc8f", + "output_sha256": "3efc1e1c193ccd790a2e6fd27fb720f4140264ee1de46ed77898807728e4101b", + "output_sha512": "3f19415f1b0aaacf45047d6dfee5245219a1ee39b6d8996eab3a7b1c3f42cea18c92d2e02281b0a984226701e0a3dcc172315a4e8d4022f1149a3d8e4d051ee6", + "output_size": 6923, "source": [], "source_content": null, "source_content_filename": null, @@ -82,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "1034869219", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "2729057441", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -103,7 +103,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract" + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" ], "sid": "WriteCodeBuildLogs" }, @@ -371,6 +371,16 @@ "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_PATH", + "type": "PLAINTEXT", + "value": "" } ], "fleet": [], @@ -476,7 +486,7 @@ "inline_policy": [ { "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" } ], "managed_policy_arns": [], @@ -509,7 +519,7 @@ "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", "role": "ctorm-opera-uat-cumulus-db-md-extract-role" }, "sensitive_attributes": [], @@ -718,7 +728,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "63b93d1d0d04eb5a9595f406ba9653cc", + "etag": "72ff56f7afe5d2511991649ddca2a839", "force_destroy": false, "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -731,7 +741,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", + "source_hash": "PvweHBk8zXkKLm/Sf7cg9BQCZO4d5G7XeJiAdyjkEBs=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 53ccb0d..4f0d2fb 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -303,8 +303,9 @@ data "aws_iam_policy_document" "cumulus_db_md_extract_upload" { effect = "Allow" actions = [ - "s3:PutObject", "s3:AbortMultipartUpload", + "s3:GetObject", + "s3:PutObject", ] resources = [ diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index f2714d5..8c00f44 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 63, + "serial": 69, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "15541977bfb6559cdcb3dce4cb03186ba760a70c", - "output_base64sha256": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", - "output_base64sha512": "5JDpSb4/VU/oaT7BC6qYuY+/ljGpjHWd5SY8B/8SFH98/DPmit3/MLGGHvjgZzhbsrOghg4nq8IQ168xlz/mpg==", + "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", "output_file_mode": null, - "output_md5": "8198f06853d8821f035afa1477cc3d3c", + "output_md5": "10648403b2277ad95dd6324dd58521c9", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "15541977bfb6559cdcb3dce4cb03186ba760a70c", - "output_sha256": "ed4bcd8457abb06013f1d7949c3363de3d829ae4778ccbbf36f9d69e80807a28", - "output_sha512": "e490e949be3f554fe8693ec10baa98b98fbf9631a98c759de5263c07ff12147f7cfc33e68addff30b1861ef8e067385bb2b3a0860e27abc210d7af31973fe6a6", - "output_size": 3346, + "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", + "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", + "output_size": 3296, "source": [], "source_content": null, "source_content_filename": null, @@ -249,9 +249,9 @@ { "schema_version": 0, "attributes": { - "id": "923578907", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "id": "3026717548", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -261,6 +261,7 @@ { "actions": [ "s3:AbortMultipartUpload", + "s3:GetObject", "s3:PutObject" ], "condition": [], @@ -676,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PROVISIONED", + "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -686,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 5, - "write_capacity": 10000 + "read_capacity": 0, + "write_capacity": 0 } ], "hash_key": "pk", @@ -703,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 5, + "read_capacity": 0, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -715,7 +716,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -724,7 +725,7 @@ "enabled": false } ], - "write_capacity": 10000 + "write_capacity": 0 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -915,7 +916,7 @@ "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", "name": "ctorm-dev-cumulus-db-md-extract-upload", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", "role": "ctorm-dev-cumulus-db-md-extract-upload" }, "sensitive_attributes": [], @@ -1194,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "8198f06853d8821f035afa1477cc3d3c", + "etag": "10648403b2277ad95dd6324dd58521c9", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", + "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 0bd8316..23ab993 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 59, + "serial": 67, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "5def5225e70ce457c014d699b2bd3e2b93a9788e", - "output_base64sha256": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", - "output_base64sha512": "yt3czhsZeaAaC4zMOD5W5zjajGbY/Z09AXYEwpFDW2hmbfqTepKrLT5xoDg5zTEU4CnE2iQyza54OPqIOOZb+w==", + "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", "output_file_mode": null, - "output_md5": "b6108bd6d78e6ebfc4b059f9b0e2f033", + "output_md5": "10648403b2277ad95dd6324dd58521c9", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "5def5225e70ce457c014d699b2bd3e2b93a9788e", - "output_sha256": "da5f4df5ac6f5c647bd90006b2d8ac57ea305bbe74620e7efd6aa95a5244aace", - "output_sha512": "cadddcce1b1979a01a0b8ccc383e56e738da8c66d8fd9d3d017604c291435b68666dfa937a92ab2d3e71a03839cd3114e029c4da2432cdae7838fa8838e65bfb", - "output_size": 3128, + "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", + "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", + "output_size": 3296, "source": [], "source_content": null, "source_content_filename": null, @@ -715,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -1194,7 +1194,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "b6108bd6d78e6ebfc4b059f9b0e2f033", + "etag": "10648403b2277ad95dd6324dd58521c9", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1206,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", + "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate index 8c6c0eb..6fae66e 100644 --- a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 25, + "serial": 48, "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -34,6 +34,38 @@ } }, "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "output_file_mode": null, + "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_path": "./.terraform/granule-md-db-loader.zip", + "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", + "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", + "output_size": 3296, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./../../granule-md-db-loader", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "archive_file", @@ -124,9 +156,9 @@ { "schema_version": 0, "attributes": { - "id": "2687090556", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "id": "1964661201", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": [\n \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\n \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -196,7 +228,8 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules" + "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules", + "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*" ], "sid": "" } @@ -216,9 +249,9 @@ { "schema_version": 0, "attributes": { - "id": "2685467495", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "id": "1452847858", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -228,6 +261,7 @@ { "actions": [ "s3:AbortMultipartUpload", + "s3:GetObject", "s3:PutObject" ], "condition": [], @@ -488,6 +522,133 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-granule-md-db-loader", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "Imports granules jsonl.gz files from S3 to DynamoDB", + "encryption_key": "arn:aws:kms:us-west-2:097260566921:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [ + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-scratch" + }, + { + "name": "TABLE_NAME", + "type": "PLAINTEXT", + "value": "ctorm-granules" + }, + { + "name": "PREFIX", + "type": "PLAINTEXT", + "value": "cumulus-granules/" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-granule-md-db-loader", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-granule-md-db-loader", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::097260566921:role/ctorm-codebuild-granule_md_db_loader-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-scratch/codebuild/granule-md-db-loader.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch", + "aws_s3_object.granule_md_db_loader_source", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_dynamodb_table", @@ -499,6 +660,14 @@ "attributes": { "arn": "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules", "attribute": [ + { + "name": "gsi1pk", + "type": "S" + }, + { + "name": "gsi1sk", + "type": "S" + }, { "name": "pk", "type": "S" @@ -510,7 +679,18 @@ ], "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, - "global_secondary_index": [], + "global_secondary_index": [ + { + "hash_key": "gsi1pk", + "name": "gsi1", + "non_key_attributes": [], + "on_demand_throughput": [], + "projection_type": "ALL", + "range_key": "gsi1sk", + "read_capacity": 0, + "write_capacity": 0 + } + ], "hash_key": "pk", "id": "ctorm-granules", "import_table": [], @@ -570,7 +750,7 @@ "inline_policy": [ { "name": "ctorm-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}" }, { "name": "ctorm-scratch-access", @@ -613,7 +793,7 @@ "inline_policy": [ { "name": "ctorm-cumulus-db-md-extract-upload", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], @@ -634,6 +814,42 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "granule_md_db_loader_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-codebuild-granule_md_db_loader-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-28T22:55:49Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-codebuild-granule_md_db_loader-role", + "inline_policy": [ + { + "name": "ctorm-codebuild-granule-md-db-loader-role-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch\",\"arn:aws:s3:::ctorm-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-codebuild-granule_md_db_loader-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROARNJJOPGEWW276Y44U" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -646,7 +862,7 @@ "id": "ctorm-cnm-sender-role:ctorm-cnm-sender-policy", "name": "ctorm-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}", "role": "ctorm-cnm-sender-role" }, "sensitive_attributes": [], @@ -663,6 +879,31 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_granule_md_db_loader_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-codebuild-granule_md_db_loader-role:ctorm-codebuild-granule-md-db-loader-role-policy", + "name": "ctorm-codebuild-granule-md-db-loader-role-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch\",\"arn:aws:s3:::ctorm-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}", + "role": "ctorm-codebuild-granule_md_db_loader-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -675,7 +916,7 @@ "id": "ctorm-cumulus-db-md-extract-upload:ctorm-cumulus-db-md-extract-upload", "name": "ctorm-cumulus-db-md-extract-upload", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", "role": "ctorm-cumulus-db-md-extract-upload" }, "sensitive_attributes": [], @@ -928,6 +1169,60 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "granule_md_db_loader_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-scratch/codebuild/granule-md-db-loader.zip", + "bucket": "ctorm-scratch", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "10648403b2277ad95dd6324dd58521c9", + "force_destroy": false, + "id": "codebuild/granule-md-db-loader.zip", + "key": "codebuild/granule-md-db-loader.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "server_side_encryption": "AES256", + "source": "./.terraform/granule-md-db-loader.zip", + "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.scratch", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", From ebf4eea9aeec87962acc6e70c7a0300066cbfde2 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Sun, 28 Jun 2026 17:50:09 -0800 Subject: [PATCH 12/15] linting, sigh --- .../granule_md_db_loader.py | 132 ++++++++++++------ 1 file changed, 89 insertions(+), 43 deletions(-) diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 6ef48c6..3a33dcf 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -11,7 +11,8 @@ from botocore.config import Config # Configure logging -logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +fmt = "%(asctime)s [%(levelname)s] %(message)s" +logging.basicConfig(level=logging.INFO, format=fmt) logger = logging.getLogger(__name__) @@ -34,7 +35,8 @@ def limit(self, pk): self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] if len(self.history[pk]) >= self.max_rate: - # Calculate sleep duration to let the oldest request roll off the 1-second window + # Calculate sleep duration to let the oldest request roll + # off the 1-second window sleep_time = 1.0 - (now - self.history[pk][0]) if sleep_time > 0: time.sleep(sleep_time) @@ -44,6 +46,53 @@ def limit(self, pk): self.history[pk].append(now) +def handle_file( + local_path, + table, + rate_limiter: PartitionRateLimiter, + records_in_file, + total_records, +): + with gzip.open(local_path, "rt", encoding="utf-8") as f: + with table.batch_writer() as batch: + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for + # DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + logger.error(f"Failed to parse JSON line: {e}") + continue + + # Add/modify fields if needed + newdate = datetime.datetime.utcnow().isoformat() + "Z" + item["imported_at"] = newdate + + # Ensure partition and sort keys are present + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: + logger.warning( + "Skipping record missing pk/sk: %s", + item.get('granule_id'), + ) + + continue + + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 + return records_in_file, total_records + + def main(): bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") table_name = os.environ.get("TABLE_NAME") @@ -55,17 +104,23 @@ def main(): s3 = boto3.client("s3") - # Configure boto3 with more aggressive retries to gracefully handle scale peaks + # Configure boto3 with more aggressive retries to gracefully + # handle scale peaks retry_config = Config( retries={ "max_attempts": 10, - "mode": "standard" + "mode": "standard", } ) dynamodb = boto3.resource("dynamodb", config=retry_config) - table = dynamodb.Table(table_name) + dyndb_table = dynamodb.Table(table_name) - logger.info(f"Starting import from s3://{bucket_name}/{prefix} into DynamoDB table {table_name}") + logger.info( + "Starting import from s3://%s/%s into DynamoDB table %s", + bucket_name, + prefix, + table_name, + ) paginator = s3.get_paginator("list_objects_v2") pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) @@ -73,7 +128,8 @@ def main(): total_files = 0 total_records = 0 - # Initialize partition-level rate limiter set to a safe threshold (850 writes/sec per pk) + # Initialize partition-level rate limiter set to a safe + # threshold (850 writes/sec per pk) rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) # Walk through the bucket objects @@ -94,52 +150,42 @@ def main(): try: s3.download_file(bucket_name, key, local_path) except Exception as e: - logger.error(f"Failed to download s3://{bucket_name}/{key}: {e}") + logger.error( + "Failed to download s3://{%s}/%s: %s", + bucket_name, + key, + e, + ) continue # Open, decompress and parse records_in_file = 0 try: - with gzip.open(local_path, "rt", encoding="utf-8") as f: - with table.batch_writer() as batch: - for line in f: - line = line.strip() - if not line: - continue - - try: - # Convert floats/doubles to Decimal for DynamoDB compatibility - item = json.loads(line, parse_float=Decimal) - except Exception as e: - logger.error(f"Failed to parse JSON line: {e}") - continue - - # Add/modify fields if needed - item["imported_at"] = datetime.datetime.utcnow().isoformat() + "Z" - - # Ensure partition and sort keys are present - pk = item.get("pk") - sk = item.get("sk") - if not pk or not sk: - logger.warning(f"Skipping record missing pk/sk: {item.get('granule_id')}") - continue - - # Apply dynamic rate limit based on the partition key - rate_limiter.limit(pk) - - # Batch insert into DynamoDB - batch.put_item(Item=item) - records_in_file += 1 - total_records += 1 - - logger.info(f"Successfully imported {records_in_file} records from {key}") + records_in_file, total_records = handle_file( + local_path, + dyndb_table, + rate_limiter, + records_in_file, + total_records + ) + + logger.info( + "Successfully imported %s records from %s", + records_in_file, + key, + + ) except Exception as e: - logger.error(f"Error processing file {key}: {e}") + logger.error("Error processing file %s: %s", key, e) finally: if os.path.exists(local_path): os.remove(local_path) - logger.info(f"Import complete. Processed {total_files} files, imported {total_records} total records.") + logger.info( + "Import complete. Processed %s files, imported %s total records.", + total_files, + total_records, + ) if __name__ == "__main__": From 05ea0d9d0a33c00f842e09ff074b11b2fbb4b2c8 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Mon, 29 Jun 2026 13:30:12 -0800 Subject: [PATCH 13/15] PR-7510 refinements to documentation, etc --- ctorm/cumulus-db-md-extract/README.md | 5 +- .../export-granules-daily.sh | 2 + ctorm/granule-md-db-loader/README.md | 3 +- .../granule_md_db_loader.py | 91 ++++++++++--------- ctorm/infra/README.md | 19 +++- ctorm/infra/terraform/terraform.tfstate | 24 ++--- .../infra/terraform/terraform.tfstate.backup | 45 ++++----- 7 files changed, 100 insertions(+), 89 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index 1fe8459..17e7950 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -128,10 +128,7 @@ echo "${BUILD_ID}" aws codebuild start-build \ --project-name "${CODEBUILD_PROJECT}" \ --region "${AWS_REGION}" \ - --environment-variables-override '[ - {"name": "SLICES_S3_URI", "value": "s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_RTC-S1_V1-20170910-20181203.tsv", "type": "PLAINTEXT"}, - {"name": "COLLECTION", "value": "OPERA_L2_RTC-S1_V1", "type": "PLAINTEXT"} - ]' \ + --environment-variables-override name=SLICES_S3_URI,value="s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_CSLC-S1_V1-20180707-20200522.tsv",type=PLAINTEXT \ --query 'build.id' \ --output text # )" diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index c28d176..d5796a9 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -1,5 +1,7 @@ set -euo pipefail +# Takes slice TSV files and creates .jsonl files out of the cumulus DB + S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" FAILED_SLICES_FILE="failed_slices_daily.tsv" diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md index 5289584..d1812f4 100644 --- a/ctorm/granule-md-db-loader/README.md +++ b/ctorm/granule-md-db-loader/README.md @@ -26,7 +26,7 @@ export CTORM_BUCKET="ctorm-dev-scratch" aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/nisar/ |grep PRE aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/opera/ |grep PRE -# If you want do them all sequentially, not recommended and not going to work with all the data: +# If you want do them all sequentially, not recommended and not going to work with all the data because throttling:: aws codebuild start-build --profile="${AWS_PROFILE}" \ --project-name "${CODEBUILD_PROJECT}" \ --region "us-west-2" \ @@ -60,6 +60,7 @@ aws codebuild start-build --profile="${AWS_PROFILE}" \ --environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DIST-ALERT-S1_V1_/,type=PLAINTEXT \ --query 'build.id' \ --output text +# nisar aws codebuild start-build --profile="${AWS_PROFILE}" \ --project-name "${CODEBUILD_PROJECT}" \ --region "us-west-2" \ diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 3a33dcf..27e020a 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -47,49 +47,51 @@ def limit(self, pk): def handle_file( - local_path, - table, - rate_limiter: PartitionRateLimiter, - records_in_file, - total_records, + local_path: str, + table, + rate_limiter: PartitionRateLimiter, + records_in_file: int, + total_records: int, ): - with gzip.open(local_path, "rt", encoding="utf-8") as f: - with table.batch_writer() as batch: - for line in f: - line = line.strip() - if not line: - continue - - try: - # Convert floats/doubles to Decimal for - # DynamoDB compatibility - item = json.loads(line, parse_float=Decimal) - except Exception as e: - logger.error(f"Failed to parse JSON line: {e}") - continue - - # Add/modify fields if needed - newdate = datetime.datetime.utcnow().isoformat() + "Z" - item["imported_at"] = newdate - - # Ensure partition and sort keys are present - pk = item.get("pk") - sk = item.get("sk") - if not pk or not sk: - logger.warning( - "Skipping record missing pk/sk: %s", - item.get('granule_id'), - ) - - continue - - # Apply dynamic rate limit based on the partition key - rate_limiter.limit(pk) - - # Batch insert into DynamoDB - batch.put_item(Item=item) - records_in_file += 1 - total_records += 1 + with ( + gzip.open(local_path, "rt", encoding="utf-8") as f, + table.batch_writer() as batch, + ): + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for + # DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + logger.error("Failed to parse JSON line: %s", e) + continue + + # Add/modify fields if needed + newdate = datetime.datetime.utcnow().isoformat() + "Z" + item["imported_at"] = newdate + + # Ensure partition and sort keys are present + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: + logger.warning( + "Skipping record missing pk/sk: %s", + item.get("granule_id"), + ) + + continue + + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 return records_in_file, total_records @@ -142,7 +144,7 @@ def main(): if not key.endswith(".jsonl.gz"): continue - logger.info(f"Processing s3://{bucket_name}/{key}") + logger.info("Processing s3://%s/%s", bucket_name, key) total_files += 1 # Download and decompress the file @@ -166,14 +168,13 @@ def main(): dyndb_table, rate_limiter, records_in_file, - total_records + total_records, ) logger.info( "Successfully imported %s records from %s", records_in_file, key, - ) except Exception as e: logger.error("Error processing file %s: %s", key, e) diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 5ae884c..e9c6d92 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -19,16 +19,25 @@ terraform plan \ terraform apply \ -var-file="${VARFILE}" +terraform destroy +``` + +## Increasing DynamoDB table capacity + +For loading the DynamoDB table with the .jsonl files, we'll need to increase the capacity of the table. +AWS will throttle drastically if we don't. There is some limitation on how often we can switch between +PROVISIONED and PAY_PER_REQUEST, so don't do it willy-nilly. + +```bash # For temporary use while loading granule metadata: terraform plan \ -var-file="${VARFILE}" \ -var="granule_table_billing_mode=PROVISIONED" \ - -var="granule_table_write_capacity=10000" + -var="granule_table_write_capacity=5000" -terraform apply \ - -var-file="${VARFILE}" \ - -var="granule_table_billing_mode=PROVISIONED" \ - -var="granule_table_write_capacity=10000" +# To set it back to the default: +terraform plan \ + -var-file="${VARFILE}" terraform destroy diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 8c00f44..f09bb23 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 69, + "serial": 80, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", - "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "id": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", + "output_base64sha256": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", + "output_base64sha512": "cYqWFwuEi6LTUM2lmqkXnWuYX+bf0oG86hslOpldD33Zq+jezzEQpTt9V3MGu8xmBy+knCvhX457ql2fPXo2qw==", "output_file_mode": null, - "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_md5": "c7918cf0c08ea6e16f8828296cd3d06a", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", - "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", - "output_size": 3296, + "output_sha": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", + "output_sha256": "b09c8d8bc03812252df7ed5e19fc6a7afea4871f07c97c4bde8db96c4ae22f96", + "output_sha512": "718a96170b848ba2d350cda59aa9179d6b985fe6dfd281bcea1b253a995d0f7dd9abe8decf3110a53b7d577306bbcc66072fa49c2be15f8e7baa5d9f3d7a36ab", + "output_size": 3481, "source": [], "source_content": null, "source_content_filename": null, @@ -793,7 +793,7 @@ "inline_policy": [ { "name": "ctorm-dev-cumulus-db-md-extract-upload", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "10648403b2277ad95dd6324dd58521c9", + "etag": "c7918cf0c08ea6e16f8828296cd3d06a", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "source_hash": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 23ab993..6c2a0ab 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 67, + "serial": 76, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", - "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "id": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", + "output_base64sha256": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", + "output_base64sha512": "/tl0guEuEuyrvuj+stzcDIneyXnGYZxXS+GHq3+Od3b3Rgyni0trdrvK2XAEb/SD0ccNAACcZ9AJxPkDFGv/EA==", "output_file_mode": null, - "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_md5": "66541e163cc3aaff54beddc162ad3aca", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", - "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", - "output_size": 3296, + "output_sha": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", + "output_sha256": "935f7853bdb8561ea66efb62eea3b49ab1f8b837c552aa0e020494b58f33996d", + "output_sha512": "fed97482e12e12ecabbee8feb2dcdc0c89dec979c6619c574be187ab7f8e7776f7460ca78b4b6b76bbcad970046ff483d1c70d00009c67d009c4f903146bff10", + "output_size": 3469, "source": [], "source_content": null, "source_content_filename": null, @@ -249,9 +249,9 @@ { "schema_version": 0, "attributes": { - "id": "923578907", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "id": "3026717548", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -261,6 +261,7 @@ { "actions": [ "s3:AbortMultipartUpload", + "s3:GetObject", "s3:PutObject" ], "condition": [], @@ -676,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PAY_PER_REQUEST", + "billing_mode": "PROVISIONED", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -686,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 0, - "write_capacity": 0 + "read_capacity": 5000, + "write_capacity": 5 } ], "hash_key": "pk", @@ -703,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 0, + "read_capacity": 5000, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -715,7 +716,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -724,7 +725,7 @@ "enabled": false } ], - "write_capacity": 0 + "write_capacity": 5 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -792,7 +793,7 @@ "inline_policy": [ { "name": "ctorm-dev-cumulus-db-md-extract-upload", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], @@ -915,7 +916,7 @@ "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", "name": "ctorm-dev-cumulus-db-md-extract-upload", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", "role": "ctorm-dev-cumulus-db-md-extract-upload" }, "sensitive_attributes": [], @@ -1194,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "10648403b2277ad95dd6324dd58521c9", + "etag": "66541e163cc3aaff54beddc162ad3aca", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "source_hash": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From e6bc6f33a7141cb65fa7995eed7a39b136cdb307 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 30 Jun 2026 11:42:25 -0800 Subject: [PATCH 14/15] PR-7510 Bug fixes to prevent hot partition --- .../terraform/terraform-nisar-prod.tfstate | 32 +++--- .../granule_md_db_loader.py | 97 ++++++++++--------- ctorm/infra/terraform/main.tf | 8 +- ctorm/infra/terraform/terraform.tfstate | 22 ++--- .../infra/terraform/terraform.tfstate.backup | 34 +++---- .../ctorm-prod/terraform.tfstate | 32 +++--- 6 files changed, 122 insertions(+), 103 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate index 71c1cb1..6ffa2ce 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 17, + "serial": 23, "lineage": "2aab81d2-395f-d5fa-becf-2772136eaa03", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", - "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", + "id": "41bd43108980d112c9deba5135abfc48dc0f4e63", + "output_base64sha256": "X7mJWPMPJp0AvEIb4z20oix4IxI8yBgrZjrukcaMtQY=", + "output_base64sha512": "nc7b+wBbmYgZU4fEDXvkye+F6a7NM51R2dqfTRyn6CZSy9YVWE3XogIuIQIguytQs1PnDCsGyEqTAY7qxHpMgg==", "output_file_mode": null, - "output_md5": "8d91e1621f990906971ffa67900b6743", + "output_md5": "d3becd0cd2b647b5c5542370840070f1", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", - "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", - "output_size": 5692, + "output_sha": "41bd43108980d112c9deba5135abfc48dc0f4e63", + "output_sha256": "5fb98958f30f269d00bc421be33db4a22c7823123cc8182b663aee91c68cb506", + "output_sha512": "9dcedbfb005b9988195387c40d7be4c9ef85e9aecd339d51d9da9f4d1ca7e82652cbd615584dd7a2022e210220bb2b50b353e70c2b06c84a93018eeac47a4c82", + "output_size": 7005, "source": [], "source_content": null, "source_content_filename": null, @@ -371,6 +371,16 @@ "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_URI", + "type": "PLAINTEXT", + "value": "" } ], "fleet": [], @@ -718,7 +728,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "8d91e1621f990906971ffa67900b6743", + "etag": "d3becd0cd2b647b5c5542370840070f1", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -731,7 +741,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "source_hash": "X7mJWPMPJp0AvEIb4z20oix4IxI8yBgrZjrukcaMtQY=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 27e020a..99deb24 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -6,6 +6,7 @@ import sys import time from decimal import Decimal +import random import boto3 from botocore.config import Config @@ -13,7 +14,7 @@ # Configure logging fmt = "%(asctime)s [%(levelname)s] %(message)s" logging.basicConfig(level=logging.INFO, format=fmt) -logger = logging.getLogger(__name__) +log = logging.getLogger(__name__) class PartitionRateLimiter: @@ -67,18 +68,18 @@ def handle_file( # DynamoDB compatibility item = json.loads(line, parse_float=Decimal) except Exception as e: - logger.error("Failed to parse JSON line: %s", e) + log.error("Failed to parse JSON line: %s", e) continue # Add/modify fields if needed - newdate = datetime.datetime.utcnow().isoformat() + "Z" + newdate = datetime.datetime.now(datetime.timezone.utc).isoformat() + "Z" item["imported_at"] = newdate # Ensure partition and sort keys are present pk = item.get("pk") sk = item.get("sk") if not pk or not sk: - logger.warning( + log.warning( "Skipping record missing pk/sk: %s", item.get("granule_id"), ) @@ -101,7 +102,7 @@ def main(): prefix = os.environ.get("PREFIX", "cumulus-granules/") if not table_name: - logger.error("TABLE_NAME environment variable is required.") + log.error("TABLE_NAME environment variable is required.") sys.exit(1) s3 = boto3.client("s3") @@ -117,7 +118,7 @@ def main(): dynamodb = boto3.resource("dynamodb", config=retry_config) dyndb_table = dynamodb.Table(table_name) - logger.info( + log.info( "Starting import from s3://%s/%s into DynamoDB table %s", bucket_name, prefix, @@ -134,55 +135,63 @@ def main(): # threshold (850 writes/sec per pk) rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) - # Walk through the bucket objects + # create random list of .json.gz objects: + # This will alleviate writing to the same partition for several files in a row. + jsongz_list = [] for page in pages: if "Contents" not in page: continue - for obj in page["Contents"]: key = obj["Key"] if not key.endswith(".jsonl.gz"): continue - - logger.info("Processing s3://%s/%s", bucket_name, key) + jsongz_list.append(key) total_files += 1 - # Download and decompress the file - local_path = "/tmp/temp.jsonl.gz" - try: - s3.download_file(bucket_name, key, local_path) - except Exception as e: - logger.error( - "Failed to download s3://{%s}/%s: %s", - bucket_name, - key, - e, - ) - continue - - # Open, decompress and parse - records_in_file = 0 - try: - records_in_file, total_records = handle_file( - local_path, - dyndb_table, - rate_limiter, - records_in_file, - total_records, - ) + total_files = len(jsongz_list) + random.shuffle(jsongz_list) - logger.info( - "Successfully imported %s records from %s", - records_in_file, - key, - ) - except Exception as e: - logger.error("Error processing file %s: %s", key, e) - finally: - if os.path.exists(local_path): - os.remove(local_path) + # Walk through the bucket objects + for key in jsongz_list: + + log.info("Processing s3://%s/%s", bucket_name, key) + + # Download and decompress the file + local_path = "/tmp/temp.jsonl.gz" + try: + s3.download_file(bucket_name, key, local_path) + except Exception as e: + log.error( + "Failed to download s3://{%s}/%s: %s", + bucket_name, + key, + e, + ) + continue - logger.info( + # Open, decompress and parse + records_in_file = 0 + try: + records_in_file, total_records = handle_file( + local_path, + dyndb_table, + rate_limiter, + records_in_file, + total_records, + ) + + log.info( + "Successfully imported %s records from %s", + records_in_file, + key, + ) + except Exception as e: + log.error("Error processing file %s: %s", key, e) + finally: + if os.path.exists(local_path): + os.remove(local_path) + + log.info( "Import complete. Processed %s files, imported %s total records.", total_files, total_records, diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 4f0d2fb..c0945fb 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -22,8 +22,8 @@ resource "aws_dynamodb_table" "granules" { # billing_mode = "PROVISIONED" # Temporarily switch to PROVISIONED for bulk load billing_mode = var.granule_table_billing_mode - read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null - write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null hash_key = "pk" range_key = "sk" @@ -53,8 +53,8 @@ resource "aws_dynamodb_table" "granules" { range_key = "gsi1sk" projection_type = "ALL" - read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null - write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null } } diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index f09bb23..1ac42bd 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 80, + "serial": 84, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", - "output_base64sha256": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", - "output_base64sha512": "cYqWFwuEi6LTUM2lmqkXnWuYX+bf0oG86hslOpldD33Zq+jezzEQpTt9V3MGu8xmBy+knCvhX457ql2fPXo2qw==", + "id": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_base64sha256": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", + "output_base64sha512": "WFh/qul/y/sdiVqJFSXxIez+r8Imr3JHAZ+r9joPIaKF/1zPmZ7tgvM6cAVP583pRXyyevCCOvB1VdL8g9oliQ==", "output_file_mode": null, - "output_md5": "c7918cf0c08ea6e16f8828296cd3d06a", + "output_md5": "fdc46333571602f7f4296ffd55d381d8", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", - "output_sha256": "b09c8d8bc03812252df7ed5e19fc6a7afea4871f07c97c4bde8db96c4ae22f96", - "output_sha512": "718a96170b848ba2d350cda59aa9179d6b985fe6dfd281bcea1b253a995d0f7dd9abe8decf3110a53b7d577306bbcc66072fa49c2be15f8e7baa5d9f3d7a36ab", - "output_size": 3481, + "output_sha": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_sha256": "8859bacbf9a50d8139553bfdcf2ca9be2d6619f433fb4e545d4ac1e3e9b88dc7", + "output_sha512": "58587faae97fcbfb1d895a891525f121ecfeafc226af7247019fabf63a0f21a285ff5ccf999eed82f33a70054fe7cde9457cb27af0823af07555d2fc83da2589", + "output_size": 3578, "source": [], "source_content": null, "source_content_filename": null, @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "c7918cf0c08ea6e16f8828296cd3d06a", + "etag": "fdc46333571602f7f4296ffd55d381d8", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", + "source_hash": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 6c2a0ab..48845ac 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 76, + "serial": 82, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", - "output_base64sha256": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", - "output_base64sha512": "/tl0guEuEuyrvuj+stzcDIneyXnGYZxXS+GHq3+Od3b3Rgyni0trdrvK2XAEb/SD0ccNAACcZ9AJxPkDFGv/EA==", + "id": "51a8bb37eb8de1dc17e0d4eb7128915bbd774f3a", + "output_base64sha256": "DHqC2CGBZ9xOueY2LbY0MjhPAdqzGLGCfdX1PGJprc0=", + "output_base64sha512": "Kl12QLhrC3g5mI7yLmsNaUMBPbKyZUBJDL9eYbDB57ejDd2qK1LQXW4I5XXtwklte6oBgEjh92ADEvdd+cjGHg==", "output_file_mode": null, - "output_md5": "66541e163cc3aaff54beddc162ad3aca", + "output_md5": "4b0eca1c7acc876b5b03c0aba84ec786", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", - "output_sha256": "935f7853bdb8561ea66efb62eea3b49ab1f8b837c552aa0e020494b58f33996d", - "output_sha512": "fed97482e12e12ecabbee8feb2dcdc0c89dec979c6619c574be187ab7f8e7776f7460ca78b4b6b76bbcad970046ff483d1c70d00009c67d009c4f903146bff10", - "output_size": 3469, + "output_sha": "51a8bb37eb8de1dc17e0d4eb7128915bbd774f3a", + "output_sha256": "0c7a82d8218167dc4eb9e6362db63432384f01dab318b1827dd5f53c6269adcd", + "output_sha512": "2a5d7640b86b0b7839988ef22e6b0d6943013db2b26540490cbf5e61b0c1e7b7a30dddaa2b52d05d6e08e575edc2496d7baa018048e1f7600312f75df9c8c61e", + "output_size": 3570, "source": [], "source_content": null, "source_content_filename": null, @@ -677,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PROVISIONED", + "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -687,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 5000, - "write_capacity": 5 + "read_capacity": 0, + "write_capacity": 0 } ], "hash_key": "pk", @@ -704,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 5000, + "read_capacity": 0, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -716,7 +716,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -725,7 +725,7 @@ "enabled": false } ], - "write_capacity": 5 + "write_capacity": 0 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "66541e163cc3aaff54beddc162ad3aca", + "etag": "4b0eca1c7acc876b5b03c0aba84ec786", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", + "source_hash": "DHqC2CGBZ9xOueY2LbY0MjhPAdqzGLGCfdX1PGJprc0=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate index 6fae66e..9abab4f 100644 --- a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 48, + "serial": 88, "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", - "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "id": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_base64sha256": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", + "output_base64sha512": "WFh/qul/y/sdiVqJFSXxIez+r8Imr3JHAZ+r9joPIaKF/1zPmZ7tgvM6cAVP583pRXyyevCCOvB1VdL8g9oliQ==", "output_file_mode": null, - "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_md5": "fdc46333571602f7f4296ffd55d381d8", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", - "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", - "output_size": 3296, + "output_sha": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_sha256": "8859bacbf9a50d8139553bfdcf2ca9be2d6619f433fb4e545d4ac1e3e9b88dc7", + "output_sha512": "58587faae97fcbfb1d895a891525f121ecfeafc226af7247019fabf63a0f21a285ff5ccf999eed82f33a70054fe7cde9457cb27af0823af07555d2fc83da2589", + "output_size": 3578, "source": [], "source_content": null, "source_content_filename": null, @@ -677,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PAY_PER_REQUEST", + "billing_mode": "PROVISIONED", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -687,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 0, - "write_capacity": 0 + "read_capacity": 5, + "write_capacity": 25000 } ], "hash_key": "pk", @@ -704,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 0, + "read_capacity": 5, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -725,7 +725,7 @@ "enabled": false } ], - "write_capacity": 0 + "write_capacity": 25000 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "10648403b2277ad95dd6324dd58521c9", + "etag": "fdc46333571602f7f4296ffd55d381d8", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "source_hash": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From 613bb79b377be81a0ac23bc024a61c941ce60bdc Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 30 Jun 2026 14:43:27 -0800 Subject: [PATCH 15/15] linting, bah --- ctorm/granule-md-db-loader/granule_md_db_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 99deb24..daf1c7f 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -3,10 +3,10 @@ import json import logging import os +import random import sys import time from decimal import Decimal -import random import boto3 from botocore.config import Config @@ -153,7 +153,6 @@ def main(): # Walk through the bucket objects for key in jsongz_list: - log.info("Processing s3://%s/%s", bucket_name, key) # Download and decompress the file