diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2877999..d9ba45b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Deploy on LocalStack +name: Deploy on LocalStack on: push: @@ -15,25 +15,26 @@ on: workflow_dispatch: permissions: - contents: write + contents: read jobs: sam: name: Setup infrastructure using SAM runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '22' - name: Setup SAM uses: aws-actions/setup-sam@v2 @@ -41,26 +42,25 @@ jobs: use-installer: true - name: Install lstk - run: npm install -g @localstack/lstk - - - name: Start LocalStack run: | - docker pull localstack/localstack-pro:latest - # Start LocalStack in the background - LOCALSTACK_DEBUG=1 lstk start + npm install -g @localstack/lstk + lstk setup aws --force --non-interactive + + # Docker Compose puts LocalStack, the Lambda containers and Neo4j on one network, + # so the function reaches the database at bolt://neo4j:7687 + - name: Start LocalStack and Neo4j env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: make start - - name: Build the Lambda Layer - run: | - cd function - pip install --target ../package/python -r requirements.txt + - name: Build and deploy the SAM app + run: make deploy - - name: Build the SAM app - run: | - lstk sam build + - name: Run tests + run: make test - - name: Deploy the SAM app + - name: Show logs + if: failure() run: | - curl http://localhost:4566/_localstack/info - lstk sam deploy --no-confirm-changeset + docker compose logs --tail 300 localstack || true + docker compose logs --tail 100 neo4j || true diff --git a/Makefile b/Makefile index 4ee46a0..697140a 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,17 @@ install: ## Install dependencies @which aws || pip install awscli @which sam || pip install aws-sam-cli -start: ## Start LocalStack and Neo4j using Docker Compose +start: ## Start LocalStack and Neo4j using Docker Compose, and wait until both are ready @test -n "${LOCALSTACK_AUTH_TOKEN}" || (echo "LOCALSTACK_AUTH_TOKEN is not set. Find your token at https://app.localstack.cloud/workspace/auth-token"; exit 1) - @LOCALSTACK_AUTH_TOKEN=$(LOCALSTACK_AUTH_TOKEN) docker-compose up -d + @LOCALSTACK_AUTH_TOKEN=$(LOCALSTACK_AUTH_TOKEN) docker compose up -d + @for i in $$(seq 1 60); do curl -sf localhost:4566/_localstack/health > /dev/null && break; sleep 2; done + @curl -sf localhost:4566/_localstack/health > /dev/null || (echo "LocalStack did not become ready"; exit 1) + @for i in $$(seq 1 60); do docker exec neo4j cypher-shell -u neo4j -p neo4j-harsh-test "RETURN 1" > /dev/null 2>&1 && break; sleep 2; done + @docker exec neo4j cypher-shell -u neo4j -p neo4j-harsh-test "RETURN 1" > /dev/null || (echo "Neo4j did not become ready"; exit 1) + @echo "LocalStack and Neo4j are ready" stop: ## Stop LocalStack and Neo4j - @docker-compose down + @docker compose down logs: ## Save the logs in a separate file @lstk logs > logs.txt @@ -26,4 +31,7 @@ deploy: ## Build and deploy the SAM application lstk sam build lstk sam deploy --no-confirm-changeset -.PHONY: usage install start stop logs deploy +test: ## Invoke the function and check it wrote to Neo4j + ./test.sh + +.PHONY: usage install start stop logs deploy test diff --git a/README.md b/README.md index 15da249..fe697f1 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This sample application will guide you through the process of integrating a loca - LocalStack with the [`lstk` CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/). - [Serverless Application Model](https://docs.localstack.cloud/user-guide/integrations/aws-sam/) with `aws-sam-cli` installed, used via the `lstk sam` proxy. - [AWS CLI](https://docs.localstack.cloud/user-guide/integrations/aws-cli/), required by `lstk aws`. -- [Python 3.10](https://www.python.org/downloads/) & `pip` +- [Python 3.12](https://www.python.org/downloads/) & `pip`, matching the Lambda runtime for `sam build` - [Docker Compose](https://docs.docker.com/compose/install/) - A valid [LocalStack for AWS license](https://localstack.cloud/pricing). Your license provides a [`LOCALSTACK_AUTH_TOKEN`](https://docs.localstack.cloud/aws/getting-started/auth-token/) to activate LocalStack. @@ -29,7 +29,7 @@ export LOCALSTACK_AUTH_TOKEN= make start ``` -The Docker Compose file will start LocalStack Pro and a local Neo4j database. The `LOCALSTACK_AUTH_TOKEN` environment variable is required to activate the LocalStack Pro features, such as Lambda Layers in this example. +The Docker Compose file will start LocalStack Pro and a local Neo4j database on a shared network, and `make start` waits until both are ready. The `LOCALSTACK_AUTH_TOKEN` environment variable is required to activate the LocalStack Pro features, such as Lambda Layers in this example. ## Instructions @@ -58,23 +58,16 @@ If you see a `Build Succeeded` message, you can proceed to the next step. To deploy the SAM application, run the following command: ```shell -lstk sam deploy --guided +lstk sam deploy --no-confirm-changeset ``` -The above command will create a new managed S3 bucket to store the artifacts of the SAM application. If you want to use an existing S3 bucket, you can use the `--s3-bucket` flag to specify the bucket name. Before being deployed, the CloudFormation changeset will be displayed in the terminal. If you want to deploy the application without confirmation, you can use the `--no-confirm-changeset` flag. +`samconfig.toml` deploys the `sam-app` stack and lets SAM create a managed S3 bucket for the artifacts. `make deploy` runs the install, build and deploy steps in one go. -### Updating the Lambda function configuration - -You need to update the Lambda function configuration to use the Neo4j environment variables. You can do this by running the following command: +The function reads the Neo4j connection from the `NEO4J_URI`, `NEO4J_USERNAME` and `NEO4J_PASSWORD` environment variables. The template sets them from parameters whose defaults match the Docker Compose setup (`bolt://neo4j:7687`, `neo4j`, `neo4j-harsh-test`). To use another database, override them: ```shell -export NEO4J_PASSWORD=neo4j-harsh-test -export NEO4J_URI=bolt://neo4j:7687 -export NEO4J_USERNAME=neo4j -FUNCTION=$(lstk aws cloudformation describe-stack-resource --stack-name sam-app --logical-resource-id function --query 'StackResourceDetail.PhysicalResourceId' --output text) -lstk aws lambda update-function-configuration \ - --function-name $FUNCTION \ - --environment "Variables={NEO4J_USERNAME=$NEO4J_USERNAME,NEO4J_PASSWORD=$NEO4J_PASSWORD,NEO4J_URI=$NEO4J_URI}" +lstk sam deploy --no-confirm-changeset \ + --parameter-overrides Neo4jUri=bolt://my-host:7687 Neo4jUsername=neo4j Neo4jPassword=my-password ``` ### Invoking the Lambda function @@ -82,6 +75,7 @@ lstk aws lambda update-function-configuration \ You can invoke the Lambda function using the following command: ```shell +FUNCTION=$(lstk aws cloudformation describe-stack-resource --stack-name sam-app --logical-resource-id function --query 'StackResourceDetail.PhysicalResourceId' --output text) lstk aws lambda invoke \ --function-name $FUNCTION \ --payload file://event.json \ @@ -103,19 +97,30 @@ You can see the following in the `out.json` file: {"TotalCodeSize": 29662331, "FunctionCount": 1} ``` +The function also stores the event as a `LambdaEvent` node in Neo4j. You can see it with `cypher-shell`: + +```shell +docker exec neo4j cypher-shell -u neo4j -p neo4j-harsh-test \ + "MATCH (e:LambdaEvent) RETURN e.created ORDER BY e.created DESC LIMIT 5" +``` + +### Testing the application + +`make test` invokes the function with a uniquely tagged event, checks it returns the account usage, and checks Neo4j has exactly one `LambdaEvent` node for that event. + ### Cleaning up To clean up the resources created by the SAM application, run the following command: ```shell -docker-compose down +make stop ``` LocalStack is ephemeral, and all the resources will be deleted once the LocalStack process is stopped. ### GitHub Action -This application sample hosts an example GitHub Action workflow that starts up LocalStack, builds the Lambda functions, and deploys the infrastructure on the runner. +This application sample hosts an example GitHub Action workflow that starts LocalStack and Neo4j with Docker Compose, builds and deploys the SAM application, and runs `make test` on the runner. You can find the workflow in the `.github/workflows/main.yml` file. To run the workflow, you can fork this repository and push a commit to the `main` branch. diff --git a/docker-compose.yml b/docker-compose.yml index 4a42521..268ad5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: localstack: container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-aws}" @@ -10,7 +8,6 @@ services: - "127.0.0.1:443:443" # LocalStack HTTPS Gateway (Pro) environment: - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN} - - LOCALSTACK_API_KEY=${LOCALSTACK_API_KEY} - DEBUG=1 - LAMBDA_RUNTIME_ENVIRONMENT_TIMEOUT=600 volumes: @@ -23,7 +20,7 @@ services: neo4j: container_name: neo4j - image: neo4j:latest + image: neo4j:5.26 ports: - 7474:7474 - 7687:7687 diff --git a/function/__pycache__/lambda_function.cpython-311.pyc b/function/__pycache__/lambda_function.cpython-311.pyc new file mode 100644 index 0000000..783d2d0 Binary files /dev/null and b/function/__pycache__/lambda_function.cpython-311.pyc differ diff --git a/function/lambda_function.py b/function/lambda_function.py index 9c03bd8..37e66da 100644 --- a/function/lambda_function.py +++ b/function/lambda_function.py @@ -38,7 +38,7 @@ def lambda_handler(event, context): logger.info("## CONNECTING TO %s" % neo4j_uri) with driver.session() as session: - session.write_transaction(write_graph_record, event) + session.execute_write(write_graph_record, event) response = client.get_account_settings() return response['AccountUsage'] diff --git a/function/requirements.txt b/function/requirements.txt index 3991b5c..346eae2 100644 --- a/function/requirements.txt +++ b/function/requirements.txt @@ -1,3 +1,3 @@ -jsonpickle==1.3 -neo4j==4.3.5 -aws-xray-sdk==2.8.0 +jsonpickle==4.1.3 +neo4j==5.28.2 +aws-xray-sdk==2.15.0 diff --git a/template.yml b/template.yml index 25a4cda..4f79d4d 100644 --- a/template.yml +++ b/template.yml @@ -1,12 +1,24 @@ AWSTemplateFormatVersion: '2010-09-09' Transform: 'AWS::Serverless-2016-10-31' Description: An AWS Lambda application that calls the Lambda API. +Parameters: + Neo4jUri: + Type: String + Default: bolt://neo4j:7687 + Description: Bolt URI of the Neo4j database, reachable from the Lambda container + Neo4jUsername: + Type: String + Default: neo4j + Neo4jPassword: + Type: String + Default: neo4j-harsh-test + NoEcho: true Resources: function: Type: AWS::Serverless::Function Properties: Handler: lambda_function.lambda_handler - Runtime: python3.10 + Runtime: python3.12 CodeUri: function/. Description: Call the AWS Lambda API Timeout: 600 @@ -15,6 +27,11 @@ Resources: - AWSLambda_ReadOnlyAccess - AWSXrayWriteOnlyAccess Tracing: Active + Environment: + Variables: + NEO4J_URI: !Ref Neo4jUri + NEO4J_USERNAME: !Ref Neo4jUsername + NEO4J_PASSWORD: !Ref Neo4jPassword Layers: - !Ref libs libs: @@ -24,4 +41,4 @@ Resources: Description: Dependencies for the aura-lambda sample app. ContentUri: package/. CompatibleRuntimes: - - python3.10 + - python3.12 diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..d21966a --- /dev/null +++ b/test.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Invoke the Lambda function with a uniquely tagged event, then check: +# 1. the invocation succeeds and returns the Lambda account usage +# 2. the function wrote the event as a LambdaEvent node to Neo4j +set -euo pipefail + +STACK_NAME=${STACK_NAME:-sam-app} +NEO4J_CONTAINER=${NEO4J_CONTAINER:-neo4j} +NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} +NEO4J_PASSWORD=${NEO4J_PASSWORD:-neo4j-harsh-test} +MARKER="ci-$(date +%s)-$RANDOM" +WORKDIR=$(mktemp -d) +trap 'rm -rf "$WORKDIR"' EXIT + +FUNCTION=$(lstk aws cloudformation describe-stack-resource --stack-name "$STACK_NAME" \ + --logical-resource-id function --query 'StackResourceDetail.PhysicalResourceId' --output text) +echo "Function: $FUNCTION" + +jq --arg marker "$MARKER" '. + {testMarker: $marker}' event.json > "$WORKDIR/event.json" + +result=$(lstk aws lambda invoke --function-name "$FUNCTION" \ + --payload "file://$WORKDIR/event.json" --cli-binary-format raw-in-base64-out \ + --cli-read-timeout 300 --query '[StatusCode, FunctionError]' --output text "$WORKDIR/out.json") +echo "Invocation: $result" +echo "Response: $(cat "$WORKDIR/out.json")" +if [ "$(echo "$result" | awk '{print $1}')" != "200" ] || [ "$(echo "$result" | awk '{print $2}')" != "None" ]; then + echo "Test failed: the invocation returned '$result'" + exit 1 +fi +if ! jq -e '.FunctionCount >= 1 and .TotalCodeSize > 0' "$WORKDIR/out.json" > /dev/null; then + echo "Test failed: the response is not the Lambda account usage" + exit 1 +fi +echo "PASS: the function returned the Lambda account usage" + +count=$(docker exec "$NEO4J_CONTAINER" cypher-shell -u "$NEO4J_USERNAME" -p "$NEO4J_PASSWORD" --format plain \ + "MATCH (e:LambdaEvent) WHERE e.event CONTAINS '$MARKER' RETURN count(e)" | tail -1) +if [ "$count" != "1" ]; then + echo "Test failed: expected 1 LambdaEvent node with marker $MARKER in Neo4j, found '$count'" + exit 1 +fi +echo "PASS: the event was written to Neo4j as a LambdaEvent node" + +echo "Test passed: the Lambda function wrote to Neo4j"