Replace the deprecated localstack CLI with lstk in Samples CI and the sample guides

.github/workflows/run-samples.yml

@@ -105,6 +105,11 @@ jobs:
     env:
       IMAGE_NAME: localstack/localstack-azure
       DEFAULT_TAG: latest
+      # lstk finds the container daemon on its own, and lstk 0.20.0 picked the socket of a
+      # co-installed Podman instead of Docker's, then reported the running emulator as absent
+      # (fixed in 0.20.1, localstack/lstk#422). DOCKER_HOST always wins over that detection,
+      # so every lstk call in this job talks to the Docker daemon the emulator runs on.
+      DOCKER_HOST: unix:///var/run/docker.sock
 
     steps:
       - name: Checkout repo
@@ -165,9 +170,34 @@ jobs:
 
       - name: Install test dependencies
         # Mirroring the localstack-pro approach: install all Python dependencies
-        # (including the localstack CLI) into a virtual environment to avoid system-level conflicts.
+        # into a virtual environment to avoid system-level conflicts.
         run: make install
 
+      - name: Install lstk CLI
+        # Starts and stops the emulator, and lets run-samples.sh route Azure CLI calls to it
+        # via `lstk az start-interception`. https://github.com/localstack/lstk
+        #
+        # Pinned, like every action in this workflow: an unpinned install means an upstream
+        # release lands in every run at once, with no commit to point at when things break.
+        # Bump deliberately.
+        env:
+          LSTK_VERSION: "1.4.0"
+        run: |
+          MAX_RETRIES=3
+          DELAY=15
+          for i in $(seq 1 $MAX_RETRIES); do
+            echo "Attempt $i/$MAX_RETRIES..."
+            npm install -g "@localstack/lstk@${LSTK_VERSION}" && break
+            if [ "$i" -eq "$MAX_RETRIES" ]; then
+              echo "All $MAX_RETRIES attempts failed"
+              exit 1
+            fi
+            echo "Retrying in ${DELAY}s..."
+            sleep $DELAY
+            DELAY=$((DELAY * 2))
+          done
+          command -v lstk
+
       - name: Login to Docker Hub
         # Mandatory login to Docker Hub to benefit from higher rate limits for authenticated pulls.
         # This prevents '429 Too Many Requests' errors during the pull of large emulator images.
@@ -189,87 +219,50 @@ jobs:
         run: docker pull ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}
 
       - name: Start LocalStack
-        # Run the emulator in detached mode using the virtual environment.
-        # The readiness wait is generous because the architecture matrix roughly doubles
-        # the number of concurrent jobs, and the contention for Docker pulls and runner
-        # I/O pushed some starts past a 120s budget - and, on the slowest runners,
-        # occasionally past 300s as well. A healthy emulator still returns as soon as
-        # it is ready, so a larger timeout costs nothing on the happy path.
-        # A start that never becomes ready is retried on a fresh container (see below).
+        # lstk starts the emulator and returns once its health endpoint answers, or fails when the
+        # timeout expires. If the container exits while starting - a rejected auth token exits with
+        # code 55 - lstk prints the last lines of its output, so the reason shows up in this step.
+        # A healthy emulator answers within seconds, so the generous timeout costs nothing on the
+        # happy path. A start that fails or never becomes ready is retried on a fresh container.
         run: |
-          source .venv/bin/activate
-          # Even at 600s the wait occasionally expires: ~80 concurrent jobs compete for runner I/O
-          # and image pulls, and a start that stalls failed the job before the sample ever ran.
-          # Give it one clean retry, and print the container's logs when the first attempt expires.
-          # A healthy start reports ready in well under a minute, so a wait that reaches the
-          # timeout means the boot has stalled rather than slowed: observed on an arm64 runner,
-          # where the container logged nothing for the last ten minutes of the wait. Three shorter
-          # attempts therefore beat two long ones - same overall budget, one more chance, and a
-          # stalled container is replaced sooner.
           attempts=3
           for attempt in $(seq 1 "${attempts}"); do
-            python -m localstack_cli.cli.main start -d
-            if python -m localstack_cli.cli.main wait -t 400; then
+            if lstk start --non-interactive --type azure --image "${IMAGE}" --timeout 400s; then
               echo "Emulator ready (attempt ${attempt}/${attempts})."
+              # The container removes itself, logs included, when it exits. Keep a copy for the
+              # Get LocalStack Logs step in case the emulator dies while the sample runs.
+              nohup docker logs --follow localstack-azure >> "${RUNNER_TEMP}/localstack-azure.log" 2>&1 < /dev/null &
               exit 0
             fi
-            echo "::warning::Emulator was not ready within 400s (attempt ${attempt}/${attempts})."
-            docker logs localstack-main --tail 100 2>&1 || true
-            python -m localstack_cli.cli.main stop || true
-            # stop does not wait for Docker to remove the container, and a start that still finds it fails with
-            # 'container named "localstack-main" already exists'.
-            docker rm -f localstack-main >/dev/null 2>&1 || true
-            for _ in $(seq 1 30); do docker container inspect localstack-main >/dev/null 2>&1 || break; sleep 1; done
+            echo "::warning::Emulator failed to start or was not ready within 400s (attempt ${attempt}/${attempts})."
+            # A timed-out start leaves the container running; one that exited is already gone.
+            docker logs localstack-azure --tail 100 2>&1 || true
+            lstk stop --non-interactive || true
+            # stop does not wait for Docker to remove the container, and lstk start reports a container
+            # that is still running as started, ready or not. Remove it and wait for the name to free up,
+            # so the next attempt boots a fresh emulator.
+            docker rm -f localstack-azure >/dev/null 2>&1 || true
+            for _ in $(seq 1 30); do docker container inspect localstack-azure >/dev/null 2>&1 || break; sleep 1; done
           done
           echo "::error::Emulator failed to become ready after ${attempts} attempts."
           exit 1
         env:
-          IMAGE_NAME: ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}
+          IMAGE: ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}
           LOCALSTACK_AUTH_TOKEN: ${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }}
+          # lstk forwards LOCALSTACK_-prefixed variables to the emulator container, and the
+          # image's entrypoint strips the prefix: LOCALSTACK_LS_LOG reaches the runtime as LS_LOG.
+          # lstk also reads LOCALSTACK_DISABLE_EVENTS itself and sends no telemetry of its own.
           # GITHUB_API_TOKEN: the emulator's package installers query api.github.com
           # (e.g. Azure Functions Core Tools releases when prebuilding the Functions
           # image); anonymous calls share the runner egress IP's 60 req/h budget,
           # which is routinely exhausted on GitHub-hosted runners. localstack-core's
           # GitHubReleaseInstaller sends this token as a Bearer for the higher
           # authenticated limit. The workflow token is ephemeral and contents:read.
-          DOCKER_FLAGS: "-e MSSQL_ACCEPT_EULA=Y -e GITHUB_API_TOKEN=${{ github.token }}"
-          LS_LOG: "DEBUG"
-          DISABLE_EVENTS: "1"
-          ACTIVATE_PRO: "1"
-          DNS_ADDRESS: "0"
-
-      - name: Install lstk CLI
-        # Required by run-samples.sh to proxy Azure CLI calls to the emulator
-        # via `lstk az` (replaces the deprecated azlocal wrapper).
-        # https://github.com/localstack/lstk
-        #
-        # Pinned, like every action in this workflow: an unpinned install means an upstream
-        # release lands in every run at once, with no commit to point at when things break.
-        # Bump deliberately.
-        env:
-          # 0.19.0, not the current 0.20.0. Since 0.20.0 was published (2026-07-30 11:10) every
-          # sample run has failed at `lstk az start-interception` with "LocalStack Azure Emulator
-          # is not running" - 6 runs across 4 unrelated branches, while the emulator was up and
-          # answering its health endpoint. Detection fails on roughly 80% of fresh runners, is
-          # decided before the first call (retries never recover it), and reproduces on neither
-          # arch locally. Every run on 0.19.0 was green.
-          # Revert to the latest release once the detection regression is fixed upstream.
-          LSTK_VERSION: "0.19.0"
-        run: |
-          MAX_RETRIES=3
-          DELAY=15
-          for i in $(seq 1 $MAX_RETRIES); do
-            echo "Attempt $i/$MAX_RETRIES..."
-            npm install -g "@localstack/lstk@${LSTK_VERSION}" && break
-            if [ "$i" -eq "$MAX_RETRIES" ]; then
-              echo "All $MAX_RETRIES attempts failed"
-              exit 1
-            fi
-            echo "Retrying in ${DELAY}s..."
-            sleep $DELAY
-            DELAY=$((DELAY * 2))
-          done
-          command -v lstk
+          LOCALSTACK_GITHUB_API_TOKEN: ${{ github.token }}
+          LOCALSTACK_MSSQL_ACCEPT_EULA: "Y"
+          LOCALSTACK_LS_LOG: "DEBUG"
+          LOCALSTACK_DISABLE_EVENTS: "1"
+          LOCALSTACK_DNS_ADDRESS: "0"
 
       - name: Install MSSQL ODBC and Tools
         # Required for the 'web-app-sql-database' sample which uses 'sqlcmd' to
@@ -305,33 +298,43 @@ jobs:
       - name: "Retry: ${{ matrix.name }}"
         # The deploy scripts are not idempotent - they fail on resources that already exist - so the
         # retry runs against a fresh emulator instead of the half-provisioned state the failed
-        # attempt left behind.
+        # attempt left behind. run-samples.sh also discards the Terraform state that attempt left,
+        # which no longer matches anything in the fresh emulator.
         if: steps.initial_run.outcome == 'failure'
         run: |
-          source .venv/bin/activate
           echo "::warning::${MATRIX_NAME} failed once; retrying on a fresh emulator."
-          python -m localstack_cli.cli.main stop || true
-          # stop does not wait for Docker to remove the container, and a start that still finds it fails with
-          # 'container named "localstack-main" already exists'.
-          docker rm -f localstack-main >/dev/null 2>&1 || true
-          for _ in $(seq 1 30); do docker container inspect localstack-main >/dev/null 2>&1 || break; sleep 1; done
-          python -m localstack_cli.cli.main start -d
-          python -m localstack_cli.cli.main wait -t 600
+          lstk stop --non-interactive || true
+          # stop does not wait for Docker to remove the container, and lstk start reports a container
+          # that is still running as started, ready or not. Remove it and wait for the name to free up,
+          # so the retry boots a fresh emulator.
+          docker rm -f localstack-azure >/dev/null 2>&1 || true
+          for _ in $(seq 1 30); do docker container inspect localstack-azure >/dev/null 2>&1 || break; sleep 1; done
+          lstk start --non-interactive --type azure --image "${IMAGE}" --timeout 600s
+          echo "===== Emulator restarted for the retry =====" >> "${RUNNER_TEMP}/localstack-azure.log"
+          nohup docker logs --follow localstack-azure >> "${RUNNER_TEMP}/localstack-azure.log" 2>&1 < /dev/null &
           make test SHARD="${SHARD}" SPLITS="${SPLITS}"
         env:
           MATRIX_NAME: ${{ matrix.name }}
           SHARD: ${{ matrix.shard }}
           SPLITS: ${{ matrix.splits }}
-          IMAGE_NAME: ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}
+          IMAGE: ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}
           LOCALSTACK_AUTH_TOKEN: ${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }}
-          DOCKER_FLAGS: "-e MSSQL_ACCEPT_EULA=Y -e GITHUB_API_TOKEN=${{ github.token }}"
-          LS_LOG: "DEBUG"
-          DISABLE_EVENTS: "1"
-          ACTIVATE_PRO: "1"
-          DNS_ADDRESS: "0"
+          LOCALSTACK_GITHUB_API_TOKEN: ${{ github.token }}
+          LOCALSTACK_MSSQL_ACCEPT_EULA: "Y"
+          LOCALSTACK_LS_LOG: "DEBUG"
+          LOCALSTACK_DISABLE_EVENTS: "1"
+          LOCALSTACK_DNS_ADDRESS: "0"
           PURGE_DOCKER: "1"
 
       - name: Get LocalStack Logs
         # Captured on failure or success to provide a detailed audit trail of the emulator's activity.
+        # The Start and Retry steps keep a copy of every emulator's logs, so a retried job shows the
+        # emulator that failed as well as the one that ran the retry, and a container that removed
+        # itself on exit still has its logs. make logs covers a job where no emulator came up.
         if: always()
-        run: make logs
+        run: |
+          if [ -s "${RUNNER_TEMP}/localstack-azure.log" ]; then
+            cat "${RUNNER_TEMP}/localstack-azure.log"
+          else
+            make logs
+          fi

Makefile

@@ -25,7 +25,7 @@ SPLITS ?= 1
 test: venv ## Run all samples
 	$(VENV_RUN); bash ./run-samples.sh $(SHARD) $(SPLITS)
 
-logs: venv ## Get LocalStack logs
-	$(VENV_RUN); localstack logs
+logs: ## Get LocalStack logs
+	lstk logs --verbose
 
 .PHONY: venv install test logs

docs/LOCALSTACK.md

@@ -18,35 +18,34 @@ docker pull localstack/localstack-azure
 ## 2. Choose a Startup Method
 You can start the Azure emulator using one of the following methods:
 
-- LocalStack CLI
+- [lstk CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/)
 - Raw `docker run`
 - Docker Compose
 
 ---
 
-### Option A: LocalStack CLI
-Make sure the `localstack` CLI is installed (`pip install localstack` or `brew install localstack/tap/localstack`).
+### Option A: lstk CLI
+Make sure the [lstk CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) is installed (`brew install localstack/tap/lstk` or `npm install -g @localstack/lstk`).
 
 ```bash
 # Set the authentication token
 export LOCALSTACK_AUTH_TOKEN=<your_auth_token>
 
 # Start the LocalStack Azure emulator
-IMAGE_NAME=localstack/localstack-azure localstack start -d
-localstack wait -t 60
+lstk start --type azure
 
 # Route all Azure CLI calls to the LocalStack Azure emulator
 lstk az start-interception
 ```
 
 This:
 - Exports your auth token
-- Overrides the default image via `IMAGE_NAME`
-- Starts LocalStack with Azure emulation enabled in the selected image
+- Starts the LocalStack Azure emulator (`--type azure`) and waits until it is ready
+- Routes all Azure CLI calls to the emulator
 
 To stop:
 ```bash
-localstack stop
+lstk stop
 ```
 
 ---
@@ -114,7 +113,7 @@ docker compose down
 
 ## 3. Verify the Emulator
 After startup, you can:
-- Check logs: `docker logs -f localstack-main`
+- Check logs: `lstk logs --follow`, or `docker logs -f localstack-main` with Docker Compose
 - Curl the edge endpoint:
   ```bash
   curl -s localhost:4566/_localstack/health | jq
@@ -140,7 +139,7 @@ $Env:LOCALSTACK_AUTH_TOKEN = "<your_auth_token>"
 ---
 
 ## 5. Cleanup
-- Stop containers (`localstack stop`, `docker stop <id>`, or `docker compose down`)
+- Stop containers (`lstk stop`, `docker stop <id>`, or `docker compose down`)
 - Remove persisted state if needed:
   ```bash
   rm -rf ~/.localstack/volume
@@ -156,7 +155,7 @@ $Env:LOCALSTACK_AUTH_TOKEN = "<your_auth_token>"
 | Auth error | Confirm token validity & export | 
 | Port already in use | Change mapping (e.g. `-p 4567:4566`) |
 | State not persisting | Verify volume mount path |
-| CLI not using Azure image | Ensure `IMAGE_NAME` env var is set before `localstack start` |
+| lstk starts another emulator | Start it with `lstk start --type azure` |
 
 ---
 

pyproject.toml

@@ -17,7 +17,6 @@ dependencies = [
     "azure-mgmt-cosmosdb",
     "azure-core",
     "python-dotenv",
-    "localstack",
 ]
 
 [project.optional-dependencies]

requirements-runtime.txt

@@ -11,4 +11,3 @@ confluent-kafka
 fastavro
 requests
 python-dotenv
-localstack

run-samples.sh

@@ -8,7 +8,6 @@ set -euo pipefail
 # - .NET 10.0+
 # - Node.js & npm
 # - Azure CLI (az)
-# - LocalStack CLI
 # - Terraform CLI
 # - lstk CLI (https://github.com/localstack/lstk)
 # - jq & zip (sudo apt-get install jq zip)
@@ -230,7 +229,6 @@ if [[ "${1:-}" == "--list" ]]; then
 fi
 
 # 3. Check for required tools
-command -v localstack >/dev/null 2>&1 || { echo >&2 "localstack CLI is required but not installed. Aborting."; exit 1; }
 command -v az >/dev/null 2>&1 || { echo >&2 "az CLI is required but not installed. Aborting."; exit 1; }
 command -v lstk >/dev/null 2>&1 || { echo >&2 "lstk is required but not installed. See https://github.com/localstack/lstk#installation. Aborting."; exit 1; }
 command -v terraform >/dev/null 2>&1 || { echo >&2 "terraform CLI is required but not installed. Aborting."; exit 1; }
@@ -241,13 +239,10 @@ if [ -z "${LOCALSTACK_AUTH_TOKEN:-}" ]; then
 fi
 
 # 4. Start LocalStack
-if ! localstack status | grep -q "running"; then
-  echo "Starting LocalStack Azure emulator..."
-  IMAGE_NAME=localstack/localstack-azure localstack start -d
-  localstack wait -t 60
-else
-  echo "LocalStack is already running."
-fi
+# lstk returns once the emulator reports ready, and succeeds without restarting it when it is
+# already running - as it is in CI, where the workflow starts it before this script.
+echo "Starting LocalStack Azure emulator..."
+lstk start --non-interactive --type azure
 
 # 5. Configure Azure CLI for LocalStack
 echo "Configuring Azure CLI for LocalStack..."
@@ -260,7 +255,7 @@ if command -v lstk >/dev/null 2>&1; then
   # lstk's emulator detection can report a running emulator as absent - `lstk status` has been
   # observed answering "LocalStack AWS Emulator is not running" against a live emulator, and CI
   # runs have failed here with "LocalStack Azure Emulator is not running" while the container was
-  # up and `localstack wait` had already returned. Retry before giving up.
+  # up and had already reported ready. Retry before giving up.
   #
   # Deliberately still fatal after the last attempt: without interception the az CLI targets real
   # Azure, so continuing would send this suite's commands to the cloud. The diagnostics below run
@@ -275,7 +270,7 @@ if command -v lstk >/dev/null 2>&1; then
       echo "[ERROR] --- lstk version ---"
       lstk --version 2>&1 || true
       echo "[ERROR] --- container ---"
-      docker ps -a --filter name=localstack-main --format '{{.Names}} | {{.Image}} | {{.Status}}' || true
+      docker ps -a --filter name=localstack-azure --format '{{.Names}} | {{.Image}} | {{.Status}}' || true
       echo "[ERROR] --- emulator health ---"
       curl -s --max-time 10 http://127.0.0.1:4566/_localstack/health || true
       echo ""
@@ -345,6 +340,14 @@ for (( i=START; i<START+COUNT; i++ )); do
 
   pushd "$path" > /dev/null
 
+  # Deploy a Terraform sample from empty state. A failed run skips the cleanup below and leaves
+  # its state behind, and the CI retry deploys the sample again on a fresh emulator: refreshing
+  # the old state there fails on resources that no longer exist (a storage table whose account is
+  # gone) before anything is deployed. The provider cache in .terraform is kept.
+  if [[ "$path" == *"/terraform" ]]; then
+    rm -f terraform.tfstate terraform.tfstate.backup tfplan
+  fi
+
   echo "Deploying..."
   eval "$deploy"