Run all samples on a Windows runner from Git Bash

.github/workflows/samples.yml

@@ -1,10 +1,10 @@
 name: Test samples on LocalStack for Azure
 
 # Creates the AKS cluster on the LocalStack for Azure emulator, deploys a sample's Python or .NET web
-# app with its numbered scripts, one job per app, and checks the app end to end with
+# app with its numbered scripts, one job per app and platform, and checks the app end to end with
 # tests/test-vacation-planner.sh. As in the Samples CI of localstack-azure-samples, a pull request or a
 # push runs the apps it changes (every app when a file all of them use changes), and a manual run can
-# choose to run every app.
+# choose to run every app, on Linux, on Windows or on both.
 
 on:
   push:
@@ -25,6 +25,15 @@ on:
           - all
           - changed
         default: changed
+      platform:
+        description: "Where the selected samples run: Linux, Windows (from Git Bash with MSYS_NO_PATHCONV=1), or all (both, in parallel jobs of the same run)"
+        required: false
+        type: choice
+        options:
+          - Windows
+          - Linux
+          - all
+        default: Linux
 
 permissions: {}
 
@@ -57,8 +66,9 @@ jobs:
       - name: Select the samples to run
         id: select
         env:
-          # Pull requests and pushes have no inputs: they run the samples they change
+          # Pull requests and pushes have no inputs: they run the samples they change, on Linux
           RUN_MODE: ${{ inputs.run_mode || 'changed' }}
+          PLATFORM: ${{ inputs.platform || 'Linux' }}
           EVENT_NAME: ${{ github.event_name }}
           PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           PUSH_BEFORE_SHA: ${{ github.event.before }}
@@ -91,25 +101,49 @@ jobs:
             fi
           fi
 
-          matrix=$(jq -n -R -c '[inputs | select(length > 0) | split("/") | {sample: .[1], language: .[2]}] | {include: .}' <<< "$selected")
-          echo "Samples to run: $(jq -r '[.include[] | "\(.sample) (\(.language))"] | if length > 0 then join(", ") else "none" end' <<< "$matrix")"
+          case $PLATFORM in
+            Linux | Windows | all) ;;
+            *) echo "::error::Unknown platform '$PLATFORM' (expected Linux, Windows or all)"; exit 1 ;;
+          esac
+
+          # One job per app and platform: the entry names the runner and the job, so a run lists only the jobs
+          # it runs, under their own names
+          matrix=$(jq -n -R -c --arg platform "$PLATFORM" '
+            [inputs | select(length > 0) | split("/") | {sample: .[1], language: .[2]}] as $apps
+            | {include: (
+                (if $platform == "Windows" then [] else
+                  [$apps[] | . + {os: "linux", runner: "ubuntu-latest", name: "\(.sample) (\(.language))"}] end)
+                + (if $platform == "Linux" then [] else
+                  [$apps[] | . + {os: "windows", runner: "windows-latest", name: "Windows (Git Bash) \(.sample) (\(.language))"}] end)
+              )}' <<< "$selected")
+          echo "Jobs to run: $(jq -r '[.include[].name] | if length > 0 then join(", ") else "none" end' <<< "$matrix")"
           {
             echo "matrix=$matrix"
             echo "has_samples=$(jq '.include | length > 0' <<< "$matrix")"
           } >> "$GITHUB_OUTPUT"
 
+  # One job per app and platform, as the setup job's matrix lists them. On Linux, lstk starts the emulator on the
+  # runner. On Windows, the job runs the samples from Git Bash with MSYS_NO_PATHCONV=1 set as the README asks
+  # Windows users to do. The emulator is a Linux image and Docker on the Windows runner runs Windows containers, so
+  # there the emulator runs in Docker Engine inside a WSL 2 distribution on the same runner; the Windows docker
+  # CLI, lstk and kubectl reach it through ports WSL forwards to the runner's localhost.
   sample:
-    name: ${{ matrix.sample }} (${{ matrix.language }})
+    name: ${{ matrix.name }}
     needs: setup
-    # Skip the job when the run changes no sample
+    # Skip the job when the run selects no sample
     if: needs.setup.outputs.has_samples == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
+    runs-on: ${{ matrix.runner }}
+    # A Windows job also sets up WSL 2 and Docker Engine before the sample
+    timeout-minutes: ${{ matrix.os == 'windows' && 90 || 60 }}
     permissions:
       contents: read
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
+    defaults:
+      run:
+        # Git Bash on Windows, whose default shell is PowerShell
+        shell: bash
     env:
       SAMPLE: ${{ matrix.sample }}
       LANGUAGE: ${{ matrix.language }}
@@ -120,13 +154,125 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Point the Windows tools at Docker Engine in WSL 2
+        # The Windows docker CLI and lstk reach the Docker Engine the next steps start in WSL 2 over TCP, and every
+        # later step runs with MSYS_NO_PATHCONV=1, what the README asks Windows users of Git Bash and MSYS2 to export
+        if: matrix.os == 'windows'
+        run: |
+          echo "DOCKER_HOST=tcp://127.0.0.1:2375" >> "$GITHUB_ENV"
+          echo "MSYS_NO_PATHCONV=1" >> "$GITHUB_ENV"
+
+      - name: Drop the carriage returns of Windows programs
+        # The README asks Windows users for `set -o igncr` and `export SHELLOPTS`, so that the bash of Git Bash
+        # and MSYS2 drops the carriage returns the Azure CLI and jq print from $(...) and read. Exporting
+        # SHELLOPTS here would also hand each step's own -e and pipefail to the sample scripts, so every bash
+        # turns the option on through BASH_ENV instead; a bash without the option skips the line.
+        if: matrix.os == 'windows'
+        run: |
+          printf '(set -o igncr) 2>/dev/null && set -o igncr\n' > "$RUNNER_TEMP/igncr.sh"
+          echo "BASH_ENV=$(cygpath -m "$RUNNER_TEMP/igncr.sh")" >> "$GITHUB_ENV"
+
+      - name: Configure WSL
+        # Once a step's wsl.exe session ends, WSL stops the idle distribution after 15 seconds, and
+        # dockerd with it; -1 keeps it running for the whole job. The memory leaves room for the AKS
+        # cluster and the sample's workloads.
+        # DNS tunneling, on by default, answers DNS for WSL itself, but the containers of the Docker Engine
+        # in WSL cannot reach its 10.255.255.254 resolver, so their lookups time out; without it, WSL
+        # hands out the NAT device on the host, which containers reach too.
+        if: matrix.os == 'windows'
+        shell: pwsh
+        run: |
+          Set-Content -Path "$env:USERPROFILE\.wslconfig" -Value "[wsl2]`nmemory=12GB`ndnsTunneling=false`n`n[general]`ninstanceIdleTimeout=-1"
+          wsl --shutdown
+
+      - name: Set up Ubuntu in WSL 2
+        id: wsl
+        if: matrix.os == 'windows'
+        uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0
+        with:
+          distribution: Ubuntu-24.04
+          wsl-version: 2
+          wsl-shell-user: root
+          additional-packages: docker.io
+
+      - name: Start Docker Engine in WSL 2
+        # Installing docker.io already started a daemon on the Unix socket alone, and a second one cannot
+        # start beside it, so it is stopped and replaced by one that also listens on TCP. The Unix socket
+        # is what the emulator container mounts to start the cluster and its backing containers; the TCP
+        # endpoint, bound to localhost, is what WSL forwards to Windows. The wait asks the TCP endpoint,
+        # so the step fails when that daemon did not start.
+        if: matrix.os == 'windows'
+        shell: wsl-bash {0}
+        run: |
+          if [ -d /run/systemd/system ]; then
+            systemctl disable --now docker.socket docker.service
+          elif [ -e /var/run/docker.pid ]; then
+            kill "$(cat /var/run/docker.pid)"
+          fi
+          for _ in $(seq 1 30); do
+            [ -e /var/run/docker.pid ] || break
+            sleep 1
+          done
+          if [ -e /var/run/docker.pid ]; then
+            echo "::error::The Docker daemon that installing docker.io started is still running"
+            exit 1
+          fi
+          setsid nohup dockerd --host unix:///var/run/docker.sock --host tcp://127.0.0.1:2375 --tls=false \
+            > /var/log/dockerd.log 2>&1 < /dev/null &
+          for _ in $(seq 1 60); do
+            docker --host tcp://127.0.0.1:2375 info > /dev/null 2>&1 && break
+            sleep 2
+          done
+          docker --host tcp://127.0.0.1:2375 info \
+            --format 'Docker {{.ServerVersion}} on {{.OperatingSystem}}, kernel {{.KernelVersion}}'
+
+      - name: Check that the Windows docker CLI reaches it
+        if: matrix.os == 'windows'
+        run: docker version
+
       - name: Install lstk
         # Pinned like actions/checkout, so an lstk release cannot change every run at once: bump it deliberately
         run: npm install -g @localstack/lstk@1.4.0
 
+      - name: Give lstk an Azure CLI path without spaces
+        # lstk sets up the Azure CLI with `az cloud register --cloud-config <JSON>`, run through az.cmd, and
+        # cmd.exe cuts that command line at the space of "C:\Program Files" once the JSON's quotes are in it
+        # ("'C:\Program' is not recognized"), so `lstk az start-interception` (lstk 1.4.0) fails wherever the
+        # Azure CLI sits in its default folder. A junction without spaces, first on the PATH, works around it
+        # until lstk passes the JSON another way; Git Bash runs the same Azure CLI through it.
+        if: matrix.os == 'windows'
+        shell: pwsh
+        run: |
+          $cliDir = Split-Path -Parent (Split-Path -Parent (Get-Command az.cmd).Source)
+          New-Item -ItemType Junction -Path C:\azure-cli -Target $cliDir | Out-Null
+          Add-Content -Path $env:GITHUB_PATH -Value 'C:\azure-cli\wbin'
+
+      - name: Install yq
+        # The deployment scripts render the Kubernetes manifests with yq, which the Windows runner image
+        # does not ship. The release and its SHA-256 are pinned.
+        if: matrix.os == 'windows'
+        env:
+          YQ_VERSION: v4.54.1
+          YQ_SHA256: b645f47ebb3a0d2fbab52998550bbd0a1706f23b5ca5f5b48e638c580bb70928
+        run: |
+          mkdir -p "$RUNNER_TEMP/bin"
+          cd "$RUNNER_TEMP/bin"
+          curl --silent --show-error --fail --location --retry 3 --output yq.exe \
+            "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_windows_amd64.exe"
+          echo "${YQ_SHA256}  yq.exe" | sha256sum --check
+          echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
+
+      - name: Put psql on the PATH
+        # The PostgreSQL and App Configuration samples seed their databases with psql, which comes with the
+        # PostgreSQL the Windows runner image installs off the PATH; the Linux runner image has it on the PATH
+        if: matrix.os == 'windows'
+        run: |
+          echo "$PGBIN" >> "$GITHUB_PATH"
+          "$PGBIN/psql" --version
+
       - name: Install sqlcmd
         # The Azure SQL Database sample seeds its database with sqlcmd, which the runner image does not ship
-        if: matrix.sample == 'web-app-sql-database'
+        if: matrix.os == 'linux' && matrix.sample == 'web-app-sql-database'
         run: |
           sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
           curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc > /dev/null
@@ -135,10 +281,24 @@ jobs:
           sudo ACCEPT_EULA=Y apt-get install -y mssql-tools18
           echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH"
 
+      - name: Install sqlcmd on Windows
+        # The Windows runner image does not ship sqlcmd either: the Go sqlcmd release, pinned with its SHA-256
+        if: matrix.os == 'windows' && matrix.sample == 'web-app-sql-database'
+        env:
+          SQLCMD_VERSION: v1.10.0
+          SQLCMD_SHA256: a4c28332fcc6e497d655e53ac8f4939f4ab170f9cfd32d0fa5081b60f4d9d691
+        run: |
+          curl --silent --show-error --fail --location --retry 3 --output "$RUNNER_TEMP/sqlcmd.zip" \
+            "https://github.com/microsoft/go-sqlcmd/releases/download/${SQLCMD_VERSION}/sqlcmd-windows-amd64.zip"
+          echo "${SQLCMD_SHA256}  $RUNNER_TEMP/sqlcmd.zip" | sha256sum --check
+          unzip -o -q -j "$RUNNER_TEMP/sqlcmd.zip" sqlcmd.exe -d "$RUNNER_TEMP/bin"
+          sqlcmd --version
+
       - name: Create the SSH key the cluster script provisions the node pools with
         run: ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa -q
 
       - name: Start the LocalStack for Azure emulator
+        if: matrix.os == 'linux'
         env:
           LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
           # lstk forwards LOCALSTACK_-prefixed variables to the emulator. The Azure SQL Database sample
@@ -152,6 +312,41 @@ jobs:
           lstk start --type azure --non-interactive
           lstk az start-interception
 
+      - name: Start the LocalStack for Azure emulator in WSL 2
+        # Started with the docker CLI rather than lstk: lstk on Windows would mount a Windows volume path,
+        # and no Docker socket, into the Linux engine. The two mounts are paths inside WSL, which
+        # MSYS_NO_PATHCONV keeps Git Bash from rewriting. lstk then finds the container by its name to
+        # route the Azure CLI to it.
+        if: matrix.os == 'windows'
+        env:
+          IMAGE: localstack/localstack-azure:latest
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+          # The emulator looks up the k3d release behind each AKS cluster on api.github.com, as on Linux
+          GITHUB_API_TOKEN: ${{ github.token }}
+        run: |
+          docker pull "$IMAGE"
+          docker run --detach --name localstack-azure \
+            --publish 127.0.0.1:4566:4566 --publish 127.0.0.1:443:443 \
+            --publish 127.0.0.1:4510-4559:4510-4559 \
+            --volume /var/run/docker.sock:/var/run/docker.sock \
+            --volume /var/lib/localstack-azure:/var/lib/localstack \
+            --env LOCALSTACK_AUTH_TOKEN --env GITHUB_API_TOKEN --env MSSQL_ACCEPT_EULA=Y \
+            "$IMAGE"
+          ready=0
+          for _ in $(seq 1 200); do
+            if curl --silent --fail --max-time 5 http://127.0.0.1:4566/_localstack/health > /dev/null; then
+              ready=1
+              break
+            fi
+            sleep 2
+          done
+          if [[ $ready != 1 ]]; then
+            echo "::error::The emulator did not answer on 127.0.0.1:4566 within 400s"
+            docker logs --tail 100 localstack-azure
+            exit 1
+          fi
+          lstk az start-interception
+
       # The script prompts for the network configuration and both node pool sizes:
       # pick option 1 (Azure network plugin and policy) and keep the default sizes
       - name: Create the AKS cluster
@@ -186,4 +381,15 @@ jobs:
             kubectl describe pods --namespace "$ns" | tail -80 || true
             kubectl logs --namespace "$ns" --all-containers --tail=100 -l "app=$ns" || true
           done
-          lstk logs | tail -300 || true
+          if [ "$RUNNER_OS" = Windows ]; then
+            # Started with the docker CLI in WSL 2, not by lstk
+            docker logs localstack-azure 2>&1 | tail -n 300 || true
+          else
+            lstk logs | tail -300 || true
+          fi
+
+      - name: Get the Docker Engine log
+        # The wsl-bash shell exists only once WSL is set up
+        if: failure() && matrix.os == 'windows' && steps.wsl.outcome == 'success'
+        shell: wsl-bash {0}
+        run: tail -n 100 /var/log/dockerd.log || true

README.md

@@ -238,6 +238,7 @@ Alternatively, you can use a terminal UI such as [k9s](https://k9scli.io/) to se
 ```
 
 The [Test samples on LocalStack for Azure](.github/workflows/samples.yml) workflow runs this against the emulator, one job per sample app, in Python and .NET. Each job creates the cluster with `scripts/01-system-assigned-managed-identity.sh`, deploys the app with its numbered scripts, and runs the test. A pull request or a push to `main` runs the apps whose folders it changes, and all 20 when it changes the cluster script, the test or the workflow. A manual run tests the changed apps by default, or all 20 with `run_mode: all`.
+It also picks where they run with `platform`: `Linux` (the default), `Windows`, from Git Bash, or `all`, which runs both at once, in parallel jobs of the same run.
 
 ## Tutorials