diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
new file mode 100644
index 0000000000..a9061244d5
--- /dev/null
+++ b/.github/workflows/docker-images.yml
@@ -0,0 +1,31 @@
+name: Check Docker images
+
+on:
+  pull_request:
+    paths:
+      - Dockerfile
+      - Dockerfile.windows
+      - Makefile
+      - 'tools/make/*'
+
+jobs:
+  # NOTE(rfratto): We only test building the Docker images for the host
+  # platform, but we're really only looking for issues that are
+  # architecture-independent.
+  linux:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build image
+        run: |
+          docker build -t alloy-test:latest -f Dockerfile .
+
+  windows:
+    runs-on: windows-2019
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build image
+        run: |
+          docker build -t alloy-test:latest -f Dockerfile.windows .
+
+
diff --git a/Dockerfile.windows b/Dockerfile.windows
index 7a123bb060..5f137b1e90 100644
--- a/Dockerfile.windows
+++ b/Dockerfile.windows
@@ -12,7 +12,7 @@ SHELL ["cmd", "/S", "/C"]
 # we can before moving on to the next step.
 RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make generate-ui && rm -rf web/ui/node_modules && yarn cache clean --all""
 
-RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} GO_TAGS='builtinassets ${GO_TAGS}' make alloy""
+RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} GO_TAGS=\"builtinassets ${GO_TAGS}\" make alloy""
 # In this case, we're separating the clean command from make alloy to avoid an issue where access to some mod cache
 # files is denied immediately after make alloy, for example:
 # "go: remove C:\go\pkg\mod\golang.org\toolchain@v0.0.1-go1.22.3.windows-amd64\bin\go.exe: Access is denied."