fix(build-and-push): build via docker context, not buildx-create-over-ssh

The 'docker buildx create --name X ssh://...' driver hangs against
docker-build.haven with 'waiting for connection: context deadline exceeded',
even though SSH, the daemon, and dial-stdio all work. Root cause is that
docker-container-driver-over-SSH path; the remote daemon's own default
buildkit (containerd-snapshotter) works fine.

Rework:
- create a docker *context* over SSH and run 'docker --context remote-builder
  buildx build --push' directly (drops docker/build-push-action, which needs
  a named builder)
- (re)install qemu/binfmt on the remote each run when an arm* platform is
  requested (binfmt is kernel state, lost on daemon/host restart -- this was
  causing 'exec format error' for arm64 after the daemon was restarted)
- gentags now emits -t args for the direct buildx invocation

Validated end-to-end on real infra: multi-arch amd64+arm64 build + push to
git.ivanch.me/ivanch/pipeline-actions-selftest:citest, manifest confirmed
multi-arch.

build-and-push/action.yaml

@@ -74,9 +74,7 @@ runs:
         # Fail fast with a real diagnostic instead of a slow buildx
         # "context deadline exceeded" if the runner can't reach / auth to the
         # remote docker daemon over SSH. A hard `timeout` wrapper guarantees we
-        # never hang: ConnectTimeout only bounds the TCP handshake, so a
-        # firewall that accepts the SYN but drops later packets would otherwise
-        # stall SSH negotiation indefinitely.
+        # never hang: ConnectTimeout only bounds the TCP handshake.
         echo "== probing ssh://${BUILDER_USER}@${BUILDER_HOST} =="
         if timeout 25 ssh \
              -o BatchMode=yes \
@@ -92,35 +90,45 @@ runs:
           rc=$?
           echo "SSH probe FAILED (exit ${rc})"
           if [ "${rc}" = "124" ]; then
-            echo "-> timed out: TCP likely opens but SSH negotiation stalls."
-            echo "-> Check firewall path runner -> ${BUILDER_HOST}:22 (VLAN), or MTU."
+            echo "-> timed out: TCP opens but SSH/daemon does not answer."
+            echo "-> Check: dockerd running on ${BUILDER_HOST}? firewall path? MTU?"
           fi
           exit "${rc}"
         fi
 
-    - name: Set up remote Docker Buildx builder
+    - name: Set up remote Docker context and buildx
       shell: bash
       env:
         BUILDER_HOST: ${{ inputs.docker_host }}
         BUILDER_USER: ${{ inputs.docker_user }}
+        PLATFORMS: ${{ inputs.platforms }}
       run: |
-        # Never set the DOCKER_HOST env var to the builder hostname: a bare
-        # value is treated as tcp://host:2375 and, more importantly, DOCKER_HOST
-        # *overrides* the active docker context/builder. Keep it unset so buildx
-        # talks to the SSH endpoint we specify below.
+        # NOTE on approach: we drive the *remote daemon's own default buildx
+        # builder* through a docker context over SSH. We deliberately do NOT use
+        # `docker buildx create --name X ssh://...` (docker-container driver over
+        # SSH): that path hangs on "waiting for connection: context deadline
+        # exceeded" against this builder host, whereas a plain docker context +
+        # the daemon's built-in buildkit (containerd-snapshotter) works and
+        # supports multi-arch. Also never set DOCKER_HOST (it overrides the
+        # active context).
         unset DOCKER_HOST || true
 
-        # Create the buildx builder directly from the SSH endpoint. We do NOT
-        # create a docker context of the same name first: buildx refuses to
-        # create a builder instance whose name collides with an existing
-        # context ("instance name already exists as context builder").
-        docker buildx rm -f remote-builder 2>/dev/null || true
         docker context rm -f remote-builder 2>/dev/null || true
-        docker buildx create \
-          --name remote-builder \
-          --use \
-          "ssh://${BUILDER_USER}@${BUILDER_HOST}"
-        docker buildx inspect --bootstrap
+        docker context create remote-builder \
+          --docker "host=ssh://${BUILDER_USER}@${BUILDER_HOST}"
+
+        # Ensure qemu/binfmt is registered on the remote for cross-arch builds.
+        # This registration lives in the kernel and is lost on daemon/host
+        # restart, so (re)install it every run. Cheap and idempotent.
+        if echo "${PLATFORMS}" | grep -q 'arm'; then
+          echo "== ensuring qemu/binfmt on remote for cross-arch =="
+          docker --context remote-builder run --privileged --rm \
+            tonistiigi/binfmt --install all >/dev/null 2>&1 || \
+            echo "WARN: binfmt install returned non-zero (may already be present)"
+        fi
+
+        # Sanity-check the remote builder answers (fast, no container spawn).
+        docker --context remote-builder buildx inspect >/dev/null
 
     - name: Generate tags list
       id: gentags
@@ -129,25 +137,31 @@ runs:
         IMAGE: ${{ inputs.image }}
         IMAGE_TAGS: ${{ inputs.image_tag }}
       run: |
-        TAGS_LIST=""
+        TAGS_ARGS=""
         IFS=',' read -ra ADDR <<< "${IMAGE_TAGS}"
         for tag in "${ADDR[@]}"; do
           trimmed_tag=$(echo "${tag}" | xargs)
           if [ -n "${trimmed_tag}" ]; then
-            TAGS_LIST="${TAGS_LIST}${IMAGE}:${trimmed_tag}"$'\n'
+            TAGS_ARGS="${TAGS_ARGS} -t ${IMAGE}:${trimmed_tag}"
           fi
         done
-        
-        echo "tags<<EOF" >> "$GITHUB_OUTPUT"
-        echo -n "${TAGS_LIST}" >> "$GITHUB_OUTPUT"
-        echo "EOF" >> "$GITHUB_OUTPUT"
+        echo "tag_args=${TAGS_ARGS}" >> "$GITHUB_OUTPUT"
 
     - name: Build and Push Multi-Arch Image
-      uses: docker/build-push-action@v6
-      with:
-        push: true
-        context: ${{ inputs.build_context }}
-        file: ${{ inputs.build_dockerfile }}
-        builder: remote-builder
-        platforms: ${{ inputs.platforms }}
-        tags: ${{ steps.gentags.outputs.tags }}
+      shell: bash
+      env:
+        BUILD_CONTEXT: ${{ inputs.build_context }}
+        BUILD_DOCKERFILE: ${{ inputs.build_dockerfile }}
+        PLATFORMS: ${{ inputs.platforms }}
+        TAG_ARGS: ${{ steps.gentags.outputs.tag_args }}
+      run: |
+        unset DOCKER_HOST || true
+        # Build on the remote daemon's default buildkit via the SSH context and
+        # push directly. Registry credentials come from the `docker login` in
+        # the first step (buildx forwards the runner's auth to buildkit).
+        docker --context remote-builder buildx build \
+          --platform "${PLATFORMS}" \
+          --file "${BUILD_CONTEXT}/${BUILD_DOCKERFILE}" \
+          ${TAG_ARGS} \
+          --push \
+          "${BUILD_CONTEXT}"