From 3fe11a16bc788b48b320330e51cd5a447627b13e Mon Sep 17 00:00:00 2001 From: Jose Henrique Date: Sun, 12 Jul 2026 13:54:52 -0300 Subject: [PATCH] fix(build-and-push): build via docker context, not buildx-create-over-ssh The 'docker buildx create --name X ssh://...' driver hangs against docker-build.haven with 'waiting for connection: context deadline exceeded', even though SSH, the daemon, and dial-stdio all work. Root cause is that docker-container-driver-over-SSH path; the remote daemon's own default buildkit (containerd-snapshotter) works fine. Rework: - create a docker *context* over SSH and run 'docker --context remote-builder buildx build --push' directly (drops docker/build-push-action, which needs a named builder) - (re)install qemu/binfmt on the remote each run when an arm* platform is requested (binfmt is kernel state, lost on daemon/host restart -- this was causing 'exec format error' for arm64 after the daemon was restarted) - gentags now emits -t args for the direct buildx invocation Validated end-to-end on real infra: multi-arch amd64+arm64 build + push to git.ivanch.me/ivanch/pipeline-actions-selftest:citest, manifest confirmed multi-arch. --- build-and-push/action.yaml | 82 ++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 34 deletions(-) diff --git a/build-and-push/action.yaml b/build-and-push/action.yaml index 4c319b9..f309e89 100644 --- a/build-and-push/action.yaml +++ b/build-and-push/action.yaml @@ -74,9 +74,7 @@ runs: # Fail fast with a real diagnostic instead of a slow buildx # "context deadline exceeded" if the runner can't reach / auth to the # remote docker daemon over SSH. A hard `timeout` wrapper guarantees we - # never hang: ConnectTimeout only bounds the TCP handshake, so a - # firewall that accepts the SYN but drops later packets would otherwise - # stall SSH negotiation indefinitely. + # never hang: ConnectTimeout only bounds the TCP handshake. echo "== probing ssh://${BUILDER_USER}@${BUILDER_HOST} ==" if timeout 25 ssh \ -o BatchMode=yes \ @@ -92,35 +90,45 @@ runs: rc=$? echo "SSH probe FAILED (exit ${rc})" if [ "${rc}" = "124" ]; then - echo "-> timed out: TCP likely opens but SSH negotiation stalls." - echo "-> Check firewall path runner -> ${BUILDER_HOST}:22 (VLAN), or MTU." + echo "-> timed out: TCP opens but SSH/daemon does not answer." + echo "-> Check: dockerd running on ${BUILDER_HOST}? firewall path? MTU?" fi exit "${rc}" fi - - name: Set up remote Docker Buildx builder + - name: Set up remote Docker context and buildx shell: bash env: BUILDER_HOST: ${{ inputs.docker_host }} BUILDER_USER: ${{ inputs.docker_user }} + PLATFORMS: ${{ inputs.platforms }} run: | - # Never set the DOCKER_HOST env var to the builder hostname: a bare - # value is treated as tcp://host:2375 and, more importantly, DOCKER_HOST - # *overrides* the active docker context/builder. Keep it unset so buildx - # talks to the SSH endpoint we specify below. + # NOTE on approach: we drive the *remote daemon's own default buildx + # builder* through a docker context over SSH. We deliberately do NOT use + # `docker buildx create --name X ssh://...` (docker-container driver over + # SSH): that path hangs on "waiting for connection: context deadline + # exceeded" against this builder host, whereas a plain docker context + + # the daemon's built-in buildkit (containerd-snapshotter) works and + # supports multi-arch. Also never set DOCKER_HOST (it overrides the + # active context). unset DOCKER_HOST || true - # Create the buildx builder directly from the SSH endpoint. We do NOT - # create a docker context of the same name first: buildx refuses to - # create a builder instance whose name collides with an existing - # context ("instance name already exists as context builder"). - docker buildx rm -f remote-builder 2>/dev/null || true docker context rm -f remote-builder 2>/dev/null || true - docker buildx create \ - --name remote-builder \ - --use \ - "ssh://${BUILDER_USER}@${BUILDER_HOST}" - docker buildx inspect --bootstrap + docker context create remote-builder \ + --docker "host=ssh://${BUILDER_USER}@${BUILDER_HOST}" + + # Ensure qemu/binfmt is registered on the remote for cross-arch builds. + # This registration lives in the kernel and is lost on daemon/host + # restart, so (re)install it every run. Cheap and idempotent. + if echo "${PLATFORMS}" | grep -q 'arm'; then + echo "== ensuring qemu/binfmt on remote for cross-arch ==" + docker --context remote-builder run --privileged --rm \ + tonistiigi/binfmt --install all >/dev/null 2>&1 || \ + echo "WARN: binfmt install returned non-zero (may already be present)" + fi + + # Sanity-check the remote builder answers (fast, no container spawn). + docker --context remote-builder buildx inspect >/dev/null - name: Generate tags list id: gentags @@ -129,25 +137,31 @@ runs: IMAGE: ${{ inputs.image }} IMAGE_TAGS: ${{ inputs.image_tag }} run: | - TAGS_LIST="" + TAGS_ARGS="" IFS=',' read -ra ADDR <<< "${IMAGE_TAGS}" for tag in "${ADDR[@]}"; do trimmed_tag=$(echo "${tag}" | xargs) if [ -n "${trimmed_tag}" ]; then - TAGS_LIST="${TAGS_LIST}${IMAGE}:${trimmed_tag}"$'\n' + TAGS_ARGS="${TAGS_ARGS} -t ${IMAGE}:${trimmed_tag}" fi done - - echo "tags<> "$GITHUB_OUTPUT" - echo -n "${TAGS_LIST}" >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" + echo "tag_args=${TAGS_ARGS}" >> "$GITHUB_OUTPUT" - name: Build and Push Multi-Arch Image - uses: docker/build-push-action@v6 - with: - push: true - context: ${{ inputs.build_context }} - file: ${{ inputs.build_dockerfile }} - builder: remote-builder - platforms: ${{ inputs.platforms }} - tags: ${{ steps.gentags.outputs.tags }} + shell: bash + env: + BUILD_CONTEXT: ${{ inputs.build_context }} + BUILD_DOCKERFILE: ${{ inputs.build_dockerfile }} + PLATFORMS: ${{ inputs.platforms }} + TAG_ARGS: ${{ steps.gentags.outputs.tag_args }} + run: | + unset DOCKER_HOST || true + # Build on the remote daemon's default buildkit via the SSH context and + # push directly. Registry credentials come from the `docker login` in + # the first step (buildx forwards the runner's auth to buildkit). + docker --context remote-builder buildx build \ + --platform "${PLATFORMS}" \ + --file "${BUILD_CONTEXT}/${BUILD_DOCKERFILE}" \ + ${TAG_ARGS} \ + --push \ + "${BUILD_CONTEXT}"