From 3a07c298f6962e9d114a2bc0023f4b90749a7f71 Mon Sep 17 00:00:00 2001 From: andwati Date: Wed, 19 Aug 2026 08:37:59 +0300 Subject: [PATCH] Harden Gitea write requests against dropped Content-Type Runs from the GitHub runner were failing every migrate with 422 "[]: Empty Content-Type" while the same script worked from a local network, so the header is being dropped in front of Gitea on the public path rather than the payload being wrong. - route POST/PATCH through one gitea_write helper: body via --data-binary @file (also keeps the PAT out of the process list), Expect: disabled, explicit charset, retry over HTTP/1.1 when the response still reports an empty Content-Type - add DEBUG_HTTP=1 to dump request/response headers, tokens redacted - move temp files into a mktemp -d cleaned by trap - add a workflow preflight that POSTs JSON over HTTP/2 and HTTP/1.1 to show whether the header survives the trip - drop the redundant chmod; the script is committed mode 100755 --- .github/workflows/mirror-to-gitea.yml | 36 +++++++++++-- mirror-github-to-gitea.sh | 73 ++++++++++++++++++++++----- 2 files changed, 91 insertions(+), 18 deletions(-) diff --git a/.github/workflows/mirror-to-gitea.yml b/.github/workflows/mirror-to-gitea.yml index be094cf..79c4ecd 100644 --- a/.github/workflows/mirror-to-gitea.yml +++ b/.github/workflows/mirror-to-gitea.yml @@ -4,7 +4,12 @@ on: schedule: # Runs at 00:00 UTC daily. Adjust if you want midnight EAT (UTC+3) instead — see note below. - cron: "0 0 * * *" - workflow_dispatch: {} # allows manual trigger from the Actions tab too + workflow_dispatch: + inputs: + debug_http: + description: "Print request/response headers for every Gitea write call" + type: boolean + default: false jobs: mirror: @@ -16,12 +21,35 @@ jobs: - name: Install jq run: sudo apt-get update && sudo apt-get install -y jq + # The mirror runs from a GitHub runner, not from the network the script is + # usually tested on, so it can reach Gitea through a different proxy path. + # This POSTs a harmless JSON body and reports whether Content-Type survived + # the trip — a 422 "Empty Content-Type" here means the header is being + # dropped in front of Gitea, not that the mirror payload is wrong. + - name: Preflight - can this runner POST JSON to Gitea? + env: + GITEA_URL: ${{ secrets.GITEA_URL }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + url="${GITEA_URL%/}" + echo "GET /api/v1/version -> $(curl -s -o /dev/null -w '%{http_code}' \ + -H "Authorization: token ${GITEA_TOKEN}" "${url}/api/v1/version")" + for proto in --http2 --http1.1; do + code=$(curl -s $proto -o /tmp/pre.json -w '%{http_code}' \ + -X POST "${url}/api/v1/markdown" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "Expect:" \ + --data-binary '{"text":"preflight","mode":"markdown"}') + echo "POST /api/v1/markdown ${proto} -> ${code}" + [ "$code" = "200" ] || head -c 300 /tmp/pre.json; echo + done + - name: Mirror all repos to Gitea env: GITHUB_TOKEN: ${{ secrets.GH_PAT }} GITEA_URL: ${{ secrets.GITEA_URL }} GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} GITEA_OWNER: ${{ secrets.GITEA_OWNER }} - run: | - chmod +x ./mirror-github-to-gitea.sh - ./mirror-github-to-gitea.sh + DEBUG_HTTP: ${{ inputs.debug_http && '1' || '' }} + run: ./mirror-github-to-gitea.sh diff --git a/mirror-github-to-gitea.sh b/mirror-github-to-gitea.sh index 4a15c26..bf6faae 100755 --- a/mirror-github-to-gitea.sh +++ b/mirror-github-to-gitea.sh @@ -12,6 +12,56 @@ MIRROR_INTERVAL="${MIRROR_INTERVAL:-8h0m0s}" # Optional: process a single repo instead of all of them, e.g. ONLY=blog ./mirror-github-to-gitea.sh # Useful for smoke-testing one repo before turning the script loose on the whole account. ONLY="${ONLY:-}" +# DEBUG_HTTP=1 prints the request/response headers of every write call (tokens redacted). +DEBUG_HTTP="${DEBUG_HTTP:-}" + +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT + +redact() { + sed -e "s/${GITEA_TOKEN}//g" -e "s/${GITHUB_TOKEN}//g" +} + +# Send a JSON body to Gitea and echo the HTTP status. +# gitea_write +# +# The body goes in from a file rather than on the command line: it keeps the PAT +# out of the process list, and it stops curl from having to guess anything about +# the payload. `Expect:` disables 100-continue, which some reverse proxies answer +# incorrectly, and an explicit charset survives proxies that normalise headers. +gitea_write() { + local method="$1" url="$2" body_file="$3" out="$4" + local code trace="${work_dir}/trace" + + _call() { + curl -s -o "$out" -w "%{http_code}" \ + -X "$method" "$url" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + -H "Accept: application/json" \ + -H "Expect:" \ + --data-binary "@${body_file}" \ + "$@" + } + + if [ -n "$DEBUG_HTTP" ]; then + code=$(_call -v 2>"$trace") + echo " --- ${method} ${url##*/api/v1} ---" >&2 + grep -E '^[<>]' "$trace" | redact >&2 + else + code=$(_call) + fi + + # Gitea answers 422 "Empty Content-Type" when the header never reached it, + # which in practice means something on the path (reverse proxy, WAF) dropped + # it. HTTP/2 header handling is the usual culprit, so try again on HTTP/1.1. + if [ "$code" != "200" ] && [ "$code" != "201" ] && grep -q "Empty Content-Type" "$out" 2>/dev/null; then + echo " Content-Type did not reach Gitea; retrying over HTTP/1.1..." >&2 + code=$(_call --http1.1) + fi + + echo "$code" +} # --- Fetch all repos for the GitHub user (paginated), including private ones --- # Uses /user/repos (authenticated, sees private repos owned by the token's user) @@ -57,17 +107,15 @@ while read -r entry; do if [ "$gitea_private" != "$private" ]; then echo " Already mirrored, but visibility differs (GitHub private=${private}, Gitea private=${gitea_private}). Syncing..." - patch_code=$(curl -s -o /tmp/gitea_patch_resp.json -w "%{http_code}" \ - -X PATCH "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${name}" \ - -H "Authorization: token ${GITEA_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "{\"private\": ${private}}") + jq -n --argjson private "$private" '{private: $private}' > "${work_dir}/patch.json" + patch_code=$(gitea_write PATCH "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${name}" \ + "${work_dir}/patch.json" "${work_dir}/patch_resp.json") if [ "$patch_code" == "200" ]; then echo " Visibility updated to private=${private}." else echo " Failed to update visibility (HTTP ${patch_code}):" - cat /tmp/gitea_patch_resp.json; echo + redact < "${work_dir}/patch_resp.json"; echo fi else echo " Already mirrored, visibility already matches (private=${gitea_private}). Skipping." @@ -75,7 +123,7 @@ while read -r entry; do continue fi - payload=$(jq -n \ + jq -n \ --arg clone_addr "https://github.com/${GITHUB_USER}/${name}.git" \ --arg repo_name "$name" \ --arg repo_owner "$GITEA_OWNER" \ @@ -91,19 +139,16 @@ while read -r entry; do private: $private, service: "github", auth_token: $auth_token - }') + }' > "${work_dir}/migrate.json" - http_code=$(curl -s -o /tmp/gitea_migrate_resp.json -w "%{http_code}" \ - -X POST "${GITEA_URL}/api/v1/repos/migrate" \ - -H "Authorization: token ${GITEA_TOKEN}" \ - -H "Content-Type: application/json" \ - --data-raw "$payload") + http_code=$(gitea_write POST "${GITEA_URL}/api/v1/repos/migrate" \ + "${work_dir}/migrate.json" "${work_dir}/migrate_resp.json") if [ "$http_code" == "201" ]; then echo " Created mirror (private=${private})." else echo " Failed (HTTP ${http_code}):" - cat /tmp/gitea_migrate_resp.json; echo + redact < "${work_dir}/migrate_resp.json"; echo fi sleep 1 # be gentle on both APIs