mirror of
https://github.com/andwati/gitea-mirror-sync.git
synced 2026-10-18 03:12:53 +00:00
Harden Gitea write requests against dropped Content-Type
Runs from the GitHub runner were failing every migrate with 422 "[]: Empty Content-Type" while the same script worked from a local network, so the header is being dropped in front of Gitea on the public path rather than the payload being wrong. - route POST/PATCH through one gitea_write helper: body via --data-binary @file (also keeps the PAT out of the process list), Expect: disabled, explicit charset, retry over HTTP/1.1 when the response still reports an empty Content-Type - add DEBUG_HTTP=1 to dump request/response headers, tokens redacted - move temp files into a mktemp -d cleaned by trap - add a workflow preflight that POSTs JSON over HTTP/2 and HTTP/1.1 to show whether the header survives the trip - drop the redundant chmod; the script is committed mode 100755
This commit is contained in:
@@ -4,7 +4,12 @@ on:
|
||||
schedule:
|
||||
# Runs at 00:00 UTC daily. Adjust if you want midnight EAT (UTC+3) instead — see note below.
|
||||
- cron: "0 0 * * *"
|
||||
workflow_dispatch: {} # allows manual trigger from the Actions tab too
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
debug_http:
|
||||
description: "Print request/response headers for every Gitea write call"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
mirror:
|
||||
@@ -16,12 +21,35 @@ jobs:
|
||||
- name: Install jq
|
||||
run: sudo apt-get update && sudo apt-get install -y jq
|
||||
|
||||
# The mirror runs from a GitHub runner, not from the network the script is
|
||||
# usually tested on, so it can reach Gitea through a different proxy path.
|
||||
# This POSTs a harmless JSON body and reports whether Content-Type survived
|
||||
# the trip — a 422 "Empty Content-Type" here means the header is being
|
||||
# dropped in front of Gitea, not that the mirror payload is wrong.
|
||||
- name: Preflight - can this runner POST JSON to Gitea?
|
||||
env:
|
||||
GITEA_URL: ${{ secrets.GITEA_URL }}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
run: |
|
||||
url="${GITEA_URL%/}"
|
||||
echo "GET /api/v1/version -> $(curl -s -o /dev/null -w '%{http_code}' \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" "${url}/api/v1/version")"
|
||||
for proto in --http2 --http1.1; do
|
||||
code=$(curl -s $proto -o /tmp/pre.json -w '%{http_code}' \
|
||||
-X POST "${url}/api/v1/markdown" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Expect:" \
|
||||
--data-binary '{"text":"preflight","mode":"markdown"}')
|
||||
echo "POST /api/v1/markdown ${proto} -> ${code}"
|
||||
[ "$code" = "200" ] || head -c 300 /tmp/pre.json; echo
|
||||
done
|
||||
|
||||
- name: Mirror all repos to Gitea
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
|
||||
GITEA_URL: ${{ secrets.GITEA_URL }}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GITEA_OWNER: ${{ secrets.GITEA_OWNER }}
|
||||
run: |
|
||||
chmod +x ./mirror-github-to-gitea.sh
|
||||
./mirror-github-to-gitea.sh
|
||||
DEBUG_HTTP: ${{ inputs.debug_http && '1' || '' }}
|
||||
run: ./mirror-github-to-gitea.sh
|
||||
|
||||
+59
-14
@@ -12,6 +12,56 @@ MIRROR_INTERVAL="${MIRROR_INTERVAL:-8h0m0s}"
|
||||
# Optional: process a single repo instead of all of them, e.g. ONLY=blog ./mirror-github-to-gitea.sh
|
||||
# Useful for smoke-testing one repo before turning the script loose on the whole account.
|
||||
ONLY="${ONLY:-}"
|
||||
# DEBUG_HTTP=1 prints the request/response headers of every write call (tokens redacted).
|
||||
DEBUG_HTTP="${DEBUG_HTTP:-}"
|
||||
|
||||
work_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$work_dir"' EXIT
|
||||
|
||||
redact() {
|
||||
sed -e "s/${GITEA_TOKEN}/<GITEA_TOKEN>/g" -e "s/${GITHUB_TOKEN}/<GITHUB_TOKEN>/g"
|
||||
}
|
||||
|
||||
# Send a JSON body to Gitea and echo the HTTP status.
|
||||
# gitea_write <METHOD> <url> <body-file> <response-file>
|
||||
#
|
||||
# The body goes in from a file rather than on the command line: it keeps the PAT
|
||||
# out of the process list, and it stops curl from having to guess anything about
|
||||
# the payload. `Expect:` disables 100-continue, which some reverse proxies answer
|
||||
# incorrectly, and an explicit charset survives proxies that normalise headers.
|
||||
gitea_write() {
|
||||
local method="$1" url="$2" body_file="$3" out="$4"
|
||||
local code trace="${work_dir}/trace"
|
||||
|
||||
_call() {
|
||||
curl -s -o "$out" -w "%{http_code}" \
|
||||
-X "$method" "$url" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json; charset=utf-8" \
|
||||
-H "Accept: application/json" \
|
||||
-H "Expect:" \
|
||||
--data-binary "@${body_file}" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
if [ -n "$DEBUG_HTTP" ]; then
|
||||
code=$(_call -v 2>"$trace")
|
||||
echo " --- ${method} ${url##*/api/v1} ---" >&2
|
||||
grep -E '^[<>]' "$trace" | redact >&2
|
||||
else
|
||||
code=$(_call)
|
||||
fi
|
||||
|
||||
# Gitea answers 422 "Empty Content-Type" when the header never reached it,
|
||||
# which in practice means something on the path (reverse proxy, WAF) dropped
|
||||
# it. HTTP/2 header handling is the usual culprit, so try again on HTTP/1.1.
|
||||
if [ "$code" != "200" ] && [ "$code" != "201" ] && grep -q "Empty Content-Type" "$out" 2>/dev/null; then
|
||||
echo " Content-Type did not reach Gitea; retrying over HTTP/1.1..." >&2
|
||||
code=$(_call --http1.1)
|
||||
fi
|
||||
|
||||
echo "$code"
|
||||
}
|
||||
|
||||
# --- Fetch all repos for the GitHub user (paginated), including private ones ---
|
||||
# Uses /user/repos (authenticated, sees private repos owned by the token's user)
|
||||
@@ -57,17 +107,15 @@ while read -r entry; do
|
||||
|
||||
if [ "$gitea_private" != "$private" ]; then
|
||||
echo " Already mirrored, but visibility differs (GitHub private=${private}, Gitea private=${gitea_private}). Syncing..."
|
||||
patch_code=$(curl -s -o /tmp/gitea_patch_resp.json -w "%{http_code}" \
|
||||
-X PATCH "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${name}" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"private\": ${private}}")
|
||||
jq -n --argjson private "$private" '{private: $private}' > "${work_dir}/patch.json"
|
||||
patch_code=$(gitea_write PATCH "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${name}" \
|
||||
"${work_dir}/patch.json" "${work_dir}/patch_resp.json")
|
||||
|
||||
if [ "$patch_code" == "200" ]; then
|
||||
echo " Visibility updated to private=${private}."
|
||||
else
|
||||
echo " Failed to update visibility (HTTP ${patch_code}):"
|
||||
cat /tmp/gitea_patch_resp.json; echo
|
||||
redact < "${work_dir}/patch_resp.json"; echo
|
||||
fi
|
||||
else
|
||||
echo " Already mirrored, visibility already matches (private=${gitea_private}). Skipping."
|
||||
@@ -75,7 +123,7 @@ while read -r entry; do
|
||||
continue
|
||||
fi
|
||||
|
||||
payload=$(jq -n \
|
||||
jq -n \
|
||||
--arg clone_addr "https://github.com/${GITHUB_USER}/${name}.git" \
|
||||
--arg repo_name "$name" \
|
||||
--arg repo_owner "$GITEA_OWNER" \
|
||||
@@ -91,19 +139,16 @@ while read -r entry; do
|
||||
private: $private,
|
||||
service: "github",
|
||||
auth_token: $auth_token
|
||||
}')
|
||||
}' > "${work_dir}/migrate.json"
|
||||
|
||||
http_code=$(curl -s -o /tmp/gitea_migrate_resp.json -w "%{http_code}" \
|
||||
-X POST "${GITEA_URL}/api/v1/repos/migrate" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-raw "$payload")
|
||||
http_code=$(gitea_write POST "${GITEA_URL}/api/v1/repos/migrate" \
|
||||
"${work_dir}/migrate.json" "${work_dir}/migrate_resp.json")
|
||||
|
||||
if [ "$http_code" == "201" ]; then
|
||||
echo " Created mirror (private=${private})."
|
||||
else
|
||||
echo " Failed (HTTP ${http_code}):"
|
||||
cat /tmp/gitea_migrate_resp.json; echo
|
||||
redact < "${work_dir}/migrate_resp.json"; echo
|
||||
fi
|
||||
|
||||
sleep 1 # be gentle on both APIs
|
||||
|
||||
Reference in New Issue
Block a user