gitea-mirror-sync/mirror-github-to-gitea.sh

#!/usr/bin/env bash
set -euo pipefail

# --- Config ---
GITHUB_USER="andwati"
GITHUB_TOKEN="${GITHUB_TOKEN:?Set GITHUB_TOKEN env var (PAT with repo read access, incl. private repos)}"
GITEA_URL="${GITEA_URL:?Set GITEA_URL env var, e.g. https://gitea.andwati.com}"
GITEA_TOKEN="${GITEA_TOKEN:?Set GITEA_TOKEN env var (Gitea access token, repo scope)}"
GITEA_OWNER="${GITEA_OWNER:-andwati}"
GITEA_URL="${GITEA_URL%/}"   # a trailing slash yields //api/v1/... which some proxies mangle
MIRROR_INTERVAL="${MIRROR_INTERVAL:-8h0m0s}"
# Optional: process a single repo instead of all of them, e.g. ONLY=blog ./mirror-github-to-gitea.sh
# Useful for smoke-testing one repo before turning the script loose on the whole account.
ONLY="${ONLY:-}"
# DEBUG_HTTP=1 prints the request/response headers of every write call (tokens redacted).
DEBUG_HTTP="${DEBUG_HTTP:-}"

# --- Sanitise the credentials ---------------------------------------------
# A secret pasted into GitHub Actions with a trailing newline is invisible in
# the logs but fatal here: curl emits `Authorization: token <value>\n` and the
# bare LF terminates the header block early, so every header curl writes after
# it - Content-Type included - lands in the request body instead. Gitea then
# rejects the call with 422 "Empty Content-Type". GETs survive because nothing
# they need comes after the Authorization header.
trim() { printf '%s' "$1" | tr -d '\r\n' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; }

for _var in GITHUB_TOKEN GITEA_TOKEN GITEA_URL GITEA_OWNER; do
  _raw="${!_var}"
  _clean=$(trim "$_raw")
  if [ "$_raw" != "$_clean" ]; then
    echo "Warning: ${_var} had surrounding whitespace or a newline; trimmed it." >&2
    echo "         Fix the secret itself - re-add it without a trailing newline." >&2
  fi
  printf -v "$_var" '%s' "$_clean"
done
unset _var _raw _clean

GITEA_URL="${GITEA_URL%/}"   # re-strip: trimming may have exposed a trailing slash

work_dir=$(mktemp -d)
trap 'rm -rf "$work_dir"' EXIT

# Literal, delimiter-free substitution - building a sed s/// out of a secret
# breaks as soon as the secret contains a slash or a newline.
redact() {
  local line
  while IFS= read -r line || [ -n "$line" ]; do
    line=${line//"$GITEA_TOKEN"/<GITEA_TOKEN>}
    line=${line//"$GITHUB_TOKEN"/<GITHUB_TOKEN>}
    printf '%s\n' "$line"
  done
}

# Send a JSON body to Gitea and echo the HTTP status.
#   gitea_write <METHOD> <url> <body-file> <response-file>
#
# The body goes in from a file rather than on the command line: it keeps the PAT
# out of the process list, and it stops curl from having to guess anything about
# the payload. `Expect:` disables 100-continue, which some reverse proxies answer
# incorrectly, and an explicit charset survives proxies that normalise headers.
gitea_write() {
  local method="$1" url="$2" body_file="$3" out="$4"
  local code trace="${work_dir}/trace"

  _call() {
    curl -s -o "$out" -w "%{http_code}" \
      -X "$method" "$url" \
      -H "Authorization: token ${GITEA_TOKEN}" \
      -H "Content-Type: application/json; charset=utf-8" \
      -H "Accept: application/json" \
      -H "Expect:" \
      --data-binary "@${body_file}" \
      "$@"
  }

  if [ -n "$DEBUG_HTTP" ]; then
    code=$(_call -v 2>"$trace")
    echo "  --- ${method} ${url##*/api/v1} ---" >&2
    grep -E '^[<>]' "$trace" | redact >&2
  else
    code=$(_call)
  fi

  # Gitea answers 422 "Empty Content-Type" when the header never reached it,
  # which in practice means something on the path (reverse proxy, WAF) dropped
  # it. HTTP/2 header handling is the usual culprit, so try again on HTTP/1.1.
  if [ "$code" != "200" ] && [ "$code" != "201" ] && grep -q "Empty Content-Type" "$out" 2>/dev/null; then
    echo "  Content-Type did not reach Gitea; retrying over HTTP/1.1..." >&2
    code=$(_call --http1.1)
  fi

  echo "$code"
}

# --- Fetch all repos for the GitHub user (paginated), including private ones ---
# Uses /user/repos (authenticated, sees private repos owned by the token's user)
# rather than /users/{user}/repos (public-only).
page=1
repos_json="[]"
while :; do
  resp=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
    "https://api.github.com/user/repos?per_page=100&page=${page}&affiliation=owner")
  count=$(echo "$resp" | jq 'length')
  [ "$count" -eq 0 ] && break
  repos_json=$(jq -s '.[0] + .[1]' <(echo "$repos_json") <(echo "$resp"))
  page=$((page + 1))
done

total=$(echo "$repos_json" | jq 'length')
echo "Found ${total} repos on GitHub for ${GITHUB_USER}"
if [ -n "$ONLY" ]; then
  echo "ONLY=${ONLY} set - skipping every other repo."
fi
matched=0

while read -r entry; do
  name=$(echo "$entry" | jq -r '.name')
  private=$(echo "$entry" | jq -r '.private')

  if [ -n "$ONLY" ] && [ "$name" != "$ONLY" ]; then
    continue
  fi
  matched=$((matched + 1))

  echo "Processing: ${name} (private=${private})"

  # Check if it already exists in Gitea
  existing_resp=$(curl -s -w "\n%{http_code}" \
    -H "Authorization: token ${GITEA_TOKEN}" \
    "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${name}")
  existing_status=$(echo "$existing_resp" | tail -n1)
  existing_body=$(echo "$existing_resp" | sed '$d')

  if [ "$existing_status" == "200" ]; then
    gitea_private=$(echo "$existing_body" | jq -r '.private')

    if [ "$gitea_private" != "$private" ]; then
      echo "  Already mirrored, but visibility differs (GitHub private=${private}, Gitea private=${gitea_private}). Syncing..."
      jq -n --argjson private "$private" '{private: $private}' > "${work_dir}/patch.json"
      patch_code=$(gitea_write PATCH "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${name}" \
        "${work_dir}/patch.json" "${work_dir}/patch_resp.json")

      if [ "$patch_code" == "200" ]; then
        echo "  Visibility updated to private=${private}."
      else
        echo "  Failed to update visibility (HTTP ${patch_code}):"
        redact < "${work_dir}/patch_resp.json"; echo
      fi
    else
      echo "  Already mirrored, visibility already matches (private=${gitea_private}). Skipping."
    fi
    continue
  fi

  jq -n \
    --arg clone_addr "https://github.com/${GITHUB_USER}/${name}.git" \
    --arg repo_name "$name" \
    --arg repo_owner "$GITEA_OWNER" \
    --arg mirror_interval "$MIRROR_INTERVAL" \
    --argjson private "$private" \
    --arg auth_token "$GITHUB_TOKEN" \
    '{
      clone_addr: $clone_addr,
      repo_name: $repo_name,
      repo_owner: $repo_owner,
      mirror: true,
      mirror_interval: $mirror_interval,
      private: $private,
      service: "github",
      auth_token: $auth_token
    }' > "${work_dir}/migrate.json"

  http_code=$(gitea_write POST "${GITEA_URL}/api/v1/repos/migrate" \
    "${work_dir}/migrate.json" "${work_dir}/migrate_resp.json")

  if [ "$http_code" == "201" ]; then
    echo "  Created mirror (private=${private})."
  else
    echo "  Failed (HTTP ${http_code}):"
    redact < "${work_dir}/migrate_resp.json"; echo
  fi

  sleep 1  # be gentle on both APIs
done < <(echo "$repos_json" | jq -c '.[] | {name, private}')

if [ -n "$ONLY" ] && [ "$matched" -eq 0 ]; then
  echo "No repo named '${ONLY}' found among the ${total} repos on GitHub. Nothing done."
  exit 1
fi

echo "Done."