Run tools in GitHub Actions

name: shiftlabs runbook
run-name: runbook ${{ inputs.runbook }}

on:
  workflow_dispatch:
    inputs:
      runbook:
        description: The runbook to run, runbooks/<runbook>.sh
        required: true
        type: string
      args:
        description: Arguments for the runbook as JSON, in SHIFTLABS_ARGS
        required: false
        default: "{}"
        type: string
      callback_url:
        description: Where the result goes. shiftlabs sets it.
        required: false
        default: ""
        type: string

permissions:
  contents: read

defaults:
  run:
    shell: bash

jobs:
  run:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      # Inputs reach the scripts through env only, never pasted into them.
      - name: Run the runbook
        id: runbook
        env:
          RUNBOOK: ${{ inputs.runbook }}
          SHIFTLABS_ARGS: ${{ inputs.args }}
        run: |
          if [[ ! "$RUNBOOK" =~ ^[a-z0-9][a-z0-9-]{0,63}$ || ! -f "runbooks/$RUNBOOK.sh" ]]; then
            echo "No runbook runbooks/$RUNBOOK.sh" | tee "$RUNNER_TEMP/output.txt"
            echo "exit_code=2" >> "$GITHUB_OUTPUT"
            exit 2
          fi
          set +e
          bash "runbooks/$RUNBOOK.sh" 2>&1 | tee "$RUNNER_TEMP/output.txt"
          code=${PIPESTATUS[0]}
          echo "exit_code=$code" >> "$GITHUB_OUTPUT"
          exit "$code"
      - name: Send the result to shiftlabs
        if: ${{ always() && inputs.callback_url != '' }}
        continue-on-error: true
        env:
          CALLBACK_URL: ${{ inputs.callback_url }}
          OUTCOME: ${{ steps.runbook.outcome }}
          EXIT_CODE: ${{ steps.runbook.outputs.exit_code }}
          SIGNING_KEY: ${{ secrets.SHIFTLABS_SIGNING_KEY }}
        run: |
          if [[ -z "$SIGNING_KEY" ]]; then
            echo "::error::Add the repository secret SHIFTLABS_SIGNING_KEY, with the value of GITHUB_RUNBOOK_SIGNING_KEY in shiftlabs."
            exit 1
          fi
          case "$OUTCOME" in
            success) conclusion=success ;;
            cancelled) conclusion=cancelled ;;
            *) conclusion=failure ;;
          esac
          [[ "$EXIT_CODE" =~ ^[0-9]+$ ]] || EXIT_CODE=""
          touch "$RUNNER_TEMP/output.txt"
          tail -c 4000 "$RUNNER_TEMP/output.txt" | jq -Rs . > "$RUNNER_TEMP/output.json"
          output_sha=$(jq -j . "$RUNNER_TEMP/output.json" | openssl dgst -sha256 | awk '{print $NF}')
          signature=$(printf '%s' "$GITHUB_RUN_ID.$conclusion.$EXIT_CODE.$output_sha" | openssl dgst -sha256 -hmac "$SIGNING_KEY" | awk '{print $NF}')
          jq -n --arg run_id "$GITHUB_RUN_ID" --arg conclusion "$conclusion" --arg exit_code "$EXIT_CODE" \
            --arg signature "$signature" --slurpfile output "$RUNNER_TEMP/output.json" \
            '{run_id: $run_id, conclusion: $conclusion, exit_code: ($exit_code | tonumber? // null), output: $output[0], signature: $signature}' |
            curl -sS --fail-with-body --retry 3 -X POST -H "content-type: application/json" --data-binary @- "$CALLBACK_URL"

# The token, with no hosts: only the connection sends it (admin key).
curl -X PUT https://api.shiftlabs.org/v1/secrets/GITHUB_ACTIONS_TOKEN \
  -H "authorization: Bearer $SHIFTLABS_API_KEY" -H "content-type: application/json" \
  -d "{\"value\": \"$GITHUB_TOKEN\"}"

# The signing key, marked hmac: code can sign with it but never read it (admin key).
curl -X PUT https://api.shiftlabs.org/v1/secrets/GITHUB_RUNBOOK_SIGNING_KEY \
  -H "authorization: Bearer $SHIFTLABS_API_KEY" -H "content-type: application/json" \
  -d "{\"value\": \"$SIGNING_KEY\", \"hmac\": true}"
gh secret set SHIFTLABS_SIGNING_KEY --repo octo-org/ops-runbooks --body "$SIGNING_KEY"

# The connection (admin key).
curl -X PUT https://api.shiftlabs.org/v1/connections/github-actions \
  -H "authorization: Bearer $SHIFTLABS_API_KEY" -H "content-type: application/json" \
  -d '{"hosts": ["api.github.com"],
       "template": {"headers": {"Authorization": "Bearer {{ secret.GITHUB_ACTIONS_TOKEN }}"}}}'

# Link it to both workflows, and name the repository (developer key).
for w in github-runbook github-runbook-runs; do
  curl -X PUT https://api.shiftlabs.org/v1/workflows/$w/connections \
    -H "authorization: Bearer $SHIFTLABS_API_KEY" -H "content-type: application/json" \
    -d '{"connections": ["github-actions"]}'
done
curl -X PUT https://api.shiftlabs.org/v1/vars/GITHUB_RUNBOOK_REPO \
  -H "authorization: Bearer $SHIFTLABS_API_KEY" -H "content-type: application/json" \
  -d '{"value": "octo-org/ops-runbooks"}'

import { workflow } from "shiftlabs";

// The GitHub workflow this starts and the branch it runs from. Change them to start another.
const WORKFLOW_FILE = "shiftlabs-runbook.yml";
const REF = "main";
// The same value as the GitHub repository secret SHIFTLABS_SIGNING_KEY, which signs each result.
const SIGNING_KEY = "GITHUB_RUNBOOK_SIGNING_KEY";
// Results that aren't this run's, or aren't signed, are skipped this many times at most before GitHub is asked.
const MAX_WAITS = 5;
const REPO = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
const RUNBOOK = /^[a-z0-9][a-z0-9-]{0,63}$/;
const HEADERS = { accept: "application/vnd.github+json", "user-agent": "shiftlabs-template", "x-github-api-version": "2022-11-28" };

type Payload = { runbook: string; args?: Record<string, string | number | boolean>; minutes?: number };
type Sent = { run_id?: unknown; conclusion?: unknown; exit_code?: unknown; output?: unknown; signature?: unknown };
type Result = { conclusion: string; exitCode: number | null; output: string };

// No Authorization header: the github-actions connection sets it.
async function github<T>(path: string, init: RequestInit = {}): Promise<T> {
  const response = await fetch(`https://api.github.com${path}`, { ...init, headers: { ...HEADERS, ...init.headers } });
  if (response.ok) return (await response.json()) as T;
  const answer = `GitHub answered ${response.status} for ${path}: ${(await response.text()).slice(0, 300)}`;
  if (response.status === 401 || response.status === 403 || response.status === 404) {
    throw new Error(`${answer}\nCheck that the github-actions connection is linked to this workflow, its token can use Actions in the repository, and ${WORKFLOW_FILE} is on ${REF}.`);
  }
  throw new Error(answer);
}

async function sha256(text: string): Promise<string> {
  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
  return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
}

function same(a: string, b: string): boolean {
  if (a.length !== b.length) return false;
  let differ = 0;
  for (let i = 0; i < a.length; i++) differ |= a.charCodeAt(i) ^ b.charCodeAt(i);
  return differ === 0;
}

function duration(ms: number): string {
  return ms < 120_000 ? `${Math.round(ms / 1_000)} seconds` : `${Math.round(ms / 60_000)} minutes`;
}

export default workflow<Payload>(async ({ payload, vars, step, log, sign }) => {
  const repo = vars.GITHUB_RUNBOOK_REPO ?? "";
  if (!REPO.test(repo)) throw new Error("set the GITHUB_RUNBOOK_REPO var to the repository with the runbooks, as owner/name");
  const runbook = String(payload.runbook ?? "");
  if (!RUNBOOK.test(runbook)) throw new Error("runbook must be 1 to 64 lowercase letters, digits and hyphens");
  const args = JSON.stringify(payload.args ?? {});
  if (args.length > 20_000) throw new Error("args must be at most 20,000 characters as JSON");
  const minutes = payload.minutes ?? 30;
  if (!Number.isInteger(minutes) || minutes < 1 || minutes > 360) throw new Error("minutes must be a whole number from 1 to 360");
  try {
    await sign(SIGNING_KEY, "check");
  } catch {
    throw new Error(`add the secret ${SIGNING_KEY}, marked hmac, with the value of the GitHub secret SHIFTLABS_SIGNING_KEY`);
  }

  const callback = step.eventUrl("result");
  // No retries: GitHub takes no idempotency key, so a retried dispatch could start a second run.
  const run = await step.run("start the GitHub run", { retries: { limit: 0, delay: "1 second" } }, async () => {
    const started = await github<{ workflow_run_id?: number; html_url: string }>(`/repos/${repo}/actions/workflows/${WORKFLOW_FILE}/dispatches`, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ ref: REF, inputs: { runbook, args, callback_url: callback }, return_run_details: true }),
    });
    if (!started.workflow_run_id) throw new Error("GitHub started the run without returning its id");
    return { id: Number(started.workflow_run_id), url: String(started.html_url) };
  });
  log.info("GitHub run started", run);

  // Anyone who can read the GitHub run can see the callback URL, so only a result signed with the key counts.
  async function accepted(sent: Sent | null): Promise<Result | null> {
    if (sent === null || typeof sent !== "object" || String(sent.run_id) !== String(run.id) || typeof sent.signature !== "string") return null;
    const exitCode = Number.isInteger(sent.exit_code) ? (sent.exit_code as number) : null;
    const output = typeof sent.output === "string" ? sent.output : "";
    const signed = `${sent.run_id}.${sent.conclusion}.${exitCode ?? ""}.${await sha256(output)}`;
    if (!same(await sign(SIGNING_KEY, signed), sent.signature)) return null;
    return { conclusion: String(sent.conclusion), exitCode, output: output.slice(-4000) };
  }

  const waitFrom = await step.run("wait from", () => Date.now());
  let result: Result | null = null;
  let skipped = 0;
  for (let wait = 0; wait < MAX_WAITS && result === null; wait++) {
    const now = wait === 0 ? waitFrom : await step.run("time waited", () => Date.now());
    const left = minutes * 60_000 - (now - waitFrom);
    if (left < 1_000) break;
    const sent = await step.waitForEvent<Sent>("result", { timeout: left });
    if (sent === null) break;
    result = await accepted(sent);
    if (result === null) {
      skipped++;
      log.warn("ignored a result that isn't this run's or isn't signed with the key");
    }
  }
  if (result !== null) return finish(result.conclusion, "callback", { exitCode: result.exitCode, output: result.output });
  const waited = skipped === MAX_WAITS ? (await step.run("time waited", () => Date.now())) - waitFrom : minutes * 60_000;

  const checked = await step.run("check the GitHub run", { retries: { limit: 2, delay: "5 seconds" } }, async () => {
    const found = await github<{ status: string; conclusion: string | null }>(`/repos/${repo}/actions/runs/${run.id}`);
    return { status: String(found.status), conclusion: String(found.conclusion) };
  });
  if (checked.status !== "completed") {
    throw new Error(`timed out: GitHub run ${run.id} is still ${checked.status} after ${duration(waited)} and sent no signed result. ${run.url}`);
  }
  return finish(checked.conclusion, "check", {});

  function finish(conclusion: string, via: "callback" | "check", more: { exitCode?: number | null; output?: string }) {
    if (conclusion !== "success") {
      const tail = more.output ? `\n${more.output.slice(-1000)}` : "";
      throw new Error(`GitHub run ${run.id} ended ${conclusion}. ${run.url}${tail}`);
    }
    return { runbook, conclusion, runId: run.id, url: run.url, via, ...more };
  }
});

import hashlib
import hmac
import json
import re
import time
from typing import Any, cast

from shiftlabs import Context, workflow
from workers import fetch

# The GitHub workflow this starts and the branch it runs from. Change them to start another.
WORKFLOW_FILE = "shiftlabs-runbook.yml"
REF = "main"
# The same value as the GitHub repository secret SHIFTLABS_SIGNING_KEY, which signs each result.
SIGNING_KEY = "GITHUB_RUNBOOK_SIGNING_KEY"
# Results that aren't this run's, or aren't signed, are skipped this many times at most before GitHub is asked.
MAX_WAITS = 5
REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
RUNBOOK = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
HEADERS = {"accept": "application/vnd.github+json", "user-agent": "shiftlabs-template", "x-github-api-version": "2022-11-28"}


# No Authorization header: the github-actions connection sets it.
async def github(path: str, body: str | None = None) -> Any:
    url = f"https://api.github.com{path}"
    if body is None:
        response = await fetch(url, headers=HEADERS)
    else:
        response = await fetch(url, method="POST", headers={**HEADERS, "content-type": "application/json"}, body=body)
    if response.ok:
        return await response.json()
    answer = f"GitHub answered {response.status} for {path}: {(await response.text())[:300]}"
    if response.status in (401, 403, 404):
        raise RuntimeError(f"{answer}\nCheck that the github-actions connection is linked to this workflow, its token can use Actions in the repository, and {WORKFLOW_FILE} is on {REF}.")
    raise RuntimeError(answer)


def duration(ms: int) -> str:
    return f"{int(ms / 1_000 + 0.5)} seconds" if ms < 120_000 else f"{int(ms / 60_000 + 0.5)} minutes"


@workflow
async def github_runbook(ctx: Context) -> dict[str, Any]:
    repo = ctx.vars.GITHUB_RUNBOOK_REPO or ""
    if not REPO.fullmatch(repo):
        raise ValueError("set the GITHUB_RUNBOOK_REPO var to the repository with the runbooks, as owner/name")
    given = ctx.payload
    payload = cast(dict[str, Any], given) if isinstance(given, dict) else {}
    runbook = str(payload.get("runbook") or "")
    if not RUNBOOK.fullmatch(runbook):
        raise ValueError("runbook must be 1 to 64 lowercase letters, digits and hyphens")
    args = json.dumps(payload.get("args") or {}, separators=(",", ":"), ensure_ascii=False)
    if len(args) > 20_000:
        raise ValueError("args must be at most 20,000 characters as JSON")
    minutes = payload.get("minutes", 30)
    if not isinstance(minutes, int) or isinstance(minutes, bool) or not 1 <= minutes <= 360:
        raise ValueError("minutes must be a whole number from 1 to 360")
    try:
        await ctx.sign(SIGNING_KEY, "check")
    except Exception:
        raise ValueError(f"add the secret {SIGNING_KEY}, marked hmac, with the value of the GitHub secret SHIFTLABS_SIGNING_KEY") from None

    callback = ctx.step.event_url("result")

    async def start() -> dict[str, Any]:
        inputs = {"runbook": runbook, "args": args, "callback_url": callback}
        body = json.dumps({"ref": REF, "inputs": inputs, "return_run_details": True})
        started = await github(f"/repos/{repo}/actions/workflows/{WORKFLOW_FILE}/dispatches", body)
        if not started.get("workflow_run_id"):
            raise RuntimeError("GitHub started the run without returning its id")
        return {"id": int(started["workflow_run_id"]), "url": str(started["html_url"])}

    # No retries: GitHub takes no idempotency key, so a retried dispatch could start a second run.
    run = await ctx.step.run("start the GitHub run", start, retries={"limit": 0, "delay": "1 second"})
    ctx.log.info("GitHub run started", run)

    def finish(conclusion: str, via: str, more: dict[str, Any]) -> dict[str, Any]:
        if conclusion != "success":
            tail = f"\n{more['output'][-1000:]}" if more.get("output") else ""
            raise RuntimeError(f"GitHub run {run['id']} ended {conclusion}. {run['url']}{tail}")
        return {"runbook": runbook, "conclusion": conclusion, "runId": run["id"], "url": run["url"], "via": via, **more}

    # Anyone who can read the GitHub run can see the callback URL, so only a result signed with the key counts.
    async def accepted(sent: Any) -> dict[str, Any] | None:
        given = cast(dict[str, Any], sent) if isinstance(sent, dict) else {}
        signature = given.get("signature")
        if str(given.get("run_id")) != str(run["id"]) or not isinstance(signature, str):
            return None
        exit_code = given.get("exit_code")
        exit_code = exit_code if isinstance(exit_code, int) and not isinstance(exit_code, bool) else None
        output = given.get("output")
        output = output if isinstance(output, str) else ""
        try:
            digest = hashlib.sha256(output.encode()).hexdigest()
        except UnicodeEncodeError:
            return None
        signed = f"{given.get('run_id')}.{given.get('conclusion')}.{'' if exit_code is None else exit_code}.{digest}"
        if not hmac.compare_digest(await ctx.sign(SIGNING_KEY, signed), signature):
            return None
        return {"conclusion": str(given.get("conclusion")), "exitCode": exit_code, "output": output[-4000:]}

    wait_from = await ctx.step.run("wait from", lambda: int(time.time() * 1000))
    result: dict[str, Any] | None = None
    skipped = 0
    for wait in range(MAX_WAITS):
        now = wait_from if wait == 0 else await ctx.step.run("time waited", lambda: int(time.time() * 1000))
        left = minutes * 60_000 - (now - wait_from)
        if left < 1_000:
            break
        sent = await ctx.step.wait_for_event("result", timeout=left)
        if sent is None:
            break
        result = await accepted(sent)
        if result is not None:
            break
        skipped += 1
        ctx.log.warn("ignored a result that isn't this run's or isn't signed with the key")
    if result is not None:
        return finish(result["conclusion"], "callback", {"exitCode": result["exitCode"], "output": result["output"]})
    waited = (await ctx.step.run("time waited", lambda: int(time.time() * 1000))) - wait_from if skipped == MAX_WAITS else minutes * 60_000

    async def check() -> dict[str, str]:
        found = await github(f"/repos/{repo}/actions/runs/{run['id']}")
        return {"status": str(found["status"]), "conclusion": str(found["conclusion"])}

    checked = await ctx.step.run("check the GitHub run", check, retries={"limit": 2, "delay": "5 seconds"})
    if checked["status"] != "completed":
        raise RuntimeError(f"timed out: GitHub run {run['id']} is still {checked['status']} after {duration(waited)} and sent no signed result. {run['url']}")
    return finish(checked["conclusion"], "check", {})