From ce0bd2221c1ee5752d9e9531b3db663a2e24f6d5 Mon Sep 17 00:00:00 2001 From: tinqs-limited Date: Tue, 26 May 2026 01:20:05 +0100 Subject: [PATCH] =?UTF-8?q?tinqs/ci=20=E2=80=94=20composite=20actions=20+?= =?UTF-8?q?=20Lambda=20dispatcher=20for=20Spot=20CI=20runners?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actions: checkout, setup-go, setup-node, setup-aws Dispatcher: Lambda → EC2 Spot (ephemeral, self-terminating) Images: base, go, node, docker, deploy, godot Co-Authored-By: Claude Opus 4.6 (1M context) --- PLAN.md | 30 ++ README.md | 128 ++++++ checkout/README.md | 33 ++ checkout/action.yml | 77 ++++ images/base/Dockerfile | 20 + images/build-all.sh | 50 +++ images/deploy/Dockerfile | 10 + images/docker/Dockerfile | 27 ++ images/go/Dockerfile | 15 + images/godot/Dockerfile | 27 ++ images/node/Dockerfile | 13 + orchestrator/dispatch/go.mod | 32 ++ orchestrator/dispatch/go.sum | 60 +++ orchestrator/dispatch/main.go | 787 ++++++++++++++++++++++++++++++++++ setup-aws/README.md | 28 ++ setup-aws/action.yml | 59 +++ setup-go/README.md | 25 ++ setup-go/action.yml | 44 ++ setup-node/README.md | 27 ++ setup-node/action.yml | 68 +++ 20 files changed, 1560 insertions(+) create mode 100644 PLAN.md create mode 100644 README.md create mode 100644 checkout/README.md create mode 100644 checkout/action.yml create mode 100644 images/base/Dockerfile create mode 100644 images/build-all.sh create mode 100644 images/deploy/Dockerfile create mode 100644 images/docker/Dockerfile create mode 100644 images/go/Dockerfile create mode 100644 images/godot/Dockerfile create mode 100644 images/node/Dockerfile create mode 100644 orchestrator/dispatch/go.mod create mode 100644 orchestrator/dispatch/go.sum create mode 100644 orchestrator/dispatch/main.go create mode 100644 setup-aws/README.md create mode 100644 setup-aws/action.yml create mode 100644 setup-go/README.md create mode 100644 setup-go/action.yml create mode 100644 setup-node/README.md create mode 100644 setup-node/action.yml diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..1513813 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,30 @@ +# tinqs/ci — Status + +## Done + +- [x] Composite actions: checkout, setup-go, setup-node, setup-aws +- [x] Lambda dispatcher with Spot instance routing +- [x] Ephemeral runners (one job, self-terminate) +- [x] Git auth for private repos (url.insteadOf) +- [x] Local action cache (pre-clone to bare repo, instant resolution) +- [x] DynamoDB run tracking + cleanup cron +- [x] Runner image Dockerfiles: base, go, node, docker, deploy, godot +- [x] Zombie runner incident resolved (25 May 2026) + +## Next + +| Priority | Task | Impact | +|----------|------|--------| +| P1 | Pre-warm Go module + build cache in AMI | -30s build time | +| P1 | Automate AMI build (Packer or script) | Repeatable, no manual SSH | +| P2 | Internal DNS for git clones | Faster than public HTTPS | +| P2 | CloudWatch agent on runner AMI | Persistent logs after instance death | +| P3 | `tinqs/ci/deploy-ecs` action | ECS update-service wrapper | +| P3 | `tinqs/ci/deploy-s3` action | S3 sync + CloudFront invalidation | +| P3 | `tinqs/ci/notify` action | Post build status to Lobster GChat | + +## Deleted (stale) + +- `tinqs-ci-exec` Lambda — never successfully ran a build, removed 26 May +- `/ecs/tinqs-runner` CloudWatch log group — from Fargate era, removed 26 May +- Fargate runner service — scaled to 0, cluster still exists for tinqs-git ECS diff --git a/README.md b/README.md new file mode 100644 index 0000000..f890ffb --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +# tinqs/ci + +CI toolchain for Tinqs Studio — composite Gitea Actions and the Lambda dispatcher that orchestrates Spot runners. + +## Architecture + +``` +Push → Gitea webhook → Lambda (tinqs-ci-dispatch) → EC2 Spot instance → act_runner → job → self-terminate +``` + +Runners are ephemeral: one job per instance, self-terminates on completion. Actions are pre-cached on the runner to avoid repeated clones. + +## Actions + +| Action | What it does | +|--------|-------------| +| `tinqs/ci/checkout@v1` | Clone a repo from tinqs.com (supports sparse checkout, depth control, token auth) | +| `tinqs/ci/setup-go@v1` | Install Go (skips if pre-baked in AMI) | +| `tinqs/ci/setup-node@v1` | Install Node.js + pnpm (skips if pre-baked) | +| `tinqs/ci/setup-aws@v1` | Install AWS CLI + optional ECR login | + +```yaml +steps: + - uses: tinqs/ci/checkout@v1 + with: + sparse: 'cmd/tstudio' + - uses: tinqs/ci/setup-go@v1 + - uses: tinqs/ci/setup-aws@v1 + with: + ecr-login: 'true' +``` + +## Dispatcher (Lambda) + +`orchestrator/dispatch/main.go` — receives Gitea webhooks, evaluates workflow triggers, launches Spot instances with the right label. + +| Label | Instance | Use | +|-------|----------|-----| +| `go` | t3.small | Go builds (tstudio, proxy, docgen) | +| `docker` | t3.medium | Docker image builds (platform, bot) | +| `deploy` | t3.micro | S3 sync, ECS update | +| `node` | t3.medium | Frontend builds | +| `godot` | t3.medium | Game exports (future) | + +## Runner Images + +Dockerfiles in `images/` — lean, purpose-built. Push to ECR with `images/build-all.sh`. + +## Deploying + +The dispatcher Lambda can't CI itself — deploy manually: + +```bash +cd orchestrator/dispatch + +# Build +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bootstrap -ldflags "-s -w" . + +# Zip (PowerShell on Windows, zip on Mac/Linux) +# Windows: +powershell -Command "Compress-Archive -Path bootstrap -DestinationPath function.zip -Force" +# Mac/Linux: +zip -j function.zip bootstrap + +# Deploy +aws lambda update-function-code --region eu-west-1 \ + --function-name tinqs-ci-dispatch \ + --zip-file fileb://function.zip + +# Verify +aws lambda invoke --region eu-west-1 --function-name tinqs-ci-dispatch \ + --payload '{}' /dev/null --query 'StatusCode' +``` + +**Lambda env vars** (configured in AWS, not in code): + +| Var | Purpose | +|-----|---------| +| `GITEA_URL` | `https://tinqs.com` | +| `GITEA_TOKEN` | API token for fetching workflows and runner git auth | +| `RUNNER_TOKEN` | act_runner registration token | +| `RUNNER_AMI` | Pre-baked AMI with Go, Node, Docker, act_runner | +| `SUBNET` | VPC subnet for Spot instances | +| `SECURITY_GROUP` | SG allowing outbound HTTPS | +| `DDB_TABLE` | DynamoDB table for run tracking | +| `INSTANCE_PROFILE` | IAM role for runner instances | + +## Contributing + +This repo is private. Only Tinqs team members contribute. + +### Adding a new action + +1. Create `/action.yml` (composite, shell: bash) +2. Keep it simple — no Node.js runtime, just bash +3. Add to the table in this README +4. Push to main — actions are resolved via `@v1` (main branch) + +### Modifying the dispatcher + +1. Edit `orchestrator/dispatch/main.go` +2. Test locally: `go build . && echo '{}' | ./dispatch` (dry run) +3. Deploy manually (see Deploying above) +4. Verify: push a change to `tinqs/studio` and check the pipeline + +### Runner images + +1. Edit `images//Dockerfile` +2. Build: `cd images && ./build-all.sh v1` +3. Requires Docker + ECR login (`aws ecr get-login-password | docker login ...`) + +## Monitoring + +```bash +# Check for zombie runners (should be 0 except during active builds) +aws ec2 describe-instances --region eu-west-1 \ + --filters "Name=tag:tinqs-ci,Values=true" "Name=instance-state-name,Values=running" \ + --query 'Reservations[].Instances[].InstanceId' + +# Lambda dispatch logs +MSYS_NO_PATHCONV=1 aws logs tail '/aws/lambda/tinqs-ci-dispatch' --region eu-west-1 + +# Build logs (via Gitea API) +curl -s "https://tinqs.com/api/v1/repos/tinqs/studio/actions/jobs//logs" \ + -H "Authorization: token $TOKEN" +``` + +Full debug guide: `tinqs/docs/.cursor/skills/ci-pipeline-discipline/SKILL.md` diff --git a/checkout/README.md b/checkout/README.md new file mode 100644 index 0000000..bba9723 --- /dev/null +++ b/checkout/README.md @@ -0,0 +1,33 @@ +# tinqs/ci/checkout + +Clones a repository from tinqs.com (self-hosted Gitea) using plain `git clone`. + +Works on any runner with git installed — Alpine, Debian, or pre-baked images. No Node.js runtime, no GitHub API dependency. + +## Usage + +```yaml +- uses: tinqs/ci/checkout@v1 +``` + +### With options + +```yaml +- uses: tinqs/ci/checkout@v1 + with: + repository: 'tinqs/engine' # default: current repo + ref: 'tinqs/main' # default: current branch + depth: '0' # default: 1 (shallow) + token: ${{ secrets.TOKEN }} # default: none (public clone) + path: 'engine' # default: . (current dir) +``` + +## Inputs + +| Input | Default | Description | +|-------|---------|-------------| +| `repository` | `${{ github.repository }}` | Repository in `owner/repo` format | +| `ref` | `${{ github.ref_name }}` | Branch or tag | +| `depth` | `1` | Clone depth (0 = full history) | +| `path` | `.` | Directory to clone into | +| `token` | `` | Gitea access token for private repos | diff --git a/checkout/action.yml b/checkout/action.yml new file mode 100644 index 0000000..dacd56c --- /dev/null +++ b/checkout/action.yml @@ -0,0 +1,77 @@ +# tinqs/ci/checkout — Tinqs Studio CI +# Clones a repo from tinqs.com (self-hosted Gitea) using plain git. +# Supports depth control, branch/tag selection, token auth, and sparse checkout. +# Composite action — runs directly on the host, no Node.js runtime needed. +# Author: Ozan + Claude Code — 2026-05-22 + +name: 'Tinqs Checkout' +description: 'Clone a Gitea repository' + +inputs: + repository: + description: 'Repository (owner/repo)' + default: '${{ github.repository }}' + ref: + description: 'Branch or tag to checkout' + default: '${{ github.ref_name }}' + depth: + description: 'Clone depth (0 = full)' + default: '1' + path: + description: 'Directory to clone into' + default: '.' + token: + description: 'Gitea access token (for private repos)' + default: '' + sparse: + description: 'Space-separated paths for sparse checkout (empty = full clone)' + default: '' + +runs: + using: 'composite' + steps: + - run: | + REPO="${{ inputs.repository }}" + REF="${{ inputs.ref }}" + DEPTH="${{ inputs.depth }}" + TARGET="${{ inputs.path }}" + TOKEN="${{ inputs.token }}" + SPARSE="${{ inputs.sparse }}" + + if [ "$DEPTH" = "0" ]; then + DEPTH_FLAG="" + else + DEPTH_FLAG="--depth $DEPTH" + fi + + if [ -n "$TOKEN" ]; then + URL="https://token:${TOKEN}@tinqs.com/${REPO}.git" + else + URL="https://tinqs.com/${REPO}.git" + fi + + if [ -n "$SPARSE" ]; then + # Sparse checkout — only fetch specified paths + mkdir -p "$TARGET" && cd "$TARGET" + git init + git remote add origin "$URL" + git config core.sparseCheckout true + for P in $SPARSE; do + echo "$P" >> .git/info/sparse-checkout + done + # Also always include go.mod/go.sum for Go builds + echo "go.mod" >> .git/info/sparse-checkout + echo "go.sum" >> .git/info/sparse-checkout + git fetch $DEPTH_FLAG origin "$REF" + git checkout FETCH_HEAD + echo "Sparse checkout ${REPO}@${REF} ($(echo $SPARSE | wc -w) paths)" + else + # Full clone + if [ "$TARGET" = "." ]; then + git clone $DEPTH_FLAG --branch "$REF" "$URL" . + else + git clone $DEPTH_FLAG --branch "$REF" "$URL" "$TARGET" + fi + echo "Checked out ${REPO}@${REF}" + fi + shell: bash diff --git a/images/base/Dockerfile b/images/base/Dockerfile new file mode 100644 index 0000000..b8200d7 --- /dev/null +++ b/images/base/Dockerfile @@ -0,0 +1,20 @@ +# tinqs/ci base — shared foundation for all runner images +# Alpine + git + bash + curl + AWS CLI + ssh +# Every other image builds FROM this. + +FROM alpine:3.23 + +RUN apk add --no-cache \ + bash git curl wget unzip tar \ + ca-certificates openssh-client \ + tzdata + +# AWS CLI (needed by almost every workflow) +RUN curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscli.zip && \ + unzip -q /tmp/awscli.zip -d /tmp && \ + /tmp/aws/install && \ + rm -rf /tmp/awscli.zip /tmp/aws + +RUN git --version && aws --version + +WORKDIR /workspace diff --git a/images/build-all.sh b/images/build-all.sh new file mode 100644 index 0000000..7f65d97 --- /dev/null +++ b/images/build-all.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Build all Tinqs CI runner images and push to ECR +set -euo pipefail + +AWS_REGION="${AWS_REGION:-eu-west-1}" +ACCOUNT_ID="${ACCOUNT_ID:-149751500842}" +ECR_BASE="${ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com" +TAG="${1:-latest}" + +echo "=== Logging into ECR ===" +aws ecr get-login-password --region "${AWS_REGION}" | \ + docker login --username AWS --password-stdin "${ECR_BASE}" + +# Ensure ECR repos exist +for NAME in tinqs-runner-base tinqs-runner-go tinqs-runner-node tinqs-runner-docker tinqs-runner-deploy tinqs-runner-godot; do + aws ecr describe-repositories --repository-names "$NAME" --region "$AWS_REGION" 2>/dev/null || \ + aws ecr create-repository --repository-name "$NAME" --region "$AWS_REGION" --no-cli-pager +done + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# 1. Base (everything depends on this) +echo "=== Building base ===" +docker build -t "${ECR_BASE}/tinqs-runner-base:${TAG}" -t "${ECR_BASE}/tinqs-runner-base:latest" "$SCRIPT_DIR/base" +docker push "${ECR_BASE}/tinqs-runner-base:${TAG}" +docker push "${ECR_BASE}/tinqs-runner-base:latest" + +BASE_ARG="--build-arg BASE_IMAGE=${ECR_BASE}/tinqs-runner-base:${TAG}" + +# 2. All others in parallel (they only depend on base) +build_and_push() { + local name=$1 + local dir=$2 + local extra_args="${3:-}" + echo "=== Building ${name} ===" + docker build $extra_args -t "${ECR_BASE}/tinqs-runner-${name}:${TAG}" -t "${ECR_BASE}/tinqs-runner-${name}:latest" "$SCRIPT_DIR/$dir" + docker push "${ECR_BASE}/tinqs-runner-${name}:${TAG}" + docker push "${ECR_BASE}/tinqs-runner-${name}:latest" + echo "=== Done ${name} ===" +} + +build_and_push "go" "go" "$BASE_ARG" & +build_and_push "node" "node" "$BASE_ARG" & +build_and_push "deploy" "deploy" "$BASE_ARG" & +build_and_push "docker" "docker" "" & +# godot is optional — uncomment when game CI is ready +# build_and_push "godot" "godot" "$BASE_ARG" & + +wait +echo "=== All images built and pushed (tag: ${TAG}) ===" diff --git a/images/deploy/Dockerfile b/images/deploy/Dockerfile new file mode 100644 index 0000000..79a1be5 --- /dev/null +++ b/images/deploy/Dockerfile @@ -0,0 +1,10 @@ +# tinqs/ci deploy — lightweight deploy-only runner +# Used by: deploy-arikigame, release +# runs-on: deploy +# Smallest image — just AWS CLI + SSH for S3 sync, CloudFront invalidation, ECS updates. + +ARG BASE_IMAGE=tinqs-runner-base:latest +FROM ${BASE_IMAGE} + +# Nothing extra needed — base already has aws, git, ssh, curl +RUN aws --version && ssh -V diff --git a/images/docker/Dockerfile b/images/docker/Dockerfile new file mode 100644 index 0000000..eca3fa0 --- /dev/null +++ b/images/docker/Dockerfile @@ -0,0 +1,27 @@ +# tinqs/ci docker — Docker builds (platform image, bot image) +# Used by: build (platform), deploy-bot +# runs-on: docker +# Based on docker:dind for Docker-in-Docker support. + +FROM docker:29-dind + +RUN apk add --no-cache \ + bash git curl wget unzip tar \ + ca-certificates openssh-client \ + build-base s6 tzdata + +# AWS CLI +RUN curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscli.zip && \ + unzip -q /tmp/awscli.zip -d /tmp && \ + /tmp/aws/install && \ + rm -rf /tmp/awscli.zip /tmp/aws + +# Go (needed for platform Docker builds that compile inside container) +ARG GO_VERSION=1.26.2 +RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz +ENV PATH="/usr/local/go/bin:/go/bin:${PATH}" +ENV GOPATH="/go" + +RUN docker --version && aws --version && go version + +WORKDIR /workspace diff --git a/images/go/Dockerfile b/images/go/Dockerfile new file mode 100644 index 0000000..45f5a97 --- /dev/null +++ b/images/go/Dockerfile @@ -0,0 +1,15 @@ +# tinqs/ci go — Go builds (platform, CLI, proxy) +# Used by: build-tstudio, deploy-proxy +# runs-on: go + +ARG BASE_IMAGE=tinqs-runner-base:latest +FROM ${BASE_IMAGE} + +ARG GO_VERSION=1.26.2 + +RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz + +ENV PATH="/usr/local/go/bin:/go/bin:${PATH}" +ENV GOPATH="/go" + +RUN go version diff --git a/images/godot/Dockerfile b/images/godot/Dockerfile new file mode 100644 index 0000000..555ef28 --- /dev/null +++ b/images/godot/Dockerfile @@ -0,0 +1,27 @@ +# tinqs/ci godot — headless Godot for game builds and export +# Used by: ariki-game build pipeline (future) +# runs-on: godot +# Headless Godot 4.6 + export templates for Windows/Linux/Web. + +ARG BASE_IMAGE=tinqs-runner-base:latest +FROM ${BASE_IMAGE} + +ARG GODOT_VERSION=4.6.2 + +# Headless Godot editor (for --headless --export-all) +RUN curl -fsSL "https://github.com/godotengine/godot-builds/releases/download/${GODOT_VERSION}-stable/Godot_v${GODOT_VERSION}-stable_linux.x86_64.zip" \ + -o /tmp/godot.zip && \ + unzip -q /tmp/godot.zip -d /tmp && \ + mv /tmp/Godot_v${GODOT_VERSION}-stable_linux.x86_64 /usr/local/bin/godot && \ + chmod +x /usr/local/bin/godot && \ + rm -f /tmp/godot.zip + +# Export templates +RUN mkdir -p /root/.local/share/godot/export_templates/${GODOT_VERSION}.stable && \ + curl -fsSL "https://github.com/godotengine/godot-builds/releases/download/${GODOT_VERSION}-stable/Godot_v${GODOT_VERSION}-stable_export_templates.tpz" \ + -o /tmp/templates.tpz && \ + unzip -q /tmp/templates.tpz -d /tmp/templates && \ + mv /tmp/templates/templates/* /root/.local/share/godot/export_templates/${GODOT_VERSION}.stable/ && \ + rm -rf /tmp/templates.tpz /tmp/templates + +RUN godot --headless --version diff --git a/images/node/Dockerfile b/images/node/Dockerfile new file mode 100644 index 0000000..fa3da71 --- /dev/null +++ b/images/node/Dockerfile @@ -0,0 +1,13 @@ +# tinqs/ci node — Node.js builds (docs, frontend) +# Used by: deploy-docs +# runs-on: node + +ARG BASE_IMAGE=tinqs-runner-base:latest +FROM ${BASE_IMAGE} + +ARG NODE_VERSION=22 + +RUN apk add --no-cache nodejs npm && \ + npm install -g pnpm + +RUN node --version && pnpm --version diff --git a/orchestrator/dispatch/go.mod b/orchestrator/dispatch/go.mod new file mode 100644 index 0000000..59c6367 --- /dev/null +++ b/orchestrator/dispatch/go.mod @@ -0,0 +1,32 @@ +module tinqs.com/tinqs/ci/orchestrator/dispatch + +go 1.23 + +require ( + github.com/aws/aws-lambda-go v1.47.0 + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.0 + github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.12 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.38.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.0 + github.com/aws/aws-sdk-go-v2/service/lambda v1.69.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect + github.com/aws/smithy-go v1.22.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) diff --git a/orchestrator/dispatch/go.sum b/orchestrator/dispatch/go.sum new file mode 100644 index 0000000..0c9266b --- /dev/null +++ b/orchestrator/dispatch/go.sum @@ -0,0 +1,60 @@ +github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= +github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= +github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.12 h1:zYf8E8zaqolHA5nQ+VmX2r3wc4K6xw5i6xKvvMjZBL0= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.12/go.mod h1:vYGIVLASk19Gb0FGwAcwES+qQF/aekD7m2G/X6mBOdQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.38.0 h1:isKhHsjpQR3CypQJ4G1g8QWx7zNpiC/xKw1zjgJYVno= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.38.0/go.mod h1:xDvUyIkwBwNtVZJdHEwAuhFly3mezwdEWkbJ5oNYwIw= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.2 h1:E7Tuo0ipWpBl0f3uThz8cZsuyD5H8jLCnbtbKR4YL2s= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.2/go.mod h1:txOfweuNPBLhHodsV+C2lvPPRTommVTWbts9SZV6Myc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.0 h1:ivPJXmGlzAjgy0jLO9naExUWE8IM8lLRcRKLPBEx6Q0= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.0/go.mod h1:00zqVNJFK6UASrTnuvjJHJuaqUdkVz5tW8Ip+VhzuNg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6 h1:nbmKXZzXPJn41CcD4HsHsGWqvKjLKz9kWu6XxvLmf1s= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6/go.mod h1:SJhcisfKfAawsdNQoZMBEjg+vyN2lH6rO6fP+T94z5Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/lambda v1.69.0 h1:BXt75frE/FYtAmEDBJRBa2HexOw+oAZWZl6QknZEFgg= +github.com/aws/aws-sdk-go-v2/service/lambda v1.69.0/go.mod h1:guz2K3x4FKSdDaoeB+TPVgJNU9oj2gftbp5cR8ela1A= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/orchestrator/dispatch/main.go b/orchestrator/dispatch/main.go new file mode 100644 index 0000000..3ef6fe0 --- /dev/null +++ b/orchestrator/dispatch/main.go @@ -0,0 +1,787 @@ +// tinqs/ci orchestrator — dispatcher Lambda +// +// Receives Gitea system webhooks, determines which workflows to run, +// and routes each job to the right execution environment based on runs-on label: +// +// go, node, docker, godot → EC2 Spot instance (pre-baked AMI, self-terminates) +// deploy → Lambda direct execution (ci-exec) +// host → skip (handled by registered always-on runner) +// +// Also handles cancel events (terminate instances) and cleanup cron. + +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + awslambda "github.com/aws/aws-sdk-go-v2/service/lambda" + "gopkg.in/yaml.v3" +) + +// --- Gitea webhook types --- + +type GiteaPushEvent struct { + Ref string `json:"ref"` + Before string `json:"before"` + After string `json:"after"` + Repository GiteaRepo `json:"repository"` + Pusher GiteaUser `json:"pusher"` + Commits []GiteaCommit `json:"commits"` +} + +type GiteaRepo struct { + ID int `json:"id"` + Name string `json:"name"` + FullName string `json:"full_name"` + CloneURL string `json:"clone_url"` + HTMLURL string `json:"html_url"` +} + +type GiteaUser struct { + Login string `json:"login"` + Email string `json:"email"` +} + +type GiteaCommit struct { + ID string `json:"id"` + Message string `json:"message"` + Added []string `json:"added"` + Removed []string `json:"removed"` + Modified []string `json:"modified"` +} + +// --- Workflow YAML (minimal parse) --- + +type Workflow struct { + Name string `yaml:"name"` + On interface{} `yaml:"on"` + Jobs map[string]WorkflowJob `yaml:"jobs"` +} + +type WorkflowJob struct { + RunsOn string `yaml:"runs-on"` +} + +// --- DynamoDB run record --- + +type RunRecord struct { + Repo string `dynamodbav:"repo"` + RunID string `dynamodbav:"run_id"` + InstanceID string `dynamodbav:"instance_id"` + Status string `dynamodbav:"status"` + Workflow string `dynamodbav:"workflow"` + Label string `dynamodbav:"label"` + StartedAt int64 `dynamodbav:"started_at"` + TTL int64 `dynamodbav:"ttl"` +} + +// --- Spot instance config per label --- + +type spotConfig struct { + InstanceType string + MaxPrice string // spot bid (empty = on-demand price cap) +} + +var labelToSpot = map[string]spotConfig{ + "go": {InstanceType: "t3.small", MaxPrice: "0.02"}, + "node": {InstanceType: "t3.medium", MaxPrice: "0.04"}, + "docker": {InstanceType: "t3.medium", MaxPrice: "0.04"}, + "deploy": {InstanceType: "t3.micro", MaxPrice: "0.01"}, + "godot": {InstanceType: "t3.medium", MaxPrice: "0.04"}, +} + +// --- Config from env --- + +type cfg struct { + GiteaURL string + GiteaToken string // API token (for fetching workflows, setting commit status) + RunnerToken string // Runner registration token (for act_runner register) + ExecFnName string + AMI string // pre-baked AMI with Go, Node, Docker, AWS CLI, act_runner + Subnet string + SecurityGroup string + DDBTable string + InstanceProfile string +} + +func loadCfg() cfg { + return cfg{ + GiteaURL: os.Getenv("GITEA_URL"), + GiteaToken: os.Getenv("GITEA_TOKEN"), + RunnerToken: os.Getenv("RUNNER_TOKEN"), + ExecFnName: os.Getenv("EXECUTOR_FUNCTION_NAME"), + AMI: os.Getenv("RUNNER_AMI"), + Subnet: os.Getenv("SUBNET"), + SecurityGroup: os.Getenv("SECURITY_GROUP"), + DDBTable: os.Getenv("DDB_TABLE"), + InstanceProfile: os.Getenv("INSTANCE_PROFILE"), + } +} + +// --- User-data script template --- +// The instance boots, registers as ephemeral runner, picks one job, then terminates itself. + +func userDataScript(c cfg, label, runnerName string) string { + script := fmt.Sprintf(`#!/bin/bash +set -euo pipefail +exec > /var/log/tinqs-ci.log 2>&1 + +echo "=== Tinqs CI Runner: %s ===" + +# Get instance metadata (IMDSv2) +TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") +INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id) +REGION=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region) + +# Self-termination: shutdown triggers terminate (InstanceInitiatedShutdownBehavior=terminate) +# No IAM needed — kernel handles it +trap 'echo "=== Job done, shutting down ===" && shutdown -h now' EXIT + +export PATH=$PATH:/usr/local/go/bin:/usr/local/bin +export HOME=/root + +# Git auth for private repo clones (checkout action uses git binary which reads this) +# Note: act_runner's internal action resolution uses go-git, which does NOT read +# gitconfig — so tinqs/ci must stay public for act_runner to clone actions. +GITEA_TOKEN="%s" +git config --global url."https://token:${GITEA_TOKEN}@tinqs.com/".insteadOf "https://tinqs.com/" +git config --global url."https://token:${GITEA_TOKEN}@git.tinqs.com/".insteadOf "https://git.tinqs.com/" + +# Create proper working directory for act_runner +mkdir -p /opt/runner && cd /opt/runner + +# Register as ephemeral runner (picks one job, then exits) +act_runner register --no-interactive \ + --instance %s \ + --token %s \ + --name %s \ + --labels "%s:host" \ + --ephemeral + +# Configure runner +cat > .runner.yaml << 'RUNCFG' +log: + level: info +runner: + capacity: 1 + timeout: 30m + envs: + HOME: /root + AWS_DEFAULT_REGION: eu-west-1 + PATH: /usr/local/go/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +host: + workdir_parent: /opt/runner/work +RUNCFG + +mkdir -p /opt/runner/work + +# Run — blocks until the job completes, then exits (ephemeral = one job only) +act_runner daemon --config .runner.yaml + +# Safety: if daemon exits without triggering trap (shouldn't happen), force shutdown +shutdown -h now + +echo "=== Runner exited, cleanup will terminate instance ===" +`, runnerName, c.GiteaToken, c.GiteaURL, c.RunnerToken, runnerName, label) + + return base64.StdEncoding.EncodeToString([]byte(script)) +} + +func handler(ctx context.Context, request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { + c := loadCfg() + + // Handle cleanup cron + if request.Body == "" || strings.Contains(request.Body, `"action":"cleanup"`) { + return handleCleanup(ctx, c) + } + + // Handle cancel webhook + eventType := request.Headers["x-gitea-event"] + if eventType == "" { + eventType = request.Headers["X-Gitea-Event"] + } + if eventType == "workflow_job" { + return handleCancel(ctx, c, request.Body) + } + + // Handle push event + var push GiteaPushEvent + if err := json.Unmarshal([]byte(request.Body), &push); err != nil { + log.Printf("Failed to parse webhook: %v", err) + return respond(400, `{"error":"invalid payload"}`) + } + + if !strings.HasPrefix(push.Ref, "refs/heads/") { + return respond(200, `{"status":"skipped","reason":"not a branch push"}`) + } + + branch := strings.TrimPrefix(push.Ref, "refs/heads/") + log.Printf("Push to %s branch=%s by=%s commits=%d", + push.Repository.FullName, branch, push.Pusher.Login, len(push.Commits)) + + changedFiles := collectChangedFiles(push.Commits) + + workflows, err := fetchWorkflows(c.GiteaURL, c.GiteaToken, push.Repository.FullName, branch) + if err != nil || len(workflows) == 0 { + return respond(200, `{"status":"no_workflows"}`) + } + + dispatched := 0 + for name, content := range workflows { + var wf Workflow + if err := yaml.Unmarshal([]byte(content), &wf); err != nil { + log.Printf("Failed to parse %s: %v", name, err) + continue + } + + if !shouldTrigger(wf, branch, changedFiles) { + log.Printf("Skipping %s (no matching trigger)", name) + continue + } + + label := "host" + for _, job := range wf.Jobs { + if job.RunsOn != "" { + label = job.RunsOn + break + } + } + + runID := fmt.Sprintf("%s-%s-%d", push.After[:7], name, time.Now().UnixMilli()) + + switch label { + case "deploy": + if err := invokeLambdaExec(ctx, c, push, name, content); err != nil { + log.Printf("Failed to invoke executor for %s: %v", name, err) + continue + } + case "host": + log.Printf("Skipping %s (runs-on: host — registered runner)", name) + continue + default: + instanceID, err := startSpotRunner(ctx, c, label, runID) + if err != nil { + log.Printf("Failed to start spot for %s: %v", name, err) + continue + } + if err := trackRun(ctx, c, push.Repository.FullName, runID, instanceID, name, label); err != nil { + log.Printf("Warning: failed to track run: %v", err) + } + } + + dispatched++ + } + + return respond(200, fmt.Sprintf( + `{"status":"dispatched","workflows":%d,"dispatched":%d,"repo":"%s","branch":"%s"}`, + len(workflows), dispatched, push.Repository.FullName, branch)) +} + +// --- EC2 Spot runner --- + +func startSpotRunner(ctx context.Context, c cfg, label, runID string) (string, error) { + awsCfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return "", fmt.Errorf("aws config: %w", err) + } + + spot, ok := labelToSpot[label] + if !ok { + return "", fmt.Errorf("unknown label: %s", label) + } + + runnerName := fmt.Sprintf("spot-%s-%s", label, runID[:12]) + userData := userDataScript(c, label, runnerName) + + client := ec2.NewFromConfig(awsCfg) + + // Docker builds need more disk (monorepo + layers) + diskSize := int32(20) + if label == "docker" { + diskSize = 40 + } + + out, err := client.RunInstances(ctx, &ec2.RunInstancesInput{ + ImageId: aws.String(c.AMI), + InstanceType: ec2types.InstanceType(spot.InstanceType), + MinCount: aws.Int32(1), + MaxCount: aws.Int32(1), + UserData: aws.String(userData), + + // Root volume — bigger for Docker builds + BlockDeviceMappings: []ec2types.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + Ebs: &ec2types.EbsBlockDevice{ + VolumeSize: aws.Int32(diskSize), + VolumeType: ec2types.VolumeTypeGp3, + DeleteOnTermination: aws.Bool(true), + }, + }}, + + // Spot request + InstanceMarketOptions: &ec2types.InstanceMarketOptionsRequest{ + MarketType: ec2types.MarketTypeSpot, + SpotOptions: &ec2types.SpotMarketOptions{ + MaxPrice: aws.String(spot.MaxPrice), + SpotInstanceType: ec2types.SpotInstanceTypeOneTime, + InstanceInterruptionBehavior: ec2types.InstanceInterruptionBehaviorTerminate, + }, + }, + + // Network + NetworkInterfaces: []ec2types.InstanceNetworkInterfaceSpecification{{ + DeviceIndex: aws.Int32(0), + SubnetId: aws.String(c.Subnet), + Groups: []string{c.SecurityGroup}, + AssociatePublicIpAddress: aws.Bool(true), + }}, + + // IAM role (for AWS CLI, ECR, S3, ECS access) + IamInstanceProfile: &ec2types.IamInstanceProfileSpecification{ + Arn: aws.String(c.InstanceProfile), + }, + + // Tags for identification and cleanup + TagSpecifications: []ec2types.TagSpecification{{ + ResourceType: ec2types.ResourceTypeInstance, + Tags: []ec2types.Tag{ + {Key: aws.String("Name"), Value: aws.String(runnerName)}, + {Key: aws.String("tinqs-ci"), Value: aws.String("true")}, + {Key: aws.String("tinqs-ci-run"), Value: aws.String(runID)}, + {Key: aws.String("tinqs-ci-label"), Value: aws.String(label)}, + }, + }}, + + // Auto-terminate safety net (instance shuts down → terminates) + InstanceInitiatedShutdownBehavior: ec2types.ShutdownBehaviorTerminate, + }) + if err != nil { + return "", fmt.Errorf("run instances: %w", err) + } + + if len(out.Instances) == 0 { + return "", fmt.Errorf("no instances launched") + } + + instanceID := *out.Instances[0].InstanceId + log.Printf("Started spot instance: %s (type=%s, label=%s, runner=%s)", + instanceID, spot.InstanceType, label, runnerName) + return instanceID, nil +} + +// --- Lambda executor (for deploy-only jobs) --- + +func invokeLambdaExec(ctx context.Context, c cfg, push GiteaPushEvent, workflow, content string) error { + if c.ExecFnName == "" { + log.Printf("[DRY RUN] Would invoke executor for %s", workflow) + return nil + } + + awsCfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return err + } + + payload, _ := json.Marshal(map[string]interface{}{ + "repo": push.Repository, + "ref": push.Ref, + "commit_sha": push.After, + "pusher": push.Pusher.Login, + "workflow_name": workflow, + "workflow_yaml": content, + "gitea_url": c.GiteaURL, + "gitea_token": c.GiteaToken, + }) + + client := awslambda.NewFromConfig(awsCfg) + _, err = client.Invoke(ctx, &awslambda.InvokeInput{ + FunctionName: aws.String(c.ExecFnName), + InvocationType: "Event", + Payload: payload, + }) + return err +} + +// --- Cancel handler --- + +func handleCancel(ctx context.Context, c cfg, body string) (events.APIGatewayProxyResponse, error) { + var event struct { + Action string `json:"action"` + Repository GiteaRepo `json:"repository"` + } + if err := json.Unmarshal([]byte(body), &event); err != nil { + return respond(400, `{"error":"invalid cancel event"}`) + } + + if event.Action != "cancelled" { + return respond(200, `{"status":"ignored","reason":"not a cancel event"}`) + } + + log.Printf("Cancel request for %s", event.Repository.FullName) + + awsCfg, _ := config.LoadDefaultConfig(ctx) + ddb := dynamodb.NewFromConfig(awsCfg) + + out, err := ddb.Query(ctx, &dynamodb.QueryInput{ + TableName: aws.String(c.DDBTable), + KeyConditionExpression: aws.String("repo = :repo"), + FilterExpression: aws.String("#s = :running"), + ExpressionAttributeNames: map[string]string{"#s": "status"}, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":repo": &types.AttributeValueMemberS{Value: event.Repository.FullName}, + ":running": &types.AttributeValueMemberS{Value: "running"}, + }, + }) + if err != nil { + return respond(500, `{"error":"db query failed"}`) + } + + ec2Client := ec2.NewFromConfig(awsCfg) + stopped := 0 + + // Collect instance IDs to terminate + var instanceIDs []string + var records []RunRecord + for _, item := range out.Items { + var rec RunRecord + if err := attributevalue.UnmarshalMap(item, &rec); err != nil || rec.InstanceID == "" { + continue + } + instanceIDs = append(instanceIDs, rec.InstanceID) + records = append(records, rec) + } + + if len(instanceIDs) > 0 { + _, err := ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: instanceIDs, + }) + if err != nil { + log.Printf("Failed to terminate instances: %v", err) + } else { + stopped = len(instanceIDs) + for _, rec := range records { + updateRunStatus(ctx, ddb, c.DDBTable, rec.Repo, rec.RunID, "cancelled") + } + log.Printf("Terminated %d instances for %s", stopped, event.Repository.FullName) + } + } + + return respond(200, fmt.Sprintf(`{"status":"cancelled","stopped":%d}`, stopped)) +} + +// --- Cleanup cron --- + +func handleCleanup(ctx context.Context, c cfg) (events.APIGatewayProxyResponse, error) { + log.Println("Running cleanup...") + + awsCfg, _ := config.LoadDefaultConfig(ctx) + ddb := dynamodb.NewFromConfig(awsCfg) + ec2Client := ec2.NewFromConfig(awsCfg) + + cutoff := time.Now().Add(-30 * time.Minute).Unix() + + out, err := ddb.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(c.DDBTable), + FilterExpression: aws.String("#s = :running AND started_at < :cutoff"), + ExpressionAttributeNames: map[string]string{"#s": "status"}, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":running": &types.AttributeValueMemberS{Value: "running"}, + ":cutoff": &types.AttributeValueMemberN{Value: fmt.Sprintf("%d", cutoff)}, + }, + }) + if err != nil { + return respond(500, fmt.Sprintf(`{"error":"%s"}`, err)) + } + + // Batch terminate stale instances + var staleIDs []string + var staleRecs []RunRecord + for _, item := range out.Items { + var rec RunRecord + if err := attributevalue.UnmarshalMap(item, &rec); err != nil || rec.InstanceID == "" { + continue + } + staleIDs = append(staleIDs, rec.InstanceID) + staleRecs = append(staleRecs, rec) + } + + killed := 0 + if len(staleIDs) > 0 { + ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: staleIDs, + }) + for _, rec := range staleRecs { + updateRunStatus(ctx, ddb, c.DDBTable, rec.Repo, rec.RunID, "timeout") + } + killed = len(staleIDs) + log.Printf("Killed %d stale instances", killed) + } + + // Also sweep any tinqs-ci tagged instances that somehow escaped DynamoDB + descOut, _ := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2types.Filter{ + {Name: aws.String("tag:tinqs-ci"), Values: []string{"true"}}, + {Name: aws.String("instance-state-name"), Values: []string{"running", "pending"}}, + }, + }) + var orphanIDs []string + for _, res := range descOut.Reservations { + for _, inst := range res.Instances { + if inst.LaunchTime != nil && time.Since(*inst.LaunchTime) > 30*time.Minute { + orphanIDs = append(orphanIDs, *inst.InstanceId) + } + } + } + if len(orphanIDs) > 0 { + ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: orphanIDs, + }) + log.Printf("Killed %d orphan instances (tag sweep)", len(orphanIDs)) + killed += len(orphanIDs) + } + + return respond(200, fmt.Sprintf(`{"status":"cleanup","killed":%d}`, killed)) +} + +// --- DynamoDB helpers --- + +func trackRun(ctx context.Context, c cfg, repo, runID, instanceID, workflow, label string) error { + awsCfg, _ := config.LoadDefaultConfig(ctx) + ddb := dynamodb.NewFromConfig(awsCfg) + + now := time.Now() + rec := RunRecord{ + Repo: repo, + RunID: runID, + InstanceID: instanceID, + Status: "running", + Workflow: workflow, + Label: label, + StartedAt: now.Unix(), + TTL: now.Add(7 * 24 * time.Hour).Unix(), + } + + item, err := attributevalue.MarshalMap(rec) + if err != nil { + return err + } + + _, err = ddb.PutItem(ctx, &dynamodb.PutItemInput{ + TableName: aws.String(c.DDBTable), + Item: item, + }) + return err +} + +func updateRunStatus(ctx context.Context, ddb *dynamodb.Client, table, repo, runID, status string) { + ddb.UpdateItem(ctx, &dynamodb.UpdateItemInput{ + TableName: aws.String(table), + Key: map[string]types.AttributeValue{ + "repo": &types.AttributeValueMemberS{Value: repo}, + "run_id": &types.AttributeValueMemberS{Value: runID}, + }, + UpdateExpression: aws.String("SET #s = :status"), + ExpressionAttributeNames: map[string]string{"#s": "status"}, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":status": &types.AttributeValueMemberS{Value: status}, + }, + }) +} + +// --- Gitea API --- + +func fetchWorkflows(baseURL, token, repoFullName, branch string) (map[string]string, error) { + url := fmt.Sprintf("%s/api/v1/repos/%s/contents/.gitea/workflows?ref=%s", + baseURL, repoFullName, branch) + + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Authorization", "token "+token) + + resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == 404 { + return nil, nil + } + + var contents []struct { + Name string `json:"name"` + Path string `json:"path"` + } + if err := json.NewDecoder(resp.Body).Decode(&contents); err != nil { + return nil, err + } + + workflows := make(map[string]string) + for _, c := range contents { + if !strings.HasSuffix(c.Name, ".yml") && !strings.HasSuffix(c.Name, ".yaml") { + continue + } + content, err := fetchRawFile(baseURL, token, repoFullName, c.Path, branch) + if err != nil { + log.Printf("Failed to fetch %s: %v", c.Path, err) + continue + } + workflows[c.Name] = content + } + return workflows, nil +} + +func fetchRawFile(baseURL, token, repoFullName, path, branch string) (string, error) { + url := fmt.Sprintf("%s/api/v1/repos/%s/raw/%s?ref=%s", baseURL, repoFullName, path, branch) + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Authorization", "token "+token) + + resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body := make([]byte, 1<<20) + n, _ := resp.Body.Read(body) + return string(body[:n]), nil +} + +// --- Trigger evaluation --- + +func shouldTrigger(wf Workflow, branch string, changedFiles []string) bool { + onMap, ok := wf.On.(map[string]interface{}) + if !ok { + return true + } + + pushCfg, ok := onMap["push"] + if !ok { + return false + } + + pushMap, ok := pushCfg.(map[string]interface{}) + if !ok { + return true + } + + if branches, ok := pushMap["branches"].([]interface{}); ok { + matched := false + for _, b := range branches { + if fmt.Sprint(b) == branch { + matched = true + break + } + } + if !matched { + return false + } + } + + if paths, ok := pushMap["paths"].([]interface{}); ok && len(changedFiles) > 0 { + matched := false + for _, p := range paths { + pattern := fmt.Sprint(p) + for _, f := range changedFiles { + if matchPath(pattern, f) { + matched = true + break + } + } + if matched { + break + } + } + if !matched { + return false + } + } + + if ignorePaths, ok := pushMap["paths-ignore"].([]interface{}); ok && len(changedFiles) > 0 { + allIgnored := true + for _, f := range changedFiles { + ignored := false + for _, p := range ignorePaths { + if matchPath(fmt.Sprint(p), f) { + ignored = true + break + } + } + if !ignored { + allIgnored = false + break + } + } + if allIgnored { + return false + } + } + + return true +} + +func matchPath(pattern, file string) bool { + if strings.HasSuffix(pattern, "/**") { + prefix := strings.TrimSuffix(pattern, "/**") + return strings.HasPrefix(file, prefix+"/") || file == prefix + } + if strings.HasPrefix(pattern, "*.") { + ext := strings.TrimPrefix(pattern, "*") + return strings.HasSuffix(file, ext) + } + return strings.HasPrefix(file, pattern) || file == pattern +} + +func collectChangedFiles(commits []GiteaCommit) []string { + seen := make(map[string]bool) + var files []string + for _, c := range commits { + for _, f := range c.Added { + if !seen[f] { + files = append(files, f) + seen[f] = true + } + } + for _, f := range c.Modified { + if !seen[f] { + files = append(files, f) + seen[f] = true + } + } + for _, f := range c.Removed { + if !seen[f] { + files = append(files, f) + seen[f] = true + } + } + } + return files +} + +func respond(status int, body string) (events.APIGatewayProxyResponse, error) { + return events.APIGatewayProxyResponse{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + }, nil +} + +func main() { + lambda.Start(handler) +} diff --git a/setup-aws/README.md b/setup-aws/README.md new file mode 100644 index 0000000..78278cb --- /dev/null +++ b/setup-aws/README.md @@ -0,0 +1,28 @@ +# tinqs/ci/setup-aws + +Installs AWS CLI and optionally logs into ECR. + +Detects Alpine (musl) vs Debian (glibc) and uses the right install method — pip on Alpine, official binary on Debian. Credentials come from the runner environment (IAM task role on Fargate, instance profile on EC2). + +## Usage + +```yaml +- uses: tinqs/ci/setup-aws@v1 +``` + +### With ECR login + +```yaml +- uses: tinqs/ci/setup-aws@v1 + with: + ecr-login: 'true' + ecr-repo: '149751500842.dkr.ecr.eu-west-1.amazonaws.com/tinqs-git' +``` + +## Inputs + +| Input | Default | Description | +|-------|---------|-------------| +| `region` | `eu-west-1` | AWS region | +| `ecr-login` | `false` | Login to ECR after install | +| `ecr-repo` | `` | ECR repository URL (required when ecr-login is true) | diff --git a/setup-aws/action.yml b/setup-aws/action.yml new file mode 100644 index 0000000..1ae2fc4 --- /dev/null +++ b/setup-aws/action.yml @@ -0,0 +1,59 @@ +# tinqs/ci/setup-aws — Tinqs Studio CI +# Installs AWS CLI and optionally logs into ECR. +# Detects Alpine (apk/pip), Debian (apt-get), and Amazon Linux/RHEL (dnf). +# Skips install if AWS CLI is already present (pre-baked AMI). +# Credentials come from the runner environment (IAM instance profile on EC2, +# task role on Fargate) — no explicit key configuration needed. +# Composite action — runs directly on the host. +# Author: Ozan + Claude Code — 2026-05-22 + +name: 'Tinqs Setup AWS' +description: 'Install AWS CLI and optionally login to ECR' + +inputs: + region: + description: 'AWS region' + default: 'eu-west-1' + ecr-login: + description: 'Login to ECR (true/false)' + default: 'false' + ecr-repo: + description: 'ECR repository URL (required if ecr-login is true)' + default: '' + +runs: + using: 'composite' + steps: + - run: | + REGION="${{ inputs.region }}" + ECR_LOGIN="${{ inputs.ecr-login }}" + ECR_REPO="${{ inputs.ecr-repo }}" + + # Install AWS CLI if missing + if ! command -v aws &>/dev/null; then + echo "Installing AWS CLI..." + if command -v apk &>/dev/null; then + # Alpine — use pip (AWS CLI v2 binary needs glibc) + apk add --no-cache python3 py3-pip + pip3 install --break-system-packages awscli 2>/dev/null || pip3 install awscli + elif command -v dnf &>/dev/null || command -v apt-get &>/dev/null; then + # Amazon Linux / Debian — official installer (glibc available) + curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscli.zip + unzip -q /tmp/awscli.zip -d /tmp/aws-install + /tmp/aws-install/aws/install + rm -rf /tmp/awscli.zip /tmp/aws-install + else + echo "ERROR: unsupported package manager" && exit 1 + fi + fi + + aws --version + + # ECR login + if [ "$ECR_LOGIN" = "true" ] && [ -n "$ECR_REPO" ]; then + echo "Logging into ECR ($REGION)..." + aws ecr get-login-password --region "$REGION" | \ + docker login --username AWS --password-stdin "$ECR_REPO" + echo "ECR login OK" + fi + shell: bash diff --git a/setup-go/README.md b/setup-go/README.md new file mode 100644 index 0000000..9ece09a --- /dev/null +++ b/setup-go/README.md @@ -0,0 +1,25 @@ +# tinqs/ci/setup-go + +Installs Go from go.dev and adds it to PATH. + +Downloads the official tarball — works on any Linux (Alpine, Debian, etc.). Skips installation if the correct version is already present in the runner image. + +## Usage + +```yaml +- uses: tinqs/ci/setup-go@v1 +``` + +### Specific version + +```yaml +- uses: tinqs/ci/setup-go@v1 + with: + go-version: '1.26.2' +``` + +## Inputs + +| Input | Default | Description | +|-------|---------|-------------| +| `go-version` | `1.26.2` | Go version to install | diff --git a/setup-go/action.yml b/setup-go/action.yml new file mode 100644 index 0000000..ec878a7 --- /dev/null +++ b/setup-go/action.yml @@ -0,0 +1,44 @@ +# tinqs/ci/setup-go — Tinqs Studio CI +# Downloads Go from go.dev and adds it to PATH. +# Works on any Linux (Alpine, Debian, etc.) — just a tarball extract. +# Skips install if the correct version is already present (pre-baked runner image). +# Composite action — runs directly on the host. +# Author: Ozan + Claude Code — 2026-05-22 + +name: 'Tinqs Setup Go' +description: 'Install Go and configure PATH' + +inputs: + go-version: + description: 'Go version to install' + default: '1.26.2' + +runs: + using: 'composite' + steps: + - run: | + GO_VERSION="${{ inputs.go-version }}" + + # Skip if already installed at correct version + if command -v go &>/dev/null; then + CURRENT=$(go version | sed 's/.*go//' | cut -d' ' -f1) + if [ "$CURRENT" = "$GO_VERSION" ]; then + echo "Go $GO_VERSION already installed" + go version + exit 0 + fi + fi + + echo "Installing Go $GO_VERSION..." + # Use wget on Alpine (no curl), curl elsewhere + if command -v curl &>/dev/null; then + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz + elif command -v wget &>/dev/null; then + wget -qO- "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz + else + apk add --no-cache curl && curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /usr/local -xz + fi + echo "/usr/local/go/bin" >> "$GITHUB_PATH" + export PATH="/usr/local/go/bin:$PATH" + go version + shell: bash diff --git a/setup-node/README.md b/setup-node/README.md new file mode 100644 index 0000000..37bf867 --- /dev/null +++ b/setup-node/README.md @@ -0,0 +1,27 @@ +# tinqs/ci/setup-node + +Installs Node.js and optionally pnpm. + +Detects Alpine vs Debian and uses the right package manager (`apk` or NodeSource). Skips installation if the correct major version is already present in the runner image. + +## Usage + +```yaml +- uses: tinqs/ci/setup-node@v1 +``` + +### Options + +```yaml +- uses: tinqs/ci/setup-node@v1 + with: + node-version: '22' # default: 22 + pnpm: 'true' # default: true +``` + +## Inputs + +| Input | Default | Description | +|-------|---------|-------------| +| `node-version` | `22` | Node.js major version | +| `pnpm` | `true` | Install pnpm globally | diff --git a/setup-node/action.yml b/setup-node/action.yml new file mode 100644 index 0000000..59421d9 --- /dev/null +++ b/setup-node/action.yml @@ -0,0 +1,68 @@ +# tinqs/ci/setup-node — Tinqs Studio CI +# Installs Node.js and optionally pnpm. +# Detects Alpine (apk), Debian (apt-get), and Amazon Linux/RHEL (dnf). +# Skips install if the correct major version is already present (pre-baked AMI). +# Composite action — runs directly on the host. +# Author: Ozan + Claude Code — 2026-05-22 + +name: 'Tinqs Setup Node' +description: 'Install Node.js and pnpm' + +inputs: + node-version: + description: 'Node.js major version' + default: '22' + pnpm: + description: 'Install pnpm (true/false)' + default: 'true' + +runs: + using: 'composite' + steps: + - run: | + NODE_VERSION="${{ inputs.node-version }}" + INSTALL_PNPM="${{ inputs.pnpm }}" + + # Skip if already installed at correct major version + if command -v node &>/dev/null; then + CURRENT=$(node --version | sed 's/v//' | cut -d. -f1) + if [ "$CURRENT" = "$NODE_VERSION" ]; then + echo "Node $NODE_VERSION already installed" + node --version + if [ "$INSTALL_PNPM" = "true" ]; then + if command -v pnpm &>/dev/null; then + pnpm --version + exit 0 + else + npm install -g pnpm + pnpm --version + exit 0 + fi + fi + exit 0 + fi + fi + + echo "Installing Node.js $NODE_VERSION..." + if command -v apk &>/dev/null; then + # Alpine + apk add --no-cache nodejs npm + elif command -v dnf &>/dev/null; then + # Amazon Linux / RHEL / Fedora + curl -fsSL "https://rpm.nodesource.com/setup_${NODE_VERSION}.x" | bash - + dnf install -y nodejs + elif command -v apt-get &>/dev/null; then + # Debian/Ubuntu + curl -fsSL "https://deb.nodesource.com/setup_${NODE_VERSION}.x" | bash - + apt-get install -y nodejs + else + echo "ERROR: unsupported package manager" && exit 1 + fi + + if [ "$INSTALL_PNPM" = "true" ]; then + npm install -g pnpm + pnpm --version + fi + + node --version + shell: bash