#!/usr/bin/env bash # deploy-main.sh - day-2 deploy of continuum `main` to the prod GCP VM. # # Ships the app source from your LOCAL clone over the IAP SSH tunnel and rebuilds # ONLY api/worker/ui. It never touches Caddy or the database, so it is safe to run # repeatedly. Both maintainers can use it: log-in user on the VM is shared (`nerd`) # and the local clone path is auto-detected (or set CONTINUUM_REPO). # # Usage: # ./deploy-main.sh deploy origin/main (app code only) # ./deploy-main.sh --with-base also ship the fasten submodule + rebuild the # continuum/fasten-core base image (needed when # the fasten pin changed) # ./deploy-main.sh --ref deploy a specific branch/tag/sha instead of main # ./deploy-main.sh --yes skip the confirmation prompt (scripted runs) # ./deploy-main.sh --dry-run print the plan and exit, change nothing # ./deploy-main.sh -h this help # # Config (env overrides shown with their defaults): # CONTINUUM_REPO auto-detected path to your local continuum clone # VM_INSTANCE continuum-app # VM_ZONE us-west1-a # GCP_PROJECT continuumstate # VM_SSH_USER nerd both maintainers log in as this user # VM_REMOTE_DIR ~/continuum the git-less deploy snapshot on the VM set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" VM_INSTANCE="${VM_INSTANCE:-continuum-app}" VM_ZONE="${VM_ZONE:-us-west1-a}" GCP_PROJECT="${GCP_PROJECT:-continuumstate}" VM_SSH_USER="${VM_SSH_USER:-nerd}" REMOTE_DIR="${VM_REMOTE_DIR:-}"; [ -n "$REMOTE_DIR" ] || REMOTE_DIR='~/continuum' REF="main" WITH_BASE=0 ASSUME_YES=0 DRY_RUN=0 # -- pretty logging ---------------------------------------------------------- if [ -t 1 ]; then B=$'\033[1m'; G=$'\033[32m'; Y=$'\033[33m'; R=$'\033[31m'; C=$'\033[36m'; Z=$'\033[0m'; else B=; G=; Y=; R=; C=; Z=; fi say() { printf '%s==>%s %s\n' "$C" "$Z" "$*"; } ok() { printf '%s ok%s %s\n' "$G" "$Z" "$*"; } warn() { printf '%swarn%s %s\n' "$Y" "$Z" "$*" >&2; } die() { printf '%sfail%s %s\n' "$R" "$Z" "$*" >&2; exit 1; } usage() { sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } # -- args -------------------------------------------------------------------- while [ $# -gt 0 ]; do case "$1" in --with-base) WITH_BASE=1 ;; --ref) REF="${2:?--ref needs a value}"; shift ;; --ref=*) REF="${1#*=}" ;; --yes|-y) ASSUME_YES=1 ;; --dry-run|-n) DRY_RUN=1 ;; -h|--help) usage 0 ;; *) die "unknown arg: $1 (try -h)" ;; esac shift done # -- resolve the local clone ------------------------------------------------- is_continuum_repo() { local d="$1" [ -n "$d" ] || return 1 [ -e "$d/.git" ] && [ -d "$d/src" ] && [ -f "$d/.gitmodules" ] \ && grep -q 'submodule "fasten"' "$d/.gitmodules" 2>/dev/null } resolve_repo() { local c for c in "${CONTINUUM_REPO:-}" \ "$SCRIPT_DIR/../continuum" \ "$HOME/continuum_workspace/continuum" \ "$HOME/continuum" \ "$HOME/src/continuum" \ "$HOME/code/continuum" \ "$PWD"; do if is_continuum_repo "$c"; then ( cd "$c" && pwd ); return 0; fi done if c="$(git -C "$PWD" rev-parse --show-toplevel 2>/dev/null)" && is_continuum_repo "$c"; then echo "$c"; return 0 fi return 1 } command -v gcloud >/dev/null 2>&1 || die "gcloud not found on this machine." command -v git >/dev/null 2>&1 || die "git not found on this machine." REPO="$(resolve_repo)" || die "Could not locate your local continuum clone. Set CONTINUUM_REPO=/path/to/continuum and re-run." cd "$REPO" say "local clone: ${B}${REPO}${Z}" # -- resolve the target commit ---------------------------------------------- say "fetching origin/${REF} ..." git fetch --quiet origin "$REF" 2>/dev/null || warn "git fetch origin ${REF} failed - using whatever is local" TARGET="$(git rev-parse --verify "origin/${REF}^{commit}" 2>/dev/null \ || git rev-parse --verify "${REF}^{commit}" 2>/dev/null)" \ || die "cannot resolve ref '${REF}' (not on origin or locally)." TARGET_LINE="$(git log -1 --format='%h %s' "$TARGET")" FASTEN_SHA="$(git rev-parse "${TARGET}:fasten" 2>/dev/null || true)" # -- compare against the last deploy ----------------------------------------- # The deployed-prod tag wins: it is shared, so it also reflects deploys made from # CI or from the other maintainer's laptop. The local state file only knows about # deploys made from THIS machine, so preferring it would show a stale baseline the # moment anyone else ships - which is exactly the drift this record exists to # prevent. The tag's tree carries the fasten gitlink, so nothing is lost. The # local file remains the offline fallback. STATE="$SCRIPT_DIR/.last-deploy.state" PREV_MAIN=""; PREV_FASTEN="" git fetch --quiet origin 'refs/tags/deployed-prod:refs/tags/deployed-prod' --force 2>/dev/null || true PREV_MAIN="$(git rev-parse -q --verify 'refs/tags/deployed-prod^{commit}' 2>/dev/null || true)" if [ -n "$PREV_MAIN" ]; then PREV_FASTEN="$(git rev-parse -q --verify 'refs/tags/deployed-prod:fasten' 2>/dev/null || true)" elif [ -f "$STATE" ]; then say "no deployed-prod tag reachable - falling back to this machine's local state" read -r PREV_MAIN PREV_FASTEN < "$STATE" || true fi echo say "deploy plan" printf ' target ref : %s\n' "$REF" printf ' target commit : %s\n' "$TARGET_LINE" printf ' fasten pin : %s\n' "${FASTEN_SHA:-}" printf ' vm : %s@%s (%s, %s) via IAP\n' "$VM_SSH_USER" "$VM_INSTANCE" "$VM_ZONE" "$GCP_PROJECT" printf ' remote dir : %s\n' "$REMOTE_DIR" printf ' rebuild services: api worker ui (caddy + db untouched)\n' printf ' rebuild base : %s\n' "$([ $WITH_BASE -eq 1 ] && echo 'yes (fasten-core)' || echo 'no')" if [ -n "$PREV_MAIN" ] && git cat-file -e "${PREV_MAIN}^{commit}" 2>/dev/null; then echo; say "commits since last deploy ($(git rev-parse --short "$PREV_MAIN")):" git log --oneline --no-decorate "${PREV_MAIN}..${TARGET}" | sed 's/^/ /' || true fi if [ -n "$PREV_FASTEN" ] && [ -n "$FASTEN_SHA" ] && [ "$PREV_FASTEN" != "$FASTEN_SHA" ] && [ $WITH_BASE -eq 0 ]; then echo warn "fasten pin changed since last deploy (${PREV_FASTEN:0:12} -> ${FASTEN_SHA:0:12})." warn "The fasten-core base image is stale. Re-run with ${B}--with-base${Z} or the new native core won't ship." fi if [ $DRY_RUN -eq 1 ]; then echo; ok "dry run - nothing changed."; exit 0; fi # -- confirm ----------------------------------------------------------------- if [ $ASSUME_YES -eq 0 ]; then if [ -t 0 ]; then echo; printf 'Proceed with deploy? [y/N] ' read -r reply; case "$reply" in y|Y|yes) ;; *) die "aborted." ;; esac else die "no TTY for confirmation - pass --yes to run non-interactively." fi fi # -- remote helper ----------------------------------------------------------- ssh_vm() { gcloud compute ssh "${VM_SSH_USER}@${VM_INSTANCE}" \ --zone="$VM_ZONE" --project="$GCP_PROJECT" --tunnel-through-iap --quiet \ --command="$1" } # -- 1. ship the app source (preserving the live deploy/local-prod overlay) -- echo; say "shipping app source (${TARGET:0:12}) to the VM ..." git archive --format=tar "$TARGET" \ | ssh_vm "set -e; cd $REMOTE_DIR && tar -x --no-same-owner --exclude='deploy/local-prod' -f -" ok "source extracted (deploy/local-prod left untouched)." # -- 2. optionally ship fasten + rebuild the base image ---------------------- if [ $WITH_BASE -eq 1 ]; then [ -n "$FASTEN_SHA" ] || die "--with-base requested but the target has no fasten submodule pin." say "preparing fasten submodule @ ${FASTEN_SHA:0:12} ..." git submodule update --init fasten >/dev/null 2>&1 || true git -C fasten fetch --quiet --all 2>/dev/null || true git -C fasten cat-file -e "${FASTEN_SHA}^{commit}" 2>/dev/null \ || die "fasten commit ${FASTEN_SHA} not in local submodule. Run: git -C \"$REPO/fasten\" fetch --all" say "shipping fasten source ..." git -C fasten archive --format=tar --prefix=fasten/ "$FASTEN_SHA" \ | ssh_vm "set -e; cd $REMOTE_DIR && tar -x --no-same-owner -f -" say "rebuilding continuum/fasten-core:latest on the VM (compiles in-container) ..." ssh_vm "set -e; cd $REMOTE_DIR && docker build -t continuum/fasten-core:latest -f Dockerfile.fasten-core ." ok "base image rebuilt." fi # -- 3. rebuild + recreate api/worker/ui (never caddy, never db) ------------- echo; say "rebuilding + recreating api worker ui ..." ssh_vm "set -e; cd $REMOTE_DIR/deploy/local-prod && ./prod-compose.sh build api worker ui" # Pre-flight BEFORE recreation: Settings() refuses env the new image considers # invalid (e.g. a stale interim PILOT_* block diverging from pilot_days); the # SELECT 1 catches a bad DATABASE_URL; the bootstrap probe resolves the DSN # exactly as db_init does (DATABASE_BOOTSTRAP_URL or DATABASE_URL) and then # probes the CAPABILITIES db_init needs - CREATE ROLE *and* CREATE EXTENSION # (timescaledb/vector are untrusted, so they need more than CREATEROLE) - # inside a rolled-back transaction, since both DDLs are transactional and # nothing should persist. Mere connectivity is a false pass (the unprivileged # app DSN connects fine and crash-loops db_init after recreation), and a # literal rolsuper check would false-fail a managed-Postgres admin role that # CAN run the DDL. IF NOT EXISTS makes the extension probe a no-op on the # live prod DB; it bites on a restored or rebuilt volume, which is exactly # where db_init would otherwise die. Any non-privilege error leaves the probe # inconclusive rather than blocking a healthy deploy. # The db container is never recreated by this script, so it is up to answer. say "pre-flight: validating the VM env against the new image ..." ssh_vm "cd $REMOTE_DIR/deploy/local-prod && ./prod-compose.sh run --rm --no-deps api python -c 'exec(\"import asyncio, os\nfrom sqlalchemy import text\nfrom api.config import get_settings\nget_settings()\nimport psycopg\ndsn = os.environ.get(\\\"DATABASE_BOOTSTRAP_URL\\\") or os.environ.get(\\\"DATABASE_URL\\\") or \\\"\\\"\nif not dsn:\n raise SystemExit(\\\"neither DATABASE_BOOTSTRAP_URL nor DATABASE_URL is set\\\")\nwith psycopg.connect(dsn.replace(\\\"postgresql+psycopg://\\\", \\\"postgresql://\\\", 1), connect_timeout=10) as c:\n try:\n c.execute(\\\"BEGIN\\\"); c.execute(\\\"CREATE ROLE __continuum_preflight_probe__\\\"); c.execute(\\\"CREATE EXTENSION IF NOT EXISTS vector\\\"); c.execute(\\\"CREATE EXTENSION IF NOT EXISTS timescaledb\\\"); c.execute(\\\"ROLLBACK\\\")\n except psycopg.errors.InsufficientPrivilege:\n raise SystemExit(\\\"bootstrap DSN role (\\\" + c.info.user + \\\") lacks the privileges db_init needs (CREATE ROLE / CREATE EXTENSION) - it would crash-loop after recreation; set DATABASE_BOOTSTRAP_URL\\\")\n except Exception as probe_exc:\n try:\n c.rollback()\n except Exception:\n pass\n print(\\\"pre-flight: privilege probe inconclusive (\\\" + type(probe_exc).__name__ + \\\"), continuing\\\")\nfrom api.database import shared_session\nasync def m():\n async with shared_session() as db:\n await db.execute(text(\\\"SELECT 1\\\"))\nasyncio.run(m())\")'" \ || die "env pre-flight failed - the new image refuses .env.prod-local (fix it, e.g. remove the interim PILOT_* block), DATABASE_URL cannot connect, or the bootstrap DSN lacks superuser (db_init would crash-loop); containers were NOT recreated." ssh_vm "set -e; cd $REMOTE_DIR/deploy/local-prod && ./prod-compose.sh up -d api worker ui" ok "services recreated." # -- 4. verify --------------------------------------------------------------- echo; say "status:" ssh_vm "cd $REMOTE_DIR/deploy/local-prod && ./prod-compose.sh ps" || true # The API's /health is NOT reachable through the public edge: ui/nginx.conf only # forwards /api/, /webhooks|waitlist and the auth callbacks, so a request to # app.continuumstate.io/health falls through to `location /` and returns the SPA's # index.html with 200 - green even when the API is dead. (The old check used # /api/v1/health, which reaches the API but isn't the route: /health is mounted # without api_prefix, so JWT middleware 401s it.) Probe the api container directly # on the VM, then confirm the edge still serves the app. Both are fatal: an # unattended deploy must not report success over a broken service. echo; say "health check:" HEALTH="$(ssh_vm "curl -fsS -m 15 http://localhost:8000/health")" \ || die "API health check failed - inspect: ./prod-compose.sh logs -f api" printf ' %s\n' "$HEALTH" case "$HEALTH" in *'"status":"ok"'*) ok "API healthy." ;; *) die "API did not report status=ok - inspect: ./prod-compose.sh logs -f api" ;; esac case "$HEALTH" in *'"worker_ok":true'*) ok "worker heartbeat fresh." ;; # Not fatal: the worker writes its heartbeat on a */5 cron, so a deploy that # restarted it can legitimately report false for a few minutes. *) warn "worker_ok=false - expected briefly after a restart; if it persists: ./prod-compose.sh logs -f worker" ;; esac ssh_vm "curl -fsS -m 15 -o /dev/null -w 'edge %{http_code}\n' https://app.continuumstate.io/" \ || die "public edge did not serve the app - inspect: ./prod-compose.sh logs -f caddy ui" ok "edge serving." # -- record for the next run ------------------------------------------------- printf '%s %s\n' "$TARGET" "${FASTEN_SHA:-}" > "$STATE" # Same record the workflow writes, so both deploy paths report the same truth. # Without this a laptop deploy leaves the tag pointing at an older commit, and # `git log -1 deployed-prod` claims something is live that isn't - which the CI # fasten-pin guard also reads, so a stale tag makes it compare the wrong base. if git push -q -f origin "${TARGET}:refs/tags/deployed-prod" 2>/dev/null; then ok "tagged deployed-prod -> ${TARGET:0:12}" else warn "could not push the deployed-prod tag (no network or no push rights)." warn "The deploy itself succeeded; re-run: git push -f origin ${TARGET:0:12}:refs/tags/deployed-prod" fi echo; ok "deployed ${TARGET_LINE}"