Kubernetes with Helm

# Downloads the release bundle, checks it, and moves into the AWS folder. The bundle holds
# the Helm chart at chart/blue and the OpenTofu module at infra/aws.
VERSION=0.2.0
curl -LO "https://github.com/BlocksOrg/blue/releases/download/v${VERSION}/blue-deployment-v${VERSION}.tar.gz"
curl -LO "https://github.com/BlocksOrg/blue/releases/download/v${VERSION}/SHA256SUMS"
grep "blue-deployment-v${VERSION}.tar.gz" SHA256SUMS | shasum -a 256 -c -
tar -xzf "blue-deployment-v${VERSION}.tar.gz"
cd "blue-deployment-${VERSION}/infra/aws"

# Writes the settings file. The names are relative to the hosted zone: with a zone for
# example.com, this gives app.example.com, api.example.com and iproxy.example.com.
export AWS_REGION=us-east-1   # every aws command and Tofu use this

cat > terraform.tfvars <<EOF
aws_region            = "$AWS_REGION"
name                  = "blue"
bootstrap_admin_email = "admin@example.com"   # the first admin's login
include_database      = true    # RDS Postgres
include_bucket        = true    # S3 buckets for sessions and packages
include_redis         = true    # ElastiCache; only used by the LiteLLM gateway
include_domain        = true
route53_zone_id       = "Z0123456789EXAMPLE"   # your hosted zone's id
dashboard_subdomain   = "app"
api_subdomain         = "api"
inference_proxy_subdomain = "iproxy"   # only used in gateway mode; free to reserve now
generate_gateway_jwt_key  = true       # same: the key gateway mode signs with, ready when needed
deletion_protection   = false   # lets tofu destroy remove the database; keep true for real use
EOF

# Creates the VPC, cluster, RDS, Redis, buckets, encryption key, IAM role and HTTPS
# certificate. Takes about 25 minutes.
tofu init
tofu apply

# Saves the values later steps need, so nothing is typed by hand.
export BLUE_KUBECONFIG_COMMAND="$(tofu output -raw kubeconfig_command)"
export BLUE_KUBE_NAMESPACE="$(tofu output -raw kubernetes_namespace)"
export BLUE_CERT_ARN="$(tofu output -raw certificate_arn)"
export BLUE_DASHBOARD_HOST="$(tofu output -raw dashboard_hostname)"
export BLUE_API_HOST="$(tofu output -raw api_hostname)"
export BLUE_SECRET_ARN="$(tofu output -raw runtime_secret_arn)"
export BLUE_CONFIG_FILE=./blue.eks.yaml
export BLUE_HELM_VALUES_FILE=./eks-values.yaml
export BLUE_HELM_GENERATED_FILE=./eks-generated-values.json
export BLUE_HELM_DEPLOY_FOLDER=../../chart/blue

# Saves the Helm settings Tofu generated: the IAM role, the bucket names, the network
# rules that let the load balancer, RDS and Redis talk to Blue, and "don't run Postgres
# or MinIO in the cluster".
tofu output -json helm_values > "$BLUE_HELM_GENERATED_FILE"

# Pins the image to an exact build. The chart refuses a moving tag in production.
export BLUE_IMAGE_DIGEST="$(docker buildx imagetools inspect ghcr.io/blocksorg/blue:${VERSION} | awk '/^Digest:/{print $2}')"
echo "$BLUE_IMAGE_DIGEST"   # must print sha256:...; if empty, stop here

export BLUE_DASHBOARD_HOST=blue.example.com
export BLUE_API_HOST=api.blue.example.com

# Asks AWS for one certificate covering both names, prints one CNAME per name to create at
# your DNS provider, then waits for AWS to see them.
export BLUE_CERT_ARN="$(aws acm request-certificate \
  --domain-name "$BLUE_DASHBOARD_HOST" --subject-alternative-names "$BLUE_API_HOST" \
  --validation-method DNS --query CertificateArn --output text)"
aws acm describe-certificate --certificate-arn "$BLUE_CERT_ARN" \
  --query 'Certificate.DomainValidationOptions[].ResourceRecord' --output table
aws acm wait certificate-validated --certificate-arn "$BLUE_CERT_ARN"

# Points kubectl at the new cluster.
eval "$BLUE_KUBECONFIG_COMMAND"

# Creates the namespace. It must match Tofu's, because the IAM role only trusts this one.
kubectl create namespace "$BLUE_KUBE_NAMESPACE"

# Tells the cluster how to build a public HTTPS load balancer with our certificate.
kubectl apply -f - <<YAML
apiVersion: eks.amazonaws.com/v1
kind: IngressClassParams
metadata: { name: blue-alb }
spec:
  scheme: internet-facing
  certificateARNs: ["$BLUE_CERT_ARN"]
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata: { name: blue-alb }
spec:
  controller: eks.amazonaws.com/alb
  parameters: { apiGroup: eks.amazonaws.com, kind: IngressClassParams, name: blue-alb }
YAML

# Tofu made the database URL, admin password and signing key. This copies them in.
SECRET_JSON="$(aws secretsmanager get-secret-value --secret-id "$BLUE_SECRET_ARN" --region "$AWS_REGION" --query SecretString --output text)"

kubectl -n "$BLUE_KUBE_NAMESPACE" create secret generic blue-runtime \
  --from-literal=HARNESS_DATABASE_URL="$(jq -r .HARNESS_DATABASE_URL <<<"$SECRET_JSON")" \
  --from-literal=BETTER_AUTH_SECRET="$(jq -r .BETTER_AUTH_SECRET <<<"$SECRET_JSON")" \
  --from-literal=HARNESS_BOOTSTRAP_ADMIN_EMAIL="$(jq -r .HARNESS_BOOTSTRAP_ADMIN_EMAIL <<<"$SECRET_JSON")" \
  --from-literal=HARNESS_BOOTSTRAP_ADMIN_PASSWORD="$(jq -r .HARNESS_BOOTSTRAP_ADMIN_PASSWORD <<<"$SECRET_JSON")" \
  --from-literal=HARNESS_PROXY_OAUTH_CLIENT_SECRET="$(openssl rand -hex 32)"
# The OAuth secret is only read in gateway mode; making it now saves a step later.

export ADMIN_EMAIL="$(jq -r .HARNESS_BOOTSTRAP_ADMIN_EMAIL <<<"$SECRET_JSON")"
export ADMIN_PW="$(jq -r .HARNESS_BOOTSTRAP_ADMIN_PASSWORD <<<"$SECRET_JSON")"

# Writes Blue's config with the final addresses. The image has none built in, so this is required.
cat > "$BLUE_CONFIG_FILE" <<EOF
control_api:
  listen: 0.0.0.0:8080
  database_url: os.environ/HARNESS_DATABASE_URL
  bootstrap: { organization_slug: dev, organization_name: Development, admin_subject: admin,
               admin_email: os.environ/HARNESS_BOOTSTRAP_ADMIN_EMAIL }
  auth: { public_url: "https://$BLUE_DASHBOARD_HOST", client_id: blue-cli,
          session_url: "http://blue-blue-dashboard:3000/api/auth/get-session",
          jwks_url: "http://blue-blue-dashboard:3000/api/auth/jwks",
          issuer: "https://$BLUE_DASHBOARD_HOST/api/auth", audience: "https://$BLUE_API_HOST" }
  identity: { mode: password }
  blob_storage: { provider: s3, bucket: os.environ/HARNESS_BLOB_BUCKET, region: $AWS_REGION,
                  auth: { mode: iam }, retention_days: 30 }
  package_artifacts: { bucket: os.environ/HARNESS_PACKAGE_BUCKET }
governance:
  revision: "eks-1"   # a label for this baseline; the server detects edits by content, not by this value
  contract_version: 3
  required_capabilities: [adapter_intervals, compiled_harness_registry, versioned_state, transactional_reconcile, gateway_inference_jwt]
  minimum_client_version: "0.1.0"
  ttl_seconds: 300
  required: true
  allowed_harnesses: [codex, claude, kimi, opencode]
  harnesses: { claude: { managed_config: { model: claude-opus-4-8 } } }
  # Uploads each finished coding session to the S3 bucket. Transcripts can hold prompts, code
  # and secrets, so agree a retention and access policy before turning this on.
  session_upload: { presign_url: "https://$BLUE_API_HOST/session-uploads/presign" }
EOF

# Uploads the config into the cluster so Blue's pods can read it.
kubectl -n "$BLUE_KUBE_NAMESPACE" create configmap blue-config --from-file=blue.yaml="$BLUE_CONFIG_FILE"

# Every Helm setting this guide adds on top of the generated ones.
cat > "$BLUE_HELM_VALUES_FILE" <<EOF
blue:
  production: true    # the chart now insists on RDS, S3, a pinned image, migrations and network policies
  existingSecret: blue-runtime              # made in step 3
  config:
    existingConfigMap: blue-config          # made in step 4
  publicUrls:
    dashboard: https://$BLUE_DASHBOARD_HOST
    controlApi: https://$BLUE_API_HOST

image:
  digest: $BLUE_IMAGE_DIGEST

# Tells the load balancer how to check each app is up. They use different paths, so
# this can't be one setting on the load balancer.
service:
  dashboard:
    annotations:
      alb.ingress.kubernetes.io/healthcheck-path: /api/health
  controlApi:
    annotations:
      alb.ingress.kubernetes.io/healthcheck-path: /ready

# One HTTPS load balancer: the dashboard name goes to the dashboard, the API name goes to
# the API, and plain HTTP redirects to HTTPS.
ingress:
  enabled: true
  className: blue-alb
  annotations:
    alb.ingress.kubernetes.io/target-type: ip
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]'
    alb.ingress.kubernetes.io/ssl-redirect: "443"
  hosts:
    - host: $BLUE_DASHBOARD_HOST
      paths: [{ path: /, pathType: Prefix, port: dashboard }]
    - host: $BLUE_API_HOST
      paths: [{ path: /, pathType: Prefix, port: control-api }]
EOF

# Runs the database migration, installs Blue and creates the load balancer. Waits until up.
helm upgrade --install blue "$BLUE_HELM_DEPLOY_FOLDER" --namespace "$BLUE_KUBE_NAMESPACE" \
  -f "$BLUE_HELM_GENERATED_FILE" -f "$BLUE_HELM_VALUES_FILE" --wait --timeout 15m

# Waits until the load balancer has an address (a few minutes).
until ALB=$(kubectl -n "$BLUE_KUBE_NAMESPACE" get ingress blue-blue -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') && [ -n "$ALB" ]; do
  sleep 10
done
echo "$ALB"

# Second Tofu run: creates the DNS records, pointing at the load balancer.
# Saves the address to the settings file so later tofu runs keep the records.
echo "alb_hostname = \"$ALB\"" >> terraform.tfvars
tofu apply

# Checks the API is up over HTTPS and can reach S3. New DNS records can take a minute.
curl -fsS "https://$BLUE_API_HOST/ready"
curl -fsS "https://$BLUE_API_HOST/health/object-storage"

# Opens the dashboard. Log in with the email and password this prints.
echo "$ADMIN_EMAIL / $ADMIN_PW"
open "https://$BLUE_DASHBOARD_HOST"

# Points your blue CLI to the deployed server. Enter https://$BLUE_API_HOST when asked.
blue setup
blue login && blue doctor

# Deletes Blue first (this removes the load balancer), then everything in AWS including
# the database and its data. Don't skip the second one.
helm uninstall blue -n "$BLUE_KUBE_NAMESPACE"
tofu destroy

kubectl -n "$BLUE_KUBE_NAMESPACE" create configmap blue-config --from-file=blue.yaml="$BLUE_CONFIG_FILE" \
  --dry-run=client -o yaml | kubectl apply -f -
kubectl -n "$BLUE_KUBE_NAMESPACE" rollout restart deployments

export BLUE_PROXY_HOST="$(tofu output -raw inference_proxy_hostname)"
export LITELLM_URL=https://litellm.example.com   # your LiteLLM, reachable from the cluster
export LITELLM_MASTER_KEY=sk-...                  # its master key

# Blue only hands out keys to people LiteLLM already knows, matched by email. Adds the admin;
# repeat for each developer.
curl -fsS -X POST "$LITELLM_URL/user/new" -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
  -H 'Content-Type: application/json' -d "{\"user_email\":\"$ADMIN_EMAIL\"}"

# Refreshes the generated Helm settings: they now name the signing-key secret.
tofu output -json helm_values > "$BLUE_HELM_GENERATED_FILE"

# Adds LiteLLM's address and master key. The proxy reads the address from here, not from the
# config file. The encryption key is ignored with KMS, but the chart refuses to start without it.
kubectl -n "$BLUE_KUBE_NAMESPACE" patch secret blue-runtime --type merge -p "{\"stringData\":{
  \"HARNESS_GATEWAY_URL\":\"$LITELLM_URL\",
  \"HARNESS_LITELLM_ADMIN_KEY\":\"$LITELLM_MASTER_KEY\",
  \"HARNESS_GATEWAY_ENCRYPTION_KEY\":\"unused-with-aws-kms\"}}"

# Copies the token-signing key Tofu made into the cluster. Only the API reads it.
JWT_JSON="$(aws secretsmanager get-secret-value --secret-id "$(tofu output -raw gateway_jwt_secret_arn)" --query SecretString --output text)"
kubectl -n "$BLUE_KUBE_NAMESPACE" create secret generic "$(jq -r .blue.inferenceJwt.secret "$BLUE_HELM_GENERATED_FILE")" \
  --from-literal=signing-key.pem="$(jq -r '."signing-key.pem"' <<<"$JWT_JSON")"

# One per cluster; skip if it is already there.
helm upgrade --install cert-manager oci://quay.io/jetstack/charts/cert-manager --version v1.18.2 \
  --namespace cert-manager --create-namespace --set crds.enabled=true --wait

# One new top-level section: where LiteLLM is, where the proxy is, how tokens are signed,
# and how stored gateway keys are encrypted (with the KMS key Tofu made).
cat >> "$BLUE_CONFIG_FILE" <<EOF
gateway:
  type: litellm
  url: $LITELLM_URL
  inference_proxy_url: https://$BLUE_PROXY_HOST
  inference_proxy_health_url: http://blue-blue-inference-proxy:8081/health
  internal_allowed_client_id: blue-inference-proxy
  inference_jwt:
    issuer: https://$BLUE_API_HOST
    audience: blue-inference-proxy
    private_key_file: /var/run/blue/gateway-jwt/signing-key.pem
  secret_encryption: { provider: aws-kms, key_id: "$(tofu output -raw kms_key_arn)" }
  provisioner: { type: builtin-litellm, reconcile_ttl_seconds: 86400 }
EOF

# Uploads the config.
kubectl -n "$BLUE_KUBE_NAMESPACE" create configmap blue-config --from-file=blue.yaml="$BLUE_CONFIG_FILE" \
  --dry-run=client -o yaml | kubectl apply -f -

# A second Helm settings file, layered on top of the first: the proxy, its public name, and
# the encrypted link to the API with its certificates issued by cert-manager.
export BLUE_HELM_GATEWAY_VALUES_FILE=./eks-gateway-values.yaml
cat > "$BLUE_HELM_GATEWAY_VALUES_FILE" <<EOF
blue:
  enableInferenceProxy: true
  gatewayType: litellm
  publicUrls:
    inferenceProxy: https://$BLUE_PROXY_HOST
  internalTransport:
    mode: mtls
    certManager: { enabled: true }   # a private authority and both certificates, renewed every 90 days
service:
  inferenceProxy:
    annotations:
      alb.ingress.kubernetes.io/healthcheck-path: /health
ingress:
  proxyHost: $BLUE_PROXY_HOST
EOF

# Upgrades Blue with the proxy, then restarts everything so all pods read the new config.
helm upgrade --install blue "$BLUE_HELM_DEPLOY_FOLDER" --namespace "$BLUE_KUBE_NAMESPACE" \
  -f "$BLUE_HELM_GENERATED_FILE" -f "$BLUE_HELM_VALUES_FILE" -f "$BLUE_HELM_GATEWAY_VALUES_FILE" \
  --wait --timeout 15m
kubectl -n "$BLUE_KUBE_NAMESPACE" rollout restart deployments

networkPolicy:
  extraEgress:
    - to: [{ ipBlock: { cidr: 10.0.0.0/8 } }]   # or a namespaceSelector for an in-cluster one
      ports: [{ protocol: TCP, port: 4000 }]

# The proxy answers on its public name.
curl -fsS "https://$BLUE_PROXY_HOST/health"

# Creates your LiteLLM key through Blue, then runs an agent whose model calls now go
# laptop -> proxy -> LiteLLM -> provider. The dashboard's Gateway page shows the key as ready.
blue login && blue gateway
blue claude