Running capacity checks for Splunk AI Tier on AWS to quickly find elusive GPU resources

1.1. Check your existing quota in your target region(s)

aws service-quotas get-service-quota \
  --service-code ec2 \
  --quota-code L-DB2E81BA \
  --region <region>

Look at the Value field in the response. If you plan to launch a g6e.12xlarge (48 vCPUs) or g7e.12xlarge (48 vCPUs), your quota needs to comfortably cover that number.

1.2. Request a quota increase (if needed)

If your quota is `0` or too low, request an increase via CLI:

aws service-quotas request-service-quota-increase \
  --service-code ec2 \
  --quota-code L-DB2E81BA \
  --desired-value 192 \
  --region <region>

Prerequisites and permissions

• Tools: AWS CLI v2, standard POSIX utilities (column, sed, mktemp).
• IAM Permissions: The IAM identity running the script needs:
  • ec2:DescribeRegions
  • ec2:DescribeInstanceTypeOfferings
  • ec2:CreateCapacityReservation
  • ec2:CancelCapacityReservation

The script: odcr-probe.sh

Save this script locally, make it executable (chmod +x odcr-probe.sh), and run it before deploying your AI Tier.

#!/usr/bin/env bash
# odcr-probe.sh
#
# Probe on-demand capacity by creating a targeted ODCR in each AZ where
# an instance type is offered, then cancelling immediately on success.
#
# A successful create means AWS had capacity at that moment. Cancelling
# releases it; this is a probe, not a hold. Interrupt (Ctrl-C) still
# attempts to cancel any in-flight reservation.
#
# Compatible with macOS Bash 3.2 (no associative arrays).
#
# Usage:
#   ./odcr-probe.sh --type g7e.12xlarge
#   ./odcr-probe.sh --type g7e.12xlarge --count 2
#   ./odcr-probe.sh --type g6e.12xlarge --regions us-east-1,eu-central-1,us-west-2
#   ./odcr-probe.sh g6e.12xlarge --profile my-aws-profile

set -euo pipefail

INSTANCE_TYPE=""
REGIONS_ARG=""
COUNT=1
PLATFORM="Linux/UNIX"
CLI_PROFILE=""
ENDPOINT_REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}"

CURRENT_CR=""
CURRENT_CR_REGION=""
ERRFILE=""
ROWSFILE=""

usage() {
  sed -n '2,18p' "$0"
  exit "${1:-0}"
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --type|--instance-type)
      INSTANCE_TYPE="$2"
      shift 2
      ;;
    --region|--regions)
      if [[ -n "${REGIONS_ARG}" ]]; then
        REGIONS_ARG="${REGIONS_ARG} $(echo "$2" | tr ',' ' ')"
      else
        REGIONS_ARG="$(echo "$2" | tr ',' ' ')"
      fi
      shift 2
      ;;
    --count)
      COUNT="$2"
      shift 2
      ;;
    --platform)
      PLATFORM="$2"
      shift 2
      ;;
    --profile)
      CLI_PROFILE="$2"
      shift 2
      ;;
    --endpoint-region)
      ENDPOINT_REGION="$2"
      shift 2
      ;;
    -h|--help) usage 0 ;;
    -*)
      echo "unknown arg: $1" >&2
      usage 1
      ;;
    *)
      if [[ -z "${INSTANCE_TYPE}" ]]; then
        INSTANCE_TYPE="$1"
        shift
      else
        echo "unexpected arg: $1" >&2
        usage 1
      fi
      ;;
  esac
done

need() { command -v "$1" >/dev/null 2>&1 || { echo "missing: $1" >&2; exit 1; }; }
need aws
need column

if [[ -z "${INSTANCE_TYPE}" ]]; then
  echo "instance type required (positional or --type)" >&2
  usage 1
fi

aws_ec2() {
  if [[ -n "${CLI_PROFILE}" ]]; then
    aws --profile "${CLI_PROFILE}" ec2 "$@"
  else
    aws ec2 "$@"
  fi
}

aws_error_code() {
  sed -n 's/.*An error occurred (\([^)]*\)).*/\1/p' "$1" | head -1
}

cleanup() {
  if [[ -n "${CURRENT_CR}" && -n "${CURRENT_CR_REGION}" ]]; then
    echo "Cleaning up leftover reservation ${CURRENT_CR} in ${CURRENT_CR_REGION}" >&2
    aws_ec2 cancel-capacity-reservation \
      --capacity-reservation-id "${CURRENT_CR}" \
      --region "${CURRENT_CR_REGION}" >/dev/null 2>&1 || \
      echo "WARN: cancel failed — release ${CURRENT_CR} in ${CURRENT_CR_REGION} manually" >&2
  fi
  rm -f "${ERRFILE}" "${ROWSFILE}"
}
trap cleanup EXIT INT TERM

ERRFILE="$(mktemp -t odcr-probe-err.XXXXXX)"
ROWSFILE="$(mktemp -t odcr-probe-rows.XXXXXX)"

if [[ -n "${REGIONS_ARG}" ]]; then
  REGIONS="${REGIONS_ARG}"
else
  REGIONS="$(aws_ec2 describe-regions --region "${ENDPOINT_REGION}" \
              --query 'Regions[].RegionName' --output text | tr '\t' '
' | sort)"
fi

echo "Instance type:  ${INSTANCE_TYPE}" >&2
echo "Count:          ${COUNT}" >&2
echo "Platform:       ${PLATFORM}" >&2
echo "Match criteria: targeted" >&2
echo >&2

echo "==> DescribeInstanceTypeOfferings" >&2
AZ_LIST=""
az_count=0
for r in ${REGIONS}; do
  azs="$(aws_ec2 describe-instance-type-offerings \
          --region "$r" \
          --location-type availability-zone \
          --filters "Name=instance-type,Values=${INSTANCE_TYPE}" \
          --query 'InstanceTypeOfferings[].Location' \
          --output text 2>/dev/null || true)"
  azs="$(echo "${azs}" | tr '\t' ' ' | sed 's/None//g; s/^[[:space:]]*//; s/[[:space:]]*$//')"
  if [[ -n "$azs" ]]; then
    echo "  ${r}: ${azs}" >&2
    for az in ${azs}; do
      AZ_LIST="${AZ_LIST}${r} ${az}"$'
'
      az_count=$((az_count + 1))
    done
  fi
done

if [[ "${az_count}" -eq 0 ]]; then
  echo "No offerings found for ${INSTANCE_TYPE} in the selected region(s)." >&2
  exit 1
fi

echo >&2
echo "Probing ${az_count} AZ(s). Each successful reservation is cancelled immediately." >&2
echo >&2

ok=0
fail=0
cancel_fail=0

while read -r r az; do
  [[ -n "${r:-}" && -n "${az:-}" ]] || continue
  : > "${ERRFILE}"
  CURRENT_CR=""
  CURRENT_CR_REGION=""

  set +e
  cr_id="$(aws_ec2 create-capacity-reservation \
            --instance-type "${INSTANCE_TYPE}" \
            --instance-platform "${PLATFORM}" \
            --availability-zone "${az}" \
            --instance-count "${COUNT}" \
            --instance-match-criteria targeted \
            --end-date-type unlimited \
            --region "${r}" \
            --query 'CapacityReservation.CapacityReservationId' \
            --output text 2>"${ERRFILE}")"
  rc=$?
  set -e

  if [[ "${rc}" -eq 0 && -n "${cr_id}" && "${cr_id}" != "None" ]]; then
    CURRENT_CR="${cr_id}"
    CURRENT_CR_REGION="${r}"
    set +e
    aws_ec2 cancel-capacity-reservation \
      --capacity-reservation-id "${cr_id}" \
      --region "${r}" >/dev/null 2>"${ERRFILE}"
    cancel_rc=$?
    set -e
    if [[ "${cancel_rc}" -eq 0 ]]; then
      CURRENT_CR=""
      CURRENT_CR_REGION=""
      printf '%s\t%s\t%s\t%s\t%s\t%s
' "$r" "$az" ok cancelled "${cr_id}" "-" >> "${ROWSFILE}"
      echo "  OK   ${r} ${az}  ${cr_id} (cancelled)" >&2
      ok=$((ok + 1))
    else
      code="$(aws_error_code "${ERRFILE}")"
      printf '%s\t%s\t%s\t%s\t%s\t%s
' "$r" "$az" ok "cancel-failed:${code:-unknown}" "${cr_id}" "cancel ${cr_id} in ${r}" >> "${ROWSFILE}"
      echo "  WARN ${r} ${az}  reserved ${cr_id} but cancel failed — stopping" >&2
      cancel_fail=$((cancel_fail + 1))
      ok=$((ok + 1))
      break
    fi
  else
    code="$(aws_error_code "${ERRFILE}")"
    [[ -n "${code}" ]] || code="error"
    printf '%s\t%s\t%s\t%s\t%s\t%s
' "$r" "$az" fail "${code}" "-" "-" >> "${ROWSFILE}"
    echo "  FAIL ${r} ${az}  ${code}" >&2
    fail=$((fail + 1))
  fi
done <<EOF
${AZ_LIST}
EOF

echo
echo "=== ODCR probe ${INSTANCE_TYPE} x${COUNT} ==="
{
  printf '%s\t%s\t%s\t%s\t%s\t%s
' REGION AZ RESULT DETAIL RESERVATION ACTION
  cat "${ROWSFILE}"
} | column -t -s "$(printf '\t')"

echo
echo "ok=${ok}  fail=${fail}  cancel_failed=${cancel_fail}"
if [[ "${cancel_fail}" -gt 0 ]]; then
  echo "Cancel failed for one or more reservations — see ACTION column and release them now." >&2
  exit 2
fi

Example output table

=== ODCR probe g6e.12xlarge x1 ===
REGION        AZ             RESULT  DETAIL                        RESERVATION   ACTION
us-east-1     us-east-1a     fail    InsufficientInstanceCapacity  -             -
us-east-1     us-east-1b     fail    InsufficientInstanceCapacity  -             -
us-east-1     us-east-1c     ok      cancelled                     cr-01234abc   -
us-west-2     us-west-2a     fail    VcpuLimitExceeded             -             -
eu-central-1  eu-central-1a  ok      cancelled                     cr-05678def   -

ok=2  fail=3  cancel_failed=0

How to interpret the findings