apiVersion: v1
kind: ConfigMap
metadata:
  name: burnin-telemetry
  namespace: kube-amd-gpu
data:
  telemetry_gate.sh: |
    #!/usr/bin/env bash
    set -euo pipefail
    WATCH_SECONDS="${WATCH_SECONDS:-86400}"
    SAMPLE_INTERVAL="${SAMPLE_INTERVAL:-10}"
    POWER_MAX_W="${POWER_MAX_W:-750}"
    POWER_TOL_PCT="${POWER_TOL_PCT:-15}"
    WARMUP="${WARMUP:-600}"
    REQUIRE_CAP_COMPLIANCE_PCT="${REQUIRE_CAP_COMPLIANCE_PCT:-99.9}"

    mkdir -p /gate /telemetry
    : > /gate/summary.txt
    echo "[INFO] Telemetry starting. cap=${POWER_MAX_W}W tol=${POWER_TOL_PCT}% warmup=${WARMUP}s" | tee -a /gate/summary.txt

    # AMD GPU Operator lays down ROCm under /opt/rocm on the host; mount is provided via hostPath in the Job
    export PATH="/opt/rocm:/opt/rocm/bin:${PATH}"

    if ! command -v amd-smi >/dev/null 2>&1; then
      echo "[ERROR] amd-smi not found even after mounting /opt/rocm" | tee -a /gate/summary.txt
      echo "amd-smi missing" > /gate/violation
      exit 2
    fi

    end=$((SECONDS + WATCH_SECONDS))
    samples=0
    sum_power=0
    max_power=0
    cap=$(awk -v p="${POWER_MAX_W}" -v t="${POWER_TOL_PCT}" 'BEGIN{printf "%.2f", p*(1+t/100.0)}')

    violation () {
      local msg="$1"
      echo "[VIOLATION] ${msg}" | tee -a /gate/summary.txt
      echo "${msg}" > /gate/violation
      exit 2
    }

    # Baseline UE before warmup
    BASE_UE=0
    RAS_BASE="$(amd-smi ras 2>&1 || true)"
    if echo "${RAS_BASE}" | grep -Ei 'UE[^0-9]*[1-9]' >/dev/null 2>&1; then
      BASE_UE="$(echo "${RAS_BASE}" | grep -Ei 'UE' | grep -Eo '[0-9]+' | tail -1 || echo 0)"
    fi
    echo "[INFO] Baseline UE=${BASE_UE}" | tee -a /gate/summary.txt

    while [ "${SECONDS}" -lt "${end}" ]; do
      ts="$(date --iso-8601=seconds)"
      METRIC="$(amd-smi metric 2>&1 || true)"
      RAS="$(amd-smi ras 2>&1 || true)"

      {
        echo "=== ${ts} METRIC ==="
        echo "${METRIC}"
      } >> /telemetry/metric.log

      {
        echo "=== ${ts} RAS ==="
        echo "${RAS}"
      } >> /telemetry/ras.log

      if echo "${METRIC}" | grep -Eiq 'thrtl|throttle|thermal.*thrott'; then
        violation "Thermal throttling detected at ${ts}"
      fi

      have_power=0
      while read -r p; do
        have_power=1
        sum_power=$(awk -v a="${sum_power}" -v b="${p}" 'BEGIN{printf "%.6f", a+b}')
        samples=$((samples+1))
        gt=$(awk -v x="${p}" -v m="${max_power}" 'BEGIN{if (x>m) print 1; else print 0}')
        [ "${gt}" = "1" ] && max_power="${p}"
        over=$(awk -v x="${p}" -v c="${cap}" 'BEGIN{if (x>c) print 1; else print 0}')
        if [ "${over}" = "1" ]; then
          violation "Power ${p}W exceeds allowed cap ${cap}W at ${ts}"
        fi
      done < <(echo "${METRIC}" | grep -Ei 'power' | grep -Eo '[0-9]+(\.[0-9]+)?W' | sed 's/W//')

      if [ "${have_power}" -eq 0 ] && [ "${SECONDS}" -gt $((end - WATCH_SECONDS + WARMUP)) ]; then
        violation "No power samples found at ${ts}"
      fi

      CUR_UE="${BASE_UE}"
      if echo "${RAS}" | grep -Ei 'UE' >/dev/null 2>&1; then
        CUR_UE="$(echo "${RAS}" | grep -Ei 'UE' | grep -Eo '[0-9]+' | tail -1 || echo "${BASE_UE}")"
      fi
      if [ "${CUR_UE}" -gt "${BASE_UE}" ]; then
        violation "UE increased from ${BASE_UE} to ${CUR_UE} at ${ts}"
      fi

      sleep "${SAMPLE_INTERVAL}"
    done

    mean=0
    if [ "${samples}" -gt 0 ]; then
      mean=$(awk -v s="${sum_power}" -v n="${samples}" 'BEGIN{printf "%.2f", s/n}')
    fi

    echo "[OK] No telemetry violations for ${WATCH_SECONDS}s" | tee -a /gate/summary.txt
    echo "[STATS] mean_power_w=${mean} max_power_w=${max_power} samples=${samples}" | tee -a /gate/summary.txt
    touch /gate/telemetry_ok
    exit 0



#apiVersion: v1
#kind: ConfigMap
#metadata:
#  name: burnin-telemetry
#  namespace: kube-amd-gpu
#data:
#  telemetry_gate.sh: |
#    #!/usr/bin/env bash
#    set -euo pipefail
#
#    WATCH_SECONDS="${WATCH_SECONDS:-86400}"
#    SAMPLE_INTERVAL="${SAMPLE_INTERVAL:-10}"
#    POWER_MAX_W="${POWER_MAX_W:-750}"
#    POWER_TOL_PCT="${POWER_TOL_PCT:-15}"
#    WARMUP="${WARMUP:-600}"
#    REQUIRE_CAP_COMPLIANCE_PCT="${REQUIRE_CAP_COMPLIANCE_PCT:-99.9}"
#
#    mkdir -p /gate /telemetry
#    : > /gate/summary.txt
#
#    echo "[INFO] Telemetry starting. cap=${POWER_MAX_W}W tol=${POWER_TOL_PCT}% warmup=${WARMUP}s" | tee -a /gate/summary.txt
#
#    # Make sure amd-smi from host /opt/rocm is visible
#    export PATH="/opt/rocm:/opt/rocm/bin:${PATH}"
#
#    if ! command -v amd-smi >/dev/null 2>&1; then
#      echo "[ERROR] amd-smi not found even after mounting /opt/rocm" | tee -a /gate/summary.txt
#      echo "amd-smi missing" > /gate/violation
#      exit 2
#    fi
#
#    end=$((SECONDS + WATCH_SECONDS))
#    samples=0
#    sum_power=0
#    max_power=0
#    cap=$(awk -v p="${POWER_MAX_W}" -v t="${POWER_TOL_PCT}" 'BEGIN{printf "%.2f", p*(1+t/100.0)}')
#
#    violation () {
#      local msg="$1"
#      echo "[VIOLATION] ${msg}" | tee -a /gate/summary.txt
#      echo "${msg}" > /gate/violation
#      exit 2
#    }
#
#    # Baseline UE before warmup
#    BASE_UE=0
#    RAS_BASE="$(amd-smi ras 2>&1 || true)"
#    if echo "${RAS_BASE}" | grep -Ei 'UE[^0-9]*[1-9]' >/dev/null 2>&1; then
#      BASE_UE="$(echo "${RAS_BASE}" | grep -Ei 'UE' | grep -Eo '[0-9]+' | tail -1 || echo 0)"
#    fi
#    echo "[INFO] Baseline UE=${BASE_UE}" | tee -a /gate/summary.txt
#
#    while [ "${SECONDS}" -lt "${end}" ]; do
#      ts="$(date --iso-8601=seconds)"
#
#      METRIC="$(amd-smi metric 2>&1 || true)"
#      RAS="$(amd-smi ras 2>&1 || true)"
#
#      {
#        echo "=== ${ts} METRIC ==="
#        echo "${METRIC}"
#      } >> /telemetry/metric.log
#
#      {
#        echo "=== ${ts} RAS ==="
#        echo "${RAS}"
#      } >> /telemetry/ras.log
#
#      # Thermal throttling detection
#      if echo "${METRIC}" | grep -Eiq 'thrtl|throttle|thermal.*thrott'; then
#        violation "Thermal throttling detected at ${ts}"
#      fi
#
#      # Power parsing
#      have_power=0
#      while read -r p; do
#        have_power=1
#        sum_power=$(awk -v a="${sum_power}" -v b="${p}" 'BEGIN{printf "%.6f", a+b}')
#        samples=$((samples+1))
#        gt=$(awk -v x="${p}" -v m="${max_power}" 'BEGIN{if (x>m) print 1; else print 0}')
#        [ "${gt}" = "1" ] && max_power="${p}"
#        over=$(awk -v x="${p}" -v c="${cap}" 'BEGIN{if (x>c) print 1; else print 0}')
#        if [ "${over}" = "1" ]; then
#          violation "Power ${p}W exceeds allowed cap ${cap}W at ${ts}"
#        fi
#      done < <(echo "${METRIC}" | grep -Ei 'power' | grep -Eo '[0-9]+(\.[0-9]+)?W' | sed 's/W//')
#
#      if [ "${have_power}" -eq 0 ] && [ "${SECONDS}" -gt $((end - WATCH_SECONDS + WARMUP)) ]; then
#        violation "No power samples found at ${ts}"
#      fi
#
#      # UE delta
#      CUR_UE="${BASE_UE}"
#      if echo "${RAS}" | grep -Ei 'UE' >/dev/null 2>&1; then
#        CUR_UE="$(echo "${RAS}" | grep -Ei 'UE' | grep -Eo '[0-9]+' | tail -1 || echo "${BASE_UE}")"
#      fi
#      if [ "${CUR_UE}" -gt "${BASE_UE}" ]; then
#        violation "UE increased from ${BASE_UE} to ${CUR_UE} at ${ts}"
#      fi
#
#      sleep "${SAMPLE_INTERVAL}"
#    done
#
#    mean=0
#    if [ "${samples}" -gt 0 ]; then
#      mean=$(awk -v s="${sum_power}" -v n="${samples}" 'BEGIN{printf "%.2f", s/n}')
#    fi
#
#    echo "[OK] No telemetry violations for ${WATCH_SECONDS}s" | tee -a /gate/summary.txt
#    echo "[STATS] mean_power_w=${mean} max_power_w=${max_power} samples=${samples}" | tee -a /gate/summary.txt
#
#    touch /gate/telemetry_ok
#    exit 0
