apiVersion: batch/v1
kind: Job
metadata:
  name: mi300-burnin-24h
  namespace: kube-amd-gpu
spec:
  ttlSecondsAfterFinished: 86400
  backoffLimit: 0
  activeDeadlineSeconds: 90000
  template:
    metadata:
      labels:
        app: mi300-burnin-24h
    spec:
      serviceAccountName: gpu-burnin
      restartPolicy: Never
      priorityClassName: burnin-priority

      nodeSelector:
        gpu.amd.com/stage: "burnin"

      tolerations:
        - key: "gpu.amd.com/stage"
          operator: "Equal"
          value: "burnin"
          effect: "NoSchedule"
        - key: "gpu.amd.com/burnin"
          operator: "Equal"
          value: "running"
          effect: "NoSchedule"

      volumes:
        - name: gate
          emptyDir: {}
        - name: rvs-burnin-conf
          configMap:
            name: rvs-burnin-conf
        - name: telemetry-cm
          configMap:
            name: burnin-telemetry
            defaultMode: 0555
        - name: rocm
          hostPath:
            path: /opt/rocm
            type: Directory
        - name: kfd
          hostPath:
            path: /dev/kfd
            type: CharDevice
        - name: dri
          hostPath:
            path: /dev/dri
            type: Directory

      containers:
        # --- 24h RVS GST burn-in runner -----------------------------------
        - name: burnin-runner
          image: docker.io/rocm/test-runner:v1.4.0
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          resources:
            limits:
              amd.com/gpu: "8"
            requests:
              amd.com/gpu: "8"
          env:
            - name: BURNIN_SECONDS
              value: "86400"   # 24 hours
          command: ["/bin/bash", "-lc"]
          args:
            - |
              set -euo pipefail
              mkdir -p /gate
              : > /gate/summary.txt

              echo "[INFO] RVS GST 24h burn-in starting for ${BURNIN_SECONDS}s" | tee -a /gate/summary.txt

              RVS_BIN="/opt/rocm/bin/rvs"
              if [ ! -x "${RVS_BIN}" ]; then
                echo "[ERROR] RVS binary not found or not executable at ${RVS_BIN}" | tee -a /gate/summary.txt
                ls -l /opt/rocm || true
                ls -l /opt/rocm/bin || true
                echo "RVS missing" > /gate/violation
                exit 1
              fi

              END=$(( $(date +%s) + ${BURNIN_SECONDS} ))
              echo "[INFO] Using ${RVS_BIN}" | tee -a /gate/summary.txt

              while [ "$(date +%s)" -lt "${END}" ]; do
                if [ -f /gate/violation ]; then
                  echo "[ERROR] Telemetry violation flag present, aborting RVS" | tee -a /gate/summary.txt
                  exit 2
                fi

                RVS_RC=0
                "${RVS_BIN}" -c /etc/rvs/gst_burnin_24h.conf -j --parallel || RVS_RC=$?

                if [ "${RVS_RC}" -ne 0 ]; then
                  echo "[VIOLATION] RVS GST 24h burn-in returned non-zero (${RVS_RC})" | tee -a /gate/summary.txt
                  echo "RVS GST failure" > /gate/violation
                  exit 2
                fi
              done

              if [ ! -f /gate/telemetry_ok ]; then
                echo "[ERROR] Telemetry never completed cleanly" | tee -a /gate/summary.txt
                echo "Telemetry incomplete" > /gate/violation
                exit 2
              fi

              echo "[OK] RVS GST 24h burn-in + telemetry met all criteria" | tee -a /gate/summary.txt
              touch /gate/success
              exit 0
          volumeMounts:
            - name: gate
              mountPath: /gate
            - name: rvs-burnin-conf
              mountPath: /etc/rvs
            - name: kfd
              mountPath: /dev/kfd
            - name: dri
              mountPath: /dev/dri

        # --- Telemetry sidecar (24h amd-smi gate) --------------------------
        - name: telemetry-gate
          image: docker.io/rocm/test-runner:v1.4.0
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          env:
            - name: WATCH_SECONDS
              value: "86400"
            - name: SAMPLE_INTERVAL
              value: "10"
            - name: POWER_MAX_W
              value: "750"
            - name: POWER_TOL_PCT
              value: "15"
            - name: WARMUP
              value: "600"
            - name: REQUIRE_CAP_COMPLIANCE_PCT
              value: "99.9"
          command: ["/bin/bash"]
          args:
            - /cfg/telemetry_gate.sh
          volumeMounts:
            - name: telemetry-cm
              mountPath: /cfg
            - name: gate
              mountPath: /gate
            - name: rocm
              mountPath: /opt/rocm
            - name: kfd
              mountPath: /dev/kfd
            - name: dri
              mountPath: /dev/dri

        # --- Judge: curl-based label (no taint) ----------------------------
        - name: burnin-judge
          image: docker.io/rocm/test-runner:v1.4.0
          imagePullPolicy: IfNotPresent
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: GUARD_SECONDS
              value: "90000"
          volumeMounts:
            - name: gate
              mountPath: /gate
          command: ["/bin/bash", "-lc"]
          args:
            - |
              set -euo pipefail

              echo "[INFO] Judge starting on node ${NODE_NAME} for 24h burn-in..."
              echo "[INFO] Waiting for /gate/success or /gate/violation..."

              if ! command -v curl >/dev/null 2>&1; then
                echo "[WARN] curl not found in image; will NOT auto label node."
                AUTO_PATCH="false"
              else
                AUTO_PATCH="true"
              fi

              SA_DIR=/var/run/secrets/kubernetes.io/serviceaccount
              TOKEN_FILE="${SA_DIR}/token"
              CA_CERT="${SA_DIR}/ca.crt"

              if [ "${AUTO_PATCH}" = "true" ]; then
                if [ ! -f "${TOKEN_FILE}" ] || [ ! -f "${CA_CERT}" ]; then
                  echo "[WARN] ServiceAccount token or CA cert missing; disabling auto patch."
                  AUTO_PATCH="false"
                fi
              fi

              API_HOST="${KUBERNETES_SERVICE_HOST:-}"
              API_PORT="${KUBERNETES_SERVICE_PORT_HTTPS:-443}"

              if [ "${AUTO_PATCH}" = "true" ] && [ -z "${API_HOST}" ]; then
                echo "[WARN] KUBERNETES_SERVICE_HOST not set; disabling auto patch."
                AUTO_PATCH="false"
              fi

              k8s_label_burnin() {
                local value="$1"
                if [ "${AUTO_PATCH}" != "true" ]; then
                  echo "[INFO] AUTO_PATCH=false, skipping label update."
                  return 0
                fi

                local token
                token="$(cat "${TOKEN_FILE}")"

                local patch_json
                patch_json="$(cat <<EOF
              [
                {
                  "op": "add",
                  "path": "/metadata/labels/gpu.amd.com~1burnin",
                  "value": "${value}"
                }
              ]
              EOF
              )"

                echo "[INFO] Setting label gpu.amd.com/burnin=${value} on ${NODE_NAME}..."
                curl -sS -X PATCH \
                  --cacert "${CA_CERT}" \
                  -H "Authorization: Bearer ${token}" \
                  -H "Content-Type: application/json-patch+json" \
                  "https://${API_HOST}:${API_PORT}/api/v1/nodes/${NODE_NAME}" \
                  -d "${patch_json}" || echo "[WARN] Label JSONPatch request failed (see above)."
              }

              GUARD_SECONDS="${GUARD_SECONDS:-90000}"
              end=$(( $(date +%s) + GUARD_SECONDS ))

              while [ ! -f /gate/success ] && [ ! -f /gate/violation ]; do
                if [ "$(date +%s)" -gt "${end}" ]; then
                  echo "[ERROR] Timeout waiting for 24h burn-in result (GUARD_SECONDS=${GUARD_SECONDS})"
                  exit 3
                fi
                sleep 30
              done

              if [ -f /gate/success ]; then
                echo "[PASS] Burn-in 24h PASSED on ${NODE_NAME}"
                k8s_label_burnin "done"
                exit 0
              else
                echo "[FAIL] Burn-in 24h FAILED on ${NODE_NAME}"
                k8s_label_burnin "failed"
                exit 2
              fi