Monitoring GPU cluster performance with NVIDIA DCGM-Exporter and Weights & Biases

import time

import tqdm
import wandb

run = wandb.init(
    project="dcgm",
    settings=wandb.Settings(
        x_stats_open_metrics_endpoints={
            "node1": "http://192.168.0.1:9400/metrics",  # ensure this is the metrics endpoint
            "node2": "http://192.168.0.2:9400/metrics",
        },
        x_stats_open_metrics_filters={
            "node1.DCGM_FI_DEV_(POWER_USAGE|MEM_COPY_UTIL|TOTAL_ENERGY_CONSUMPTION|GPU_TEMP|MEMORY_TEMP)": {
                "gpu": "[0,1]",
            },
            "node2.DCGM_FI_DEV_(POWER_USAGE|MEM_COPY_UTIL|TOTAL_ENERGY_CONSUMPTION|GPU_TEMP|MEMORY_TEMP)": {
                "gpu": ".*",
            },
        },
        # optional headers in case, for example, the endpoints sit behind a proxy requiring authentication
        # x_stats_open_metrics_http_headers={"Authorization": "Bearer MEDVED"}
    ),
)

for i in tqdm.tqdm(range(300)):
    time.sleep(1)
    run.log({"loss": 1.0 / (i + 1)})

run.finish()

srun --nodes=2 --gpus=4 --cpus-per-gpu=12 --job-name=wandb python3 /<path>/<to>/srun-dcgm-multinode.py

import os
import re
import time
from typing import List

import tqdm
import wandb


def unpack_node_list(regex_string: str) -> List[str]:
    """Unpacks a regex with the SLURM node list string into a list of strings."""
    pattern = re.compile(r"\[([\d,-]+)\]")
    match = pattern.search(regex_string)

    if match:
        segments = match.group(1).split(',')
        unpacked_strings = []

        for segment in segments:
            if '-' in segment:
                start_number, end_number = [int(x) for x in segment.split('-')]
                for number in range(start_number, end_number + 1):
                    unpacked_strings.append(
                        regex_string[: match.start()] + str(number) + regex_string[match.end():]
                    )
            else:
                unpacked_strings.append(
                    regex_string[: match.start()] + segment + regex_string[match.end():]
                )

        return unpacked_strings

    return [regex_string]

def main() -> None:
    # simulate creating a wandb run only on the rank 0 node
    if os.environ.get("SLURM_NODEID") != "0":
        return

    node_list = unpack_node_list(os.environ["SLURM_NODELIST"])
    # drop this node:
    node_list = [node for node in node_list if node != os.environ["SLURMD_NODENAME"]]

    run = wandb.init(
        project="dcgm",
        settings=wandb.Settings(
            x_stats_sampling_interval=1,
            x_stats_open_metrics_endpoints={
                **{
                    "node1": "http://localhost:9400/metrics",  # see note above
                },
                **{
                    f"node{n+2}": f"http://{node}:9400/metrics" for n, node in enumerate(node_list)
                }
            },
            x_stats_open_metrics_filters={
                ".*DCGM_FI_DEV_(POWER_USAGE|MEM_COPY_UTIL|TOTAL_ENERGY_CONSUMPTION|GPU_TEMP|MEMORY_TEMP)": {
                    "gpu": ".*",
                },
            },
        ),
    )

    for i in tqdm.tqdm(range(300)):
        time.sleep(1)
        run.log({"loss": 1 / (i + 1)})
    run.finish()

srun --nodes=1 --gpus=2 --cpus-per-gpu=12 --job-name=wandb /admin/home-dimaduev/run.sh

#!/usr/bin/bash

export WANDB_X_STATS_OPEN_METRICS_ENDPOINTS='{"node1": "http://localhost:9400/metrics"}'

WANDB_X_STATS_OPEN_METRICS_FILTERS='{"node1.DCGM_FI_DEV_(POWER_USAGE|MEM_COPY_UTIL|TOTAL_ENERGY_CONSUMPTION|GPU_TEMP|MEMORY_TEMP)": {"gpu": "[__CUDA_VISIBLE_DEVICES__]"}}'
export WANDB_X_STATS_OPEN_METRICS_FILTERS=$(echo "$WANDB__STATS_OPEN_METRICS_FILTERS" | sed "s/__CUDA_VISIBLE_DEVICES__/$CUDA_VISIBLE_DEVICES/")
export WANDB_X_STATS_SAMPLING_INTERVAL=15

/usr/bin/python3 srun-dcgm.py

import time

import tqdm
import wandb


def main() -> None:
    run = wandb.init(project="dcgm")
    for _ in tqdm.tqdm(range(300)):
        time.sleep(1)
        run.log({"x": _})
    run.finish()


if __name__ == "__main__":
    main()

export ZONE="us-central1-f"
export CLUSTER_NAME="gke-dcgm"

gcloud beta container clusters create $CLUSTER_NAME \
  --zone $ZONE \
  --machine-type=n1-standard-8 \
  --accelerator=type=nvidia-tesla-t4,count=2 \
  --num-nodes=2 \
  --enable-managed-prometheus

gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE

kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded.yaml

kubectl get pods -n kube-system | grep nvidia-gpu-device-plugin

kubectl create namespace gpu-monitoring-system
kubectl apply -f dcgm.yml  # see below for the file content

kubectl apply -f dcgm_loadtest.yml  # see below for the file content

kubectl apply -f clusterip_service.yml  # see below for the file content

kubectl apply -f ubuntu.yml  # see below for the file content
kubectl get pods --all-namespaces
kubectl exec -it --namespace=gpu-monitoring-system ubuntu -- /bin/bash

apt update && apt install curl
curl http://dcgm-exporter-svc:9400/metrics