Infrastructure & Deployment

# Check NVMe-oF multipath status
$ nvme list-subsys
nvme-subsys0 - NQN=nqn.2024-01.com.vendor:array01
\
 +- nvme0 rdma traddr=192.168.1.10 trsvcid=4420 live optimized
 +- nvme1 rdma traddr=192.168.1.11 trsvcid=4420 live non-optimized

# ANA states:
# - optimized: preferred path, lowest latency
# - non-optimized: functional but higher latency
# - inaccessible: path down, don't use

# Linux native multipath (dm-multipath not needed)
$ cat /sys/class/nvme-subsystem/nvme-subsys0/iopolicy
round-robin   # or numa, queue-depth

// cuFile.json configuration for fabric storage
{
  "logging": { "level": 2 },
  "nvfs": {
    "rdma": {
      "enable": true,
      "devices": ["mlx5_0", "mlx5_1"],
      "poll_mode": true,
      "max_direct_io_size_kb": 16384
    }
  },
  "properties": {
    "max_device_cache_size_kb": 131072,
    "max_device_pinned_mem_size_kb": 33554432
  }
}

// io_uring with registered buffers for GPU-like access patterns

struct io_uring ring;
io_uring_queue_init(256, &ring, IORING_SETUP_SQPOLL);  // Kernel polling

// Register fixed buffers (avoids per-I/O buffer registration)
struct iovec iovecs[16];
for (int i = 0; i < 16; i++) {
    iovecs[i].iov_base = aligned_alloc(4096, BUFFER_SIZE);
    iovecs[i].iov_len = BUFFER_SIZE;
}
io_uring_register_buffers(&ring, iovecs, 16);

// Submit batched reads
for (int i = 0; i < batch_size; i++) {
    struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
    io_uring_prep_read_fixed(sqe, fd, iovecs[i % 16].iov_base, 
                              size, offset, i % 16);
    sqe->user_data = i;
}
io_uring_submit(&ring);  // Single syscall for all

# GDS-enabled AI training pod
apiVersion: v1
kind: Pod
metadata:
  name: gds-training-pod
spec:
  containers:
  - name: trainer
    image: nvcr.io/nvidia/pytorch:24.01-py3
    resources:
      limits:
        nvidia.com/gpu: 8
        hugepages-2Mi: 4Gi         # Required for GDS DMA
        rdma/rdma_shared_device_a: 1  # GPUDirect RDMA
    volumeMounts:
    - name: training-data
      mountPath: /data
    - name: nvme-direct        # Raw NVMe for GDS
      mountPath: /dev/nvme0n1
    securityContext:
      privileged: true         # Required for device access
      capabilities:
        add: [SYS_ADMIN, IPC_LOCK]
  
  volumes:
  - name: training-data
    persistentVolumeClaim:
      claimName: gds-pvc
  - name: nvme-direct
    hostPath:
      path: /dev/nvme0n1
      type: BlockDevice

# StorageClass optimized for GDS
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gds-nvme
provisioner: csi.nvidia.com
parameters:
  type: nvme-local
  fsType: xfs
  mkfsOptions: "-K"           # Don't discard on mkfs
mountOptions:
  - noatime                        # Don't update access times
  - nodiratime
  - nobarrier                      # SSD doesn't need barriers
  - logbufs=8                      # XFS log buffers
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true

// O_DIRECT requirements
int fd = open(path, O_RDONLY | O_DIRECT);

// Alignment requirements (typically 4KB)
size_t alignment = 4096;
void* buffer;
posix_memalign(&buffer, alignment, size);

// Size must be multiple of alignment
size = (size + alignment - 1) & ~(alignment - 1);

# Lustre tuning for GDS
lctl set_param llite.*.max_read_ahead_mb=0      # Disable readahead (GDS handles)
lctl set_param osc.*.max_pages_per_rpc=1024     # Larger RPCs
lctl set_param osc.*.max_rpcs_in_flight=32      # More parallelism

# XFS tuning
mkfs.xfs -d su=1m,sw=4 /dev/nvme0n1              # Stripe unit/width for RAID
mount -o noatime,nodiratime,logbufs=8 /dev/nvme0n1 /data