WORKING_DIR=/opt/docker/fastapi-dls/cert
mkdir -p $WORKING_DIR
cd $WORKING_DIR
openssl genrsa -out $WORKING_DIR/instance.private.pem 2048
openssl rsa -in $WORKING_DIR/instance.private.pem -outform PEM -pubout -out $WORKING_DIR/instance.public.pem
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout $WORKING_DIR/webserver.key -out $WORKING_DIR/webserver.crt
x-dls-variables: &dls-variables
TZ: Europe/London # REQUIRED, set your timezone correctly on fastapi-dls AND YOUR CLIENTS !!!
DLS_URL: nvidia-cls.hostname.com
DLS_PORT: 443
LEASE_EXPIRE_DAYS: 90 # 90 days is maximum
DATABASE: sqlite:////app/database/db.sqlite
DEBUG: false
services:
dls:
image: collinwebdesigns/fastapi-dls:latest
restart: always
environment:
<<: *dls-variables
volumes:
- /opt/docker/fastapi-dls/cert:/app/cert
- dls-db:/app/database
logging: # optional, for those who do not need logs
driver: "json-file"
options:
max-file: 10
max-size: 10m
volumes:
dls-db:
external: true
535.161.05.patch
NVIDIA-Linux-x86_64-535.161.05-vgpu-kvm.run
Shouldn't need this since kernel 8.2.2
sed -i 's/$/ intel_iommu=on/' /etc/kernel/cmdline
cat /etc/kernel/cmdline
proxmox-boot-tool refresh
(If using grub: edit /etc/default/grub ## update-grub)
echo -e "vfio\nvfio_iommu_type1\nvfio_pci" >> /etc/modules
cat /etc/modules
echo "blacklist acpi_power_meter" >> /etc/modprobe.d/hwmon.conf
echo "blacklist nouveau" >> /etc/modprobe.d/blacklist.conf
echo "blacklist nvidia*" >> /etc/modprobe.d/blacklist.conf
cat /etc/modprobe.d/blacklist.conf
update-initramfs -u -k all
reboot
lsmod | grep vfio
dmesg | grep -e DMAR -e IOMMU -e AMD-Vi
dmesg | grep 'remapping'
lspci -nnk | grep -B 2 "nvidia"
apt install --no-install-recommends git build-essential dkms pve-headers mdevctl
curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal
source $HOME/.cargo/env
cd /etc
git clone https://github.com/mbilker/vgpu_unlock-rs.git
cd vgpu_unlock-rs
cargo build --release
mkdir /etc/vgpu_unlock
touch /etc/vgpu_unlock/profile_override.toml
echo "unlock = false" >> /etc/vgpu_unlock/config.toml
cat /etc/vgpu_unlock/config.toml
mkdir /etc/systemd/system/{nvidia-vgpud.service.d,nvidia-vgpu-mgr.service.d}
echo -e "[Service]\nEnvironment=LD_PRELOAD=/etc/vgpu_unlock-rs/target/release/libvgpu_unlock_rs.so" > /etc/systemd/system/nvidia-vgpud.service.d/vgpu_unlock.conf
echo -e "[Service]\nEnvironment=LD_PRELOAD=/etc/vgpu_unlock-rs/target/release/libvgpu_unlock_rs.so" > /etc/systemd/system/nvidia-vgpu-mgr.service.d/vgpu_unlock.conf
ls /etc/systemd/system/nvidia-*
systemctl daemon-reload
cd /root/
mkdir nvidia-drivers
(transfer NVIDIA-Linux-x86_64-535.161.05-vgpu-kvm.run & 535.161.05.patch)
cd nvidia-drivers
chmod +x NVIDIA-Linux-x86_64-535.161.05-vgpu-kvm.run
chmod +x 535.161.05.patch
./NVIDIA-Linux-x86_64-535.161.05-vgpu-kvm.run --apply-patch 535.161.05.patch
./NVIDIA-Linux-x86_64-535.161.05-vgpu-kvm-custom.run --dkms -m=kernel
shutdown -r now
-or-
reboot
Possible issues with IOMMU
Check IOMMU isolation to ensure GPU is not sharing a group with anything else!
pvesh get /nodes/<node_hostname>/hardware/pci --pci-class-blacklist ""
nvidia-smi can be ran on host and clients
nvidia-smi -q (querys card)
nvidia-smi -q | grep "License" (run on client machine to check license status)
mdevctl types
systemctl status nvidia-vgpu-mgr
journalctl -b -u nvidia-vgpu-mgr (add -f to follow output)
systemctl status nvidia-vgpud.service
journalctl -b -u nvidia-vgpud (add -f to follow output)
./NVIDIA-Linux-x86_64-535.161.05-vgpu-kvm.run --unistall
apt install --no-install-recommends build-essential -y
apt install --no-install-recommends linux-headers-`uname -r` -y
apt install --no-install-recommends dkms -y
mkdir -p /opt/nvidia
cd /opt/nvidia
wget https://storage.googleapis.com/nvidia-drivers-us-public/GRID/vGPU16.4/NVIDIA-Linux-x86_64-535.161.07-grid.run
chmod +x NVIDIA-Linux-x86_64-535.161.07-grid.run
./NVIDIA-Linux-x86_64-535.161.07-grid.run --dkms -m=kernel
curl -L -X GET https://nvidia-cls.hostname.com/-/client-token -o /etc/nvidia/ClientConfigToken/client_configuration_token_$(date '+%d-%m-%Y-%H-%M-%S').tok
service nvidia-gridd restart
nvidia-smi -q | grep "License"
cd /opt/nvidia
nano license-update.sh
#!/bin/bash
sudo curl -L -X GET https://nvidia-cls.hostname.com/-/client-token -o /etc/nvidia/ClientConfigToken/client_configuration_token_$(date '+%d-%m-%Y-%H-%M-%S').tok
sudo service nvidia-gridd restart
CTRL + X to exit the editor
Configure permissions for the script to be ran:
chmod +x license-update.sh
crontab -e
You'll be prompted which text editor to use, choose your favourite (I'd recommend nano)
Scroll down to the bottom of the Crontab file, and paste the following:
@monthly /opt/nvidia/driver-update.sh
Save the file and you've automated your license renewal so they'll never expire! Congratulations!
Run Windows Powershell as an Administrator
curl.exe -L -X GET https://nvidia-cls.hostname.com-/client-token -o "C:\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken\client_configuration_token_$($(Get-Date).tostring('dd-MM-yy-hh-mm-ss')).tok"
Restart-Service NVDisplay.ContainerLocalSystem
& 'nvidia-smi' -q | Select-String "License"