From 59675ea59a9ba73c50b4a0b76f94a3566801c37b Mon Sep 17 00:00:00 2001 From: Egor Date: Mon, 3 Nov 2025 18:49:42 -0800 Subject: [PATCH] fix CUDA devices not being listed under system hardware, calcuate our own auto VRAM layers instead of relying on kcpp --- package.json | 10 +- .../Launch/CommandLineArgumentsModal.tsx | 2 +- .../Launch/GeneralTab/BackendSelector.tsx | 90 +++++++++- src/components/settings/GeneralTab.tsx | 2 +- src/hooks/useLaunchLogic.ts | 6 +- src/main/ipc.ts | 18 ++ src/preload/index.ts | 13 ++ src/types/electron.d.ts | 15 ++ src/utils/node/vram.ts | 122 ++++++++++++++ src/utils/systemInfo.ts | 12 +- yarn.lock | 154 +++++++++--------- 11 files changed, 351 insertions(+), 93 deletions(-) create mode 100644 src/utils/node/vram.ts diff --git a/package.json b/package.json index ec95702..c6008d3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "gerbil", "productName": "Gerbil", - "version": "1.8.3", + "version": "1.8.4", "description": "Run Large Language Models locally", "main": "out/main/index.js", "homepage": "./", @@ -38,19 +38,19 @@ }, "license": "AGPL-3.0-or-later", "devDependencies": { - "@eslint/js": "^9.39.0", + "@eslint/js": "^9.39.1", "@types/node": "^24.10.0", "@types/react": "^19.2.2", "@types/react-dom": "^19.2.2", "@types/yauzl": "^2.10.3", - "@typescript-eslint/eslint-plugin": "^8.46.2", - "@typescript-eslint/parser": "^8.46.2", + "@typescript-eslint/eslint-plugin": "^8.46.3", + "@typescript-eslint/parser": "^8.46.3", "@vitejs/plugin-react": "^5.1.0", "cross-env": "^10.1.0", "electron": "^38.5.0", "electron-builder": "^26.0.12", "electron-vite": "^4.0.1", - "eslint": "^9.39.0", + "eslint": "^9.39.1", "eslint-plugin-import": "^2.32.0", "eslint-plugin-no-comments": "^1.1.10", "eslint-plugin-promise": "^7.2.1", diff --git a/src/components/screens/Launch/CommandLineArgumentsModal.tsx b/src/components/screens/Launch/CommandLineArgumentsModal.tsx index 69eab54..8ebbe90 100644 --- a/src/components/screens/Launch/CommandLineArgumentsModal.tsx +++ b/src/components/screens/Launch/CommandLineArgumentsModal.tsx @@ -610,7 +610,7 @@ const COMMAND_LINE_ARGUMENTS = [ { flag: '--sdvaeauto', description: - 'Uses a built-in VAE via TAE SD, which is very fast, and fixed bad VAEs.', + 'Uses a built-in VAE via TAE SD, which is very fast and fixed bad VAEs.', type: 'boolean', category: 'Image Generation', }, diff --git a/src/components/screens/Launch/GeneralTab/BackendSelector.tsx b/src/components/screens/Launch/GeneralTab/BackendSelector.tsx index 32cb741..eca3aff 100644 --- a/src/components/screens/Launch/GeneralTab/BackendSelector.tsx +++ b/src/components/screens/Launch/GeneralTab/BackendSelector.tsx @@ -12,6 +12,10 @@ export const BackendSelector = () => { backend, gpuLayers, autoGpuLayers, + model, + contextSize, + gpuDeviceSelection, + flashattention, handleBackendChange, handleGpuLayersChange, handleAutoGpuLayersChange, @@ -21,6 +25,7 @@ export const BackendSelector = () => { [] ); const [isLoadingBackends, setIsLoadingBackends] = useState(false); + const [isCalculatingLayers, setIsCalculatingLayers] = useState(false); const hasInitialized = useRef(false); useEffect(() => { @@ -62,6 +67,76 @@ export const BackendSelector = () => { } }, [availableBackends, backend, handleBackendChange]); + useEffect(() => { + const calculateLayers = async () => { + if ( + !autoGpuLayers || + !model || + !contextSize || + backend === 'cpu' || + isLoadingBackends + ) { + return; + } + + try { + setIsCalculatingLayers(true); + + const gpuMemory = await window.electronAPI.kobold.detectGPUMemory(); + if (!gpuMemory || gpuMemory.length === 0) { + return; + } + + const selectedDeviceIndices = gpuDeviceSelection + .split(',') + .map((d) => parseInt(d.trim(), 10)) + .filter((d) => !isNaN(d)); + + const availableVramGB = selectedDeviceIndices.reduce( + (total, deviceIndex) => { + const device = gpuMemory[deviceIndex]; + const vramGB = device?.totalMemoryGB + ? parseFloat(device.totalMemoryGB) + : 0; + return total + vramGB; + }, + 0 + ); + + if (availableVramGB === 0) { + return; + } + + const result = await window.electronAPI.kobold.calculateOptimalLayers( + model, + contextSize, + availableVramGB, + flashattention + ); + + handleGpuLayersChange(result.recommendedLayers); + } catch (error) { + window.electronAPI.logs.logError( + 'Failed to calculate optimal GPU layers', + error as Error + ); + } finally { + setIsCalculatingLayers(false); + } + }; + + calculateLayers(); + }, [ + autoGpuLayers, + model, + contextSize, + backend, + gpuDeviceSelection, + flashattention, + isLoadingBackends, + handleGpuLayersChange, + ]); + return (
@@ -70,7 +145,7 @@ export const BackendSelector = () => { Backend - +