allow analyzing local .gguf model files

This commit is contained in:
Egor 2025-10-24 23:31:13 -07:00
parent eb921c93b3
commit 273d1b690f
12 changed files with 408 additions and 52 deletions

View file

@ -1,7 +1,7 @@
{
"name": "gerbil",
"productName": "Gerbil",
"version": "1.7.3",
"version": "1.8.0",
"description": "Run Large Language Models locally",
"main": "out/main/index.js",
"homepage": "./",
@ -45,7 +45,7 @@
"@types/yauzl": "^2.10.3",
"@typescript-eslint/eslint-plugin": "^8.46.2",
"@typescript-eslint/parser": "^8.46.2",
"@vitejs/plugin-react": "^5.0.4",
"@vitejs/plugin-react": "^5.1.0",
"cross-env": "^10.1.0",
"electron": "^38.4.0",
"electron-builder": "^26.0.12",
@ -55,7 +55,7 @@
"eslint-plugin-no-comments": "^1.1.10",
"eslint-plugin-promise": "^7.2.1",
"eslint-plugin-react": "^7.37.5",
"eslint-plugin-react-hooks": "^7.0.0",
"eslint-plugin-react-hooks": "^7.0.1",
"eslint-plugin-sonarjs": "^3.0.5",
"globals": "^16.4.0",
"jiti": "^2.6.1",
@ -69,12 +69,13 @@
"@codemirror/theme-one-dark": "^6.1.3",
"@codemirror/view": "^6.38.6",
"@fontsource/inter": "^5.2.8",
"@huggingface/gguf": "^0.3.2",
"@mantine/core": "^8.3.5",
"@mantine/hooks": "^8.3.5",
"@uiw/react-codemirror": "^4.25.2",
"electron-updater": "^6.6.2",
"execa": "^9.6.0",
"lucide-react": "^0.546.0",
"lucide-react": "^0.548.0",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-error-boundary": "^6.0.0",

View file

@ -42,8 +42,8 @@ export const GeneralTab = ({ configLoaded = true }: GeneralTabProps) => {
tooltip="Select a GGUF text generation model file for chat and completion tasks."
onChange={handleModelChange}
onSelectFile={handleSelectModelFile}
showSearchHF
searchUrl="https://huggingface.co/models?pipeline_tag=text-generation&library=gguf&sort=trending"
showAnalyze
/>
<div>

View file

@ -75,8 +75,8 @@ export const ImageGenerationTab = () => {
tooltip="The primary image generation model. This is the main model that will generate images."
onChange={handleSdmodelChange}
onSelectFile={handleSelectSdmodelFile}
showSearchHF
searchUrl="https://huggingface.co/models?pipeline_tag=text-to-image&library=gguf&sort=trending"
showAnalyze
/>
<ModelFileField

View file

@ -0,0 +1,135 @@
import { Modal } from '@/components/Modal';
import { Stack, Group, Text, Divider, Alert, rem } from '@mantine/core';
import { Info } from 'lucide-react';
import type { ModelAnalysis } from '@/types';
interface ModelAnalysisModalProps {
opened: boolean;
onClose: () => void;
analysis: ModelAnalysis | null;
loading?: boolean;
error?: string;
}
interface InfoRowProps {
label: string;
value?: string | number;
}
const InfoRow = ({ label, value }: InfoRowProps) => {
if (!value) return null;
return (
<Group gap="md" wrap="nowrap">
<Text size="sm" c="dimmed" style={{ minWidth: rem(160) }}>
{label}:
</Text>
<Text size="sm" fw={500}>
{value}
</Text>
</Group>
);
};
export const ModelAnalysisModal = ({
opened,
onClose,
analysis,
loading = false,
error,
}: ModelAnalysisModalProps) => (
<Modal
opened={opened}
onClose={onClose}
title={
<Group gap="xs">
<Info size={20} />
<Text>Model Analysis</Text>
</Group>
}
size="lg"
showCloseButton
>
{loading && (
<Text size="sm" c="dimmed">
Analyzing model...
</Text>
)}
{error && (
<Alert color="red" title="Analysis Failed">
{error}
</Alert>
)}
{analysis && (
<Stack gap="md">
<div>
<Text size="sm" fw={600} mb="xs">
General Information
</Text>
<Stack gap="xs">
<InfoRow
label="Architecture"
value={analysis.general.architecture}
/>
{analysis.general.name && (
<InfoRow label="Model Name" value={analysis.general.name} />
)}
{analysis.general.parameterCount && (
<InfoRow
label="Parameters"
value={analysis.general.parameterCount}
/>
)}
<InfoRow label="File Size" value={analysis.general.fileSize} />
{analysis.architecture.layers && (
<InfoRow label="Layers" value={analysis.architecture.layers} />
)}
{analysis.architecture.expertCount && (
<InfoRow
label="Expert Count (MoE)"
value={analysis.architecture.expertCount}
/>
)}
</Stack>
</div>
{analysis.context.maxContextLength && (
<>
<Divider />
<div>
<Text size="sm" fw={600} mb="xs">
Context
</Text>
<Stack gap="xs">
<InfoRow
label="Max Context Length"
value={analysis.context.maxContextLength}
/>
</Stack>
</div>
</>
)}
<Divider />
<div>
<Text size="sm" fw={600} mb="xs">
Memory Estimates
</Text>
<Stack gap="xs">
<InfoRow label="Full VRAM" value={analysis.estimates.fullGpuVram} />
<InfoRow label="Full RAM" value={analysis.estimates.systemRam} />
{analysis.estimates.vramPerLayer && (
<InfoRow
label="VRAM per Layer"
value={analysis.estimates.vramPerLayer}
/>
)}
</Stack>
</div>
</Stack>
)}
</Modal>
);

View file

@ -1,7 +1,11 @@
import { Group, TextInput, Button } from '@mantine/core';
import { File, Search } from 'lucide-react';
import { Group, TextInput, ActionIcon, Tooltip, Button } from '@mantine/core';
import { useState } from 'react';
import { File, Search, Info } from 'lucide-react';
import { LabelWithTooltip } from '@/components/LabelWithTooltip';
import { ModelAnalysisModal } from '@/components/screens/Launch/ModelAnalysisModal';
import { getInputValidationState } from '@/utils/validation';
import { logError } from '@/utils/logger';
import type { ModelAnalysis } from '@/types';
interface ModelFileFieldProps {
label: string;
@ -10,8 +14,8 @@ interface ModelFileFieldProps {
tooltip?: string;
onChange: (value: string) => void;
onSelectFile: () => void;
showSearchHF?: boolean;
searchUrl?: string;
showAnalyze?: boolean;
}
export const ModelFileField = ({
@ -21,10 +25,16 @@ export const ModelFileField = ({
tooltip,
onChange,
onSelectFile,
showSearchHF = false,
searchUrl = 'https://huggingface.co/models?pipeline_tag=text-to-image&library=gguf&sort=trending',
searchUrl,
showAnalyze = false,
}: ModelFileFieldProps) => {
const validationState = getInputValidationState(value);
const [analysisModalOpened, setAnalysisModalOpened] = useState(false);
const [modelAnalysis, setModelAnalysis] = useState<ModelAnalysis | null>(
null
);
const [analysisLoading, setAnalysisLoading] = useState(false);
const [analysisError, setAnalysisError] = useState<string>();
const getHelperText = () => {
if (!value.trim()) return undefined;
@ -36,6 +46,29 @@ export const ModelFileField = ({
return undefined;
};
const isLocalFile = value.trim() && validationState === 'local';
const handleAnalyzeModel = async () => {
if (!value.trim()) return;
setAnalysisModalOpened(true);
setAnalysisLoading(true);
setAnalysisError(undefined);
setModelAnalysis(null);
try {
const analysis = await window.electronAPI.kobold.analyzeModel(value);
setModelAnalysis(analysis);
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : 'Failed to analyze model';
setAnalysisError(errorMessage);
logError('Failed to analyze model:', error as Error);
} finally {
setAnalysisLoading(false);
}
};
return (
<div>
<LabelWithTooltip label={label} tooltip={tooltip} />
@ -55,16 +88,38 @@ export const ModelFileField = ({
>
Browse
</Button>
{showSearchHF && (
<Button
onClick={() => window.electronAPI.app.openExternal(searchUrl!)}
variant="outline"
leftSection={<Search size={16} />}
>
Search HF
</Button>
{searchUrl && (
<Tooltip label="Search Hugging Face">
<ActionIcon
onClick={() => window.electronAPI.app.openExternal(searchUrl)}
variant="outline"
size="lg"
>
<Search size={16} />
</ActionIcon>
</Tooltip>
)}
{showAnalyze && isLocalFile && (
<Tooltip label="Analyze model">
<ActionIcon
onClick={handleAnalyzeModel}
variant="light"
color="blue"
size="lg"
>
<Info size={16} />
</ActionIcon>
</Tooltip>
)}
</Group>
<ModelAnalysisModal
opened={analysisModalOpened}
onClose={() => setAnalysisModalOpened(false)}
analysis={modelAnalysis}
loading={analysisLoading}
error={analysisError}
/>
</div>
);
};

View file

@ -21,6 +21,7 @@ import {
selectModelFile,
selectInstallDirectory,
} from '@/main/modules/koboldcpp/config';
import { analyzeGGUFModel } from '@/main/modules/koboldcpp/analyze';
import {
get as getConfig,
set as setConfig,
@ -156,6 +157,10 @@ export function setupIPCHandlers() {
selectModelFile(title)
);
ipcMain.handle('kobold:analyzeModel', async (_, filePath: string) =>
analyzeGGUFModel(filePath)
);
ipcMain.handle('config:get', (_, key) => getConfig(key));
ipcMain.on('config:set', (_, key, value) => setConfig(key, value));

View file

@ -0,0 +1,110 @@
import { gguf } from '@huggingface/gguf';
import { stat } from 'fs/promises';
import { logError } from '@/utils/node/logging';
import { formatBytes } from '@/utils/format';
function estimateMemoryRequirements(fileSize: number) {
const vramOverhead = 1.1;
const fullGpuVram = fileSize * vramOverhead;
const ramOverhead = 1.2;
const systemRam = fileSize * ramOverhead;
return {
fullGpuVram: formatBytes(fullGpuVram),
systemRam: formatBytes(systemRam),
};
}
function estimateVramPerLayer(fileSize: number, layers?: number) {
if (!layers || layers === 0) return undefined;
const perLayer = fileSize / layers;
return formatBytes(perLayer);
}
function formatParameterCount(params?: number) {
if (!params) return undefined;
if (params >= 1e9) return `${(params / 1e9).toFixed(1)}B`;
if (params >= 1e6) return `${(params / 1e6).toFixed(1)}M`;
if (params >= 1e3) return `${(params / 1e3).toFixed(1)}K`;
return params.toString();
}
function formatContextLength(length?: number) {
if (!length) return undefined;
if (length >= 1e6) return `${(length / 1e6).toFixed(1)}M`;
if (length >= 1e3) return `${(length / 1e3).toFixed(0)}K`;
return length.toString();
}
function getMetadataValue(metadata: Record<string, unknown>, key: string) {
return metadata[key];
}
export async function analyzeGGUFModel(filePath: string) {
try {
const stats = await stat(filePath);
const fileSize = stats.size;
const { metadata } = await gguf(filePath, {
allowLocalFile: true,
});
const metadataRecord = metadata as Record<string, unknown>;
const architecture = getMetadataValue(
metadataRecord,
'general.architecture'
) as string;
const name = getMetadataValue(metadataRecord, 'general.name') as
| string
| undefined;
const paramCount = getMetadataValue(
metadataRecord,
'general.parameter_count'
) as number | undefined;
const contextLength = getMetadataValue(
metadataRecord,
`${architecture}.context_length`
) as number | undefined;
const blockCount = getMetadataValue(
metadataRecord,
`${architecture}.block_count`
) as number | undefined;
const expertCount = getMetadataValue(
metadataRecord,
`${architecture}.expert_count`
) as number | undefined;
const memoryEstimates = estimateMemoryRequirements(fileSize);
const vramPerLayer = estimateVramPerLayer(fileSize, blockCount);
return {
general: {
architecture,
name,
fileSize: formatBytes(fileSize),
parameterCount: formatParameterCount(paramCount),
},
context: {
maxContextLength: formatContextLength(contextLength),
},
architecture: {
layers: blockCount,
expertCount,
},
estimates: {
fullGpuVram: memoryEstimates.fullGpuVram,
systemRam: memoryEstimates.systemRam,
vramPerLayer,
},
};
} catch (error) {
logError('Error analyzing GGUF model:', error as Error);
throw new Error(
`Failed to analyze model: ${error instanceof Error ? error.message : 'Unknown error'}`
);
}
}

View file

@ -51,6 +51,8 @@ const koboldAPI: KoboldAPI = {
ipcRenderer.invoke('kobold:parseConfigFile', filePath),
selectModelFile: (title) =>
ipcRenderer.invoke('kobold:selectModelFile', title),
analyzeModel: (filePath) =>
ipcRenderer.invoke('kobold:analyzeModel', filePath),
stopKoboldCpp: () => ipcRenderer.invoke('kobold:stopKoboldCpp'),
onDownloadProgress: (callback) => {
const handler = (_: IpcRendererEvent, progress: number) =>

View file

@ -5,7 +5,12 @@ import type {
GPUMemoryInfo,
SystemMemoryInfo,
} from '@/types/hardware';
import type { BackendOption, BackendSupport, Screen } from '@/types';
import type {
BackendOption,
BackendSupport,
Screen,
ModelAnalysis,
} from '@/types';
import type { MantineColorScheme } from '@mantine/core';
import type {
CpuMetrics,
@ -142,6 +147,7 @@ export interface KoboldAPI {
setSelectedConfig: (configName: string) => Promise<boolean>;
parseConfigFile: (filePath: string) => Promise<KoboldConfig | null>;
selectModelFile: (title?: string) => Promise<string | null>;
analyzeModel: (filePath: string) => Promise<ModelAnalysis>;
stopKoboldCpp: () => void;
onDownloadProgress: (callback: (progress: number) => void) => () => void;
onInstallDirChanged: (callback: (newPath: string) => void) => () => void;

21
src/types/index.d.ts vendored
View file

@ -86,3 +86,24 @@ export interface BackendSupport {
failsafe: boolean;
cuda: boolean;
}
export interface ModelAnalysis {
general: {
architecture: string;
name?: string;
fileSize: string;
parameterCount?: string;
};
context: {
maxContextLength?: string;
};
architecture: {
layers?: number;
expertCount?: number;
};
estimates: {
fullGpuVram: string;
systemRam: string;
vramPerLayer?: string;
};
}

View file

@ -21,9 +21,11 @@ const isValidFilePath = (path: string) => {
export const getInputValidationState = (path: string) => {
if (!path.trim()) return 'neutral';
if (isValidUrl(path) || isValidFilePath(path)) {
return 'valid';
}
const isUrl = isValidUrl(path);
const isFile = isValidFilePath(path);
if (isUrl) return 'valid';
if (isFile) return 'local';
return 'invalid';
};

View file

@ -803,6 +803,24 @@ __metadata:
languageName: node
linkType: hard
"@huggingface/gguf@npm:^0.3.2":
version: 0.3.2
resolution: "@huggingface/gguf@npm:0.3.2"
dependencies:
"@huggingface/tasks": "npm:^0.19.47"
bin:
gguf-view: dist/cli.js
checksum: 10c0/f39bfc6b468bdd4f9ee1da548dde504dd1cb6ceee75c6c8c2c5473ed48bed789cbf75604f3e04a1b91cf9b3714e4fa081a3fb8f2189196b659bfce4824e558f9
languageName: node
linkType: hard
"@huggingface/tasks@npm:^0.19.47":
version: 0.19.58
resolution: "@huggingface/tasks@npm:0.19.58"
checksum: 10c0/3349a674bba49dfd729285ae6bb5698291748ba1cc80c8eb1e390b6627d7e97acad52114dbf196b1215a4e962d50c4cad6ec1cf91887301fd7c1115fa8ac4217
languageName: node
linkType: hard
"@humanfs/core@npm:^0.19.1":
version: 0.19.1
resolution: "@humanfs/core@npm:0.19.1"
@ -1053,10 +1071,10 @@ __metadata:
languageName: node
linkType: hard
"@rolldown/pluginutils@npm:1.0.0-beta.38":
version: 1.0.0-beta.38
resolution: "@rolldown/pluginutils@npm:1.0.0-beta.38"
checksum: 10c0/8353ec2528349f79e27d1a3193806725b85830da334e935cbb606d88c1177c58ea6519c578e4e93e5f677f5b22aecb8738894dbed14603e14b6bffe3facf1002
"@rolldown/pluginutils@npm:1.0.0-beta.43":
version: 1.0.0-beta.43
resolution: "@rolldown/pluginutils@npm:1.0.0-beta.43"
checksum: 10c0/1c17a0b16c277a0fdbab080fd22ef91e37c1f0d710ecfdacb6a080068062eb14ff030d0e9d2ec2325a1d4246dba0c49625755c82c0090f6cbf98d16e80183e02
languageName: node
linkType: hard
@ -1636,19 +1654,19 @@ __metadata:
languageName: node
linkType: hard
"@vitejs/plugin-react@npm:^5.0.4":
version: 5.0.4
resolution: "@vitejs/plugin-react@npm:5.0.4"
"@vitejs/plugin-react@npm:^5.1.0":
version: 5.1.0
resolution: "@vitejs/plugin-react@npm:5.1.0"
dependencies:
"@babel/core": "npm:^7.28.4"
"@babel/plugin-transform-react-jsx-self": "npm:^7.27.1"
"@babel/plugin-transform-react-jsx-source": "npm:^7.27.1"
"@rolldown/pluginutils": "npm:1.0.0-beta.38"
"@rolldown/pluginutils": "npm:1.0.0-beta.43"
"@types/babel__core": "npm:^7.20.5"
react-refresh: "npm:^0.17.0"
react-refresh: "npm:^0.18.0"
peerDependencies:
vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0
checksum: 10c0/bb9360a4b4c0abf064d22211756b999faf23889ac150de490590ca7bd029b0ef7f4cd8ba3a32b86682a62d46fb7bebd75b3fa9835c57c78123f4a646de2e0136
checksum: 10c0/e192a12e2b854df109eafb1d06c0bc848e8e2b162c686aa6b999b1048658983e72674b2068ccc37562fcce44d32ad92b65f3a4e1897a0cb7859c2ee69cc63eac
languageName: node
linkType: hard
@ -3191,18 +3209,18 @@ __metadata:
languageName: node
linkType: hard
"eslint-plugin-react-hooks@npm:^7.0.0":
version: 7.0.0
resolution: "eslint-plugin-react-hooks@npm:7.0.0"
"eslint-plugin-react-hooks@npm:^7.0.1":
version: 7.0.1
resolution: "eslint-plugin-react-hooks@npm:7.0.1"
dependencies:
"@babel/core": "npm:^7.24.4"
"@babel/parser": "npm:^7.24.4"
hermes-parser: "npm:^0.25.1"
zod: "npm:^3.22.4 || ^4.0.0"
zod-validation-error: "npm:^3.0.3 || ^4.0.0"
zod: "npm:^3.25.0 || ^4.0.0"
zod-validation-error: "npm:^3.5.0 || ^4.0.0"
peerDependencies:
eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0
checksum: 10c0/911c9efdd9b102ce2eabac247dff8c217ecb8d6972aaf3b7eecfb1cfc293d4d902766355993ff7a37a33c0abde3e76971f43bc1c8ff36d6c123310e5680d0423
checksum: 10c0/1e711d1a9d1fa9cfc51fa1572500656577201199c70c795c6a27adfc1df39e5c598f69aab6aa91117753d23cc1f11388579a2bed14921cf9a4efe60ae8618496
languageName: node
linkType: hard
@ -3750,6 +3768,7 @@ __metadata:
"@codemirror/view": "npm:^6.38.6"
"@eslint/js": "npm:^9.38.0"
"@fontsource/inter": "npm:^5.2.8"
"@huggingface/gguf": "npm:^0.3.2"
"@mantine/core": "npm:^8.3.5"
"@mantine/hooks": "npm:^8.3.5"
"@types/node": "npm:^24.9.1"
@ -3759,7 +3778,7 @@ __metadata:
"@typescript-eslint/eslint-plugin": "npm:^8.46.2"
"@typescript-eslint/parser": "npm:^8.46.2"
"@uiw/react-codemirror": "npm:^4.25.2"
"@vitejs/plugin-react": "npm:^5.0.4"
"@vitejs/plugin-react": "npm:^5.1.0"
cross-env: "npm:^10.1.0"
electron: "npm:^38.4.0"
electron-builder: "npm:^26.0.12"
@ -3770,12 +3789,12 @@ __metadata:
eslint-plugin-no-comments: "npm:^1.1.10"
eslint-plugin-promise: "npm:^7.2.1"
eslint-plugin-react: "npm:^7.37.5"
eslint-plugin-react-hooks: "npm:^7.0.0"
eslint-plugin-react-hooks: "npm:^7.0.1"
eslint-plugin-sonarjs: "npm:^3.0.5"
execa: "npm:^9.6.0"
globals: "npm:^16.4.0"
jiti: "npm:^2.6.1"
lucide-react: "npm:^0.546.0"
lucide-react: "npm:^0.548.0"
prettier: "npm:^3.6.2"
react: "npm:^19.2.0"
react-dom: "npm:^19.2.0"
@ -4850,12 +4869,12 @@ __metadata:
languageName: node
linkType: hard
"lucide-react@npm:^0.546.0":
version: 0.546.0
resolution: "lucide-react@npm:0.546.0"
"lucide-react@npm:^0.548.0":
version: 0.548.0
resolution: "lucide-react@npm:0.548.0"
peerDependencies:
react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0
checksum: 10c0/42ee0bd358517f012297aefb69b54da0b3c62f1ac8485ffa24141b63f05c9f4a682eacc2637c4b13f597aed11f9a8c79627af0c17717400bebb25581daeaad80
checksum: 10c0/4b3416982927622a8aad49edd3ed53c7a7c202e7357188f56b8dd582e8f22d33b30fda736440a2e640e0faf9baa724a06d0d3c1de5ecf42d2844eb6b24ddefdd
languageName: node
linkType: hard
@ -5693,10 +5712,10 @@ __metadata:
languageName: node
linkType: hard
"react-refresh@npm:^0.17.0":
version: 0.17.0
resolution: "react-refresh@npm:0.17.0"
checksum: 10c0/002cba940384c9930008c0bce26cac97a9d5682bc623112c2268ba0c155127d9c178a9a5cc2212d560088d60dfd503edd808669a25f9b377f316a32361d0b23c
"react-refresh@npm:^0.18.0":
version: 0.18.0
resolution: "react-refresh@npm:0.18.0"
checksum: 10c0/34a262f7fd803433a534f50deb27a148112a81adcae440c7d1cbae7ef14d21ea8f2b3d783e858cb7698968183b77755a38b4d4b5b1d79b4f4689c2f6d358fff2
languageName: node
linkType: hard
@ -7388,7 +7407,7 @@ __metadata:
languageName: node
linkType: hard
"zod-validation-error@npm:^3.0.3 || ^4.0.0":
"zod-validation-error@npm:^3.5.0 || ^4.0.0":
version: 4.0.2
resolution: "zod-validation-error@npm:4.0.2"
peerDependencies:
@ -7397,7 +7416,7 @@ __metadata:
languageName: node
linkType: hard
"zod@npm:^3.22.4 || ^4.0.0":
"zod@npm:^3.25.0 || ^4.0.0":
version: 4.1.12
resolution: "zod@npm:4.1.12"
checksum: 10c0/b64c1feb19e99d77075261eaf613e0b2be4dfcd3551eff65ad8b4f2a079b61e379854d066f7d447491fcf193f45babd8095551a9d47973d30b46b6d8e2c46774