diff --git a/README.md b/README.md index 3a50b41..f28dd7a 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Gerbil supports the following command-line arguments: ### Considerations -You might want to run CLI Mode if you're looking to use a different frontend, such as OpenWebUI, than the ones bundled (eg. KoboldAI Lite, Stable UI) with KoboldCpp AND you're looking to minimize any resource utilization of this app. Note that at the time of this writing, Gerbil only takes about ~200MB of RAM and ~100MB of VRAM for its Chromium-based UI. When running in CLI Mode, Gerbil will still take about 1/3 of those RAM and VRAM numbers. +You might want to run CLI Mode if you're looking to use a different frontend, such as OpenWebUI, than the ones bundled (eg. KoboldAI Lite, llama.cpp, Stable UI) with KoboldCpp AND you're looking to minimize any resource utilization of this app. Note that at the time of this writing, Gerbil only takes about ~200MB of RAM and ~100MB of VRAM for its Chromium-based UI. When running in CLI Mode, Gerbil will still take about 1/3 of those RAM and VRAM numbers. ### Usage diff --git a/package.json b/package.json index 6eb5d1e..74a1bba 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "gerbil", "productName": "Gerbil", - "version": "1.11.1", + "version": "1.12.0", "description": "Run Large Language Models locally", "main": "out/main/index.js", "homepage": "./", @@ -48,7 +48,7 @@ "@typescript-eslint/parser": "^8.48.0", "@vitejs/plugin-react": "^5.1.1", "cross-env": "^10.1.0", - "electron": "^38.7.1", + "electron": "^38.7.2", "electron-builder": "^26.0.12", "electron-vite": "^4.0.1", "eslint": "^9.39.1", @@ -76,7 +76,7 @@ "@uiw/react-codemirror": "^4.25.3", "electron-updater": "^6.6.2", "execa": "^9.6.0", - "lucide-react": "^0.554.0", + "lucide-react": "^0.555.0", "mime-types": "^3.0.2", "react": "^19.2.0", "react-dom": "^19.2.0", diff --git a/src/components/screens/Launch/CommandLineArgumentsModal.tsx b/src/components/screens/Launch/CommandLineArgumentsModal.tsx index cb3f0fd..96951ef 100644 --- a/src/components/screens/Launch/CommandLineArgumentsModal.tsx +++ b/src/components/screens/Launch/CommandLineArgumentsModal.tsx @@ -66,6 +66,7 @@ const UI_COVERED_ARGS = new Set([ '--sdflashattention', '--tensor_split', '--debugmode', + '--lowvram', ] as const) as ReadonlySet; const IGNORED_ARGS = new Set([ @@ -80,6 +81,13 @@ const IGNORED_ARGS = new Set([ '--exporttemplate', '--nomodel', '--singleinstance', + '--hordeconfig', + '--sdconfig', + '--noblas', + '--nommap', + '--no-mmap', + '--sdnotile', + '--testmemory', ] as const) as ReadonlySet; const COMMAND_LINE_ARGUMENTS = [ @@ -364,14 +372,6 @@ const COMMAND_LINE_ARGUMENTS = [ default: 0, category: 'Performance', }, - { - flag: '--lowvram', - aliases: ['-nkvo', '--no-kv-offload'], - description: - 'If supported by the backend, do not offload KV to GPU (lowvram mode). Not recommended, will be slow.', - type: 'boolean', - category: 'Performance', - }, { flag: '--defaultgenamt', description: @@ -418,6 +418,21 @@ const COMMAND_LINE_ARGUMENTS = [ default: 'AutoGuess', category: 'Advanced', }, + { + flag: '--jinja', + description: + 'Enables using jinja chat template formatting for chat completions endpoint. Other endpoints are unaffected. Tool calls are done without jinja.', + type: 'boolean', + category: 'Advanced', + }, + { + flag: '--jinja_tools', + aliases: ['--jinja-tools'], + description: + 'Enables using jinja chat template formatting for chat completions endpoint. Other endpoints are unaffected. Tool calls are done with jinja.', + type: 'boolean', + category: 'Advanced', + }, { flag: '--forceversion', description: diff --git a/src/components/settings/FrontendInterfaceSelector.tsx b/src/components/settings/FrontendInterfaceSelector.tsx index 19b9ba6..350e4ed 100644 --- a/src/components/settings/FrontendInterfaceSelector.tsx +++ b/src/components/settings/FrontendInterfaceSelector.tsx @@ -16,7 +16,7 @@ interface FrontendRequirement { } interface FrontendConfig { - value: string; + value: FrontendPreference; label: string; requirements?: FrontendRequirement[]; requirementCheck?: () => Promise; @@ -42,9 +42,13 @@ export const FrontendInterfaceSelector = ({ const frontendConfigs: FrontendConfig[] = useMemo( () => [ + { + value: 'llamacpp', + label: FRONTENDS.LLAMA_CPP, + }, { value: 'koboldcpp', - label: 'Built-in', + label: FRONTENDS.KOBOLDAI_LITE, }, { value: 'sillytavern', @@ -93,7 +97,7 @@ export const FrontendInterfaceSelector = ({ (config) => config.value === frontendPreference ); if (currentFrontendConfig && !requirementResults.get(frontendPreference)) { - setFrontendPreference('koboldcpp'); + setFrontendPreference('llamacpp'); } }, [frontendConfigs, frontendPreference, setFrontendPreference]); diff --git a/src/constants/index.ts b/src/constants/index.ts index 999d3f6..9fff1d3 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -78,6 +78,7 @@ export const ROCM = { export const FRONTENDS = { KOBOLDAI_LITE: 'KoboldAI Lite', + LLAMA_CPP: 'llama.cpp', STABLE_UI: 'Stable UI', SILLYTAVERN: 'SillyTavern', OPENWEBUI: 'Open WebUI', diff --git a/src/types/index.d.ts b/src/types/index.d.ts index e8356cd..d0633fe 100644 --- a/src/types/index.d.ts +++ b/src/types/index.d.ts @@ -12,7 +12,11 @@ export type ChatMode = 'text' | 'image'; export type SdConvDirectMode = 'off' | 'vaeonly' | 'full'; -export type FrontendPreference = 'koboldcpp' | 'sillytavern' | 'openwebui'; +export type FrontendPreference = + | 'koboldcpp' + | 'llamacpp' + | 'sillytavern' + | 'openwebui'; export type ImageGenerationFrontendPreference = 'match' | 'builtin'; diff --git a/src/utils/interface.ts b/src/utils/interface.ts index 2a4c617..a83ca56 100644 --- a/src/utils/interface.ts +++ b/src/utils/interface.ts @@ -56,10 +56,20 @@ export function getAvailableInterfaceOptions({ label: FRONTENDS.KOBOLDAI_LITE, }); } + } else if (frontendPreference === 'llamacpp') { + if (isTextMode) { + chatItems.push({ + value: 'chat-text', + label: FRONTENDS.LLAMA_CPP, + }); + } } if (isImageGenerationMode) { - if (effectiveImageFrontend === 'koboldcpp') { + if ( + effectiveImageFrontend === 'koboldcpp' || + effectiveImageFrontend === 'llamacpp' + ) { chatItems.push({ value: 'chat-image', label: FRONTENDS.STABLE_UI, @@ -151,6 +161,13 @@ export function getServerInterfaceInfo({ }; } + if (frontendPreference === 'llamacpp') { + return { + url: isImageGenerationMode ? `${proxyUrl}/sdui` : `${proxyUrl}/lcpp`, + title: isImageGenerationMode ? FRONTENDS.STABLE_UI : FRONTENDS.LLAMA_CPP, + }; + } + return { url: isImageGenerationMode ? `${proxyUrl}/sdui` : proxyUrl, title: isImageGenerationMode diff --git a/yarn.lock b/yarn.lock index 211b0ea..1233ad9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2792,16 +2792,16 @@ __metadata: languageName: node linkType: hard -"electron@npm:^38.7.1": - version: 38.7.1 - resolution: "electron@npm:38.7.1" +"electron@npm:^38.7.2": + version: 38.7.2 + resolution: "electron@npm:38.7.2" dependencies: "@electron/get": "npm:^2.0.0" "@types/node": "npm:^22.7.7" extract-zip: "npm:^2.0.1" bin: electron: cli.js - checksum: 10c0/ef5471b046c34c24cddc97f3dab5d67ef06fef051cc61f1676a6b2e160dc84154cc59572931335bcbaae57d254989e671925eff32a34a11c781a058ccba6e2dd + checksum: 10c0/b79fc6bf44086649c8ff0ffd3572bb1e6ba65993a6a09a32c42f246cd69aaedd5864fac7d3ed4aacdedb03f2d6c21dee9b92d0982c20617e288c185d847b570b languageName: node linkType: hard @@ -3720,7 +3720,7 @@ __metadata: "@uiw/react-codemirror": "npm:^4.25.3" "@vitejs/plugin-react": "npm:^5.1.1" cross-env: "npm:^10.1.0" - electron: "npm:^38.7.1" + electron: "npm:^38.7.2" electron-builder: "npm:^26.0.12" electron-updater: "npm:^6.6.2" electron-vite: "npm:^4.0.1" @@ -3734,7 +3734,7 @@ __metadata: execa: "npm:^9.6.0" globals: "npm:^16.5.0" jiti: "npm:^2.6.1" - lucide-react: "npm:^0.554.0" + lucide-react: "npm:^0.555.0" mime-types: "npm:^3.0.2" prettier: "npm:^3.6.2" react: "npm:^19.2.0" @@ -4794,12 +4794,12 @@ __metadata: languageName: node linkType: hard -"lucide-react@npm:^0.554.0": - version: 0.554.0 - resolution: "lucide-react@npm:0.554.0" +"lucide-react@npm:^0.555.0": + version: 0.555.0 + resolution: "lucide-react@npm:0.555.0" peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 - checksum: 10c0/35b11d6f11e4b00047175fcc52f531156be66e5ae7c1f689d0d18bc7bbf06d9e0bf33c767f449e25d99f105acf420399346787491aafe0b55dc68b8d45937e87 + checksum: 10c0/18f0870307fe2a37cf62473de88845a78107295de1efcf599a8662be3af4239c0b661a57f8e29815e424773e4aad04d199ad4d94d7ae977b7ce7a61be7f64e04 languageName: node linkType: hard