diff --git a/.gitignore b/.gitignore index 7497115..5d338e1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ ae.safetensors clip_l.safetensors t5xxl_fp8_e4m3fn.safetensors flux1-kontext-dev-Q3_K_S.gguf +gemma-3-4b-it.Q8_0.gguf diff --git a/README.md b/README.md index 77fa6e0..df7a088 100644 --- a/README.md +++ b/README.md @@ -90,28 +90,34 @@ The `--cli` argument allows you to use the Friendly Kobold binary as a proxy to ### Considerations -You might want to run CLI Mode if you're looking to use a different frontend, such as SillyTavern or OpenWebUI, than the ones bundled (eg. KoboldAI Lite, Stable UI) with KoboldCpp AND you're looking to minimize any resource utilization of this app. Note that at the time of this writing, Friendly Kobold only takes about ~200MB of RAM and ~100MB of VRAM for its Chromium-based UI. When running in CLI Mode, Friendly Kobold will still take about 1/3 of those RAM and VRAM numbers. +You might want to run CLI Mode if you're looking to use a different frontend, such as OpenWebUI, than the ones bundled (eg. KoboldAI Lite, Stable UI) with KoboldCpp AND you're looking to minimize any resource utilization of this app. Note that at the time of this writing, Friendly Kobold only takes about ~200MB of RAM and ~100MB of VRAM for its Chromium-based UI. When running in CLI Mode, Friendly Kobold will still take about 1/3 of those RAM and VRAM numbers. ### Usage **Linux/macOS:** ```bash -# Basic usage - launch KoboldCpp with no arguments -./friendly-kobold --cli +# Basic usage - launch KoboldCpp launcher with no arguments +friendly-kobold --cli # Pass arguments to KoboldCpp -./friendly-kobold --cli --help -./friendly-kobold --cli --port 5001 --model /path/to/model.gguf +friendly-kobold --cli --help +friendly-kobold --cli --port 5001 --model /path/to/model.gguf # Any KoboldCpp arguments are supported -./friendly-kobold --cli --model /path/to/model.gguf --port 5001 --host 0.0.0.0 --multiuser 2 +friendly-kobold --cli --model /path/to/model.gguf --port 5001 --host 0.0.0.0 --multiuser 2 + +# CLI inception (friendly kobold CLI calling KoboldCpp CLI mode) +# This is the ideal way to run a custom frontend +friendly-kobold --cli --cli --model /path/to/model.gguf --gpulayers 57 --contextsize 8192 --port 5001 --multiuser 1 --flashattention --usemmap --usevulkan ``` **Windows:** CLI mode will only work correctly on Windows if you install Friendly Kobold using the Setup.exe from the github releases. Otherwise there is currently a technical limitation with the Windows portable .exe which will cause it to not display the terminal output correctly nor will it be killable through the standard terminal (Ctrl+C) commands. +You can use the CLI mode on Windows in exactly the same way as in the Linux/macOS examples above, except you'll be calling the "Friendly Kobold.exe". Note that it will not be on your system PATH by default, so you'll need to manually specify the full path to it when callig it from the Windows terminal. + ## For Local Dev ### Prerequisites @@ -139,10 +145,6 @@ CLI mode will only work correctly on Windows if you install Friendly Kobold usin yarn dev ``` -### Future considerations - -It would make a lot of sense to transition this project to Tauri from Electron. The app size should drop from ~110MB to ~10MB; however, users on obsolete OSes (with outdated WebViews) will very likely encounter issues. In addition, I would need to learn Rust to rewrite the BE (Electron main code), but at least we can re-use all the React code. The app would be much smaller, faster and memory efficient, but not work for some users. I think it's a worthy tradeoff. - ## License AGPL v3 License - see LICENSE file for details diff --git a/eslint.config.ts b/eslint.config.ts index 75ec841..41e2890 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -56,6 +56,10 @@ const config = [ rules: { ...reactHooks.configs.recommended.rules, ...react.configs.recommended.rules, + ...importPlugin.configs.recommended.rules, + ...importPlugin.configs.typescript.rules, + ...tseslint.configs.recommended.rules, + ...tseslint.configs.stylistic.rules, '@typescript-eslint/no-unused-vars': [ 'error', @@ -102,6 +106,11 @@ const config = [ 'import/no-default-export': 'error', 'import/prefer-default-export': 'off', + 'import/no-unresolved': 'off', + 'import/no-relative-parent-imports': 'error', + + '@typescript-eslint/array-type': ['error', { default: 'array' }], + '@typescript-eslint/no-require-imports': 'error', 'arrow-body-style': ['error', 'as-needed'], 'prefer-arrow-callback': ['error', { allowNamedFunctions: false }], diff --git a/package.json b/package.json index 00725dc..e5dffd0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "friendly-kobold", "productName": "Friendly Kobold", - "version": "0.8.1", + "version": "0.9.0", "description": "A desktop app for running Large Language Models locally", "main": "out/main/index.js", "homepage": "./", @@ -54,14 +54,13 @@ "devDependencies": { "@eslint/js": "^9.34.0", "@types/node": "^24.3.0", - "@types/react": "^19.1.11", + "@types/react": "^19.1.12", "@types/react-dom": "^19.1.8", - "@types/strip-ansi": "^5.2.1", "@typescript-eslint/eslint-plugin": "^8.41.0", "@typescript-eslint/parser": "^8.41.0", - "@vitejs/plugin-react": "^5.0.1", + "@vitejs/plugin-react": "^5.0.2", "cross-env": "^10.0.0", - "electron": "^37.3.1", + "electron": "^37.4.0", "electron-builder": "^26.0.12", "electron-vite": "^4.0.0", "eslint": "^9.34.0", @@ -86,9 +85,9 @@ "execa": "^9.6.0", "got": "^14.4.7", "lucide-react": "^0.542.0", + "npm": "^11.5.2", "react": "^19.1.1", "react-dom": "^19.1.1", - "strip-ansi": "^7.1.0", "systeminformation": "^5.27.8", "winston": "^3.17.0", "winston-daily-rotate-file": "^5.0.0", diff --git a/src/App.tsx b/src/App.tsx index da64b54..f437c71 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -13,9 +13,7 @@ import { useUpdateChecker } from '@/hooks/useUpdateChecker'; import { useKoboldVersions } from '@/hooks/useKoboldVersions'; import { UI } from '@/constants'; import type { DownloadItem } from '@/types/electron'; -import type { InterfaceTab } from '@/types'; - -type Screen = 'welcome' | 'download' | 'launch' | 'interface'; +import type { InterfaceTab, FrontendPreference, Screen } from '@/types'; export const App = () => { const [currentScreen, setCurrentScreen] = useState(null); @@ -25,6 +23,8 @@ export const App = () => { const [activeInterfaceTab, setActiveInterfaceTab] = useState('terminal'); const [isImageGenerationMode, setIsImageGenerationMode] = useState(false); + const [frontendPreference, setFrontendPreference] = + useState('koboldcpp'); const { updateInfo: binaryUpdateInfo, @@ -46,15 +46,19 @@ export const App = () => { useEffect(() => { const checkInstallation = async () => { try { - const [versions, currentBinaryPath, hasSeenWelcome] = await Promise.all( - [ + const [versions, currentBinaryPath, hasSeenWelcome, preference] = + await Promise.all([ window.electronAPI.kobold.getInstalledVersions(), window.electronAPI.config.get( 'currentKoboldBinary' ) as Promise, window.electronAPI.config.get('hasSeenWelcome') as Promise, - ] - ); + window.electronAPI.config.get( + 'frontendPreference' + ) as Promise, + ]); + + setFrontendPreference(preference || 'koboldcpp'); if (!hasSeenWelcome) { setCurrentScreenWithTransition('welcome'); @@ -214,106 +218,118 @@ export const App = () => { }; return ( - <> - + {currentScreen !== 'welcome' && ( + setSettingsOpened(true)} + /> + )} + - {currentScreen !== 'welcome' && ( - setSettingsOpened(true)} + {currentScreen === null ? ( +
+ + + + Loading... + + +
+ ) : ( + <> + + + + + + + + + + + + + + + + + )} + + {showUpdateModal && binaryUpdateInfo && ( + )} - - {currentScreen === null ? ( -
- - - - Loading... - - -
- ) : ( - <> - - - - - - - - - - - - - - - - - )} - - {showUpdateModal && binaryUpdateInfo && ( - - )} - - setSettingsOpened(false)} - currentScreen={currentScreen || undefined} - /> - setShowEjectModal(false)} - onConfirm={handleEjectConfirm} - /> - - + + { + setSettingsOpened(false); + try { + const preference = (await window.electronAPI.config.get( + 'frontendPreference' + )) as FrontendPreference; + setFrontendPreference(preference || 'koboldcpp'); + } catch (error) { + window.electronAPI.logs.logError( + 'Failed to load frontend preference:', + error as Error + ); + } + }} + currentScreen={currentScreen || undefined} + /> + setShowEjectModal(false)} + onConfirm={handleEjectConfirm} + /> + ); }; diff --git a/src/components/AppHeader.tsx b/src/components/AppHeader.tsx index e683ade..f5435a3 100644 --- a/src/components/AppHeader.tsx +++ b/src/components/AppHeader.tsx @@ -12,16 +12,16 @@ import { } from '@mantine/core'; import { Settings, ArrowLeft } from 'lucide-react'; import { soundAssets, playSound, initializeAudio } from '@/utils'; -import type { InterfaceTab } from '@/types'; import iconUrl from '/icon.png'; - -type Screen = 'welcome' | 'download' | 'launch' | 'interface'; +import { FRONTENDS } from '@/constants'; +import type { InterfaceTab, FrontendPreference, Screen } from '@/types'; interface AppHeaderProps { currentScreen: Screen | null; activeInterfaceTab: InterfaceTab; setActiveInterfaceTab: (tab: InterfaceTab) => void; isImageGenerationMode: boolean; + frontendPreference: FrontendPreference; onEject: () => void; onSettingsOpen: () => void; } @@ -31,6 +31,7 @@ export const AppHeader = ({ activeInterfaceTab, setActiveInterfaceTab, isImageGenerationMode, + frontendPreference, onEject, onSettingsOpen, }: AppHeaderProps) => { @@ -117,7 +118,12 @@ export const AppHeader = ({ data={[ { value: 'chat', - label: isImageGenerationMode ? 'Stable UI' : 'KoboldAI Lite', + label: + frontendPreference === 'sillytavern' + ? FRONTENDS.SILLYTAVERN + : isImageGenerationMode + ? FRONTENDS.STABLE_UI + : FRONTENDS.KOBOLDAI_LITE, }, { value: 'terminal', label: 'Terminal' }, ]} diff --git a/src/components/screens/Interface/ServerTab.tsx b/src/components/screens/Interface/ServerTab.tsx index a369b22..7a92e1a 100644 --- a/src/components/screens/Interface/ServerTab.tsx +++ b/src/components/screens/Interface/ServerTab.tsx @@ -1,18 +1,20 @@ import { useRef } from 'react'; import { Box, Text, Stack } from '@mantine/core'; -import { UI } from '@/constants'; -import type { ServerTabMode } from '@/types'; +import { UI, SILLYTAVERN, FRONTENDS } from '@/constants'; +import type { ServerTabMode, FrontendPreference } from '@/types'; interface ServerTabProps { serverUrl?: string; isServerReady?: boolean; mode: ServerTabMode; + frontendPreference?: FrontendPreference; } export const ServerTab = ({ serverUrl, isServerReady, mode, + frontendPreference = 'koboldcpp', }: ServerTabProps) => { const iframeRef = useRef(null); @@ -29,7 +31,7 @@ export const ServerTab = ({ > - Waiting for KoboldCpp server to start... + Waiting for the KoboldCpp server to start... The {mode === 'chat' ? 'chat' : 'image generation'} interface will @@ -40,12 +42,19 @@ export const ServerTab = ({ ); } - const iframeUrl = - mode === 'image-generation' ? `${serverUrl}/sdui` : serverUrl; - const title = - mode === 'image-generation' - ? 'Stable UI Interface' - : 'KoboldAI Lite Interface'; + let iframeUrl: string; + let title: string; + + if (frontendPreference === 'sillytavern') { + iframeUrl = SILLYTAVERN.PROXY_URL; + title = FRONTENDS.SILLYTAVERN; + } else { + iframeUrl = mode === 'image-generation' ? `${serverUrl}/sdui` : serverUrl; + title = + mode === 'image-generation' + ? FRONTENDS.STABLE_UI + : FRONTENDS.KOBOLDAI_LITE; + } return ( void; + onServerReady: (url: string) => void; } export const TerminalTab = ({ onServerReady }: TerminalTabProps) => { + const { host, port } = useLaunchConfigStore(); const computedColorScheme = useComputedColorScheme('light', { getInitialValueInEffect: false, }); @@ -57,16 +59,19 @@ export const TerminalTab = ({ onServerReady }: TerminalTabProps) => { setTerminalContent((prev) => { const newData = data.toString(); - if ( - onServerReady && - newData.includes('Please connect to custom endpoint at ') - ) { - const match = newData.match( - /Please connect to custom endpoint at (http:\/\/[^\s]+)/ - ); - if (match) { - const serverUrl = match[1]; - setTimeout(() => onServerReady(serverUrl), 1500); + if (onServerReady) { + if ( + newData.includes('Please connect to custom endpoint at') || + newData.includes( + 'Now running KoboldCpp in Interactive Terminal Chat mode' + ) + ) { + const serverHost = host || 'localhost'; + const serverPort = port || 5001; + setTimeout( + () => onServerReady(`http://${serverHost}:${serverPort}`), + 1500 + ); } } @@ -75,7 +80,7 @@ export const TerminalTab = ({ onServerReady }: TerminalTabProps) => { }); return cleanup; - }, [onServerReady]); + }, [onServerReady, host, port]); const scrollToBottom = () => { if (viewportRef.current) { @@ -126,9 +131,10 @@ export const TerminalTab = ({ onServerReady }: TerminalTabProps) => { whiteSpace: 'pre-wrap', wordBreak: 'break-word', }} - > - {terminalContent} - + dangerouslySetInnerHTML={{ + __html: processTerminalContent(terminalContent), + }} + /> )} diff --git a/src/components/screens/Interface/index.tsx b/src/components/screens/Interface/index.tsx index 1455286..fbce9ed 100644 --- a/src/components/screens/Interface/index.tsx +++ b/src/components/screens/Interface/index.tsx @@ -1,7 +1,7 @@ -import { useState, useCallback } from 'react'; +import { useState, useCallback, useEffect } from 'react'; import { ServerTab } from '@/components/screens/Interface/ServerTab'; import { TerminalTab } from '@/components/screens/Interface/TerminalTab'; -import type { InterfaceTab } from '@/types'; +import type { InterfaceTab, FrontendPreference } from '@/types'; interface InterfaceScreenProps { activeTab?: InterfaceTab | null; @@ -16,6 +16,26 @@ export const InterfaceScreen = ({ }: InterfaceScreenProps) => { const [serverUrl, setServerUrl] = useState(''); const [isServerReady, setIsServerReady] = useState(false); + const [frontendPreference, setFrontendPreference] = + useState('koboldcpp'); + + useEffect(() => { + const loadFrontendPreference = async () => { + try { + const frontendPreference = (await window.electronAPI.config.get( + 'frontendPreference' + )) as FrontendPreference; + setFrontendPreference(frontendPreference || 'koboldcpp'); + } catch (error) { + window.electronAPI.logs.logError( + 'Failed to load frontend preference:', + error as Error + ); + } + }; + + loadFrontendPreference(); + }, []); const handleServerReady = useCallback( (url: string) => { @@ -47,6 +67,7 @@ export const InterfaceScreen = ({ serverUrl={serverUrl} isServerReady={isServerReady} mode={isImageGenerationMode ? 'image-generation' : 'chat'} + frontendPreference={frontendPreference} />
; export const BackendSelectItem = ({ label, diff --git a/src/components/screens/Launch/GeneralTab/BackendSelector.tsx b/src/components/screens/Launch/GeneralTab/BackendSelector.tsx index 1d95917..c9f1ae2 100644 --- a/src/components/screens/Launch/GeneralTab/BackendSelector.tsx +++ b/src/components/screens/Launch/GeneralTab/BackendSelector.tsx @@ -4,6 +4,7 @@ import { InfoTooltip } from '@/components/InfoTooltip'; import { BackendSelectItem } from '@/components/screens/Launch/GeneralTab/BackendSelectItem'; import { GpuDeviceSelector } from '@/components/screens/Launch/GeneralTab/GpuDeviceSelector'; import { useLaunchConfig } from '@/hooks/useLaunchConfig'; +import type { BackendOption } from '@/types'; export const BackendSelector = () => { const { @@ -15,88 +16,16 @@ export const BackendSelector = () => { handleAutoGpuLayersChange, } = useLaunchConfig(); - const [availableBackends, setAvailableBackends] = useState< - Array<{ - value: string; - label: string; - devices?: string[]; - disabled?: boolean; - }> - >([]); + const [availableBackends, setAvailableBackends] = useState( + [] + ); const hasInitialized = useRef(false); useEffect(() => { const loadBackends = async () => { try { - const [cpuCapabilitiesResult, binarySupport, gpuCapabilities] = - await Promise.all([ - window.electronAPI.kobold.detectCPU(), - window.electronAPI.kobold.detectBackendSupport(), - window.electronAPI.kobold.detectGPUCapabilities(), - ]); - - if (!binarySupport) { - setAvailableBackends([]); - hasInitialized.current = true; - - return; - } - - const backends: Array<{ - value: string; - label: string; - devices?: string[]; - disabled?: boolean; - }> = []; - - if (binarySupport.cuda) { - backends.push({ - value: 'cuda', - label: 'CUDA', - devices: gpuCapabilities.cuda.devices, - disabled: !gpuCapabilities.cuda.supported, - }); - } - - if (binarySupport.rocm) { - backends.push({ - value: 'rocm', - label: 'ROCm', - devices: gpuCapabilities.rocm.devices, - disabled: !gpuCapabilities.rocm.supported, - }); - } - - if (binarySupport.vulkan) { - backends.push({ - value: 'vulkan', - label: 'Vulkan', - devices: gpuCapabilities.vulkan.devices, - disabled: !gpuCapabilities.vulkan.supported, - }); - } - - if (binarySupport.clblast) { - backends.push({ - value: 'clblast', - label: 'CLBlast', - devices: gpuCapabilities.clblast.devices, - disabled: !gpuCapabilities.clblast.supported, - }); - } - - backends.push({ - value: 'cpu', - label: 'CPU', - devices: cpuCapabilitiesResult.devices, - disabled: false, - }); - - backends.sort((a, b) => { - if (a.disabled === b.disabled) return 0; - return a.disabled ? 1 : -1; - }); - + const backends = + await window.electronAPI.kobold.getAvailableBackends(true); setAvailableBackends(backends); hasInitialized.current = true; } catch (error) { diff --git a/src/components/screens/Launch/GeneralTab/GpuDeviceSelector.tsx b/src/components/screens/Launch/GeneralTab/GpuDeviceSelector.tsx index fa691b6..d75b42a 100644 --- a/src/components/screens/Launch/GeneralTab/GpuDeviceSelector.tsx +++ b/src/components/screens/Launch/GeneralTab/GpuDeviceSelector.tsx @@ -1,13 +1,10 @@ import { Text, Group, Select, TextInput } from '@mantine/core'; import { InfoTooltip } from '@/components/InfoTooltip'; import { useLaunchConfig } from '@/hooks/useLaunchConfig'; +import type { BackendOption } from '@/types'; interface GpuDeviceSelectorProps { - availableBackends: Array<{ - value: string; - label: string; - devices?: string[]; - }>; + availableBackends: BackendOption[]; } export const GpuDeviceSelector = ({ diff --git a/src/components/settings/AppearanceTab.tsx b/src/components/settings/AppearanceTab.tsx index 41f7368..400cbc6 100644 --- a/src/components/settings/AppearanceTab.tsx +++ b/src/components/settings/AppearanceTab.tsx @@ -1,5 +1,12 @@ -import { Stack, Text, Group, SegmentedControl, rem } from '@mantine/core'; -import { useMantineColorScheme, useComputedColorScheme } from '@mantine/core'; +import { + Stack, + Text, + Group, + SegmentedControl, + rem, + useMantineColorScheme, + useComputedColorScheme, +} from '@mantine/core'; import { Sun, Moon, Monitor } from 'lucide-react'; export const AppearanceTab = () => { diff --git a/src/components/settings/GeneralTab.tsx b/src/components/settings/GeneralTab.tsx index 3e67221..27e6160 100644 --- a/src/components/settings/GeneralTab.tsx +++ b/src/components/settings/GeneralTab.tsx @@ -1,13 +1,26 @@ import { useState, useEffect } from 'react'; -import { Stack, Text, Group, TextInput, Button, rem } from '@mantine/core'; -import { Folder, FolderOpen } from 'lucide-react'; +import { + Stack, + Text, + Group, + TextInput, + Button, + rem, + Select, +} from '@mantine/core'; +import { Folder, FolderOpen, Monitor } from 'lucide-react'; import styles from '@/styles/layout.module.css'; +import type { FrontendPreference } from '@/types'; +import { FRONTENDS } from '@/constants'; export const GeneralTab = () => { const [installDir, setInstallDir] = useState(''); + const [FrontendPreference, setFrontendPreference] = + useState('koboldcpp'); useEffect(() => { loadCurrentInstallDir(); + loadFrontendPreference(); }, []); const loadCurrentInstallDir = async () => { @@ -22,6 +35,20 @@ export const GeneralTab = () => { } }; + const loadFrontendPreference = async () => { + try { + const frontendPreference = (await window.electronAPI.config.get( + 'frontendPreference' + )) as FrontendPreference; + setFrontendPreference(frontendPreference || 'koboldcpp'); + } catch (error) { + window.electronAPI.logs.logError( + 'Failed to load frontend preference:', + error as Error + ); + } + }; + const handleSelectInstallDir = async () => { try { const selectedDir = @@ -38,6 +65,20 @@ export const GeneralTab = () => { } }; + const handleFrontendPreferenceChange = async (value: string | null) => { + if (!value || (value !== 'koboldcpp' && value !== 'sillytavern')) return; + + try { + await window.electronAPI.config.set('frontendPreference', value); + setFrontendPreference(value); + } catch (error) { + window.electronAPI.logs.logError( + 'Failed to save frontend preference:', + error as Error + ); + } + }; + return (
@@ -66,6 +107,27 @@ export const GeneralTab = () => {
+ +
+ + Frontend Interface + + + Choose which frontend interface to use for interacting with models + +