Skip to content

Commit 6494f5a

Browse files
thecodacusthecodacus
authored
fix: updated logger and model caching minor bugfix #release (#895)
* fix: updated logger and model caching * usage token stream issue fix * minor changes * updated starter template change to fix the app title * starter template bigfix * fixed hydretion errors and raw logs * removed raw log * made auto select template false by default * more cleaner logs and updated logic to call dynamicModels only if not found in static models * updated starter template instructions * browser console log improved for firefox * provider icons fix icons
1 parent 389eedc commit 6494f5a

File tree

23 files changed

+473
-562
lines changed

23 files changed

+473
-562
lines changed

‎app/components/chat/BaseChat.tsx‎

Lines changed: 48 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -168,30 +168,32 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
168168
}, []);
169169

170170
useEffect(() => {
171-
const providerSettings = getProviderSettings();
172-
let parsedApiKeys: Record<string, string> | undefined = {};
171+
if (typeof window !== 'undefined') {
172+
const providerSettings = getProviderSettings();
173+
let parsedApiKeys: Record<string, string> | undefined = {};
173174

174-
try {
175-
parsedApiKeys = getApiKeysFromCookies();
176-
setApiKeys(parsedApiKeys);
177-
} catch (error) {
178-
console.error('Error loading API keys from cookies:', error);
175+
try {
176+
parsedApiKeys = getApiKeysFromCookies();
177+
setApiKeys(parsedApiKeys);
178+
} catch (error) {
179+
console.error('Error loading API keys from cookies:', error);
179180

180-
// Clear invalid cookie data
181-
Cookies.remove('apiKeys');
181+
// Clear invalid cookie data
182+
Cookies.remove('apiKeys');
183+
}
184+
setIsModelLoading('all');
185+
initializeModelList({ apiKeys: parsedApiKeys, providerSettings })
186+
.then((modelList) => {
187+
// console.log('Model List: ', modelList);
188+
setModelList(modelList);
189+
})
190+
.catch((error) => {
191+
console.error('Error initializing model list:', error);
192+
})
193+
.finally(() => {
194+
setIsModelLoading(undefined);
195+
});
182196
}
183-
setIsModelLoading('all');
184-
initializeModelList({ apiKeys: parsedApiKeys, providerSettings })
185-
.then((modelList) => {
186-
console.log('Model List: ', modelList);
187-
setModelList(modelList);
188-
})
189-
.catch((error) => {
190-
console.error('Error initializing model list:', error);
191-
})
192-
.finally(() => {
193-
setIsModelLoading(undefined);
194-
});
195197
}, [providerList]);
196198

197199
const onApiKeysChange = async (providerName: string, apiKey: string) => {
@@ -401,28 +403,32 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
401403
<rect className={classNames(styles.PromptShine)} x="48" y="24" width="70" height="1"></rect>
402404
</svg>
403405
<div>
404-
<div className={isModelSettingsCollapsed ? 'hidden' : ''}>
405-
<ModelSelector
406-
key={provider?.name + ':' + modelList.length}
407-
model={model}
408-
setModel={setModel}
409-
modelList={modelList}
410-
provider={provider}
411-
setProvider={setProvider}
412-
providerList={providerList || (PROVIDER_LIST as ProviderInfo[])}
413-
apiKeys={apiKeys}
414-
modelLoading={isModelLoading}
415-
/>
416-
{(providerList || []).length > 0 && provider && (
417-
<APIKeyManager
418-
provider={provider}
419-
apiKey={apiKeys[provider.name] || ''}
420-
setApiKey={(key) => {
421-
onApiKeysChange(provider.name, key);
422-
}}
423-
/>
406+
<ClientOnly>
407+
{() => (
408+
<div className={isModelSettingsCollapsed ? 'hidden' : ''}>
409+
<ModelSelector
410+
key={provider?.name + ':' + modelList.length}
411+
model={model}
412+
setModel={setModel}
413+
modelList={modelList}
414+
provider={provider}
415+
setProvider={setProvider}
416+
providerList={providerList || (PROVIDER_LIST as ProviderInfo[])}
417+
apiKeys={apiKeys}
418+
modelLoading={isModelLoading}
419+
/>
420+
{(providerList || []).length > 0 && provider && (
421+
<APIKeyManager
422+
provider={provider}
423+
apiKey={apiKeys[provider.name] || ''}
424+
setApiKey={(key) => {
425+
onApiKeysChange(provider.name, key);
426+
}}
427+
/>
428+
)}
429+
</div>
424430
)}
425-
</div>
431+
</ClientOnly>
426432
</div>
427433
<FilePreview
428434
files={uploadedFiles}

‎app/components/chat/Chat.client.tsx‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ export const ChatImpl = memo(
168168
});
169169
useEffect(() => {
170170
const prompt = searchParams.get('prompt');
171-
console.log(prompt, searchParams, model, provider);
171+
172+
// console.log(prompt, searchParams, model, provider);
172173

173174
if (prompt) {
174175
setSearchParams({});
@@ -289,14 +290,14 @@ export const ChatImpl = memo(
289290

290291
// reload();
291292

292-
const template = await selectStarterTemplate({
293+
const { template, title } = await selectStarterTemplate({
293294
message: messageInput,
294295
model,
295296
provider,
296297
});
297298

298299
if (template !== 'blank') {
299-
const temResp = await getTemplates(template);
300+
const temResp = await getTemplates(template, title);
300301

301302
if (temResp) {
302303
const { assistantMessage, userMessage } = temResp;

‎app/components/settings/providers/ProvidersTab.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,10 @@ import type { IProviderConfig } from '~/types/model';
66
import { logStore } from '~/lib/stores/logs';
77

88
// Import a default fallback icon
9-
import DefaultIcon from '/icons/Default.svg'; // Adjust the path as necessary
109
import { providerBaseUrlEnvKeys } from '~/utils/constants';
1110

11+
const DefaultIcon = '/icons/Default.svg'; // Adjust the path as necessary
12+
1213
export default function ProvidersTab() {
1314
const { providers, updateProviderSettings, isLocalModel } = useSettings();
1415
const [filteredProviders, setFilteredProviders] = useState<IProviderConfig[]>([]);

‎app/entry.server.tsx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import { renderToReadableStream } from 'react-dom/server';
55
import { renderHeadToString } from 'remix-island';
66
import { Head } from './root';
77
import { themeStore } from '~/lib/stores/theme';
8-
import { initializeModelList } from '~/utils/constants';
98

109
export default async function handleRequest(
1110
request: Request,
@@ -14,7 +13,7 @@ export default async function handleRequest(
1413
remixContext: EntryContext,
1514
_loadContext: AppLoadContext,
1615
) {
17-
await initializeModelList({});
16+
// await initializeModelList({});
1817

1918
const readable = await renderToReadableStream(<RemixServer context={remixContext} url={request.url} />, {
2019
signal: request.signal,

‎app/lib/.server/llm/stream-text.ts‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { getSystemPrompt } from '~/lib/common/prompts/prompts';
44
import {
55
DEFAULT_MODEL,
66
DEFAULT_PROVIDER,
7-
getModelList,
87
MODEL_REGEX,
98
MODIFICATIONS_TAG_NAME,
109
PROVIDER_LIST,
@@ -15,6 +14,8 @@ import ignore from 'ignore';
1514
import type { IProviderSetting } from '~/types/model';
1615
import { PromptLibrary } from '~/lib/common/prompt-library';
1716
import { allowedHTMLElements } from '~/utils/markdown';
17+
import { LLMManager } from '~/lib/modules/llm/manager';
18+
import { createScopedLogger } from '~/utils/logger';
1819

1920
interface ToolResult<Name extends string, Args, Result> {
2021
toolCallId: string;
@@ -142,6 +143,8 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid
142143
return { model, provider, content: cleanedContent };
143144
}
144145

146+
const logger = createScopedLogger('stream-text');
147+
145148
export async function streamText(props: {
146149
messages: Messages;
147150
env: Env;
@@ -158,15 +161,10 @@ export async function streamText(props: {
158161

159162
let currentModel = DEFAULT_MODEL;
160163
let currentProvider = DEFAULT_PROVIDER.name;
161-
const MODEL_LIST = await getModelList({ apiKeys, providerSettings, serverEnv: serverEnv as any });
162164
const processedMessages = messages.map((message) => {
163165
if (message.role === 'user') {
164166
const { model, provider, content } = extractPropertiesFromMessage(message);
165-
166-
if (MODEL_LIST.find((m) => m.name === model)) {
167-
currentModel = model;
168-
}
169-
167+
currentModel = model;
170168
currentProvider = provider;
171169

172170
return { ...message, content };
@@ -183,11 +181,36 @@ export async function streamText(props: {
183181
return message;
184182
});
185183

186-
const modelDetails = MODEL_LIST.find((m) => m.name === currentModel);
184+
const provider = PROVIDER_LIST.find((p) => p.name === currentProvider) || DEFAULT_PROVIDER;
185+
const staticModels = LLMManager.getInstance().getStaticModelListFromProvider(provider);
186+
let modelDetails = staticModels.find((m) => m.name === currentModel);
187+
188+
if (!modelDetails) {
189+
const modelsList = [
190+
...(provider.staticModels || []),
191+
...(await LLMManager.getInstance().getModelListFromProvider(provider, {
192+
apiKeys,
193+
providerSettings,
194+
serverEnv: serverEnv as any,
195+
})),
196+
];
197+
198+
if (!modelsList.length) {
199+
throw new Error(`No models found for provider ${provider.name}`);
200+
}
187201

188-
const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS;
202+
modelDetails = modelsList.find((m) => m.name === currentModel);
189203

190-
const provider = PROVIDER_LIST.find((p) => p.name === currentProvider) || DEFAULT_PROVIDER;
204+
if (!modelDetails) {
205+
// Fallback to first model
206+
logger.warn(
207+
`MODEL [${currentModel}] not found in provider [${provider.name}]. Falling back to first model. ${modelsList[0].name}`,
208+
);
209+
modelDetails = modelsList[0];
210+
}
211+
}
212+
213+
const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS;
191214

192215
let systemPrompt =
193216
PromptLibrary.getPropmtFromLibrary(promptId || 'default', {
@@ -201,6 +224,8 @@ export async function streamText(props: {
201224
systemPrompt = `${systemPrompt}\n\n ${codeContext}`;
202225
}
203226

227+
logger.info(`Sending llm call to ${provider.name} with model ${modelDetails.name}`);
228+
204229
return _streamText({
205230
model: provider.getModelInstance({
206231
model: currentModel,

‎app/lib/modules/llm/base-provider.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ export abstract class BaseProvider implements ProviderInfo {
88
abstract name: string;
99
abstract staticModels: ModelInfo[];
1010
abstract config: ProviderConfig;
11+
cachedDynamicModels?: {
12+
cacheId: string;
13+
models: ModelInfo[];
14+
};
1115

1216
getApiKeyLink?: string;
1317
labelForGetApiKey?: string;
@@ -49,6 +53,54 @@ export abstract class BaseProvider implements ProviderInfo {
4953
apiKey,
5054
};
5155
}
56+
getModelsFromCache(options: {
57+
apiKeys?: Record<string, string>;
58+
providerSettings?: Record<string, IProviderSetting>;
59+
serverEnv?: Record<string, string>;
60+
}): ModelInfo[] | null {
61+
if (!this.cachedDynamicModels) {
62+
// console.log('no dynamic models',this.name);
63+
return null;
64+
}
65+
66+
const cacheKey = this.cachedDynamicModels.cacheId;
67+
const generatedCacheKey = this.getDynamicModelsCacheKey(options);
68+
69+
if (cacheKey !== generatedCacheKey) {
70+
// console.log('cache key mismatch',this.name,cacheKey,generatedCacheKey);
71+
this.cachedDynamicModels = undefined;
72+
return null;
73+
}
74+
75+
return this.cachedDynamicModels.models;
76+
}
77+
getDynamicModelsCacheKey(options: {
78+
apiKeys?: Record<string, string>;
79+
providerSettings?: Record<string, IProviderSetting>;
80+
serverEnv?: Record<string, string>;
81+
}) {
82+
return JSON.stringify({
83+
apiKeys: options.apiKeys?.[this.name],
84+
providerSettings: options.providerSettings?.[this.name],
85+
serverEnv: options.serverEnv,
86+
});
87+
}
88+
storeDynamicModels(
89+
options: {
90+
apiKeys?: Record<string, string>;
91+
providerSettings?: Record<string, IProviderSetting>;
92+
serverEnv?: Record<string, string>;
93+
},
94+
models: ModelInfo[],
95+
) {
96+
const cacheId = this.getDynamicModelsCacheKey(options);
97+
98+
// console.log('caching dynamic models',this.name,cacheId);
99+
this.cachedDynamicModels = {
100+
cacheId,
101+
models,
102+
};
103+
}
52104

53105
// Declare the optional getDynamicModels method
54106
getDynamicModels?(

0 commit comments

Comments
 (0)