1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.llm;
17
18 import java.util.List;
19 import java.util.Map;
20 import java.util.concurrent.CopyOnWriteArrayList;
21
22 import org.apache.logging.log4j.LogManager;
23 import org.apache.logging.log4j.Logger;
24 import org.codelibs.fess.Constants;
25 import org.codelibs.fess.util.ComponentUtil;
26
27
28
29
30
31
32
33
34 public class LlmClientManager {
35
36 private static final Logger logger = LogManager.getLogger(LlmClientManager.class);
37
38
39 protected final List<LlmClient> clientList = new CopyOnWriteArrayList<>();
40
41
42
43
44 public LlmClientManager() {
45
46 }
47
48
49
50
51
52
53 public boolean available() {
54 final String llmType = getLlmType();
55 if (Constants.NONE.equals(llmType)) {
56 if (logger.isTraceEnabled()) {
57 logger.trace("[LLM] LLM not available. llmType=none");
58 }
59 return false;
60 }
61 if (!isRagChatEnabled()) {
62 if (logger.isTraceEnabled()) {
63 logger.trace("[LLM] LLM not available. ragChatEnabled=false");
64 }
65 return false;
66 }
67 final LlmClient client = getClient();
68 final boolean isAvailable = client != null && client.isAvailable();
69 if (logger.isTraceEnabled()) {
70 logger.trace("[LLM] LLM availability check. llmType={}, clientFound={}, isAvailable={}", llmType, client != null, isAvailable);
71 }
72 return isAvailable;
73 }
74
75
76
77
78
79
80 public LlmClient getClient() {
81 final String llmType = getLlmType();
82 final String name = llmType + "LlmClient";
83 if (ComponentUtil.hasComponent(name)) {
84 final LlmClient client = ComponentUtil.getComponent(name);
85 if (logger.isTraceEnabled()) {
86 logger.trace("[LLM] LlmClient found via DI. componentName={}, clientName={}, available={}", name, client.getName(),
87 client.isAvailable());
88 }
89 return client;
90 }
91
92 for (final LlmClient client : clientList) {
93 if (llmType.equals(client.getName())) {
94 if (logger.isTraceEnabled()) {
95 logger.trace("[LLM] LlmClient found via registration. name={}", client.getName());
96 }
97 return client;
98 }
99 }
100 logger.warn("[LLM] LlmClient not found. componentName={}", name);
101 return null;
102 }
103
104
105
106
107
108
109 protected String getLlmType() {
110 return ComponentUtil.getFessConfig().getSystemProperty("rag.llm.name", "ollama");
111 }
112
113
114
115
116
117
118 protected boolean isRagChatEnabled() {
119 return ComponentUtil.getFessConfig().isRagChatEnabled();
120 }
121
122
123
124
125
126
127 public LlmClient[] getClients() {
128 return clientList.toArray(new LlmClient[clientList.size()]);
129 }
130
131
132
133
134
135
136 public void register(final LlmClient client) {
137 if (logger.isDebugEnabled()) {
138 logger.debug("Loaded LlmClient: {}", client.getClass().getSimpleName());
139 }
140 clientList.add(client);
141 }
142
143
144
145
146
147
148
149
150 public LlmChatResponse chat(final LlmChatRequest request) {
151 final long startTime = System.currentTimeMillis();
152 final String llmType = getLlmType();
153 if (logger.isDebugEnabled()) {
154 logger.debug("[LLM] Starting LLM chat request. llmType={}, messageCount={}", llmType, request.getMessages().size());
155 for (final LlmMessage msg : request.getMessages()) {
156 logger.debug("[LLM] message: role={}, content={}", msg.getRole(), msg.getContent());
157 }
158 }
159 try {
160 final LlmClient client = getAvailableClient();
161 if (logger.isDebugEnabled()) {
162 logger.debug("[LLM] Using LLM client. clientName={}", client.getName());
163 }
164 final LlmChatResponse response = client.chat(request);
165 if (logger.isDebugEnabled()) {
166 logger.debug("[LLM] LLM chat request completed. llmType={}", llmType);
167 }
168 return response;
169 } catch (final LlmException e) {
170 logger.warn("[LLM] Chat request failed. llmType={}, error={}, elapsedTime={}ms", llmType, e.getMessage(),
171 System.currentTimeMillis() - startTime);
172 throw e;
173 } catch (final Exception e) {
174 logger.warn("[LLM] Chat request failed with unexpected error. llmType={}, error={}, elapsedTime={}ms", llmType, e.getMessage(),
175 System.currentTimeMillis() - startTime, e);
176 throw new LlmException("LLM chat request failed", e);
177 }
178 }
179
180
181
182
183
184
185
186
187 public void streamChat(final LlmChatRequest request, final LlmStreamCallback callback) {
188 final long startTime = System.currentTimeMillis();
189 final String llmType = getLlmType();
190 if (logger.isDebugEnabled()) {
191 logger.debug("[LLM] Starting LLM streaming chat request. llmType={}, messageCount={}", llmType, request.getMessages().size());
192 for (final LlmMessage msg : request.getMessages()) {
193 logger.debug("[LLM] message: role={}, content={}", msg.getRole(), msg.getContent());
194 }
195 }
196 try {
197 final LlmClient client = getAvailableClient();
198 if (logger.isDebugEnabled()) {
199 logger.debug("[LLM] Using LLM client for streaming. clientName={}", client.getName());
200 }
201 client.streamChat(request, callback);
202 if (logger.isDebugEnabled()) {
203 logger.debug("[LLM] LLM streaming chat request completed. llmType={}", llmType);
204 }
205 } catch (final LlmException e) {
206 logger.warn("[LLM] Stream chat request failed. llmType={}, error={}, elapsedTime={}ms", llmType, e.getMessage(),
207 System.currentTimeMillis() - startTime);
208 throw e;
209 } catch (final Exception e) {
210 logger.warn("[LLM] Stream chat request failed with unexpected error. llmType={}, error={}, elapsedTime={}ms", llmType,
211 e.getMessage(), System.currentTimeMillis() - startTime, e);
212 throw new LlmException("LLM streaming chat request failed", e);
213 }
214 }
215
216
217
218
219
220
221
222 protected LlmClient getAvailableClient() {
223 final String llmType = getLlmType();
224 if (Constants.NONE.equals(llmType)) {
225 throw new LlmException("LLM client is not available");
226 }
227 if (!isRagChatEnabled()) {
228 throw new LlmException("LLM client is not available");
229 }
230 final LlmClient client = getClient();
231 if (client == null || !client.isAvailable()) {
232 throw new LlmException("LLM client is not available");
233 }
234 return client;
235 }
236
237
238
239
240
241
242
243
244
245
246 public IntentDetectionResult detectIntent(final String userMessage) {
247 if (logger.isDebugEnabled()) {
248 logger.debug("[LLM] Delegating detectIntent. llmType={}", getLlmType());
249 }
250 return getAvailableClient().detectIntent(userMessage);
251 }
252
253
254
255
256
257
258
259
260
261 public IntentDetectionResult detectIntent(final String userMessage, final List<LlmMessage> history) {
262 if (logger.isDebugEnabled()) {
263 logger.debug("[LLM] Delegating detectIntent with history. llmType={}, historySize={}", getLlmType(),
264 history != null ? history.size() : 0);
265 }
266 return getAvailableClient().detectIntent(userMessage, history);
267 }
268
269
270
271
272
273
274
275
276
277
278 public RelevanceEvaluationResult evaluateResults(final String userMessage, final String query,
279 final List<Map<String, Object>> searchResults) {
280 if (logger.isDebugEnabled()) {
281 logger.debug("[LLM] Delegating evaluateResults. llmType={}", getLlmType());
282 }
283 return getAvailableClient().evaluateResults(userMessage, query, searchResults);
284 }
285
286
287
288
289
290
291
292
293
294
295 public LlmChatResponse generateAnswer(final String userMessage, final List<Map<String, Object>> documents,
296 final List<LlmMessage> history) {
297 if (logger.isDebugEnabled()) {
298 logger.debug("[LLM] Delegating generateAnswer. llmType={}", getLlmType());
299 }
300 return getAvailableClient().generateAnswer(userMessage, documents, history);
301 }
302
303
304
305
306
307
308
309
310
311
312
313 public String regenerateQuery(final String userMessage, final String failedQuery, final String failureReason,
314 final List<LlmMessage> history) {
315 if (logger.isDebugEnabled()) {
316 logger.debug("[LLM] Delegating regenerateQuery. llmType={}", getLlmType());
317 }
318 return getAvailableClient().regenerateQuery(userMessage, failedQuery, failureReason, history);
319 }
320
321
322
323
324
325
326
327
328
329
330 public void streamGenerateAnswer(final String userMessage, final List<Map<String, Object>> documents, final List<LlmMessage> history,
331 final LlmStreamCallback callback) {
332 if (logger.isDebugEnabled()) {
333 logger.debug("[LLM] Delegating streamGenerateAnswer. llmType={}", getLlmType());
334 }
335 getAvailableClient().streamGenerateAnswer(userMessage, documents, history, callback);
336 }
337
338
339
340
341
342
343
344
345
346 public void generateUnclearIntentResponse(final String userMessage, final List<LlmMessage> history, final LlmStreamCallback callback) {
347 if (logger.isDebugEnabled()) {
348 logger.debug("[LLM] Delegating generateUnclearIntentResponse. llmType={}", getLlmType());
349 }
350 getAvailableClient().generateUnclearIntentResponse(userMessage, history, callback);
351 }
352
353
354
355
356
357
358
359
360
361 public void generateNoResultsResponse(final String userMessage, final List<LlmMessage> history, final LlmStreamCallback callback) {
362 if (logger.isDebugEnabled()) {
363 logger.debug("[LLM] Delegating generateNoResultsResponse. llmType={}", getLlmType());
364 }
365 getAvailableClient().generateNoResultsResponse(userMessage, history, callback);
366 }
367
368
369
370
371
372
373
374
375
376
377 public void generateDocumentNotFoundResponse(final String userMessage, final String documentUrl, final List<LlmMessage> history,
378 final LlmStreamCallback callback) {
379 if (logger.isDebugEnabled()) {
380 logger.debug("[LLM] Delegating generateDocumentNotFoundResponse. llmType={}", getLlmType());
381 }
382 getAvailableClient().generateDocumentNotFoundResponse(userMessage, documentUrl, history, callback);
383 }
384
385
386
387
388
389
390
391
392
393
394 public void generateSummaryResponse(final String userMessage, final List<Map<String, Object>> documents, final List<LlmMessage> history,
395 final LlmStreamCallback callback) {
396 if (logger.isDebugEnabled()) {
397 logger.debug("[LLM] Delegating generateSummaryResponse. llmType={}", getLlmType());
398 }
399 getAvailableClient().generateSummaryResponse(userMessage, documents, history, callback);
400 }
401
402
403
404
405
406
407
408
409
410
411 public void generateFaqAnswerResponse(final String userMessage, final List<Map<String, Object>> documents,
412 final List<LlmMessage> history, final LlmStreamCallback callback) {
413 if (logger.isDebugEnabled()) {
414 logger.debug("[LLM] Delegating generateFaqAnswerResponse. llmType={}", getLlmType());
415 }
416 getAvailableClient().generateFaqAnswerResponse(userMessage, documents, history, callback);
417 }
418
419
420
421
422
423
424
425
426
427 public void generateDirectAnswer(final String userMessage, final List<LlmMessage> history, final LlmStreamCallback callback) {
428 if (logger.isDebugEnabled()) {
429 logger.debug("[LLM] Delegating generateDirectAnswer. llmType={}", getLlmType());
430 }
431 getAvailableClient().generateDirectAnswer(userMessage, history, callback);
432 }
433 }