1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.opensearch.client;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20 import static org.codelibs.opensearch.runner.OpenSearchRunner.newConfigs;
21 import static org.opensearch.core.action.ActionListener.wrap;
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.net.InetAddress;
26 import java.net.UnknownHostException;
27 import java.nio.charset.StandardCharsets;
28 import java.text.SimpleDateFormat;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collections;
32 import java.util.Date;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.Map.Entry;
37 import java.util.Set;
38 import java.util.function.BiConsumer;
39 import java.util.function.BiFunction;
40 import java.util.function.Function;
41 import java.util.function.UnaryOperator;
42 import java.util.regex.Pattern;
43 import java.util.stream.Collectors;
44
45 import org.apache.logging.log4j.LogManager;
46 import org.apache.logging.log4j.Logger;
47 import org.codelibs.core.beans.util.BeanUtil;
48 import org.codelibs.core.exception.ResourceNotFoundRuntimeException;
49 import org.codelibs.core.io.FileUtil;
50 import org.codelibs.core.io.ResourceUtil;
51 import org.codelibs.core.lang.StringUtil;
52 import org.codelibs.core.lang.ThreadUtil;
53 import org.codelibs.curl.CurlResponse;
54 import org.codelibs.fesen.client.EngineInfo;
55 import org.codelibs.fesen.client.HttpClient;
56 import org.codelibs.fess.Constants;
57 import org.codelibs.fess.entity.FacetInfo;
58 import org.codelibs.fess.entity.GeoInfo;
59 import org.codelibs.fess.entity.HighlightInfo;
60 import org.codelibs.fess.entity.PingResponse;
61 import org.codelibs.fess.entity.QueryContext;
62 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
63 import org.codelibs.fess.exception.FessSystemException;
64 import org.codelibs.fess.exception.InvalidQueryException;
65 import org.codelibs.fess.exception.ResultOffsetExceededException;
66 import org.codelibs.fess.exception.SearchQueryException;
67 import org.codelibs.fess.helper.DocumentHelper;
68 import org.codelibs.fess.helper.QueryHelper;
69 import org.codelibs.fess.helper.SystemHelper;
70 import org.codelibs.fess.mylasta.direction.FessConfig;
71 import org.codelibs.fess.query.QueryFieldConfig;
72 import org.codelibs.fess.util.BooleanFunction;
73 import org.codelibs.fess.util.ComponentUtil;
74 import org.codelibs.fess.util.DocMap;
75 import org.codelibs.fess.util.IpAddressUtil;
76 import org.codelibs.fess.util.SearchEngineUtil;
77 import org.codelibs.fess.util.SystemUtil;
78 import org.codelibs.opensearch.runner.OpenSearchRunner;
79 import org.codelibs.opensearch.runner.OpenSearchRunner.Configs;
80 import org.codelibs.opensearch.runner.net.OpenSearchCurl;
81 import org.dbflute.exception.IllegalBehaviorStateException;
82 import org.dbflute.optional.OptionalEntity;
83 import org.lastaflute.core.message.UserMessages;
84 import org.lastaflute.di.exception.ContainerInitFailureException;
85 import org.opensearch.OpenSearchException;
86 import org.opensearch.OpenSearchStatusException;
87 import org.opensearch.action.ActionRequest;
88 import org.opensearch.action.ActionType;
89 import org.opensearch.action.DocWriteRequest.OpType;
90 import org.opensearch.action.DocWriteResponse.Result;
91 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
92 import org.opensearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
93 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
94 import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse;
95 import org.opensearch.action.admin.indices.flush.FlushResponse;
96 import org.opensearch.action.admin.indices.get.GetIndexResponse;
97 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
98 import org.opensearch.action.admin.indices.refresh.RefreshResponse;
99 import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse;
100 import org.opensearch.action.admin.indices.segments.PitSegmentsRequest;
101 import org.opensearch.action.bulk.BulkRequest;
102 import org.opensearch.action.bulk.BulkRequestBuilder;
103 import org.opensearch.action.bulk.BulkResponse;
104 import org.opensearch.action.delete.DeleteRequest;
105 import org.opensearch.action.delete.DeleteRequestBuilder;
106 import org.opensearch.action.delete.DeleteResponse;
107 import org.opensearch.action.explain.ExplainRequest;
108 import org.opensearch.action.explain.ExplainRequestBuilder;
109 import org.opensearch.action.explain.ExplainResponse;
110 import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest;
111 import org.opensearch.action.fieldcaps.FieldCapabilitiesRequestBuilder;
112 import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse;
113 import org.opensearch.action.get.GetRequest;
114 import org.opensearch.action.get.GetRequestBuilder;
115 import org.opensearch.action.get.GetResponse;
116 import org.opensearch.action.get.MultiGetRequest;
117 import org.opensearch.action.get.MultiGetRequestBuilder;
118 import org.opensearch.action.get.MultiGetResponse;
119 import org.opensearch.action.index.IndexRequest;
120 import org.opensearch.action.index.IndexRequestBuilder;
121 import org.opensearch.action.index.IndexResponse;
122 import org.opensearch.action.search.ClearScrollRequest;
123 import org.opensearch.action.search.ClearScrollRequestBuilder;
124 import org.opensearch.action.search.ClearScrollResponse;
125 import org.opensearch.action.search.CreatePitRequest;
126 import org.opensearch.action.search.CreatePitResponse;
127 import org.opensearch.action.search.DeletePitRequest;
128 import org.opensearch.action.search.DeletePitResponse;
129 import org.opensearch.action.search.GetAllPitNodesRequest;
130 import org.opensearch.action.search.GetAllPitNodesResponse;
131 import org.opensearch.action.search.MultiSearchRequest;
132 import org.opensearch.action.search.MultiSearchRequestBuilder;
133 import org.opensearch.action.search.MultiSearchResponse;
134 import org.opensearch.action.search.SearchPhaseExecutionException;
135 import org.opensearch.action.search.SearchRequest;
136 import org.opensearch.action.search.SearchRequestBuilder;
137 import org.opensearch.action.search.SearchResponse;
138 import org.opensearch.action.search.SearchScrollRequest;
139 import org.opensearch.action.search.SearchScrollRequestBuilder;
140 import org.opensearch.action.support.WriteRequest.RefreshPolicy;
141 import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
142 import org.opensearch.action.termvectors.MultiTermVectorsRequest;
143 import org.opensearch.action.termvectors.MultiTermVectorsRequestBuilder;
144 import org.opensearch.action.termvectors.MultiTermVectorsResponse;
145 import org.opensearch.action.termvectors.TermVectorsRequest;
146 import org.opensearch.action.termvectors.TermVectorsRequestBuilder;
147 import org.opensearch.action.termvectors.TermVectorsResponse;
148 import org.opensearch.action.update.UpdateRequest;
149 import org.opensearch.action.update.UpdateRequestBuilder;
150 import org.opensearch.action.update.UpdateResponse;
151 import org.opensearch.cluster.metadata.MappingMetadata;
152 import org.opensearch.common.action.ActionFuture;
153 import org.opensearch.common.document.DocumentField;
154 import org.opensearch.common.settings.Settings;
155 import org.opensearch.common.settings.Settings.Builder;
156 import org.opensearch.common.unit.TimeValue;
157 import org.opensearch.common.xcontent.XContentType;
158 import org.opensearch.core.action.ActionListener;
159 import org.opensearch.core.action.ActionResponse;
160 import org.opensearch.core.rest.RestStatus;
161 import org.opensearch.index.query.InnerHitBuilder;
162 import org.opensearch.index.query.QueryBuilder;
163 import org.opensearch.index.query.QueryBuilders;
164 import org.opensearch.index.reindex.UpdateByQueryRequest;
165 import org.opensearch.script.Script;
166 import org.opensearch.script.ScriptType;
167 import org.opensearch.search.SearchHit;
168 import org.opensearch.search.SearchHits;
169 import org.opensearch.search.aggregations.AggregationBuilders;
170 import org.opensearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
171 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
172 import org.opensearch.search.collapse.CollapseBuilder;
173 import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
174 import org.opensearch.threadpool.ThreadPool;
175 import org.opensearch.transport.client.AdminClient;
176 import org.opensearch.transport.client.Client;
177
178 import com.fasterxml.jackson.core.type.TypeReference;
179 import com.fasterxml.jackson.databind.ObjectMapper;
180 import com.google.common.io.BaseEncoding;
181
182 import jakarta.annotation.PostConstruct;
183 import jakarta.annotation.PreDestroy;
184
185
186
187
188
189 public class SearchEngineClient implements Client {
190
191
192
193
194 public SearchEngineClient() {
195
196 }
197
198 private static final Logger logger = LogManager.getLogger(SearchEngineClient.class);
199
200 private static final String DOC_INDEX = "fess";
201
202 private static final String LOG_INDEX_PREFIX = "fess_log";
203
204 private static final String USER_INDEX_PREFIX = "fess_user";
205
206 private static final String CONFIG_INDEX_PREFIX = "fess_config";
207
208
209 protected OpenSearchRunner runner;
210
211
212 protected Client client;
213
214
215 protected Map<String, String> settings;
216
217
218 protected String indexConfigPath = "fess_indices";
219
220
221 protected List<String> indexConfigList = new ArrayList<>();
222
223
224 protected Map<String, List<String>> configListMap = new HashMap<>();
225
226
227 protected String scrollForSearch = "1m";
228
229
230 protected int sizeForDelete = 100;
231
232
233 protected String scrollForDelete = "1m";
234
235
236 protected int sizeForUpdate = 100;
237
238
239 protected String scrollForUpdate = "1m";
240
241
242 protected int maxConfigSyncStatusRetry = 10;
243
244
245 protected int maxEsStatusRetry = 60;
246
247
248 protected String clusterName = "fesen";
249
250
251 protected final List<UnaryOperator<String>> docSettingRewriteRuleList = new ArrayList<>();
252
253
254 protected final List<UnaryOperator<String>> docMappingRewriteRuleList = new ArrayList<>();
255
256
257 protected boolean usePipeline = false;
258
259
260
261
262
263
264 public void addIndexConfig(final String path) {
265 indexConfigList.add(path);
266 }
267
268
269
270
271
272
273
274 public void addConfigFile(final String index, final String path) {
275 configListMap.computeIfAbsent(index, k -> new ArrayList<>()).add(path);
276 }
277
278
279
280
281
282
283 public void setSettings(final Map<String, String> settings) {
284 this.settings = settings;
285 }
286
287
288
289
290
291
292 public String getStatus() {
293 return admin().cluster()
294 .prepareHealth()
295 .execute()
296 .actionGet(ComponentUtil.getFessConfig().getIndexHealthTimeout())
297 .getStatus()
298 .name();
299 }
300
301
302
303
304
305
306 public void setRunner(final OpenSearchRunner runner) {
307 this.runner = runner;
308 }
309
310
311
312
313
314
315 public boolean isEmbedded() {
316 return runner != null;
317 }
318
319
320
321
322 public void usePipeline() {
323 usePipeline = true;
324 }
325
326
327
328
329
330
331
332
333 protected InetAddress getInetAddressByName(final String host) {
334 try {
335 return InetAddress.getByName(host);
336 } catch (final UnknownHostException e) {
337 throw new FessSystemException("Failed to resolve the hostname: " + host, e);
338 }
339 }
340
341
342
343
344
345 @PostConstruct
346 public void open() {
347 if (logger.isDebugEnabled()) {
348 logger.debug("Initializing {}", this.getClass().getSimpleName());
349 }
350 final FessConfig fessConfig = ComponentUtil.getFessConfig();
351
352 if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) {
353 String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
354 if (StringUtil.isBlank(dictionaryPath)) {
355 System.setProperty("fess.dictionary.path", fessConfig.getIndexDictionaryPrefix() + "/");
356 } else {
357 if (!dictionaryPath.endsWith("/")) {
358 dictionaryPath = dictionaryPath + "/";
359 }
360 System.setProperty("fess.dictionary.path", dictionaryPath + fessConfig.getIndexDictionaryPrefix() + "/");
361 }
362 }
363
364 String httpAddress = SystemUtil.getSearchEngineHttpAddress();
365 if (StringUtil.isBlank(httpAddress) && runner == null) {
366 switch (fessConfig.getFesenType()) {
367 case Constants.FESEN_TYPE_CLOUD:
368 case Constants.FESEN_TYPE_AWS:
369 httpAddress = org.codelibs.fess.util.ResourceUtil.getFesenHttpUrl();
370 break;
371 default:
372 runner = new OpenSearchRunner();
373 final Configs config = newConfigs().clusterName(clusterName).numOfNode(1).useLogger();
374 final String esDir = System.getProperty("fess.es.dir");
375 if (esDir != null) {
376 config.basePath(esDir);
377 }
378 config.disableESLogger();
379 runner.onBuild((number, settingsBuilder) -> {
380 final File moduleDir = new File(esDir, "modules");
381 if (moduleDir.isDirectory()) {
382 settingsBuilder.put("path.modules", moduleDir.getAbsolutePath());
383 } else {
384 settingsBuilder.put("path.modules", new File(System.getProperty("user.dir"), "modules").getAbsolutePath());
385 }
386 final File pluginDir = new File(esDir, "plugins");
387 if (pluginDir.isDirectory()) {
388 settingsBuilder.put("path.plugins", pluginDir.getAbsolutePath());
389 } else {
390 settingsBuilder.put("path.plugins", new File(System.getProperty("user.dir"), "plugins").getAbsolutePath());
391 }
392 if (settings != null) {
393 settingsBuilder.putProperties(settings, s -> s);
394 }
395 });
396 runner.build(config);
397
398 final int port = runner.node().settings().getAsInt("http.port", 9200);
399 try {
400 final InetAddress localhost = InetAddress.getByName("localhost");
401 httpAddress = IpAddressUtil.buildUrl("http", localhost, port, "");
402 } catch (final UnknownHostException e) {
403 httpAddress = "http://localhost:" + port; // Fallback
404 }
405 logger.warn("Embedded OpenSearch is running. This configuration is not recommended for production use.");
406 break;
407 }
408 }
409 client = createHttpClient(fessConfig, httpAddress);
410
411 if (StringUtil.isNotBlank(httpAddress)) {
412 System.setProperty(Constants.FESS_SEARCH_ENGINE_HTTP_ADDRESS, httpAddress);
413 }
414
415 waitForYellowStatus(fessConfig);
416
417 indexConfigList.forEach(configName -> {
418 final String[] values = configName.split("/");
419 if (values.length == 2) {
420 final String configIndex = values[0];
421 final String configType = values[1];
422
423 final boolean isFessIndex = DOC_INDEX.equals(configIndex);
424 final String indexName;
425 if (isFessIndex) {
426 final boolean exists = existsIndex(fessConfig.getIndexDocumentUpdateIndex());
427 if (!exists) {
428 indexName = generateNewIndexName(configIndex);
429 createIndex(configIndex, indexName);
430 createAlias(configIndex, indexName);
431 } else {
432 client.admin()
433 .cluster()
434 .prepareHealth(fessConfig.getIndexDocumentUpdateIndex())
435 .setWaitForYellowStatus()
436 .execute()
437 .actionGet(fessConfig.getIndexIndicesTimeout());
438 final GetIndexResponse response = client.admin()
439 .indices()
440 .prepareGetIndex()
441 .addIndices(fessConfig.getIndexDocumentUpdateIndex())
442 .execute()
443 .actionGet(fessConfig.getIndexIndicesTimeout());
444 final String[] indices = response.indices();
445 if (indices.length == 1) {
446 indexName = indices[0];
447 } else {
448 indexName = configIndex;
449 }
450 }
451 } else {
452 if (configIndex.startsWith(CONFIG_INDEX_PREFIX)) {
453 final String name = fessConfig.getIndexConfigIndex();
454 indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
455 } else if (configIndex.startsWith(USER_INDEX_PREFIX)) {
456 final String name = fessConfig.getIndexUserIndex();
457 indexName = configIndex.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
458 } else if (configIndex.startsWith(LOG_INDEX_PREFIX)) {
459 final String name = fessConfig.getIndexLogIndex();
460 indexName = configIndex.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
461 } else {
462 throw new FessSystemException("Unknown config index: " + configIndex);
463 }
464 final boolean exists = existsIndex(indexName);
465 if (!exists) {
466 createIndex(configIndex, indexName);
467 createAlias(configIndex, indexName);
468 }
469 }
470
471 addMapping(configIndex, configType, indexName);
472 } else {
473 logger.warn("Invalid index config name: configName={}", configName);
474 }
475 });
476 }
477
478
479
480
481
482
483
484
485 protected Client createHttpClient(final FessConfig fessConfig, final String host) {
486 final String[] hosts =
487 split(host, ",").get(stream -> stream.map(String::trim).filter(StringUtil::isNotEmpty).toArray(n -> new String[n]));
488 final Builder builder = Settings.builder()
489 .putList("http.hosts", hosts)
490 .put("processors", fessConfig.availableProcessors())
491 .put("http.heartbeat_interval", fessConfig.getFesenHeartbeatInterval());
492 final String username = fessConfig.getFesenUsername();
493 final String password = fessConfig.getFesenPassword();
494 if (StringUtil.isNotBlank(username) && StringUtil.isNotBlank(password)) {
495 builder.put(Constants.FESEN_USERNAME, username);
496 builder.put(Constants.FESEN_PASSWORD, password);
497 }
498 final String authorities = fessConfig.getFesenHttpSslCertificateAuthorities();
499 if (StringUtil.isNotBlank(authorities)) {
500 builder.put("http.ssl.certificate_authorities", authorities);
501 }
502 return new HttpClient(builder.build(), null);
503 }
504
505
506
507
508
509
510
511 public boolean existsIndex(final String indexName) {
512 final FessConfig fessConfig = ComponentUtil.getFessConfig();
513 boolean exists = false;
514 try {
515 final IndicesExistsResponse response =
516 client.admin().indices().prepareExists(indexName).execute().actionGet(fessConfig.getIndexSearchTimeout());
517 exists = response.isExists();
518 } catch (final Exception e) {
519 logger.debug("Failed to check index status: indexName={}", indexName, e);
520 }
521 return exists;
522 }
523
524
525
526
527
528
529
530 public long getDocumentCount(final String indexName) {
531 final FessConfig fessConfig = ComponentUtil.getFessConfig();
532 try {
533 client.admin().indices().prepareRefresh(indexName).execute().actionGet(fessConfig.getIndexIndicesTimeout());
534 try (CurlResponse response = ComponentUtil.getCurlHelper().get("/" + indexName + "/_count").execute()) {
535 if (response.getHttpStatusCode() == 200) {
536 final Map<String, Object> contentMap = response.getContent(OpenSearchCurl.jsonParser());
537 final Object count = contentMap.get("count");
538 if (count instanceof Number) {
539 return ((Number) count).longValue();
540 }
541 }
542 }
543 } catch (final Exception e) {
544 logger.debug("Failed to get document count: indexName={}", indexName, e);
545 }
546 return -1;
547 }
548
549
550
551
552
553
554
555 public int getAliasCount(final String indexName) {
556 try (CurlResponse response =
557 ComponentUtil.getCurlHelper().get("/_cat/aliases").param("format", "json").param("h", "alias,index").execute()) {
558 if (response.getHttpStatusCode() == 200) {
559 final String content = response.getContentAsString();
560 final ObjectMapper mapper = new ObjectMapper();
561 final List<Map<String, String>> aliases = mapper.readValue(content, new TypeReference<List<Map<String, String>>>() {
562 });
563 int count = 0;
564 for (final Map<String, String> entry : aliases) {
565 if (indexName.equals(entry.get("index"))) {
566 count++;
567 }
568 }
569 return count;
570 }
571 } catch (final Exception e) {
572 logger.debug("Failed to check aliases: indexName={}", indexName, e);
573 }
574 return 0;
575 }
576
577
578
579
580
581
582
583
584
585 public boolean copyDocIndex(final String fromIndex, final String toIndex, final boolean waitForCompletion) {
586 final FessConfig fessConfig = ComponentUtil.getFessConfig();
587 final String source = fessConfig.getIndexReindexBody()
588 .replace("__SOURCE_INDEX__", fromIndex)
589 .replace("__SIZE__", fessConfig.getIndexReindexSize())
590 .replace("__DEST_INDEX__", toIndex)
591 .replace("__SCRIPT_SOURCE__", ComponentUtil.getLanguageHelper().getReindexScriptSource());
592 return reindex(fromIndex, toIndex, source, waitForCompletion);
593 }
594
595
596
597
598
599
600
601
602
603 public boolean reindex(final String fromIndex, final String toIndex, final boolean waitForCompletion) {
604 final String template = """
605 {"source":{"index":"__SOURCE_INDEX__","size":__SIZE__},"dest":{"index":"__DEST_INDEX__"}}
606 """;
607 final FessConfig fessConfig = ComponentUtil.getFessConfig();
608 final String source = template
609 .replace("__SOURCE_INDEX__", fromIndex)
610 .replace("__SIZE__", fessConfig.getIndexReindexSize())
611 .replace("__DEST_INDEX__", toIndex);
612 return reindex(fromIndex, toIndex, source, waitForCompletion);
613 }
614
615
616
617
618
619
620
621
622
623
624 protected boolean reindex(final String fromIndex, final String toIndex, final String source, final boolean waitForCompletion) {
625 final FessConfig fessConfig = ComponentUtil.getFessConfig();
626 final String refresh = StringUtil.isNotBlank(fessConfig.getIndexReindexRefresh()) ? fessConfig.getIndexReindexRefresh() : null;
627 final String requestsPerSecond = getReindexRequestsPerSecound(fessConfig);
628 final String scroll = StringUtil.isNotBlank(fessConfig.getIndexReindexScroll()) ? fessConfig.getIndexReindexScroll() : null;
629 final String maxDocs = StringUtil.isNotBlank(fessConfig.getIndexReindexMaxDocs()) ? fessConfig.getIndexReindexMaxDocs() : null;
630 try (CurlResponse response = ComponentUtil.getCurlHelper()
631 .post("/_reindex")
632 .param("refresh", refresh)
633 .param("requests_per_second", requestsPerSecond)
634 .param("scroll", scroll)
635 .param("max_docs", maxDocs)
636 .param("wait_for_completion", Boolean.toString(waitForCompletion))
637 .body(source)
638 .execute()) {
639 if (response.getHttpStatusCode() == 200) {
640 return true;
641 }
642 logger.warn("Failed to reindex: fromIndex={}, toIndex={}, response={}", fromIndex, toIndex, response.getContentAsString());
643 } catch (final IOException e) {
644 logger.warn("Failed to reindex from {} to {}", fromIndex, toIndex, e);
645 }
646 return false;
647 }
648
649
650
651
652
653
654
655 protected String getReindexRequestsPerSecound(final FessConfig fessConfig) {
656 if (StringUtil.isBlank(fessConfig.getIndexReindexRequestsPerSecond())) {
657 return null;
658 }
659 final String value = fessConfig.getIndexReindexRequestsPerSecond();
660 if ("adaptive".equalsIgnoreCase(value)) {
661 if (fessConfig.availableProcessors() >= 4) {
662 return null;
663 }
664 final String requestsPerSecond = String.valueOf(fessConfig.getIndexReindexSizeAsInteger() * fessConfig.availableProcessors());
665 logger.info("Set requests_per_second: value={}", requestsPerSecond);
666 return requestsPerSecond;
667 }
668 return value;
669 }
670
671
672
673
674
675
676
677
678 public boolean createIndex(final String index, final String indexName) {
679 final FessConfig fessConfig = ComponentUtil.getFessConfig();
680 return createIndex(index, indexName, fessConfig.getIndexNumberOfShards(), fessConfig.getIndexAutoExpandReplicas(), true);
681 }
682
683
684
685
686
687
688
689
690
691
692
693 public boolean createIndex(final String index, final String indexName, final String numberOfShards, final String autoExpandReplicas,
694 final boolean uploadConfig) {
695 final FessConfig fessConfig = ComponentUtil.getFessConfig();
696
697 final String fesenType = fessConfig.getFesenType();
698 if (uploadConfig) {
699 switch (fesenType) {
700 case Constants.FESEN_TYPE_CLOUD:
701 case Constants.FESEN_TYPE_AWS:
702
703 break;
704 default:
705 waitForConfigSyncStatus();
706 sendConfigFiles(index);
707 break;
708 }
709 }
710
711 final String indexConfigFile = getResourcePath(indexConfigPath, fesenType, "/" + index + ".json");
712 try {
713 final String source = readIndexSetting(fesenType, indexConfigFile, numberOfShards, autoExpandReplicas);
714 final CreateIndexResponse indexResponse = client.admin()
715 .indices()
716 .prepareCreate(indexName)
717 .setSource(source, XContentType.JSON)
718 .execute()
719 .actionGet(fessConfig.getIndexIndicesTimeout());
720 if (indexResponse.isAcknowledged()) {
721 logger.info("Created index: indexName={}", indexName);
722 return true;
723 }
724 if (logger.isDebugEnabled()) {
725 logger.debug("Failed to create index: indexName={}", indexName);
726 }
727 } catch (final Exception e) {
728 logger.warn("Index config file not found: path={}", indexConfigFile, e);
729 }
730
731 return false;
732 }
733
734
735
736
737
738
739
740 public boolean deleteIndex(final String indexName) {
741 final FessConfig fessConfig = ComponentUtil.getFessConfig();
742 try {
743 final AcknowledgedResponse response =
744 client.admin().indices().prepareDelete(indexName).execute().actionGet(fessConfig.getIndexIndicesTimeout());
745 return response.isAcknowledged();
746 } catch (final Exception e) {
747 logger.warn("Failed to delete index: indexName={}", indexName, e);
748 }
749 return false;
750 }
751
752
753
754
755
756
757
758
759
760
761
762 public boolean reindexConfigIndices(final boolean loadBulkData, final Set<String> targetPrefixes) {
763 final FessConfig fessConfig = ComponentUtil.getFessConfig();
764 final String timestamp = new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date());
765 boolean success = true;
766
767 for (final String configName : indexConfigList) {
768 final String[] values = configName.split("/");
769 if (values.length != 2) {
770 continue;
771 }
772
773 final String configIndex = values[0];
774 final String configType = values[1];
775
776 if (DOC_INDEX.equals(configIndex)) {
777 continue;
778 }
779
780 final String indexName;
781 if (configIndex.startsWith(CONFIG_INDEX_PREFIX)) {
782 if (!targetPrefixes.contains(CONFIG_INDEX_PREFIX)) {
783 continue;
784 }
785 indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), fessConfig.getIndexConfigIndex());
786 } else if (configIndex.startsWith(USER_INDEX_PREFIX)) {
787 if (!targetPrefixes.contains(USER_INDEX_PREFIX)) {
788 continue;
789 }
790 indexName = configIndex.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), fessConfig.getIndexUserIndex());
791 } else if (configIndex.startsWith(LOG_INDEX_PREFIX)) {
792 if (!targetPrefixes.contains(LOG_INDEX_PREFIX)) {
793 continue;
794 }
795 indexName = configIndex.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), fessConfig.getIndexLogIndex());
796 } else {
797 logger.warn("[Rebuild] Unknown config index: {}", configIndex);
798 success = false;
799 continue;
800 }
801
802 if (!existsIndex(indexName)) {
803 logger.info("[Rebuild] Creating new index: {}", indexName);
804 if (!createIndex(configIndex, indexName)) {
805 logger.warn("[Rebuild] Failed to create index: {}", indexName);
806 success = false;
807 continue;
808 }
809 try {
810 addMapping(configIndex, configType, indexName, loadBulkData);
811 createAlias(configIndex, indexName);
812 } catch (final Exception e) {
813 logger.warn("[Rebuild] Failed to set up index: {}", indexName, e);
814 success = false;
815 }
816 continue;
817 }
818
819 final String backupIndex = indexName + ".backup." + timestamp;
820 logger.info("[Rebuild] Starting rebuild for {}", indexName);
821
822 try {
823 final long sourceCount = getDocumentCount(indexName);
824
825
826 if (!createIndex(configIndex, backupIndex)) {
827 logger.warn("[Rebuild] Failed to create backup index: {}", backupIndex);
828 success = false;
829 continue;
830 }
831 addMapping(configIndex, configType, backupIndex, false);
832
833
834 if (!reindex(indexName, backupIndex, true)) {
835 logger.warn("[Rebuild] Failed to reindex from {} to {}", indexName, backupIndex);
836 deleteIndex(backupIndex);
837 success = false;
838 continue;
839 }
840 if (sourceCount >= 0) {
841 final long backupCount = getDocumentCount(backupIndex);
842 if (backupCount != sourceCount) {
843 logger.warn("[Rebuild] Document count mismatch after reindex: source={}, backup={} for {}", sourceCount,
844 backupCount, indexName);
845 deleteIndex(backupIndex);
846 success = false;
847 continue;
848 }
849 }
850
851
852 deleteIndex(indexName);
853 if (!createIndex(configIndex, indexName)) {
854 logger.warn("[Rebuild] Failed to recreate index: {}. Keeping backup: {}", indexName, backupIndex);
855 success = false;
856 continue;
857 }
858 addMapping(configIndex, configType, indexName, false);
859
860
861 if (!reindex(backupIndex, indexName, true)) {
862 logger.warn("[Rebuild] Failed to reindex from {} to {}. Cleaning up.", backupIndex, indexName);
863 deleteIndex(backupIndex);
864 success = false;
865 continue;
866 }
867 if (sourceCount >= 0) {
868 final long rebuiltCount = getDocumentCount(indexName);
869 if (rebuiltCount != sourceCount) {
870 logger.warn("[Rebuild] Document count mismatch after rebuild: expected={}, actual={} for {}", sourceCount,
871 rebuiltCount, indexName);
872 deleteIndex(backupIndex);
873 success = false;
874 continue;
875 }
876 }
877
878
879 if (loadBulkData) {
880 final String dataPath =
881 getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + configIndex + "/" + configType + ".bulk");
882 if (ResourceUtil.isExist(dataPath)) {
883 insertBulkData(fessConfig, indexName, dataPath, true);
884 }
885 split(fessConfig.getAppExtensionNames(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(name -> {
886 final String bulkPath = getResourcePath(indexConfigPath, fessConfig.getFesenType(),
887 "/" + configIndex + "/" + configType + "_" + name + ".bulk");
888 if (ResourceUtil.isExist(bulkPath)) {
889 insertBulkData(fessConfig, indexName, bulkPath, true);
890 }
891 }));
892 }
893
894
895 createAlias(configIndex, indexName);
896
897
898 deleteIndex(backupIndex);
899
900 logger.info("[Rebuild] Completed rebuild for {}", indexName);
901 } catch (final Exception e) {
902 logger.warn("[Rebuild] Failed to rebuild index: {}", indexName, e);
903 if (existsIndex(backupIndex)) {
904 logger.info("[Rebuild] Backup index {} remains for recovery", backupIndex);
905 }
906 success = false;
907 }
908 }
909 return success;
910 }
911
912
913
914
915
916
917
918
919
920
921
922 protected boolean switchAliases(final String configIndex, final String oldIndexName, final String newIndexName) {
923 final FessConfig fessConfig = ComponentUtil.getFessConfig();
924 final String aliasConfigDirPath = getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + configIndex + "/alias");
925 try {
926 final File aliasConfigDir = ResourceUtil.getResourceAsFile(aliasConfigDirPath);
927 if (aliasConfigDir.isDirectory()) {
928 final IndicesAliasesRequestBuilder builder = client.admin().indices().prepareAliases();
929 stream(aliasConfigDir.listFiles((dir, name) -> name.endsWith(".json"))).of(stream -> stream.forEach(f -> {
930 String aliasName = f.getName().replaceFirst(".json$", "");
931 if (configIndex.startsWith(CONFIG_INDEX_PREFIX)) {
932 final String name = fessConfig.getIndexConfigIndex();
933 if ("fess_basic_config".equals(aliasName) && !CONFIG_INDEX_PREFIX.equals(name)) {
934 aliasName = aliasName.replaceFirst("fess_basic_config", "basic_" + name);
935 } else {
936 aliasName = aliasName.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
937 }
938 } else if (configIndex.startsWith(USER_INDEX_PREFIX)) {
939 final String name = fessConfig.getIndexUserIndex();
940 aliasName = aliasName.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
941 } else if (configIndex.startsWith(LOG_INDEX_PREFIX)) {
942 final String name = fessConfig.getIndexLogIndex();
943 aliasName = aliasName.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
944 }
945 String source = FileUtil.readUTF8(f);
946 if ("{}".equals(source.trim())) {
947 source = null;
948 }
949 logger.info("[Rebuild] Alias action: remove alias={} from index={}, add alias={} to index={}", aliasName, oldIndexName,
950 aliasName, newIndexName);
951 builder.removeAlias(oldIndexName, aliasName);
952 if (source != null) {
953 builder.addAlias(newIndexName, aliasName, source);
954 } else {
955 builder.addAlias(newIndexName, aliasName);
956 }
957 }));
958 final AcknowledgedResponse response = builder.execute().actionGet(fessConfig.getIndexIndicesTimeout());
959 if (response.isAcknowledged()) {
960 logger.info("[Rebuild] Switched aliases from {} to {}", oldIndexName, newIndexName);
961 return true;
962 }
963 logger.warn("[Rebuild] Alias switch not acknowledged from {} to {}", oldIndexName, newIndexName);
964 }
965 } catch (final ResourceNotFoundRuntimeException e) {
966
967 logger.warn("[Rebuild] No alias config found for {}, falling back to createAlias (old aliases NOT removed)", configIndex);
968 createAlias(configIndex, newIndexName);
969 return true;
970 } catch (final Exception e) {
971 logger.warn("[Rebuild] Failed to switch aliases from {} to {}", oldIndexName, newIndexName, e);
972 }
973 return false;
974 }
975
976
977
978
979
980
981
982
983
984
985 protected String readIndexSetting(final String fesenType, final String indexConfigFile, final String numberOfShards,
986 final String autoExpandReplicas) {
987 final FessConfig fessConfig = ComponentUtil.getFessConfig();
988 String source = FileUtil.readUTF8(indexConfigFile);
989 String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
990 if (StringUtil.isNotBlank(dictionaryPath) && !dictionaryPath.endsWith("/")) {
991 dictionaryPath = dictionaryPath + "/";
992 }
993 source = source.replaceAll(Pattern.quote("${fess.dictionary.path}"), dictionaryPath)
994 .replaceAll(Pattern.quote("${fess.index.codec}"), fessConfig.getIndexCodec())
995 .replaceAll(Pattern.quote("${fess.index.number_of_shards}"), numberOfShards)
996 .replaceAll(Pattern.quote("${fess.index.auto_expand_replicas}"), autoExpandReplicas);
997 for (final UnaryOperator<String> rule : docSettingRewriteRuleList) {
998 source = rule.apply(source);
999 }
1000 return source;
1001 }
1002
1003
1004
1005
1006
1007
1008 public void addDocumentSettingRewriteRule(final UnaryOperator<String> rule) {
1009 docSettingRewriteRuleList.add(rule);
1010 }
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 protected String getResourcePath(final String basePath, final String type, final String path) {
1021 final String target = basePath + "/_" + type + path;
1022 if (ResourceUtil.getResourceNoException(target) != null) {
1023 return target;
1024 }
1025 return basePath + path;
1026 }
1027
1028
1029
1030
1031
1032
1033
1034
1035 public void addMapping(final String index, final String docType, final String indexName) {
1036 addMapping(index, docType, indexName, true);
1037 }
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047 public void addMapping(final String index, final String docType, final String indexName, final boolean loadBulkData) {
1048 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1049
1050 final GetMappingsResponse getMappingsResponse =
1051 client.admin().indices().prepareGetMappings(indexName).execute().actionGet(fessConfig.getIndexIndicesTimeout());
1052 final Map<String, MappingMetadata> indexMappings = getMappingsResponse.mappings();
1053 if (indexMappings == null || !indexMappings.containsKey("properties")) {
1054 String source = null;
1055 final String mappingFile = getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + index + "/" + docType + ".json");
1056 try {
1057 source = FileUtil.readUTF8(mappingFile);
1058 if (DOC_INDEX.equals(index)) {
1059 for (final UnaryOperator<String> rule : docMappingRewriteRuleList) {
1060 source = rule.apply(source);
1061 }
1062 }
1063 } catch (final Exception e) {
1064 logger.warn("{} is not found.", mappingFile, e);
1065 }
1066 try {
1067 final AcknowledgedResponse putMappingResponse = client.admin()
1068 .indices()
1069 .preparePutMapping(indexName)
1070 .setSource(source, XContentType.JSON)
1071 .execute()
1072 .actionGet(fessConfig.getIndexIndicesTimeout());
1073 if (putMappingResponse.isAcknowledged()) {
1074 logger.info("Created {}/{} mapping.", indexName, docType);
1075 } else {
1076 logger.warn("Failed to create {}/{} mapping.", indexName, docType);
1077 }
1078
1079 if (loadBulkData) {
1080 final String dataPath =
1081 getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + index + "/" + docType + ".bulk");
1082 if (ResourceUtil.isExist(dataPath)) {
1083 insertBulkData(fessConfig, indexName, dataPath);
1084 }
1085 split(fessConfig.getAppExtensionNames(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(name -> {
1086 final String bulkPath = getResourcePath(indexConfigPath, fessConfig.getFesenType(),
1087 "/" + index + "/" + docType + "_" + name + ".bulk");
1088 if (ResourceUtil.isExist(bulkPath)) {
1089 insertBulkData(fessConfig, indexName, bulkPath);
1090 }
1091 }));
1092 }
1093 } catch (final Exception e) {
1094 logger.warn("Failed to create {}/{} mapping.", indexName, docType, e);
1095 }
1096 } else if (logger.isDebugEnabled()) {
1097 logger.debug("{}/{} mapping exists.", indexName, docType);
1098 }
1099 }
1100
1101
1102
1103
1104
1105
1106 public void addDocumentMappingRewriteRule(final UnaryOperator<String> rule) {
1107 docMappingRewriteRuleList.add(rule);
1108 }
1109
1110
1111
1112
1113
1114
1115
1116 public boolean updateAlias(final String newIndex) {
1117 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1118 final String updateAlias = fessConfig.getIndexDocumentUpdateIndex();
1119 final String searchAlias = fessConfig.getIndexDocumentSearchIndex();
1120 final GetIndexResponse response1 =
1121 client.admin().indices().prepareGetIndex().addIndices(updateAlias).execute().actionGet(fessConfig.getIndexIndicesTimeout());
1122 final String[] updateIndices = response1.indices();
1123 final GetIndexResponse response2 =
1124 client.admin().indices().prepareGetIndex().addIndices(searchAlias).execute().actionGet(fessConfig.getIndexIndicesTimeout());
1125 final String[] searchIndices = response2.indices();
1126
1127 final IndicesAliasesRequestBuilder builder =
1128 client.admin().indices().prepareAliases().addAlias(newIndex, updateAlias).addAlias(newIndex, searchAlias);
1129 for (final String index : updateIndices) {
1130 builder.removeAlias(index, updateAlias);
1131 }
1132 for (final String index : searchIndices) {
1133 builder.removeAlias(index, searchAlias);
1134 }
1135 final AcknowledgedResponse response = builder.execute().actionGet(fessConfig.getIndexIndicesTimeout());
1136 return response.isAcknowledged();
1137 }
1138
1139
1140
1141
1142
1143
1144
1145 protected void createAlias(final String index, final String createdIndexName) {
1146 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1147
1148 final String aliasConfigDirPath = getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + index + "/alias");
1149 try {
1150 final File aliasConfigDir = ResourceUtil.getResourceAsFile(aliasConfigDirPath);
1151 if (aliasConfigDir.isDirectory()) {
1152 stream(aliasConfigDir.listFiles((dir, name) -> name.endsWith(".json"))).of(stream -> stream.forEach(f -> {
1153 String aliasName = f.getName().replaceFirst(".json$", "");
1154 if (index.equals(DOC_INDEX)) {
1155 if ("fess.search".equals(aliasName)) {
1156 aliasName = fessConfig.getIndexDocumentSearchIndex();
1157 } else if ("fess.update".equals(aliasName)) {
1158 aliasName = fessConfig.getIndexDocumentUpdateIndex();
1159 }
1160 } else if (index.startsWith(CONFIG_INDEX_PREFIX)) {
1161 final String name = fessConfig.getIndexConfigIndex();
1162 if ("fess_basic_config".equals(aliasName) && !CONFIG_INDEX_PREFIX.equals(name)) {
1163 aliasName = aliasName.replaceFirst("fess_basic_config", "basic_" + name);
1164 } else {
1165 aliasName = aliasName.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
1166 }
1167 } else if (index.startsWith(USER_INDEX_PREFIX)) {
1168 final String name = fessConfig.getIndexUserIndex();
1169 aliasName = aliasName.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
1170 } else if (index.startsWith(LOG_INDEX_PREFIX)) {
1171 final String name = fessConfig.getIndexLogIndex();
1172 aliasName = aliasName.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
1173 }
1174 String source = FileUtil.readUTF8(f);
1175 if ("{}".equals(source.trim())) {
1176 source = null;
1177 }
1178 final AcknowledgedResponse response = client.admin()
1179 .indices()
1180 .prepareAliases()
1181 .addAlias(createdIndexName, aliasName, source)
1182 .execute()
1183 .actionGet(fessConfig.getIndexIndicesTimeout());
1184 if (response.isAcknowledged()) {
1185 logger.info("Created {} alias for {}", aliasName, createdIndexName);
1186 } else if (logger.isDebugEnabled()) {
1187 logger.debug("Failed to create {} alias for {}", aliasName, createdIndexName);
1188 }
1189 }));
1190 }
1191 } catch (final ResourceNotFoundRuntimeException e) {
1192
1193 } catch (final Exception e) {
1194 logger.warn("{} is not found.", aliasConfigDirPath, e);
1195 }
1196 }
1197
1198
1199
1200
1201
1202
1203 protected void sendConfigFiles(final String index) {
1204 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1205 configListMap.getOrDefault(index, Collections.emptyList()).forEach(path -> {
1206 String source = null;
1207 final String filePath = indexConfigPath + "/" + index + "/" + path;
1208 final String dictionaryPath;
1209 if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) {
1210 dictionaryPath = fessConfig.getIndexDictionaryPrefix() + "/" + path;
1211 } else {
1212 dictionaryPath = path;
1213 }
1214 try {
1215 source = FileUtil.readUTF8(filePath);
1216 try (CurlResponse response =
1217 ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictionaryPath).body(source).execute()) {
1218 if (response.getHttpStatusCode() == 200) {
1219 logger.info("Register {} to {}", path, index);
1220 } else if (response.getContentException() != null) {
1221 logger.warn("Invalid request for {}.", path, response.getContentException());
1222 } else {
1223 logger.warn("Invalid request for {}. The response is {}", path, response.getContentAsString());
1224 }
1225 }
1226 } catch (final Exception e) {
1227 logger.warn("Failed to register {}", filePath, e);
1228 }
1229 });
1230 try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_configsync/flush").execute()) {
1231 if (response.getHttpStatusCode() == 200) {
1232 logger.info("Flushed config files.");
1233 } else {
1234 logger.warn("Failed to flush config files.");
1235 }
1236 } catch (final Exception e) {
1237 logger.warn("Failed to flush config files.", e);
1238 }
1239 }
1240
1241
1242
1243
1244
1245
1246 public void flushConfigFiles(final Runnable callback) {
1247 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1248
1249 final String fesenType = fessConfig.getFesenType();
1250 switch (fesenType) {
1251 case Constants.FESEN_TYPE_CLOUD:
1252 case Constants.FESEN_TYPE_AWS:
1253 if (logger.isDebugEnabled()) {
1254 logger.debug("Skipped configsync flush: {}", fesenType);
1255 }
1256 callback.run();
1257 break;
1258 default:
1259 ComponentUtil.getCurlHelper().post("/_configsync/flush").execute(response -> {
1260 if (logger.isDebugEnabled()) {
1261 logger.debug("Flushed config files: {} => {}", fesenType, response.getContentAsString());
1262 }
1263 callback.run();
1264 }, e -> {
1265 logger.warn("Failed to flush config files.", e);
1266 callback.run();
1267 });
1268 break;
1269 }
1270 }
1271
1272
1273
1274
1275
1276
1277
1278 protected String generateNewIndexName(final String configIndex) {
1279 return configIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date());
1280 }
1281
1282
1283
1284
1285
1286
1287
1288
1289 protected void insertBulkData(final FessConfig fessConfig, final String configIndex, final String dataPath) {
1290 insertBulkData(fessConfig, configIndex, dataPath, false);
1291 }
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301 protected void insertBulkData(final FessConfig fessConfig, final String configIndex, final String dataPath, final boolean createOnly) {
1302 try {
1303 final BulkRequestBuilder builder = client.prepareBulk();
1304 final ObjectMapper mapper = new ObjectMapper();
1305 final String userIndex = fessConfig.getIndexUserIndex() + ".user";
1306 Arrays.stream(FileUtil.readUTF8(dataPath).split("\n"))
1307 .map(line -> line
1308 .replace("\"_index\":\"fess_config.", "\"_index\":\"" + fessConfig.getIndexConfigIndex() + ".")
1309 .replace("\"_index\":\"fess_user.", "\"_index\":\"" + fessConfig.getIndexUserIndex() + ".")
1310 .replace("\"_index\":\"fess_log.", "\"_index\":\"" + fessConfig.getIndexLogIndex() + "."))
1311 .reduce((prev, line) -> {
1312 try {
1313 if (StringUtil.isBlank(prev)) {
1314 final Map<String, Map<String, String>> result =
1315 mapper.readValue(line, new TypeReference<Map<String, Map<String, String>>>() {
1316 });
1317 if (result.containsKey("index") || result.containsKey("update")) {
1318 return line;
1319 }
1320 if (result.containsKey("delete")) {
1321 return StringUtil.EMPTY;
1322 }
1323 } else {
1324 final Map<String, Map<String, String>> result =
1325 mapper.readValue(prev, new TypeReference<Map<String, Map<String, String>>>() {
1326 });
1327 if (result.containsKey("index")) {
1328 String source = line;
1329 if (userIndex.equals(configIndex)) {
1330 source = source.replace("${fess.index.initial_password}",
1331 ComponentUtil.getPasswordHashHelper().encode(fessConfig.getIndexUserInitialPassword()));
1332 }
1333 final IndexRequestBuilder requestBuilder = client.prepareIndex()
1334 .setIndex(configIndex)
1335 .setId(result.get("index").get("_id"))
1336 .setSource(source, XContentType.JSON);
1337 if (createOnly) {
1338 requestBuilder.setOpType(OpType.CREATE);
1339 }
1340 builder.add(requestBuilder);
1341 }
1342 }
1343 } catch (final Exception e) {
1344 logger.warn("Failed to parse {}", dataPath, e);
1345 }
1346 return StringUtil.EMPTY;
1347 });
1348 final BulkResponse response = builder.execute().actionGet(fessConfig.getIndexBulkTimeout());
1349 if (response.hasFailures()) {
1350 if (createOnly) {
1351 final long realFailures = Arrays.stream(response.getItems())
1352 .filter(item -> item.isFailed() && item.getFailure().getStatus() != RestStatus.CONFLICT)
1353 .count();
1354 if (realFailures > 0) {
1355 logger.warn("Failed to register {}: {}", dataPath, response.buildFailureMessage());
1356 } else if (logger.isDebugEnabled()) {
1357 logger.debug("Skipped existing documents in {}", dataPath);
1358 }
1359 } else {
1360 logger.warn("Failed to register {}: {}", dataPath, response.buildFailureMessage());
1361 }
1362 }
1363 } catch (final Exception e) {
1364 logger.warn("Failed to create {} mapping.", configIndex, e);
1365 }
1366 }
1367
1368
1369
1370
1371
1372
1373
1374 protected void waitForYellowStatus(final FessConfig fessConfig) {
1375 Exception cause = null;
1376 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
1377 final long startTime = systemHelper.getCurrentTimeAsLong();
1378 for (int i = 0; i < maxEsStatusRetry; i++) {
1379 try {
1380 final ClusterHealthResponse response = client.admin()
1381 .cluster()
1382 .prepareHealth()
1383 .setWaitForYellowStatus()
1384 .execute()
1385 .actionGet(fessConfig.getIndexHealthTimeout());
1386 if (logger.isDebugEnabled()) {
1387 logger.debug("Fesen Cluster Status: {}", response.getStatus());
1388 }
1389 return;
1390 } catch (final Exception e) {
1391 cause = e;
1392 }
1393 if (cause instanceof OpenSearchStatusException) {
1394 final RestStatus status = ((OpenSearchStatusException) cause).status();
1395 switch (status) {
1396 case UNAUTHORIZED:
1397 logger.warn("[{}] Unauthorized access: {}", i, SystemUtil.getSearchEngineHttpAddress(), cause);
1398 break;
1399 default:
1400 logger.debug("[{}][{}] Failed to access to Fesen ({})", i, status, SystemUtil.getSearchEngineHttpAddress(), cause);
1401 break;
1402 }
1403 } else if (logger.isDebugEnabled()) {
1404 logger.debug("[{}] Failed to access to Fesen ({})", i, SystemUtil.getSearchEngineHttpAddress(), cause);
1405 }
1406 ThreadUtil.sleep(1000L);
1407 }
1408 final String message =
1409 "Fesen (" + SystemUtil.getSearchEngineHttpAddress() + ") is not available. Check the state of your Fesen cluster ("
1410 + clusterName + ") in " + (systemHelper.getCurrentTimeAsLong() - startTime) + "ms.";
1411 throw new ContainerInitFailureException(message, cause);
1412 }
1413
1414
1415
1416
1417
1418
1419 protected void waitForConfigSyncStatus() {
1420 FessSystemException cause = null;
1421 for (int i = 0; i < maxConfigSyncStatusRetry; i++) {
1422 try (CurlResponse response = ComponentUtil.getCurlHelper().get("/_configsync/wait").param("status", "green").execute()) {
1423 final int httpStatusCode = response.getHttpStatusCode();
1424 if (httpStatusCode == 200) {
1425 logger.info("ConfigSync is ready.");
1426 return;
1427 }
1428 final String message = "Configsync is not available. HTTP Status is " + httpStatusCode;
1429 if (response.getContentException() != null) {
1430 throw new FessSystemException(message, response.getContentException());
1431 }
1432 throw new FessSystemException(message);
1433 } catch (final Exception e) {
1434 cause = new FessSystemException("Configsync is not available.", e);
1435 }
1436 if (logger.isDebugEnabled()) {
1437 logger.debug("Failed to access to configsync:{}", i, cause);
1438 }
1439 ThreadUtil.sleep(1000L);
1440 }
1441 throw cause;
1442 }
1443
1444 @Override
1445 @PreDestroy
1446 public void close() {
1447 if (runner != null) {
1448 try {
1449 client.admin()
1450 .indices()
1451 .prepareFlush()
1452 .setForce(true)
1453 .execute()
1454 .actionGet(ComponentUtil.getFessConfig().getIndexIndicesTimeout());
1455 } catch (final Exception e) {
1456 logger.warn("Failed to flush indices.", e);
1457 }
1458 }
1459 try {
1460 client.close();
1461 } catch (final OpenSearchException e) {
1462 logger.warn("Failed to close Client: {}", client, e);
1463 }
1464 }
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474 public long updateByQuery(final String index, final Function<SearchRequestBuilder, SearchRequestBuilder> option,
1475 final BiFunction<UpdateRequestBuilder, SearchHit, UpdateRequestBuilder> builder) {
1476
1477 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1478 SearchResponse response = option.apply(client.prepareSearch(index)
1479 .setScroll(scrollForUpdate)
1480 .setSize(sizeForUpdate)
1481 .setPreference(Constants.SEARCH_PREFERENCE_LOCAL)).execute().actionGet(fessConfig.getIndexScrollSearchTimeout());
1482
1483 int count = 0;
1484 String scrollId = response.getScrollId();
1485 try {
1486 while (scrollId != null) {
1487 final SearchHits searchHits = response.getHits();
1488 final SearchHit[] hits = searchHits.getHits();
1489 if (hits.length == 0) {
1490 break;
1491 }
1492
1493 final BulkRequestBuilder bulkRequest = client.prepareBulk();
1494 for (final SearchHit hit : hits) {
1495 final UpdateRequestBuilder requestBuilder =
1496 builder.apply(client.prepareUpdate().setIndex(index).setId(hit.getId()), hit);
1497 if (requestBuilder != null) {
1498 bulkRequest.add(requestBuilder);
1499 }
1500 count++;
1501 }
1502 final BulkResponse bulkResponse = bulkRequest.execute().actionGet(fessConfig.getIndexBulkTimeout());
1503 if (bulkResponse.hasFailures()) {
1504 throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
1505 }
1506
1507 response = client.prepareSearchScroll(scrollId)
1508 .setScroll(scrollForUpdate)
1509 .execute()
1510 .actionGet(fessConfig.getIndexBulkTimeout());
1511 if (!scrollId.equals(response.getScrollId())) {
1512 deleteScrollContext(scrollId);
1513 }
1514 scrollId = response.getScrollId();
1515 }
1516 } finally {
1517 deleteScrollContext(scrollId);
1518 }
1519 return count;
1520 }
1521
1522
1523
1524
1525
1526
1527
1528
1529 public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
1530
1531 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1532 SearchResponse response = client.prepareSearch(index)
1533 .setScroll(scrollForDelete)
1534 .setSize(sizeForDelete)
1535 .setFetchSource(new String[] { fessConfig.getIndexFieldId() }, null)
1536 .setQuery(queryBuilder)
1537 .setPreference(Constants.SEARCH_PREFERENCE_LOCAL)
1538 .execute()
1539 .actionGet(fessConfig.getIndexScrollSearchTimeout());
1540
1541 int count = 0;
1542 String scrollId = response.getScrollId();
1543 try {
1544 while (scrollId != null) {
1545 final SearchHits searchHits = response.getHits();
1546 final SearchHit[] hits = searchHits.getHits();
1547 if (hits.length == 0) {
1548 break;
1549 }
1550
1551 final BulkRequestBuilder bulkRequest = client.prepareBulk();
1552 for (final SearchHit hit : hits) {
1553 bulkRequest.add(client.prepareDelete().setIndex(index).setId(hit.getId()));
1554 count++;
1555 }
1556 final BulkResponse bulkResponse = bulkRequest.execute().actionGet(fessConfig.getIndexBulkTimeout());
1557 if (bulkResponse.hasFailures()) {
1558 throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
1559 }
1560
1561 response = client.prepareSearchScroll(scrollId)
1562 .setScroll(scrollForDelete)
1563 .execute()
1564 .actionGet(fessConfig.getIndexBulkTimeout());
1565 if (!scrollId.equals(response.getScrollId())) {
1566 deleteScrollContext(scrollId);
1567 }
1568 scrollId = response.getScrollId();
1569 }
1570 } finally {
1571 deleteScrollContext(scrollId);
1572 }
1573 return count;
1574 }
1575
1576
1577
1578
1579
1580
1581 protected void deleteScrollContext(final String scrollId) {
1582 if (scrollId != null) {
1583 client.prepareClearScroll()
1584 .addScrollId(scrollId)
1585 .execute(wrap(res -> {}, e -> logger.warn("Failed to clear the scroll context.", e)));
1586 }
1587 }
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599 protected <T> T get(final String index, final String id, final SearchCondition<GetRequestBuilder> condition,
1600 final SearchResult<T, GetRequestBuilder, GetResponse> searchResult) {
1601 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
1602 final long startTime = systemHelper.getCurrentTimeAsLong();
1603
1604 GetResponse response = null;
1605 final GetRequestBuilder requestBuilder = client.prepareGet(index, id);
1606 if (condition.build(requestBuilder)) {
1607 response = requestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout());
1608 }
1609 final long execTime = systemHelper.getCurrentTimeAsLong() - startTime;
1610
1611 return searchResult.build(requestBuilder, execTime, OptionalEntity.ofNullable(response, () -> {}));
1612 }
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624 public <T> T search(final String index, final SearchCondition<SearchRequestBuilder> condition,
1625 final SearchResult<T, SearchRequestBuilder, SearchResponse> searchResult) {
1626 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
1627 final long startTime = systemHelper.getCurrentTimeAsLong();
1628
1629 SearchResponse searchResponse = null;
1630 final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
1631 if (condition.build(searchRequestBuilder)) {
1632
1633 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1634 final long queryTimeout = fessConfig.getQueryTimeoutAsInteger().longValue();
1635 if (queryTimeout >= 0) {
1636 searchRequestBuilder.setTimeout(TimeValue.timeValueMillis(queryTimeout));
1637 }
1638
1639 try {
1640 if (logger.isDebugEnabled()) {
1641 logger.debug("Query DSL: {}", searchRequestBuilder);
1642 }
1643 searchResponse = searchRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout());
1644 } catch (final SearchPhaseExecutionException e) {
1645 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
1646 "Invalid query: " + searchRequestBuilder, e);
1647 } catch (final OpenSearchException e) {
1648 if (logger.isDebugEnabled()) {
1649 logger.debug("Cannot process {}", searchRequestBuilder, e);
1650 }
1651 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryCannotProcess(UserMessages.GLOBAL_PROPERTY_KEY),
1652 "Failed query: " + searchRequestBuilder, e);
1653 }
1654 }
1655 final long execTime = systemHelper.getCurrentTimeAsLong() - startTime;
1656
1657 return searchResult.build(searchRequestBuilder, execTime, OptionalEntity.ofNullable(searchResponse, () -> {}));
1658 }
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669 public long scrollSearch(final String index, final SearchCondition<SearchRequestBuilder> condition,
1670 final BooleanFunction<Map<String, Object>> cursor) {
1671 return scrollSearch(index, condition, getDefaultEntityCreator(), cursor);
1672 }
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685 public <T> long scrollSearch(final String index, final SearchCondition<SearchRequestBuilder> condition,
1686 final EntityCreator<T, SearchResponse, SearchHit> creator, final BooleanFunction<T> cursor) {
1687 long count = 0;
1688
1689 final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setScroll(scrollForSearch);
1690 if (condition.build(searchRequestBuilder)) {
1691 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1692
1693 String scrollId = null;
1694 try {
1695 if (logger.isDebugEnabled()) {
1696 logger.debug("Query DSL: {}", searchRequestBuilder);
1697 }
1698 SearchResponse response = searchRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout());
1699
1700 scrollId = response.getScrollId();
1701 while (scrollId != null) {
1702 final SearchHits searchHits = response.getHits();
1703 final SearchHit[] hits = searchHits.getHits();
1704 if (hits.length == 0) {
1705 break;
1706 }
1707
1708 for (final SearchHit hit : hits) {
1709 count++;
1710 if (!cursor.apply(creator.build(response, hit))) {
1711 break;
1712 }
1713 }
1714
1715 response = client.prepareSearchScroll(scrollId)
1716 .setScroll(scrollForDelete)
1717 .execute()
1718 .actionGet(fessConfig.getIndexBulkTimeout());
1719 if (!scrollId.equals(response.getScrollId())) {
1720 deleteScrollContext(scrollId);
1721 }
1722 scrollId = response.getScrollId();
1723 }
1724 } catch (final SearchPhaseExecutionException e) {
1725 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
1726 "Invalid query: " + searchRequestBuilder, e);
1727 } finally {
1728 deleteScrollContext(scrollId);
1729 }
1730 }
1731
1732 return count;
1733 }
1734
1735
1736
1737
1738
1739
1740
1741
1742 public OptionalEntity<Map<String, Object>> getDocument(final String index, final SearchCondition<SearchRequestBuilder> condition) {
1743 return getDocument(index, condition, (response, hit) -> {
1744 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1745 final Map<String, Object> source = hit.getSourceAsMap();
1746 if (source != null) {
1747 final Map<String, Object> docMap = new HashMap<>(source);
1748 docMap.put(fessConfig.getIndexFieldId(), hit.getId());
1749 docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
1750 docMap.put(fessConfig.getIndexFieldSeqNo(), hit.getSeqNo());
1751 docMap.put(fessConfig.getIndexFieldPrimaryTerm(), hit.getPrimaryTerm());
1752 return docMap;
1753 }
1754 final Map<String, DocumentField> fields = hit.getFields();
1755 if (fields != null) {
1756 final Map<String, Object> docMap = fields.entrySet()
1757 .stream()
1758 .collect(Collectors.toMap(Entry<String, DocumentField>::getKey, e -> (Object) e.getValue().getValues()));
1759 docMap.put(fessConfig.getIndexFieldId(), hit.getId());
1760 docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
1761 docMap.put(fessConfig.getIndexFieldSeqNo(), hit.getSeqNo());
1762 docMap.put(fessConfig.getIndexFieldPrimaryTerm(), hit.getPrimaryTerm());
1763 return docMap;
1764 }
1765 return null;
1766 });
1767 }
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778 protected <T> OptionalEntity<T> getDocument(final String index, final SearchCondition<SearchRequestBuilder> condition,
1779 final EntityCreator<T, SearchResponse, SearchHit> creator) {
1780 return search(index, searchRequestBuilder -> {
1781 searchRequestBuilder.setVersion(true);
1782 return condition.build(searchRequestBuilder);
1783 }, (queryBuilder, execTime, searchResponse) -> searchResponse.map(response -> {
1784 final SearchHit[] hits = response.getHits().getHits();
1785 if (hits.length > 0) {
1786 return creator.build(response, hits[0]);
1787 }
1788 return null;
1789 }));
1790 }
1791
1792
1793
1794
1795
1796
1797
1798
1799 public List<Map<String, Object>> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition) {
1800 return getDocumentList(index, condition, getDefaultEntityCreator());
1801 }
1802
1803
1804
1805
1806
1807
1808 protected EntityCreator<Map<String, Object>, SearchResponse, SearchHit> getDefaultEntityCreator() {
1809 return (response, hit) -> {
1810 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1811 final Map<String, Object> source = hit.getSourceAsMap();
1812 if (source != null) {
1813 final Map<String, Object> docMap = new HashMap<>(source);
1814 docMap.put(fessConfig.getIndexFieldId(), hit.getId());
1815 return docMap;
1816 }
1817 final Map<String, DocumentField> fields = hit.getFields();
1818 if (fields != null) {
1819 final Map<String, Object> docMap = fields.entrySet()
1820 .stream()
1821 .collect(Collectors.toMap(Entry<String, DocumentField>::getKey, e -> (Object) e.getValue().getValues()));
1822 docMap.put(fessConfig.getIndexFieldId(), hit.getId());
1823 return docMap;
1824 }
1825 return null;
1826 };
1827 }
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838 protected <T> List<T> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition,
1839 final EntityCreator<T, SearchResponse, SearchHit> creator) {
1840 return search(index, condition, (searchRequestBuilder, execTime, searchResponse) -> {
1841 final List<T> list = new ArrayList<>();
1842 searchResponse.ifPresent(response -> response.getHits().forEach(hit -> {
1843 list.add(creator.build(response, hit));
1844 }));
1845 return list;
1846 });
1847 }
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859 public boolean update(final String index, final String id, final String field, final Object value) {
1860
1861 if (usePipeline) {
1862 return updateByIdWithScript(index, id, field, value);
1863 }
1864 try {
1865 final Result result = client.prepareUpdate()
1866 .setIndex(index)
1867 .setId(id)
1868 .setDoc(field, value)
1869 .execute()
1870 .actionGet(ComponentUtil.getFessConfig().getIndexIndexTimeout())
1871 .getResult();
1872 return result == Result.CREATED || result == Result.UPDATED;
1873 } catch (final OpenSearchException e) {
1874 throw new SearchEngineClientException("[" + index + "] Failed to set " + value + " to " + field + " for doc " + id, e);
1875 }
1876 }
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888 protected boolean updateByIdWithScript(final String index, final String id, final String field, final Object value) {
1889 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1890 final UpdateByQueryRequest request = new UpdateByQueryRequest(index).setQuery(QueryBuilders.idsQuery().addIds(id))
1891 .setScript(new Script(ScriptType.INLINE, "painless",
1892 "ctx._source[params.f]=params.v;" + ComponentUtil.getLanguageHelper().getReindexScriptSource(),
1893 Map.of("f", field, "v", value)));
1894 try {
1895 final String source = SearchEngineUtil.getXContentString(request, XContentType.JSON);
1896 if (logger.isDebugEnabled()) {
1897 logger.debug("update script by id: {}", source);
1898 }
1899 final String refresh = StringUtil.isNotBlank(fessConfig.getIndexReindexRefresh()) ? fessConfig.getIndexReindexRefresh() : null;
1900 try (CurlResponse response = ComponentUtil.getCurlHelper()
1901 .post("/" + index + "/_update_by_query")
1902 .param("refresh", refresh)
1903 .param("max_docs", "1")
1904 .body(source)
1905 .execute()) {
1906 if (response.getHttpStatusCode() == 200) {
1907 return true;
1908 }
1909 return false;
1910 }
1911 } catch (final IOException e) {
1912 throw new SearchEngineClientException("[" + index + "] Failed to set " + value + " to " + field + " for doc " + id, e);
1913 }
1914 }
1915
1916
1917
1918
1919
1920
1921 public void refresh(final String... indices) {
1922 client.admin().indices().prepareRefresh(indices).execute(new ActionListener<RefreshResponse>() {
1923 @Override
1924 public void onResponse(final RefreshResponse response) {
1925 if (logger.isDebugEnabled()) {
1926 logger.debug(() -> "Refreshed " + stream(indices).get(stream -> stream.collect(Collectors.joining(", "))));
1927 }
1928 }
1929
1930 @Override
1931 public void onFailure(final Exception e) {
1932 logger.error(() -> "Failed to refresh " + stream(indices).get(stream -> stream.collect(Collectors.joining(", "))), e);
1933 }
1934 });
1935
1936 }
1937
1938
1939
1940
1941
1942
1943 public void flush(final String... indices) {
1944 client.admin().indices().prepareFlush(indices).execute(new ActionListener<FlushResponse>() {
1945
1946 @Override
1947 public void onResponse(final FlushResponse response) {
1948 if (logger.isDebugEnabled()) {
1949 logger.debug(() -> "Flushed " + stream(indices).get(stream -> stream.collect(Collectors.joining(", "))));
1950 }
1951 }
1952
1953 @Override
1954 public void onFailure(final Exception e) {
1955 logger.error(() -> "Failed to flush " + stream(indices).get(stream -> stream.collect(Collectors.joining(", "))), e);
1956 }
1957 });
1958
1959 }
1960
1961
1962
1963
1964
1965
1966
1967 public PingResponse ping() {
1968 try {
1969 final ClusterHealthResponse response =
1970 client.admin().cluster().prepareHealth().execute().actionGet(ComponentUtil.getFessConfig().getIndexHealthTimeout());
1971 return new PingResponse(response);
1972 } catch (final OpenSearchException e) {
1973 throw new SearchEngineClientException("Failed to process a ping request.", e);
1974 }
1975 }
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985 public BulkResponse addAll(final String index, final List<Map<String, Object>> docList,
1986 final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
1987 final FessConfig fessConfig = ComponentUtil.getFessConfig();
1988 final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
1989 for (final Map<String, Object> doc : docList) {
1990 final Object id = doc.remove(fessConfig.getIndexFieldId());
1991 final IndexRequestBuilder builder = client.prepareIndex().setIndex(index).setId(id.toString()).setSource(new DocMap(doc));
1992 options.accept(doc, builder);
1993 bulkRequestBuilder.add(builder);
1994 }
1995 return bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
1996 }
1997
1998
1999
2000
2001 public static class SearchConditionBuilder {
2002
2003 protected final SearchRequestBuilder searchRequestBuilder;
2004
2005 protected String query;
2006
2007 protected String[] responseFields;
2008
2009 protected int offset = Constants.DEFAULT_START_COUNT;
2010
2011 protected int size = Constants.DEFAULT_PAGE_SIZE;
2012
2013 protected GeoInfo geoInfo;
2014
2015 protected FacetInfo facetInfo;
2016
2017 protected HighlightInfo highlightInfo;
2018
2019 protected String similarDocHash;
2020
2021 protected SearchRequestType searchRequestType = SearchRequestType.SEARCH;
2022
2023 protected boolean isScroll = false;
2024
2025 protected String trackTotalHits = null;
2026
2027 protected Float minScore = null;
2028
2029
2030
2031
2032
2033
2034
2035 public static SearchConditionBuilder builder(final SearchRequestBuilder searchRequestBuilder) {
2036 return new SearchConditionBuilder(searchRequestBuilder);
2037 }
2038
2039
2040
2041
2042
2043
2044 SearchConditionBuilder(final SearchRequestBuilder searchRequestBuilder) {
2045 this.searchRequestBuilder = searchRequestBuilder;
2046 }
2047
2048
2049
2050
2051
2052
2053 public Map<String, Object> condition() {
2054 final Map<String, Object> params = new HashMap<>();
2055 params.put("query", query);
2056 params.put("responseFields", responseFields);
2057 params.put("offset", offset);
2058 params.put("size", size);
2059
2060
2061
2062 params.put("similarDocHash", similarDocHash);
2063 return params;
2064 }
2065
2066
2067
2068
2069
2070
2071
2072 public SearchConditionBuilder query(final String query) {
2073 this.query = query;
2074 return this;
2075 }
2076
2077
2078
2079
2080
2081
2082
2083 public SearchConditionBuilder searchRequestType(final SearchRequestType searchRequestType) {
2084 this.searchRequestType = searchRequestType;
2085 return this;
2086 }
2087
2088
2089
2090
2091
2092
2093
2094 public SearchConditionBuilder responseFields(final String[] responseFields) {
2095 this.responseFields = responseFields;
2096 return this;
2097 }
2098
2099
2100
2101
2102
2103
2104
2105 public SearchConditionBuilder offset(final int offset) {
2106 this.offset = offset;
2107 return this;
2108 }
2109
2110
2111
2112
2113
2114
2115
2116 public SearchConditionBuilder size(final int size) {
2117 this.size = size;
2118 return this;
2119 }
2120
2121
2122
2123
2124
2125
2126
2127 public SearchConditionBuilder geoInfo(final GeoInfo geoInfo) {
2128 this.geoInfo = geoInfo;
2129 return this;
2130 }
2131
2132
2133
2134
2135
2136
2137
2138 public SearchConditionBuilder highlightInfo(final HighlightInfo highlightInfo) {
2139 this.highlightInfo = highlightInfo;
2140 return this;
2141 }
2142
2143
2144
2145
2146
2147
2148
2149 public SearchConditionBuilder similarDocHash(final String similarDocHash) {
2150 if (StringUtil.isNotBlank(similarDocHash)) {
2151 this.similarDocHash = similarDocHash;
2152 }
2153 return this;
2154 }
2155
2156
2157
2158
2159
2160
2161
2162 public SearchConditionBuilder facetInfo(final FacetInfo facetInfo) {
2163 this.facetInfo = facetInfo;
2164 return this;
2165 }
2166
2167
2168
2169
2170
2171
2172 public SearchConditionBuilder scroll() {
2173 isScroll = true;
2174 return this;
2175 }
2176
2177
2178
2179
2180
2181
2182
2183 public SearchConditionBuilder trackTotalHits(final String trackTotalHits) {
2184 this.trackTotalHits = trackTotalHits;
2185 return this;
2186 }
2187
2188
2189
2190
2191
2192
2193
2194 public SearchConditionBuilder minScore(final Float minScore) {
2195 this.minScore = minScore;
2196 return this;
2197 }
2198
2199
2200
2201
2202
2203
2204
2205
2206 public boolean build() {
2207 if (StringUtil.isBlank(query)) {
2208 return false;
2209 }
2210
2211 final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
2212 final QueryFieldConfig queryFieldConfig = ComponentUtil.getQueryFieldConfig();
2213 final FessConfig fessConfig = ComponentUtil.getFessConfig();
2214
2215 if (offset > fessConfig.getQueryMaxSearchResultOffsetAsInteger()) {
2216 throw new ResultOffsetExceededException("The number of result size is exceeded.");
2217 }
2218
2219 final QueryContext queryContext = buildQueryContext(queryHelper, queryFieldConfig, fessConfig);
2220
2221 searchRequestBuilder.setFrom(offset).setSize(size);
2222
2223 buildTrackTotalHits(fessConfig);
2224 buildMinScore(fessConfig);
2225
2226 if (responseFields != null) {
2227 searchRequestBuilder.setFetchSource(responseFields, null);
2228 }
2229
2230
2231 buildRescorer(queryHelper, queryFieldConfig, fessConfig);
2232
2233
2234 buildSort(queryContext, queryFieldConfig, fessConfig);
2235
2236
2237 if (highlightInfo != null) {
2238 buildHighlighter(queryHelper, queryFieldConfig, fessConfig);
2239 }
2240
2241
2242 if (facetInfo != null) {
2243 buildFacet(queryHelper, queryFieldConfig, fessConfig);
2244 }
2245
2246 if (!SearchRequestType.ADMIN_SEARCH.equals(searchRequestType) && !isScroll && fessConfig.isResultCollapsed()
2247 && similarDocHash == null) {
2248 searchRequestBuilder.setCollapse(getCollapseBuilder(fessConfig));
2249 }
2250
2251 searchRequestBuilder.setQuery(queryContext.getQueryBuilder());
2252 return true;
2253 }
2254
2255
2256
2257
2258
2259
2260 protected void buildMinScore(final FessConfig fessConfig) {
2261 if (minScore != null) {
2262 searchRequestBuilder.setMinScore(minScore);
2263 }
2264 }
2265
2266
2267
2268
2269
2270
2271 protected void buildTrackTotalHits(final FessConfig fessConfig) {
2272 if (isScroll) {
2273 return;
2274 }
2275 if (StringUtil.isNotBlank(trackTotalHits)) {
2276 if (Constants.TRUE.equalsIgnoreCase(trackTotalHits) || Constants.FALSE.equalsIgnoreCase(trackTotalHits)) {
2277 searchRequestBuilder.setTrackTotalHits(Boolean.parseBoolean(trackTotalHits));
2278 return;
2279 }
2280 try {
2281 searchRequestBuilder.setTrackTotalHitsUpTo(Integer.parseInt(trackTotalHits));
2282 return;
2283 } catch (final NumberFormatException e) {
2284
2285 }
2286 }
2287 final Object trackTotalHitsValue = fessConfig.getQueryTrackTotalHitsValue();
2288 if (trackTotalHitsValue instanceof Boolean) {
2289 searchRequestBuilder.setTrackTotalHits((Boolean) trackTotalHitsValue);
2290 } else if (trackTotalHitsValue instanceof Number) {
2291 searchRequestBuilder.setTrackTotalHitsUpTo(((Number) trackTotalHitsValue).intValue());
2292 }
2293 }
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303 protected void buildFacet(final QueryHelper queryHelper, final QueryFieldConfig queryFieldConfig, final FessConfig fessConfig) {
2304 stream(facetInfo.field).of(stream -> stream.forEach(f -> {
2305 if (!queryFieldConfig.isFacetField(f)) {
2306 throw new SearchQueryException("Invalid facet field: " + f);
2307 }
2308 final String encodedField = BaseEncoding.base64().encode(f.getBytes(StandardCharsets.UTF_8));
2309 final TermsAggregationBuilder termsBuilder =
2310 AggregationBuilders.terms(Constants.FACET_FIELD_PREFIX + encodedField).field(f);
2311 termsBuilder.order(facetInfo.getBucketOrder());
2312 if (facetInfo.size != null) {
2313 termsBuilder.size(facetInfo.size);
2314 }
2315 if (facetInfo.minDocCount != null) {
2316 termsBuilder.minDocCount(facetInfo.minDocCount);
2317 }
2318 if (facetInfo.missing != null) {
2319 termsBuilder.missing(facetInfo.missing);
2320 }
2321 searchRequestBuilder.addAggregation(termsBuilder);
2322 }));
2323 stream(facetInfo.query).of(stream -> stream.forEach(fq -> {
2324 final QueryContext facetContext = new QueryContext(fq, false);
2325 queryHelper.buildBaseQuery(facetContext, c -> {});
2326 final String encodedFacetQuery = BaseEncoding.base64().encode(fq.getBytes(StandardCharsets.UTF_8));
2327 final FilterAggregationBuilder filterBuilder =
2328 AggregationBuilders.filter(Constants.FACET_QUERY_PREFIX + encodedFacetQuery, facetContext.getQueryBuilder());
2329 searchRequestBuilder.addAggregation(filterBuilder);
2330 }));
2331 }
2332
2333
2334
2335
2336
2337
2338
2339
2340 protected void buildHighlighter(final QueryHelper queryHelper, final QueryFieldConfig queryFieldConfig,
2341 final FessConfig fessConfig) {
2342 final String highlighterType = highlightInfo.getType();
2343 final int fragmentSize = highlightInfo.getFragmentSize();
2344 final int numOfFragments = highlightInfo.getNumOfFragments();
2345 final int fragmentOffset = highlightInfo.getFragmentOffset();
2346 final char[] boundaryChars = fessConfig.getQueryHighlightBoundaryCharsAsArray();
2347 final int boundaryMaxScan = fessConfig.getQueryHighlightBoundaryMaxScanAsInteger();
2348 final String boundaryScannerType = fessConfig.getQueryHighlightBoundaryScanner();
2349 final boolean forceSource = fessConfig.isQueryHighlightForceSource();
2350 final String fragmenter = fessConfig.getQueryHighlightFragmenter();
2351 final int noMatchSize = fessConfig.getQueryHighlightNoMatchSizeAsInteger();
2352 final String order = fessConfig.getQueryHighlightOrder();
2353 final int phraseLimit = fessConfig.getQueryHighlightPhraseLimitAsInteger();
2354 final String encoder = fessConfig.getQueryHighlightEncoder();
2355 final HighlightBuilder highlightBuilder = new HighlightBuilder();
2356 final String[] preTags = highlightInfo.getPreTags();
2357 final String[] postTags = highlightInfo.getPostTags();
2358 if (preTags != null) {
2359 highlightBuilder.preTags(preTags);
2360 }
2361 if (postTags != null) {
2362 highlightBuilder.postTags(postTags);
2363 }
2364 queryFieldConfig.highlightedFields(
2365 stream -> stream.forEach(hf -> highlightBuilder.field(new HighlightBuilder.Field(hf).highlighterType(highlighterType)
2366 .fragmentSize(fragmentSize)
2367 .numOfFragments(numOfFragments)
2368 .boundaryChars(boundaryChars)
2369 .boundaryMaxScan(boundaryMaxScan)
2370 .boundaryScannerType(boundaryScannerType)
2371 .forceSource(forceSource)
2372 .fragmenter(fragmenter)
2373 .fragmentOffset(fragmentOffset)
2374 .noMatchSize(noMatchSize)
2375 .order(order)
2376 .phraseLimit(phraseLimit)).encoder(encoder)));
2377 searchRequestBuilder.highlighter(highlightBuilder);
2378 }
2379
2380
2381
2382
2383
2384
2385
2386
2387 protected void buildSort(final QueryContext queryContext, final QueryFieldConfig queryFieldConfig, final FessConfig fessConfig) {
2388 queryContext.sortBuilders().forEach(sortBuilder -> searchRequestBuilder.addSort(sortBuilder));
2389 }
2390
2391
2392
2393
2394
2395
2396
2397
2398 protected void buildRescorer(final QueryHelper queryHelper, final QueryFieldConfig queryFieldConfig, final FessConfig fessConfig) {
2399 stream(queryHelper.getRescorers(condition())).of(stream -> stream.forEach(searchRequestBuilder::addRescorer));
2400 }
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410 protected QueryContext buildQueryContext(final QueryHelper queryHelper, final QueryFieldConfig queryFieldConfig,
2411 final FessConfig fessConfig) {
2412 return queryHelper.build(searchRequestType, query, context -> {
2413 if (SearchRequestType.ADMIN_SEARCH.equals(searchRequestType)) {
2414 context.skipRoleQuery();
2415 } else if (similarDocHash != null) {
2416 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
2417 context.addQuery(boolQuery -> {
2418 boolQuery.filter(QueryBuilders.termQuery(fessConfig.getIndexFieldContentMinhashBits(),
2419 documentHelper.decodeSimilarDocHash(similarDocHash)));
2420 });
2421 }
2422
2423 if (geoInfo != null && geoInfo.toQueryBuilder() != null) {
2424 context.addQuery(boolQuery -> boolQuery.filter(geoInfo.toQueryBuilder()));
2425 }
2426 });
2427 }
2428
2429
2430
2431
2432
2433
2434
2435 protected CollapseBuilder getCollapseBuilder(final FessConfig fessConfig) {
2436 final InnerHitBuilder innerHitBuilder = new InnerHitBuilder().setName(fessConfig.getQueryCollapseInnerHitsName())
2437 .setSize(fessConfig.getQueryCollapseInnerHitsSizeAsInteger());
2438 fessConfig.getQueryCollapseInnerHitsSortBuilders()
2439 .ifPresent(builders -> stream(builders).of(stream -> stream.forEach(innerHitBuilder::addSort)));
2440 return new CollapseBuilder(fessConfig.getIndexFieldContentMinhashBits())
2441 .setMaxConcurrentGroupRequests(fessConfig.getQueryCollapseMaxConcurrentGroupResultsAsInteger())
2442 .setInnerHits(innerHitBuilder);
2443 }
2444 }
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454 public boolean store(final String index, final Object obj) {
2455 final FessConfig fessConfig = ComponentUtil.getFessConfig();
2456 @SuppressWarnings("unchecked")
2457 final Map<String, Object> source = obj instanceof Map ? (Map<String, Object>) obj : BeanUtil.copyBeanToNewMap(obj);
2458 final String id = (String) source.remove(fessConfig.getIndexFieldId());
2459 source.remove(fessConfig.getIndexFieldVersion());
2460 final Number seqNo = (Number) source.remove(fessConfig.getIndexFieldSeqNo());
2461 final Number primaryTerm = (Number) source.remove(fessConfig.getIndexFieldPrimaryTerm());
2462 IndexResponse response;
2463 try {
2464 if (id == null) {
2465
2466
2467 response = client.prepareIndex()
2468 .setIndex(index)
2469 .setSource(new DocMap(source))
2470 .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
2471 .setOpType(OpType.CREATE)
2472 .execute()
2473 .actionGet(fessConfig.getIndexIndexTimeout());
2474 } else {
2475
2476 final IndexRequestBuilder builder = client.prepareIndex()
2477 .setIndex(index)
2478 .setId(id)
2479 .setSource(new DocMap(source))
2480 .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
2481 .setOpType(OpType.INDEX);
2482 if (seqNo != null) {
2483 builder.setIfSeqNo(seqNo.longValue());
2484 }
2485 if (primaryTerm != null) {
2486 builder.setIfPrimaryTerm(primaryTerm.longValue());
2487 }
2488 response = builder.execute().actionGet(fessConfig.getIndexIndexTimeout());
2489 }
2490 final Result result = response.getResult();
2491 return result == Result.CREATED || result == Result.UPDATED;
2492 } catch (final OpenSearchException e) {
2493 throw new SearchEngineClientException("Failed to store: " + obj, e);
2494 }
2495 }
2496
2497
2498
2499
2500
2501
2502
2503
2504 public boolean delete(final String index, final String id) {
2505 return delete(index, id, null, null);
2506 }
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518 public boolean delete(final String index, final String id, final Number seqNo, final Number primaryTerm) {
2519 try {
2520 final DeleteRequestBuilder builder = client.prepareDelete().setIndex(index).setId(id).setRefreshPolicy(RefreshPolicy.IMMEDIATE);
2521 if (seqNo != null) {
2522 builder.setIfSeqNo(seqNo.longValue());
2523 }
2524 if (primaryTerm != null) {
2525 builder.setIfPrimaryTerm(primaryTerm.longValue());
2526 }
2527 final DeleteResponse response = builder.execute().actionGet(ComponentUtil.getFessConfig().getIndexDeleteTimeout());
2528 return response.getResult() == Result.DELETED;
2529 } catch (final OpenSearchException e) {
2530 throw new SearchEngineClientException("Failed to delete: " + index + "/" + id + "@" + seqNo + ":" + primaryTerm, e);
2531 }
2532 }
2533
2534
2535
2536
2537
2538
2539 public void setIndexConfigPath(final String indexConfigPath) {
2540 this.indexConfigPath = indexConfigPath;
2541 }
2542
2543
2544
2545
2546
2547
2548 public interface SearchCondition<B> {
2549
2550
2551
2552
2553
2554
2555 boolean build(B requestBuilder);
2556 }
2557
2558
2559
2560
2561
2562
2563
2564
2565 public interface SearchResult<T, B, R> {
2566
2567
2568
2569
2570
2571
2572
2573
2574 T build(B requestBuilder, long execTime, OptionalEntity<R> response);
2575 }
2576
2577
2578
2579
2580
2581
2582
2583
2584 public interface EntityCreator<T, R, H> {
2585
2586
2587
2588
2589
2590
2591
2592 T build(R response, H hit);
2593 }
2594
2595
2596
2597
2598
2599
2600 public void setClusterName(final String clusterName) {
2601 this.clusterName = clusterName;
2602 }
2603
2604
2605
2606
2607
2608
2609
2610 public EngineInfo getEngineInfo() {
2611 if (client instanceof final HttpClient httpClient) {
2612 return httpClient.getEngineInfo();
2613 }
2614 throw new SearchEngineClientException("client is not HttpClient.");
2615 }
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626 @Override
2627 public ThreadPool threadPool() {
2628 return client.threadPool();
2629 }
2630
2631
2632
2633
2634
2635
2636 @Override
2637 public AdminClient admin() {
2638 return client.admin();
2639 }
2640
2641
2642
2643
2644
2645
2646
2647 @Override
2648 public ActionFuture<IndexResponse> index(final IndexRequest request) {
2649 return client.index(request);
2650 }
2651
2652
2653
2654
2655
2656
2657
2658 @Override
2659 public void index(final IndexRequest request, final ActionListener<IndexResponse> listener) {
2660 client.index(request, listener);
2661 }
2662
2663
2664
2665
2666
2667
2668 @Override
2669 public IndexRequestBuilder prepareIndex() {
2670 return client.prepareIndex();
2671 }
2672
2673
2674
2675
2676
2677
2678
2679 @Override
2680 public ActionFuture<UpdateResponse> update(final UpdateRequest request) {
2681 return client.update(request);
2682 }
2683
2684
2685
2686
2687
2688
2689
2690 @Override
2691 public void update(final UpdateRequest request, final ActionListener<UpdateResponse> listener) {
2692 client.update(request, listener);
2693 }
2694
2695
2696
2697
2698
2699
2700 @Override
2701 public UpdateRequestBuilder prepareUpdate() {
2702 return client.prepareUpdate();
2703 }
2704
2705
2706
2707
2708
2709
2710
2711
2712 @Override
2713 public UpdateRequestBuilder prepareUpdate(final String index, final String id) {
2714 return client.prepareUpdate(index, id);
2715 }
2716
2717
2718
2719
2720
2721
2722
2723 @Override
2724 public IndexRequestBuilder prepareIndex(final String index) {
2725 return client.prepareIndex(index);
2726 }
2727
2728
2729
2730
2731
2732
2733
2734 @Override
2735 public ActionFuture<DeleteResponse> delete(final DeleteRequest request) {
2736 return client.delete(request);
2737 }
2738
2739
2740
2741
2742
2743
2744
2745 @Override
2746 public void delete(final DeleteRequest request, final ActionListener<DeleteResponse> listener) {
2747 client.delete(request, listener);
2748 }
2749
2750
2751
2752
2753
2754
2755 @Override
2756 public DeleteRequestBuilder prepareDelete() {
2757 return client.prepareDelete();
2758 }
2759
2760
2761
2762
2763
2764
2765
2766
2767 @Override
2768 public DeleteRequestBuilder prepareDelete(final String index, final String id) {
2769 return client.prepareDelete(index, id);
2770 }
2771
2772
2773
2774
2775
2776
2777
2778 @Override
2779 public ActionFuture<BulkResponse> bulk(final BulkRequest request) {
2780 return client.bulk(request);
2781 }
2782
2783
2784
2785
2786
2787
2788
2789 @Override
2790 public void bulk(final BulkRequest request, final ActionListener<BulkResponse> listener) {
2791 client.bulk(request, listener);
2792 }
2793
2794
2795
2796
2797
2798
2799 @Override
2800 public BulkRequestBuilder prepareBulk() {
2801 return client.prepareBulk();
2802 }
2803
2804
2805
2806
2807
2808
2809
2810 @Override
2811 public ActionFuture<GetResponse> get(final GetRequest request) {
2812 return client.get(request);
2813 }
2814
2815
2816
2817
2818
2819
2820
2821 @Override
2822 public void get(final GetRequest request, final ActionListener<GetResponse> listener) {
2823 client.get(request, listener);
2824 }
2825
2826
2827
2828
2829
2830
2831 @Override
2832 public GetRequestBuilder prepareGet() {
2833 return client.prepareGet();
2834 }
2835
2836
2837
2838
2839
2840
2841
2842
2843 @Override
2844 public GetRequestBuilder prepareGet(final String index, final String id) {
2845 return client.prepareGet(index, id);
2846 }
2847
2848
2849
2850
2851
2852
2853
2854 @Override
2855 public ActionFuture<MultiGetResponse> multiGet(final MultiGetRequest request) {
2856 return client.multiGet(request);
2857 }
2858
2859
2860
2861
2862
2863
2864
2865 @Override
2866 public void multiGet(final MultiGetRequest request, final ActionListener<MultiGetResponse> listener) {
2867 client.multiGet(request, listener);
2868 }
2869
2870
2871
2872
2873
2874
2875 @Override
2876 public MultiGetRequestBuilder prepareMultiGet() {
2877 return client.prepareMultiGet();
2878 }
2879
2880
2881
2882
2883
2884
2885
2886 @Override
2887 public ActionFuture<SearchResponse> search(final SearchRequest request) {
2888 return client.search(request);
2889 }
2890
2891
2892
2893
2894
2895
2896
2897 @Override
2898 public void search(final SearchRequest request, final ActionListener<SearchResponse> listener) {
2899 client.search(request, listener);
2900 }
2901
2902
2903
2904
2905
2906
2907
2908 @Override
2909 public SearchRequestBuilder prepareSearch(final String... indices) {
2910 return client.prepareSearch(indices);
2911 }
2912
2913
2914
2915
2916
2917
2918
2919 @Override
2920 public SearchRequestBuilder prepareStreamSearch(final String... indices) {
2921 return client.prepareStreamSearch(indices);
2922 }
2923
2924
2925
2926
2927
2928
2929
2930 @Override
2931 public ActionFuture<SearchResponse> searchScroll(final SearchScrollRequest request) {
2932 return client.searchScroll(request);
2933 }
2934
2935
2936
2937
2938
2939
2940
2941 @Override
2942 public void searchScroll(final SearchScrollRequest request, final ActionListener<SearchResponse> listener) {
2943 client.searchScroll(request, listener);
2944 }
2945
2946
2947
2948
2949
2950
2951
2952 @Override
2953 public SearchScrollRequestBuilder prepareSearchScroll(final String scrollId) {
2954 return client.prepareSearchScroll(scrollId);
2955 }
2956
2957
2958
2959
2960
2961
2962
2963 @Override
2964 public ActionFuture<MultiSearchResponse> multiSearch(final MultiSearchRequest request) {
2965 return client.multiSearch(request);
2966 }
2967
2968
2969
2970
2971
2972
2973
2974 @Override
2975 public void multiSearch(final MultiSearchRequest request, final ActionListener<MultiSearchResponse> listener) {
2976 client.multiSearch(request, listener);
2977 }
2978
2979
2980
2981
2982
2983
2984 @Override
2985 public MultiSearchRequestBuilder prepareMultiSearch() {
2986 return client.prepareMultiSearch();
2987 }
2988
2989
2990
2991
2992
2993
2994
2995
2996 @Override
2997 public ExplainRequestBuilder prepareExplain(final String index, final String id) {
2998 return client.prepareExplain(index, id);
2999 }
3000
3001
3002
3003
3004
3005
3006
3007 @Override
3008 public ActionFuture<ExplainResponse> explain(final ExplainRequest request) {
3009 return client.explain(request);
3010 }
3011
3012
3013
3014
3015
3016
3017
3018 @Override
3019 public void explain(final ExplainRequest request, final ActionListener<ExplainResponse> listener) {
3020 client.explain(request, listener);
3021 }
3022
3023
3024
3025
3026
3027
3028 @Override
3029 public ClearScrollRequestBuilder prepareClearScroll() {
3030 return client.prepareClearScroll();
3031 }
3032
3033
3034
3035
3036
3037
3038
3039 @Override
3040 public ActionFuture<ClearScrollResponse> clearScroll(final ClearScrollRequest request) {
3041 return client.clearScroll(request);
3042 }
3043
3044
3045
3046
3047
3048
3049
3050 @Override
3051 public void clearScroll(final ClearScrollRequest request, final ActionListener<ClearScrollResponse> listener) {
3052 client.clearScroll(request, listener);
3053 }
3054
3055
3056
3057
3058
3059
3060 @Override
3061 public Settings settings() {
3062 return client.settings();
3063 }
3064
3065
3066
3067
3068
3069
3070
3071 @Override
3072 public ActionFuture<TermVectorsResponse> termVectors(final TermVectorsRequest request) {
3073 return client.termVectors(request);
3074 }
3075
3076
3077
3078
3079
3080
3081
3082 @Override
3083 public void termVectors(final TermVectorsRequest request, final ActionListener<TermVectorsResponse> listener) {
3084 client.termVectors(request, listener);
3085 }
3086
3087
3088
3089
3090
3091
3092 @Override
3093 public TermVectorsRequestBuilder prepareTermVectors() {
3094 return client.prepareTermVectors();
3095 }
3096
3097
3098
3099
3100
3101
3102
3103
3104 @Override
3105 public TermVectorsRequestBuilder prepareTermVectors(final String index, final String id) {
3106 return client.prepareTermVectors(index, id);
3107 }
3108
3109
3110
3111
3112
3113
3114
3115 @Override
3116 public ActionFuture<MultiTermVectorsResponse> multiTermVectors(final MultiTermVectorsRequest request) {
3117 return client.multiTermVectors(request);
3118 }
3119
3120
3121
3122
3123
3124
3125
3126 @Override
3127 public void multiTermVectors(final MultiTermVectorsRequest request, final ActionListener<MultiTermVectorsResponse> listener) {
3128 client.multiTermVectors(request, listener);
3129 }
3130
3131
3132
3133
3134
3135
3136 @Override
3137 public MultiTermVectorsRequestBuilder prepareMultiTermVectors() {
3138 return client.prepareMultiTermVectors();
3139 }
3140
3141
3142
3143
3144
3145
3146 public void setSizeForUpdate(final int sizeForUpdate) {
3147 this.sizeForUpdate = sizeForUpdate;
3148 }
3149
3150
3151
3152
3153
3154
3155 public void setScrollForUpdate(final String scrollForUpdate) {
3156 this.scrollForUpdate = scrollForUpdate;
3157 }
3158
3159
3160
3161
3162
3163
3164 public void setSizeForDelete(final int sizeForDelete) {
3165 this.sizeForDelete = sizeForDelete;
3166 }
3167
3168
3169
3170
3171
3172
3173 public void setScrollForDelete(final String scrollForDelete) {
3174 this.scrollForDelete = scrollForDelete;
3175 }
3176
3177
3178
3179
3180
3181
3182 public void setScrollForSearch(final String scrollForSearch) {
3183 this.scrollForSearch = scrollForSearch;
3184 }
3185
3186
3187
3188
3189
3190
3191 public void setMaxConfigSyncStatusRetry(final int maxConfigSyncStatusRetry) {
3192 this.maxConfigSyncStatusRetry = maxConfigSyncStatusRetry;
3193 }
3194
3195
3196
3197
3198
3199
3200 public void setMaxEsStatusRetry(final int maxEsStatusRetry) {
3201 this.maxEsStatusRetry = maxEsStatusRetry;
3202 }
3203
3204
3205
3206
3207
3208
3209
3210 @Override
3211 public Client filterWithHeader(final Map<String, String> headers) {
3212 return client.filterWithHeader(headers);
3213 }
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224 @Override
3225 public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(
3226 final ActionType<Response> action, final Request request) {
3227 return client.execute(action, request);
3228 }
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239 @Override
3240 public <Request extends ActionRequest, Response extends ActionResponse> void execute(final ActionType<Response> action,
3241 final Request request, final ActionListener<Response> listener) {
3242 client.execute(action, request, listener);
3243 }
3244
3245
3246
3247
3248
3249
3250
3251 @Override
3252 public FieldCapabilitiesRequestBuilder prepareFieldCaps(final String... indices) {
3253 return client.prepareFieldCaps(indices);
3254 }
3255
3256
3257
3258
3259
3260
3261
3262 @Override
3263 public ActionFuture<FieldCapabilitiesResponse> fieldCaps(final FieldCapabilitiesRequest request) {
3264 return client.fieldCaps(request);
3265 }
3266
3267
3268
3269
3270
3271
3272
3273 @Override
3274 public void fieldCaps(final FieldCapabilitiesRequest request, final ActionListener<FieldCapabilitiesResponse> listener) {
3275 client.fieldCaps(request, listener);
3276 }
3277
3278
3279
3280
3281
3282
3283
3284 @Override
3285 public BulkRequestBuilder prepareBulk(final String globalIndex) {
3286 return client.prepareBulk(globalIndex);
3287 }
3288
3289
3290
3291
3292
3293
3294
3295 @Override
3296 public void createPit(final CreatePitRequest createPITRequest, final ActionListener<CreatePitResponse> listener) {
3297 client.createPit(createPITRequest, listener);
3298 }
3299
3300
3301
3302
3303
3304
3305
3306 @Override
3307 public void deletePits(final DeletePitRequest deletePITRequest, final ActionListener<DeletePitResponse> listener) {
3308 client.deletePits(deletePITRequest, listener);
3309 }
3310
3311
3312
3313
3314
3315
3316
3317 @Override
3318 public void getAllPits(final GetAllPitNodesRequest getAllPitNodesRequest, final ActionListener<GetAllPitNodesResponse> listener) {
3319 client.getAllPits(getAllPitNodesRequest, listener);
3320 }
3321
3322
3323
3324
3325
3326
3327
3328 @Override
3329 public void pitSegments(final PitSegmentsRequest pitSegmentsRequest, final ActionListener<IndicesSegmentResponse> listener) {
3330 client.pitSegments(pitSegmentsRequest, listener);
3331 }
3332
3333
3334
3335
3336
3337
3338
3339
3340 @Override
3341 public void searchView(org.opensearch.action.admin.indices.view.SearchViewAction.Request request,
3342 ActionListener<SearchResponse> listener) {
3343 throw new UnsupportedOperationException("Not implemented yet");
3344 }
3345
3346
3347
3348
3349
3350
3351
3352
3353 @Override
3354 public ActionFuture<SearchResponse> searchView(org.opensearch.action.admin.indices.view.SearchViewAction.Request request) {
3355 throw new UnsupportedOperationException("Not implemented yet");
3356 }
3357
3358
3359
3360
3361
3362
3363
3364
3365 @Override
3366 public void listViewNames(org.opensearch.action.admin.indices.view.ListViewNamesAction.Request request,
3367 ActionListener<org.opensearch.action.admin.indices.view.ListViewNamesAction.Response> listener) {
3368 throw new UnsupportedOperationException("Not implemented yet");
3369 }
3370
3371
3372
3373
3374
3375
3376
3377
3378 @Override
3379 public ActionFuture<org.opensearch.action.admin.indices.view.ListViewNamesAction.Response> listViewNames(
3380 org.opensearch.action.admin.indices.view.ListViewNamesAction.Request request) {
3381 throw new UnsupportedOperationException("Not implemented yet");
3382 }
3383 }