1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import java.util.ArrayList;
19 import java.util.List;
20 import java.util.Map;
21 import java.util.Optional;
22 import java.util.concurrent.ConcurrentHashMap;
23 import java.util.function.Consumer;
24
25 import org.apache.logging.log4j.LogManager;
26 import org.apache.logging.log4j.Logger;
27 import org.codelibs.core.lang.StringUtil;
28 import org.codelibs.core.timer.TimeoutManager;
29 import org.codelibs.core.timer.TimeoutTask;
30 import org.codelibs.curl.CurlResponse;
31 import org.codelibs.fess.mylasta.direction.FessConfig;
32 import org.codelibs.fess.util.ComponentUtil;
33
34 import com.fasterxml.jackson.core.type.TypeReference;
35 import com.fasterxml.jackson.databind.ObjectMapper;
36
37 import jakarta.annotation.PostConstruct;
38 import jakarta.annotation.PreDestroy;
39
40
41
42
43
44
45 public class CoordinatorHelper {
46
47 private static final Logger logger = LogManager.getLogger(CoordinatorHelper.class);
48
49 private static final String INDEX_NAME = "fess_config.coordinator";
50
51 private static final String TYPE_HEARTBEAT = "heartbeat";
52
53 private static final String TYPE_OPERATION = "operation";
54
55 private static final String TYPE_EVENT = "event";
56
57 private static final String STATUS_ACTIVE = "active";
58
59 private static final String STATUS_RUNNING = "running";
60
61 private static final String TARGET_ALL = "*";
62
63 private final ObjectMapper objectMapper = new ObjectMapper();
64
65 private String instanceId;
66
67 private TimeoutTask pollTask;
68
69 private long lastEventCheckTime;
70
71 private final Map<String, List<Consumer<EventInfo>>> eventHandlers = new ConcurrentHashMap<>();
72
73
74
75
76 public CoordinatorHelper() {
77
78 }
79
80
81
82
83 @PostConstruct
84 public void init() {
85 instanceId = ComponentUtil.getSystemHelper().getInstanceId();
86 lastEventCheckTime = System.currentTimeMillis();
87
88 sendHeartbeat();
89
90 final FessConfig fessConfig = ComponentUtil.getFessConfig();
91 final int interval = fessConfig.getCoordinatorPollIntervalAsInteger();
92 pollTask = TimeoutManager.getInstance().addTimeoutTarget(this::poll, interval, true);
93
94 if (logger.isInfoEnabled()) {
95 logger.info("CoordinatorHelper started: instanceId={}", instanceId);
96 }
97 }
98
99
100
101
102 @PreDestroy
103 public void destroy() {
104 if (pollTask != null) {
105 pollTask.cancel();
106 }
107 try {
108 removeHeartbeat();
109 } catch (final Exception e) {
110 logger.debug("Failed to remove heartbeat on shutdown.", e);
111 }
112 if (logger.isInfoEnabled()) {
113 logger.info("CoordinatorHelper stopped: instanceId={}", instanceId);
114 }
115 }
116
117
118
119
120
121
122
123
124 public void sendHeartbeat() {
125 final long now = System.currentTimeMillis();
126 final long ttl = ComponentUtil.getFessConfig().getCoordinatorHeartbeatTtlAsInteger().longValue();
127 final String hostname = ComponentUtil.getSystemHelper().getHostname();
128 final String targetName = ComponentUtil.getFessConfig().getSchedulerTargetName();
129
130 final String body = toJson(Map.of(
131 "type", TYPE_HEARTBEAT,
132 "instanceId", instanceId,
133 "hostname", hostname,
134 "name", StringUtil.isNotBlank(targetName) ? targetName : hostname,
135 "status", STATUS_ACTIVE,
136 "createdTime", now,
137 "expiredTime", now + ttl));
138
139 try (CurlResponse response = ComponentUtil.getCurlHelper()
140 .put("/" + getIndexName() + "/_doc/" + instanceId + "?refresh=true")
141 .body(body)
142 .execute()) {
143 if (response.getHttpStatusCode() != 200 && response.getHttpStatusCode() != 201) {
144 logger.warn("Failed to send heartbeat: status={}", response.getHttpStatusCode());
145 }
146 } catch (final Exception e) {
147 logger.debug("Failed to send heartbeat.", e);
148 }
149 }
150
151
152
153
154 protected void removeHeartbeat() {
155 try (CurlResponse response = ComponentUtil.getCurlHelper()
156 .delete("/" + getIndexName() + "/_doc/" + instanceId + "?refresh=true")
157 .execute()) {
158 if (logger.isDebugEnabled()) {
159 logger.debug("Removed heartbeat: status={}", response.getHttpStatusCode());
160 }
161 } catch (final Exception e) {
162 logger.debug("Failed to remove heartbeat.", e);
163 }
164 }
165
166
167
168
169
170
171 public List<InstanceInfo> getActiveInstances() {
172 final List<InstanceInfo> instances = new ArrayList<>();
173 final long now = System.currentTimeMillis();
174 final String query = toJson(Map.of(
175 "query", Map.of("bool", Map.of("must", List.of(
176 Map.of("term", Map.of("type", TYPE_HEARTBEAT)),
177 Map.of("range", Map.of("expiredTime", Map.of("gte", now)))))),
178 "size", 100));
179
180 try (CurlResponse response = ComponentUtil.getCurlHelper()
181 .post("/" + getIndexName() + "/_search")
182 .body(query)
183 .execute()) {
184 if (response.getHttpStatusCode() == 200) {
185 final Map<String, Object> result = parseJson(response.getContentAsString());
186 final Map<String, Object> hits = getMapValue(result, "hits");
187 if (hits != null) {
188 final List<Map<String, Object>> hitList = getListValue(hits, "hits");
189 if (hitList != null) {
190 for (final Map<String, Object> hit : hitList) {
191 final Map<String, Object> source = getMapValue(hit, "_source");
192 if (source != null) {
193 final InstanceInfo info = new InstanceInfo();
194 info.instanceId = getStringValue(source, "instanceId");
195 info.hostname = getStringValue(source, "hostname");
196 info.name = getStringValue(source, "name");
197 info.lastSeen = getLongValue(source, "createdTime");
198 instances.add(info);
199 }
200 }
201 }
202 }
203 }
204 } catch (final Exception e) {
205 logger.warn("Failed to get active instances.", e);
206 }
207 return instances;
208 }
209
210
211
212
213
214
215
216 public boolean isInstanceActive(final String targetInstanceId) {
217 return getActiveInstances().stream().anyMatch(i -> i.instanceId.equals(targetInstanceId));
218 }
219
220
221
222
223
224
225
226
227
228
229
230 public boolean tryStartOperation(final String operationName) {
231 return tryStartOperation(operationName, null);
232 }
233
234
235
236
237
238
239
240
241 public boolean tryStartOperation(final String operationName, final String data) {
242 final int maxRetry = ComponentUtil.getFessConfig().getCoordinatorOperationRetryAsInteger();
243 return tryStartOperation(operationName, data, maxRetry);
244 }
245
246
247
248
249
250
251
252
253
254 protected boolean tryStartOperation(final String operationName, final String data, final int remainingRetries) {
255 final long now = System.currentTimeMillis();
256 final long ttl = ComponentUtil.getFessConfig().getCoordinatorOperationTtlAsInteger().longValue();
257 final String hostname = ComponentUtil.getSystemHelper().getHostname();
258
259 final Map<String, Object> bodyMap = new java.util.LinkedHashMap<>();
260 bodyMap.put("type", TYPE_OPERATION);
261 bodyMap.put("name", operationName);
262 bodyMap.put("instanceId", instanceId);
263 bodyMap.put("hostname", hostname);
264 bodyMap.put("status", STATUS_RUNNING);
265 bodyMap.put("createdTime", now);
266 bodyMap.put("expiredTime", now + ttl);
267 if (data != null) {
268 bodyMap.put("data", data);
269 }
270
271 try (CurlResponse response = ComponentUtil.getCurlHelper()
272 .put("/" + getIndexName() + "/_create/" + operationName + "?refresh=true")
273 .body(toJson(bodyMap))
274 .execute()) {
275 if (response.getHttpStatusCode() == 201) {
276 if (logger.isInfoEnabled()) {
277 logger.info("Acquired operation lock: operation={}, instanceId={}", operationName, instanceId);
278 }
279 return true;
280 }
281 } catch (final Exception e) {
282 logger.debug("Failed to create operation document: operation={}", operationName, e);
283 }
284
285 if (remainingRetries <= 0) {
286 if (logger.isDebugEnabled()) {
287 logger.debug("No remaining retries for operation lock: operation={}", operationName);
288 }
289 return false;
290 }
291
292
293 return tryCleanupAndRetry(operationName, data, remainingRetries);
294 }
295
296
297
298
299
300
301
302
303
304 protected boolean tryCleanupAndRetry(final String operationName, final String data, final int remainingRetries) {
305 try (CurlResponse response = ComponentUtil.getCurlHelper()
306 .get("/" + getIndexName() + "/_doc/" + operationName)
307 .execute()) {
308 if (response.getHttpStatusCode() != 200) {
309 return false;
310 }
311
312 final Map<String, Object> result = parseJson(response.getContentAsString());
313 final Map<String, Object> source = getMapValue(result, "_source");
314 if (source == null) {
315 return false;
316 }
317
318 final long expiredTime = getLongValue(source, "expiredTime");
319 final String ownerInstanceId = getStringValue(source, "instanceId");
320 final long now = System.currentTimeMillis();
321
322
323 if (expiredTime < now || !isInstanceActive(ownerInstanceId)) {
324 final long seqNo = getLongValue(result, "_seq_no");
325 final long primaryTerm = getLongValue(result, "_primary_term");
326
327
328 try (CurlResponse deleteResponse = ComponentUtil.getCurlHelper()
329 .delete("/" + getIndexName() + "/_doc/" + operationName
330 + "?refresh=true&if_seq_no=" + seqNo + "&if_primary_term=" + primaryTerm)
331 .execute()) {
332 if (deleteResponse.getHttpStatusCode() == 200) {
333 if (logger.isInfoEnabled()) {
334 logger.info("Cleaned up stale operation: operation={}, previousOwner={}", operationName, ownerInstanceId);
335 }
336
337 return tryStartOperation(operationName, data, remainingRetries - 1);
338 }
339 }
340 }
341 } catch (final Exception e) {
342 logger.warn("Failed to check existing operation: operation={}", operationName, e);
343 }
344 return false;
345 }
346
347
348
349
350
351
352
353 public void completeOperation(final String operationName) {
354 try (CurlResponse getResponse = ComponentUtil.getCurlHelper()
355 .get("/" + getIndexName() + "/_doc/" + operationName)
356 .execute()) {
357 if (getResponse.getHttpStatusCode() != 200) {
358 logger.debug("Operation document not found: operation={}", operationName);
359 return;
360 }
361 final Map<String, Object> result = parseJson(getResponse.getContentAsString());
362 final Map<String, Object> source = getMapValue(result, "_source");
363 if (source == null) {
364 return;
365 }
366
367
368 final String ownerInstanceId = getStringValue(source, "instanceId");
369 if (!instanceId.equals(ownerInstanceId)) {
370 logger.warn("Cannot release operation lock owned by another instance: operation={}, owner={}", operationName,
371 ownerInstanceId);
372 return;
373 }
374
375
376 final long seqNo = getLongValue(result, "_seq_no");
377 final long primaryTerm = getLongValue(result, "_primary_term");
378 try (CurlResponse deleteResponse = ComponentUtil.getCurlHelper()
379 .delete("/" + getIndexName() + "/_doc/" + operationName
380 + "?refresh=true&if_seq_no=" + seqNo + "&if_primary_term=" + primaryTerm)
381 .execute()) {
382 if (deleteResponse.getHttpStatusCode() == 200) {
383 if (logger.isInfoEnabled()) {
384 logger.info("Released operation lock: operation={}, instanceId={}", operationName, instanceId);
385 }
386 } else {
387 logger.warn("Failed to release operation lock: operation={}, status={}", operationName,
388 deleteResponse.getHttpStatusCode());
389 }
390 }
391 } catch (final Exception e) {
392 logger.warn("Failed to release operation lock: operation={}", operationName, e);
393 }
394 }
395
396
397
398
399
400
401
402 public boolean isOperationRunning(final String operationName) {
403 try (CurlResponse response = ComponentUtil.getCurlHelper()
404 .get("/" + getIndexName() + "/_doc/" + operationName)
405 .execute()) {
406 if (response.getHttpStatusCode() != 200) {
407 return false;
408 }
409
410 final Map<String, Object> result = parseJson(response.getContentAsString());
411 final Boolean found = (Boolean) result.get("found");
412 if (found == null || !found) {
413 return false;
414 }
415
416 final Map<String, Object> source = getMapValue(result, "_source");
417 if (source == null) {
418 return false;
419 }
420
421 final long expiredTime = getLongValue(source, "expiredTime");
422 if (expiredTime < System.currentTimeMillis()) {
423 return false;
424 }
425
426 final String ownerInstanceId = getStringValue(source, "instanceId");
427 return isInstanceActive(ownerInstanceId);
428 } catch (final Exception e) {
429 logger.debug("Failed to check operation status: operation={}", operationName, e);
430 }
431 return false;
432 }
433
434
435
436
437
438
439
440 public Optional<OperationInfo> getOperationInfo(final String operationName) {
441 try (CurlResponse response = ComponentUtil.getCurlHelper()
442 .get("/" + getIndexName() + "/_doc/" + operationName)
443 .execute()) {
444 if (response.getHttpStatusCode() == 200) {
445 final Map<String, Object> result = parseJson(response.getContentAsString());
446 final Boolean found = (Boolean) result.get("found");
447 if (found != null && found) {
448 final Map<String, Object> source = getMapValue(result, "_source");
449 if (source != null) {
450 final OperationInfo info = new OperationInfo();
451 info.name = getStringValue(source, "name");
452 info.instanceId = getStringValue(source, "instanceId");
453 info.hostname = getStringValue(source, "hostname");
454 info.status = getStringValue(source, "status");
455 info.createdTime = getLongValue(source, "createdTime");
456 info.data = getStringValue(source, "data");
457 return Optional.of(info);
458 }
459 }
460 }
461 } catch (final Exception e) {
462 logger.debug("Failed to get operation info: operation={}", operationName, e);
463 }
464 return Optional.empty();
465 }
466
467
468
469
470
471
472
473
474
475
476
477 public void publishEvent(final String eventName, final String data) {
478 publishEvent(eventName, TARGET_ALL, data);
479 }
480
481
482
483
484
485
486
487
488 public void publishEvent(final String eventName, final String targetInstanceId, final String data) {
489 final long now = System.currentTimeMillis();
490 final long ttl = ComponentUtil.getFessConfig().getCoordinatorEventTtlAsInteger().longValue();
491
492 final Map<String, Object> bodyMap = new java.util.LinkedHashMap<>();
493 bodyMap.put("type", TYPE_EVENT);
494 bodyMap.put("name", eventName);
495 bodyMap.put("instanceId", instanceId);
496 bodyMap.put("targetInstanceId", targetInstanceId);
497 bodyMap.put("createdTime", now);
498 bodyMap.put("expiredTime", now + ttl);
499 if (data != null) {
500 bodyMap.put("data", data);
501 }
502
503 try (CurlResponse response = ComponentUtil.getCurlHelper()
504 .post("/" + getIndexName() + "/_doc?refresh=true")
505 .body(toJson(bodyMap))
506 .execute()) {
507 if (response.getHttpStatusCode() != 201) {
508 logger.warn("Failed to publish event: eventName={}, status={}", eventName, response.getHttpStatusCode());
509 }
510 } catch (final Exception e) {
511 logger.warn("Failed to publish event: eventName={}", eventName, e);
512 }
513 }
514
515
516
517
518
519
520
521 public void addEventHandler(final String eventName, final Consumer<EventInfo> handler) {
522 eventHandlers.computeIfAbsent(eventName, k -> new ArrayList<>()).add(handler);
523 }
524
525
526
527
528
529
530 protected List<EventInfo> fetchNewEvents() {
531 final List<EventInfo> events = new ArrayList<>();
532 final String query = toJson(Map.of(
533 "query", Map.of("bool", Map.of(
534 "must", List.of(
535 Map.of("term", Map.of("type", TYPE_EVENT)),
536 Map.of("range", Map.of("createdTime", Map.of("gt", lastEventCheckTime)))),
537 "should", List.of(
538 Map.of("term", Map.of("targetInstanceId", TARGET_ALL)),
539 Map.of("term", Map.of("targetInstanceId", instanceId))),
540 "minimum_should_match", 1,
541 "must_not", List.of(
542 Map.of("term", Map.of("instanceId", instanceId))))),
543 "size", 100,
544 "sort", List.of(Map.of("createdTime", "asc"))));
545
546 try (CurlResponse response = ComponentUtil.getCurlHelper()
547 .post("/" + getIndexName() + "/_search")
548 .body(query)
549 .execute()) {
550 if (response.getHttpStatusCode() == 200) {
551 final Map<String, Object> result = parseJson(response.getContentAsString());
552 final Map<String, Object> hits = getMapValue(result, "hits");
553 if (hits != null) {
554 final List<Map<String, Object>> hitList = getListValue(hits, "hits");
555 if (hitList != null) {
556 for (final Map<String, Object> hit : hitList) {
557 final Map<String, Object> source = getMapValue(hit, "_source");
558 if (source != null) {
559 final EventInfo info = new EventInfo();
560 info.name = getStringValue(source, "name");
561 info.instanceId = getStringValue(source, "instanceId");
562 info.targetInstanceId = getStringValue(source, "targetInstanceId");
563 info.createdTime = getLongValue(source, "createdTime");
564 info.data = getStringValue(source, "data");
565 events.add(info);
566
567 if (info.createdTime >= lastEventCheckTime) {
568 lastEventCheckTime = info.createdTime + 1;
569 }
570 }
571 }
572 }
573 }
574 }
575 } catch (final Exception e) {
576 logger.debug("Failed to fetch events.", e);
577 }
578 return events;
579 }
580
581
582
583
584
585
586
587
588 protected void poll() {
589 try {
590 sendHeartbeat();
591 } catch (final Exception e) {
592 logger.debug("Failed to send heartbeat in poll.", e);
593 }
594
595 try {
596 final List<EventInfo> events = fetchNewEvents();
597 for (final EventInfo event : events) {
598 dispatchEvent(event);
599 }
600 } catch (final Exception e) {
601 logger.debug("Failed to process events in poll.", e);
602 }
603
604 try {
605 cleanupExpiredDocuments();
606 } catch (final Exception e) {
607 logger.debug("Failed to cleanup expired documents in poll.", e);
608 }
609 }
610
611
612
613
614
615
616 protected void dispatchEvent(final EventInfo event) {
617 final List<Consumer<EventInfo>> handlers = eventHandlers.get(event.name);
618 if (handlers != null) {
619 for (final Consumer<EventInfo> handler : handlers) {
620 try {
621 handler.accept(event);
622 } catch (final Exception e) {
623 logger.warn("Failed to handle event: eventName={}", event.name, e);
624 }
625 }
626 }
627 }
628
629
630
631
632 protected void cleanupExpiredDocuments() {
633 final long now = System.currentTimeMillis();
634 final String query = toJson(Map.of(
635 "query", Map.of("range", Map.of("expiredTime", Map.of("lt", now)))));
636
637 try (CurlResponse response = ComponentUtil.getCurlHelper()
638 .post("/" + getIndexName() + "/_delete_by_query?refresh=true")
639 .body(query)
640 .execute()) {
641 if (response.getHttpStatusCode() == 200) {
642 final Map<String, Object> result = parseJson(response.getContentAsString());
643 final Number deleted = (Number) result.get("deleted");
644 if (deleted != null && deleted.longValue() > 0 && logger.isDebugEnabled()) {
645 logger.debug("Cleaned up expired documents: count={}", deleted);
646 }
647 }
648 } catch (final Exception e) {
649 logger.debug("Failed to cleanup expired documents.", e);
650 }
651 }
652
653
654
655
656
657
658
659
660
661
662 protected String getIndexName() {
663 final FessConfig fessConfig = ComponentUtil.getFessConfig();
664 return INDEX_NAME.replaceFirst("fess_config", fessConfig.getIndexConfigIndex());
665 }
666
667
668
669
670
671
672
673
674
675
676
677 protected String toJson(final Map<String, Object> map) {
678 try {
679 return objectMapper.writeValueAsString(map);
680 } catch (final Exception e) {
681 logger.warn("Failed to serialize JSON.", e);
682 return "{}";
683 }
684 }
685
686
687
688
689
690
691
692 protected Map<String, Object> parseJson(final String json) {
693 try {
694 return objectMapper.readValue(json, new TypeReference<Map<String, Object>>() {
695 });
696 } catch (final Exception e) {
697 logger.warn("Failed to parse JSON.", e);
698 return Map.of();
699 }
700 }
701
702
703
704
705
706
707
708
709 @SuppressWarnings("unchecked")
710 protected Map<String, Object> getMapValue(final Map<String, Object> map, final String key) {
711 final Object value = map.get(key);
712 if (value instanceof Map) {
713 return (Map<String, Object>) value;
714 }
715 return null;
716 }
717
718
719
720
721
722
723
724
725 @SuppressWarnings("unchecked")
726 protected List<Map<String, Object>> getListValue(final Map<String, Object> map, final String key) {
727 final Object value = map.get(key);
728 if (value instanceof List) {
729 return (List<Map<String, Object>>) value;
730 }
731 return null;
732 }
733
734
735
736
737
738
739
740
741 protected String getStringValue(final Map<String, Object> map, final String key) {
742 final Object value = map.get(key);
743 if (value != null) {
744 return value.toString();
745 }
746 return null;
747 }
748
749
750
751
752
753
754
755
756 protected long getLongValue(final Map<String, Object> map, final String key) {
757 final Object value = map.get(key);
758 if (value instanceof Number) {
759 return ((Number) value).longValue();
760 }
761 return 0L;
762 }
763
764
765
766
767
768
769
770
771 public static class InstanceInfo {
772
773 public InstanceInfo() {
774 }
775
776
777 public String instanceId;
778
779 public String hostname;
780
781 public String name;
782
783 public long lastSeen;
784 }
785
786
787
788
789 public static class OperationInfo {
790
791 public OperationInfo() {
792 }
793
794
795 public String name;
796
797 public String instanceId;
798
799 public String hostname;
800
801 public String status;
802
803 public long createdTime;
804
805 public String data;
806 }
807
808
809
810
811 public static class EventInfo {
812
813 public EventInfo() {
814 }
815
816
817 public String name;
818
819 public String instanceId;
820
821 public String targetInstanceId;
822
823 public long createdTime;
824
825 public String data;
826 }
827 }