1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.api.admin.documents;
17
18 import java.util.Arrays;
19 import java.util.Date;
20 import java.util.HashMap;
21 import java.util.List;
22 import java.util.Map;
23
24 import org.apache.logging.log4j.LogManager;
25 import org.apache.logging.log4j.Logger;
26 import org.codelibs.fess.app.web.admin.searchlist.AdminSearchlistAction;
27 import org.codelibs.fess.app.web.api.ApiResult;
28 import org.codelibs.fess.app.web.api.ApiResult.ApiBulkResponse;
29 import org.codelibs.fess.app.web.api.ApiResult.Status;
30 import org.codelibs.fess.app.web.api.admin.FessApiAdminAction;
31 import org.codelibs.fess.app.web.api.admin.searchlist.ApiAdminSearchlistAction;
32 import org.codelibs.fess.helper.CrawlingConfigHelper;
33 import org.codelibs.fess.helper.CrawlingInfoHelper;
34 import org.codelibs.fess.helper.LanguageHelper;
35 import org.codelibs.fess.opensearch.client.SearchEngineClient;
36 import org.codelibs.fess.thumbnail.ThumbnailManager;
37 import org.codelibs.fess.util.ComponentUtil;
38 import org.lastaflute.web.Execute;
39 import org.lastaflute.web.response.JsonResponse;
40 import org.opensearch.action.bulk.BulkResponse;
41
42 import jakarta.annotation.Resource;
43
44
45
46
47
48
49 public class ApiAdminDocumentsAction extends FessApiAdminAction {
50
51
52
53
54 private static final Logger logger = LogManager.getLogger(ApiAdminSearchlistAction.class);
55
56
57
58
59
60
61
62
63 public ApiAdminDocumentsAction() {
64 super();
65 }
66
67
68
69
70
71 @Resource
72 protected SearchEngineClient searchEngineClient;
73
74
75
76
77
78
79
80
81
82
83
84
85
86 @Execute
87 public JsonResponse<ApiResult> put$bulk(final BulkBody body) {
88 validateApi(body, messages -> {});
89 if (body.documents == null) {
90 throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, "documents is required."));
91 }
92 if (body.documents.isEmpty()) {
93 throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, "documents is empty."));
94 }
95 final String indexFieldId = fessConfig.getIndexFieldId();
96 final String indexFieldDocId = fessConfig.getIndexFieldDocId();
97 final String indexFieldContentLength = fessConfig.getIndexFieldContentLength();
98 final String indexFieldTitle = fessConfig.getIndexFieldTitle();
99 final String indexFieldContent = fessConfig.getIndexFieldContent();
100 final String indexFieldFavoriteCount = fessConfig.getIndexFieldFavoriteCount();
101 final String indexFieldClickCount = fessConfig.getIndexFieldClickCount();
102 final String indexFieldBoost = fessConfig.getIndexFieldBoost();
103 final String indexFieldRole = fessConfig.getIndexFieldRole();
104 final String indexFieldLastModified = fessConfig.getIndexFieldLastModified();
105 final String indexFieldTimestamp = fessConfig.getIndexFieldTimestamp();
106 final String indexFieldLang = fessConfig.getIndexFieldLang();
107 final List<String> guestRoleList = fessConfig.getSearchGuestRoleList();
108 final Date now = systemHelper.getCurrentTime();
109 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
110 final LanguageHelper languageHelper = ComponentUtil.getLanguageHelper();
111 final List<Map<String, Object>> docList = body.documents.stream().map(doc -> {
112 if (!doc.containsKey(indexFieldContentLength)) {
113 long contentLength = 0;
114 if (doc.get(indexFieldTitle) instanceof final String title) {
115 contentLength += title.length();
116 }
117 if (doc.get(indexFieldContent) instanceof final String content) {
118 contentLength += content.length();
119 }
120 doc.put(indexFieldContentLength, contentLength);
121 }
122 if (!doc.containsKey(indexFieldFavoriteCount)) {
123 doc.put(indexFieldFavoriteCount, 0L);
124 }
125 if (!doc.containsKey(indexFieldClickCount)) {
126 doc.put(indexFieldClickCount, 0L);
127 }
128 if (!doc.containsKey(indexFieldBoost)) {
129 doc.put(indexFieldBoost, 1.0f);
130 }
131 if (!doc.containsKey(indexFieldRole)) {
132 doc.put(indexFieldRole, guestRoleList);
133 }
134 if (!doc.containsKey(indexFieldLastModified)) {
135 doc.put(indexFieldLastModified, now);
136 }
137 if (!doc.containsKey(indexFieldTimestamp)) {
138 doc.put(indexFieldTimestamp, now);
139 }
140 AdminSearchlistAction.validateFields(doc, this::throwValidationErrorApi);
141 final Map<String, Object> newDoc = fessConfig.convertToStorableDoc(doc);
142 newDoc.put(indexFieldId, crawlingInfoHelper.generateId(newDoc));
143 newDoc.put(indexFieldDocId, systemHelper.generateDocId(newDoc));
144 if (newDoc.get(indexFieldLang) instanceof final List<?> langList) {
145 if (langList.contains("auto")) {
146 newDoc.remove(indexFieldLang);
147 }
148 languageHelper.updateDocument(newDoc);
149 }
150 return newDoc;
151 }).toList();
152 if (fessConfig.isThumbnailCrawlerEnabled()) {
153 final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
154 final String thumbnailField = fessConfig.getIndexFieldThumbnail();
155 docList.stream().forEach(doc -> {
156 if (!thumbnailManager.offer(doc)) {
157 if (logger.isDebugEnabled()) {
158 logger.debug("Removing {}={} from doc[{}]", thumbnailField, doc.get(thumbnailField),
159 doc.get(fessConfig.getIndexFieldUrl()));
160 }
161 doc.remove(thumbnailField);
162 }
163 });
164 }
165
166 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
167 final BulkResponse response = searchEngineClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), docList, (doc, builder) -> {
168 if (doc.get(fessConfig.getIndexFieldConfigId()) instanceof final String configId) {
169 crawlingConfigHelper.getPipeline(configId).ifPresent(s -> builder.setPipeline(s));
170 }
171 });
172 return asJson(new ApiBulkResponse().items(Arrays.stream(response.getItems()).map(item -> {
173 final Map<String, Object> itemMap = new HashMap<>();
174 itemMap.put("result", item.status().name());
175 if (item.isFailed()) {
176 itemMap.put("message", item.getFailureMessage());
177 } else {
178 itemMap.put("id", item.getId());
179 }
180 return itemMap;
181 }).toList()).status(response.hasFailures() ? Status.FAILED : Status.OK).result());
182
183 }
184
185 }