1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.admin.backup;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedReader;
21 import java.io.BufferedWriter;
22 import java.io.ByteArrayInputStream;
23 import java.io.ByteArrayOutputStream;
24 import java.io.File;
25 import java.io.FileInputStream;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.io.Writer;
33 import java.nio.file.Files;
34 import java.nio.file.Path;
35 import java.time.LocalDateTime;
36 import java.time.ZoneId;
37 import java.time.format.DateTimeFormatter;
38 import java.util.Arrays;
39 import java.util.HashMap;
40 import java.util.List;
41 import java.util.Map;
42 import java.util.function.Consumer;
43 import java.util.stream.Collectors;
44
45 import javax.annotation.Resource;
46
47 import org.apache.commons.text.StringEscapeUtils;
48 import org.apache.logging.log4j.LogManager;
49 import org.apache.logging.log4j.Logger;
50 import org.codelibs.core.exception.IORuntimeException;
51 import org.codelibs.core.io.CopyUtil;
52 import org.codelibs.core.lang.StringUtil;
53 import org.codelibs.core.misc.Pair;
54 import org.codelibs.curl.CurlResponse;
55 import org.codelibs.fess.Constants;
56 import org.codelibs.fess.annotation.Secured;
57 import org.codelibs.fess.app.web.base.FessAdminAction;
58 import org.codelibs.fess.es.config.exbhv.FileConfigBhv;
59 import org.codelibs.fess.es.config.exbhv.LabelTypeBhv;
60 import org.codelibs.fess.es.config.exbhv.WebConfigBhv;
61 import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
62 import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv;
63 import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
64 import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
65 import org.codelibs.fess.mylasta.direction.FessConfig;
66 import org.codelibs.fess.util.ComponentUtil;
67 import org.codelibs.fess.util.GsaConfigParser;
68 import org.codelibs.fess.util.RenderDataUtil;
69 import org.codelibs.fess.util.ResourceUtil;
70 import org.codelibs.fess.util.SearchEngineUtil;
71 import org.lastaflute.core.magic.async.AsyncManager;
72 import org.lastaflute.web.Execute;
73 import org.lastaflute.web.response.ActionResponse;
74 import org.lastaflute.web.response.HtmlResponse;
75 import org.lastaflute.web.response.StreamResponse;
76 import org.lastaflute.web.ruts.process.ActionRuntime;
77 import org.xml.sax.InputSource;
78
79 import com.fasterxml.jackson.core.type.TypeReference;
80 import com.fasterxml.jackson.databind.ObjectMapper;
81
82
83
84
85 public class AdminBackupAction extends FessAdminAction {
86
87 public static final String ROLE = "admin-backup";
88
89 private static final Logger logger = LogManager.getLogger(AdminBackupAction.class);
90
91 public static final String NDJSON_EXTENTION = ".ndjson";
92
93 private static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
94
95 @Resource
96 private AsyncManager asyncManager;
97
98 @Resource
99 private WebConfigBhv webConfigBhv;
100
101 @Resource
102 private FileConfigBhv fileConfigBhv;
103
104 @Resource
105 private LabelTypeBhv labelTypeBhv;
106
107 @Override
108 protected void setupHtmlData(final ActionRuntime runtime) {
109 super.setupHtmlData(runtime);
110 runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameBackup()));
111 }
112
113 @Override
114 protected String getActionRole() {
115 return ROLE;
116 }
117
118 @Execute
119 @Secured({ ROLE, ROLE + VIEW })
120 public HtmlResponse index() {
121 saveToken();
122 return asListHtml();
123 }
124
125 @Execute
126 @Secured({ ROLE })
127 public HtmlResponse upload(final UploadForm form) {
128 validate(form, messages -> {}, this::asListHtml);
129 verifyToken(this::asListHtml);
130 final String fileName = form.bulkFile.getFileName();
131 final File tempFile = ComponentUtil.getSystemHelper().createTempFile("fess_restore_", ".tmp");
132 try (final InputStream in = form.bulkFile.getInputStream(); final OutputStream out = new FileOutputStream(tempFile)) {
133 CopyUtil.copy(in, out);
134 asyncImport(fileName, tempFile);
135 } catch (final IOException e) {
136 logger.warn("Failed to create a temp file.", e);
137 if (tempFile.exists() && !tempFile.delete()) {
138 logger.warn("Failed to delete {}.", tempFile.getAbsolutePath());
139 }
140 throwValidationError(messages -> messages.addErrorsFileIsNotSupported(GLOBAL, fileName), this::asListHtml);
141 }
142 saveInfo(messages -> messages.addSuccessBulkProcessStarted(GLOBAL));
143 return redirect(getClass());
144 }
145
146 protected void asyncImport(final String fileName, final File tempFile) {
147 final int fileType;
148 if (fileName.startsWith("system") && fileName.endsWith(".properties")) {
149 fileType = 1;
150 } else if (fileName.startsWith("gsa") && fileName.endsWith(".xml")) {
151 fileType = 2;
152 } else if (fileName.endsWith(".bulk")) {
153 fileType = 3;
154 } else if (fileName.startsWith("fess") && fileName.endsWith(".json")) {
155 fileType = 4;
156 } else if (fileName.startsWith("doc") && fileName.endsWith(".json")) {
157 fileType = 5;
158 } else {
159 throwValidationError(messages -> messages.addErrorsFileIsNotSupported(GLOBAL, fileName), this::asListHtml);
160 return;
161 }
162
163 asyncManager.async(() -> {
164 switch (fileType) {
165 case 1:
166 importSystemProperties(fileName, tempFile);
167 break;
168 case 2:
169 importGsaXml(fileName, tempFile);
170 break;
171 case 3:
172 importBulk(fileName, tempFile);
173 break;
174 case 4:
175 importFessJson(fileName, tempFile);
176 break;
177 case 5:
178 importDocJson(fileName, tempFile);
179 break;
180 default:
181 break;
182 }
183 });
184 }
185
186 private void importBulk(final String fileName, final File tempFile) {
187 final ObjectMapper mapper = new ObjectMapper();
188 try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_bulk").onConnect((req, con) -> {
189 con.setDoOutput(true);
190 try (final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(tempFile)));
191 final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(con.getOutputStream(), Constants.CHARSET_UTF_8))) {
192 String line;
193 while ((line = br.readLine()) != null) {
194 if (StringUtil.isNotBlank(line)) {
195 final Map<String, Map<String, String>> dataObj;
196 if (line.contains("_type")) {
197 dataObj = parseObject(mapper, line);
198 } else {
199 dataObj = null;
200 }
201 if (dataObj != null) {
202 final Map<String, String> indexObj = dataObj.get("index");
203 if (indexObj != null && indexObj.containsKey("_type")) {
204 indexObj.remove("_type");
205 bw.write(mapper.writeValueAsString(dataObj));
206 } else {
207 bw.write(line);
208 }
209 } else {
210 bw.write(line);
211 }
212 }
213 bw.write("\n");
214 }
215 bw.flush();
216 } catch (IOException e) {
217 throw new IORuntimeException(e);
218 }
219 }).execute()) {
220 if (logger.isDebugEnabled()) {
221 logger.debug("Bulk Response:\n{}", response.getContentAsString());
222 }
223 systemHelper.reloadConfiguration();
224 } catch (final Exception e) {
225 logger.warn("Failed to process bulk file: {}", fileName, e);
226 } finally {
227 deleteTempFile(tempFile);
228 }
229 }
230
231 private void importGsaXml(final String fileName, final File tempFile) {
232 final GsaConfigParser configParser = ComponentUtil.getComponent(GsaConfigParser.class);
233 try (final InputStream in = new FileInputStream(tempFile)) {
234 configParser.parse(new InputSource(in));
235 } catch (final IOException e) {
236 logger.warn("Failed to process gsa.xml file: {}", fileName, e);
237 } finally {
238 deleteTempFile(tempFile);
239 }
240 configParser.getWebConfig().ifPresent(c -> webConfigBhv.insert(c));
241 configParser.getFileConfig().ifPresent(c -> fileConfigBhv.insert(c));
242 labelTypeBhv.batchInsert(Arrays.stream(configParser.getLabelTypes()).collect(Collectors.toList()));
243 }
244
245 private void importSystemProperties(final String fileName, final File tempFile) {
246 try (final InputStream in = new FileInputStream(tempFile)) {
247 ComponentUtil.getSystemProperties().load(in);
248 } catch (final IOException e) {
249 logger.warn("Failed to process system.properties file: {}", fileName, e);
250 } finally {
251 deleteTempFile(tempFile);
252 }
253 }
254
255 private void importFessJson(final String fileName, final File tempFile) {
256 try (final InputStream in = new FileInputStream(tempFile); final OutputStream out = Files.newOutputStream(getFessJsonPath())) {
257 CopyUtil.copy(in, out);
258 } catch (final IOException e) {
259 logger.warn("Failed to process fess.json file: {}", fileName, e);
260 } finally {
261 deleteTempFile(tempFile);
262 }
263 }
264
265 private void importDocJson(final String fileName, final File tempFile) {
266 try (final InputStream in = new FileInputStream(tempFile); final OutputStream out = Files.newOutputStream(getDocJsonPath())) {
267 CopyUtil.copy(in, out);
268 } catch (final IOException e) {
269 logger.warn("Failed to process doc.json file: {}", fileName, e);
270 } finally {
271 deleteTempFile(tempFile);
272 }
273 }
274
275 private Map<String, Map<String, String>> parseObject(final ObjectMapper mapper, final String line) {
276 try {
277 return mapper.readValue(line, new TypeReference<Map<String, Map<String, String>>>() {
278 });
279 } catch (final Exception e) {
280 if (logger.isDebugEnabled()) {
281 logger.debug("Failed to parse {}", line, e);
282 }
283 return null;
284 }
285 }
286
287 @Execute
288 @Secured({ ROLE, ROLE + VIEW })
289 public ActionResponse download(final String id) {
290 if (stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
291 if ("system.properties".equals(id)) {
292 return asStream(id).contentTypeOctetStream().stream(out -> {
293 try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
294 ComponentUtil.getSystemProperties().store(baos, id);
295 try (final InputStream in = new ByteArrayInputStream(baos.toByteArray())) {
296 out.write(in);
297 }
298 }
299 });
300 }
301 if (id.endsWith(NDJSON_EXTENTION)) {
302 final String name = id.substring(0, id.length() - NDJSON_EXTENTION.length());
303 if ("search_log".equals(name)) {
304 return writeNdjsonResponse(id, getSearchLogNdjsonWriteCall());
305 }
306 if ("user_info".equals(name)) {
307 return writeNdjsonResponse(id, getUserInfoNdjsonWriteCall());
308 } else if ("click_log".equals(name)) {
309 return writeNdjsonResponse(id, getClickLogNdjsonWriteCall());
310 } else if ("favorite_log".equals(name)) {
311 return writeNdjsonResponse(id, getFavoriteLogNdjsonWriteCall());
312 }
313 } else if ("fess.json".equals(id)) {
314 return asStream(id).contentTypeOctetStream().stream(out -> {
315 final Path fessJsonPath = getFessJsonPath();
316 try (final InputStream in = Files.newInputStream(fessJsonPath)) {
317 out.write(in);
318 }
319 });
320 } else if ("doc.json".equals(id)) {
321 return asStream(id).contentTypeOctetStream().stream(out -> {
322 final Path fessJsonPath = getDocJsonPath();
323 try (final InputStream in = Files.newInputStream(fessJsonPath)) {
324 out.write(in);
325 }
326 });
327 } else {
328 final String index;
329 final String filename;
330 if (id.endsWith(".bulk")) {
331 index = id.substring(0, id.length() - 5);
332 filename = id;
333 } else {
334 index = id;
335 filename = id + ".bulk";
336 }
337 return asStream(filename).contentTypeOctetStream().stream(out -> {
338 try (final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out.stream(), Constants.CHARSET_UTF_8))) {
339 SearchEngineUtil.scroll(index, hit -> {
340 try {
341 writer.write("{\"index\":{\"_index\":\"" + hit.getIndex() + "\",\"_id\":\""
342 + StringEscapeUtils.escapeJson(hit.getId()) + "\"}}\n");
343 writer.write(hit.getSourceAsString());
344 writer.write("\n");
345 } catch (final IOException e) {
346 throw new IORuntimeException(e);
347 }
348 return true;
349 });
350 writer.flush();
351 }
352 });
353 }
354 }
355 throwValidationError(messages -> messages.addErrorsCouldNotFindBackupIndex(GLOBAL), this::asListHtml);
356 return redirect(getClass());
357 }
358
359 private Path getDocJsonPath() {
360 return ResourceUtil.getClassesPath("fess_indices", "fess", "doc.json");
361 }
362
363 private Path getFessJsonPath() {
364 return ResourceUtil.getClassesPath("fess_indices", "fess.json");
365 }
366
367 private StreamResponse writeNdjsonResponse(final String id, final Consumer<Writer> writeCall) {
368 return asStream(id)
369 .header("Pragma", "no-cache")
370 .header("Cache-Control", "no-cache")
371 .header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT")
372 .header("Content-Type", "application/x-ndjson")
373 .stream(out -> {
374 try (final Writer writer = new BufferedWriter(new OutputStreamWriter(out.stream(), Constants.CHARSET_UTF_8))) {
375 writeCall.accept(writer);
376 writer.flush();
377 } catch (final Exception e) {
378 logger.warn("Failed to write {} to response.", id, e);
379 }
380 });
381 }
382
383 private static StringBuilder appendJson(final String field, final Object value, final StringBuilder buf) {
384 buf.append('"').append(StringEscapeUtils.escapeJson(field)).append('"').append(':');
385 if (value == null) {
386 buf.append("null");
387 } else if (value instanceof LocalDateTime) {
388 final String format =
389 ((LocalDateTime) value).atZone(ZoneId.systemDefault()).withZoneSameInstant(ZoneId.of("UTC")).format(ISO_8601_FORMATTER);
390 buf.append('"').append(StringEscapeUtils.escapeJson(format)).append('"');
391 } else if (value instanceof String[]) {
392 final String json = Arrays.stream((String[]) value).map(s -> "\"" + StringEscapeUtils.escapeJson(s) + "\"")
393 .collect(Collectors.joining(","));
394 buf.append('[').append(json).append(']');
395 } else if (value instanceof List) {
396 final String json = ((List<?>) value).stream().map(s -> "\"" + StringEscapeUtils.escapeJson(s.toString()) + "\"")
397 .collect(Collectors.joining(","));
398 buf.append('[').append(json).append(']');
399 } else if (value instanceof Map) {
400 buf.append('{');
401 final String json = ((Map<?, ?>) value).entrySet().stream().map(e -> {
402 final StringBuilder tempBuf = new StringBuilder();
403 appendJson(e.getKey().toString(), e.getValue(), tempBuf);
404 return tempBuf.toString();
405 }).collect(Collectors.joining(","));
406 buf.append(json);
407 buf.append('}');
408 } else if (value instanceof Long || value instanceof Integer) {
409 buf.append(((Number) value).longValue());
410 } else if (value instanceof Number) {
411 buf.append(((Number) value).doubleValue());
412 } else {
413 buf.append('"').append(StringEscapeUtils.escapeJson(value.toString())).append('"');
414 }
415 return buf;
416 }
417
418 public static Consumer<Writer> getSearchLogNdjsonWriteCall() {
419 return writer -> {
420 final SearchLogBhv bhv = ComponentUtil.getComponent(SearchLogBhv.class);
421 bhv.selectCursor(cb -> {
422 cb.query().matchAll();
423 cb.query().addOrderBy_RequestedAt_Asc();
424 }, entity -> {
425 final StringBuilder buf = new StringBuilder();
426 buf.append('{');
427 appendJson("id", entity.getId(), buf).append(',');
428 appendJson("query-id", entity.getQueryId(), buf).append(',');
429 appendJson("user-info-id", entity.getUserInfoId(), buf).append(',');
430 appendJson("user-session-id", entity.getUserSessionId(), buf).append(',');
431 appendJson("user", entity.getUser(), buf).append(',');
432 appendJson("search-word", entity.getSearchWord(), buf).append(',');
433 appendJson("hit-count", entity.getHitCount(), buf).append(',');
434 appendJson("query-page-size", entity.getQueryPageSize(), buf).append(',');
435 appendJson("query-offset", entity.getQueryOffset(), buf).append(',');
436 appendJson("referer", entity.getReferer(), buf).append(',');
437 appendJson("languages", entity.getLanguages(), buf).append(',');
438 appendJson("roles", entity.getRoles(), buf).append(',');
439 appendJson("user-agent", entity.getUserAgent(), buf).append(',');
440 appendJson("client-ip", entity.getClientIp(), buf).append(',');
441 appendJson("access-type", entity.getAccessType(), buf).append(',');
442 appendJson("query-time", entity.getQueryTime(), buf).append(',');
443 appendJson("response-time", entity.getResponseTime(), buf).append(',');
444 appendJson("requested-at", entity.getRequestedAt(), buf).append(',');
445 final Map<String, List<String>> searchFieldMap = entity.getSearchFieldLogList().stream()
446 .collect(Collectors.groupingBy(Pair::getFirst, Collectors.mapping(Pair::getSecond, Collectors.toList())));
447 appendJson("search-field", searchFieldMap, buf);
448 buf.append('}');
449 buf.append('\n');
450 try {
451 writer.write(buf.toString());
452 } catch (final IOException e) {
453 throw new IORuntimeException(e);
454 }
455 });
456 };
457 }
458
459 public static Consumer<Writer> getUserInfoNdjsonWriteCall() {
460 return writer -> {
461 final UserInfoBhv bhv = ComponentUtil.getComponent(UserInfoBhv.class);
462 bhv.selectCursor(cb -> {
463 cb.query().matchAll();
464 cb.query().addOrderBy_CreatedAt_Asc();
465 }, entity -> {
466 final StringBuilder buf = new StringBuilder();
467 buf.append('{');
468 appendJson("id", entity.getId(), buf).append(',');
469 appendJson("created-at", entity.getCreatedAt(), buf).append(',');
470 appendJson("updated-at", entity.getUpdatedAt(), buf);
471 buf.append('}');
472 buf.append('\n');
473 try {
474 writer.write(buf.toString());
475 } catch (final IOException e) {
476 throw new IORuntimeException(e);
477 }
478 });
479 };
480 }
481
482 public static Consumer<Writer> getFavoriteLogNdjsonWriteCall() {
483 return writer -> {
484 final FavoriteLogBhv bhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
485 bhv.selectCursor(cb -> {
486 cb.query().matchAll();
487 cb.query().addOrderBy_CreatedAt_Asc();
488 }, entity -> {
489 final StringBuilder buf = new StringBuilder();
490 buf.append('{');
491 appendJson("id", entity.getId(), buf).append(',');
492 appendJson("created-at", entity.getCreatedAt(), buf).append(',');
493 appendJson("query-id", entity.getQueryId(), buf).append(',');
494 appendJson("user-info-id", entity.getUserInfoId(), buf).append(',');
495 appendJson("doc-id", entity.getDocId(), buf).append(',');
496 appendJson("url", entity.getUrl(), buf);
497 buf.append('}');
498 buf.append('\n');
499 try {
500 writer.write(buf.toString());
501 } catch (final IOException e) {
502 throw new IORuntimeException(e);
503 }
504 });
505 };
506 }
507
508 public static Consumer<Writer> getClickLogNdjsonWriteCall() {
509 return writer -> {
510 final ClickLogBhv bhv = ComponentUtil.getComponent(ClickLogBhv.class);
511 bhv.selectCursor(cb -> {
512 cb.query().matchAll();
513 cb.query().addOrderBy_RequestedAt_Asc();
514 }, entity -> {
515 final StringBuilder buf = new StringBuilder();
516 buf.append('{');
517 appendJson("id", entity.getId(), buf).append(',');
518 appendJson("query-id", entity.getQueryId(), buf).append(',');
519 appendJson("user-session-id", entity.getUserSessionId(), buf).append(',');
520 appendJson("doc-id", entity.getDocId(), buf).append(',');
521 appendJson("url", entity.getUrl(), buf).append(',');
522 appendJson("order", entity.getOrder(), buf).append(',');
523 appendJson("query-requested-at", entity.getQueryRequestedAt(), buf).append(',');
524 appendJson("requested-at", entity.getRequestedAt(), buf);
525 buf.append('}');
526 buf.append('\n');
527 try {
528 writer.write(buf.toString());
529 } catch (final IOException e) {
530 throw new IORuntimeException(e);
531 }
532 });
533 };
534 }
535
536 public static List<Map<String, String>> getBackupItems() {
537 final FessConfig fessConfig = ComponentUtil.getFessConfig();
538 return stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.map(name -> {
539 final Map<String, String> map = new HashMap<>();
540 map.put("id", name);
541 map.put("name", name);
542 return map;
543 }).collect(Collectors.toList()));
544 }
545
546 private HtmlResponse asListHtml() {
547 return asHtml(path_AdminBackup_AdminBackupJsp).useForm(UploadForm.class)
548 .renderWith(data -> RenderDataUtil.register(data, "backupItems", getBackupItems()));
549 }
550
551 private void deleteTempFile(final File tempFile) {
552 if (tempFile != null && !tempFile.delete()) {
553 logger.warn("Failed to delete {}", tempFile.getAbsolutePath());
554 }
555 }
556
557 }