1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.admin.backup;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedWriter;
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.OutputStream;
26 import java.io.OutputStreamWriter;
27 import java.time.LocalDateTime;
28 import java.time.format.DateTimeFormatter;
29 import java.util.ArrayList;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.function.Consumer;
34 import java.util.stream.Collectors;
35
36 import javax.annotation.Resource;
37
38 import org.codelibs.core.exception.IORuntimeException;
39 import org.codelibs.core.io.CopyUtil;
40 import org.codelibs.core.lang.StringUtil;
41 import org.codelibs.elasticsearch.runner.net.Curl;
42 import org.codelibs.elasticsearch.runner.net.CurlResponse;
43 import org.codelibs.fess.app.web.base.FessAdminAction;
44 import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
45 import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv;
46 import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
47 import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
48 import org.codelibs.fess.mylasta.direction.FessConfig;
49 import org.codelibs.fess.util.ComponentUtil;
50 import org.codelibs.fess.util.RenderDataUtil;
51 import org.codelibs.fess.util.ResourceUtil;
52 import org.lastaflute.core.magic.async.AsyncManager;
53 import org.lastaflute.web.Execute;
54 import org.lastaflute.web.response.ActionResponse;
55 import org.lastaflute.web.response.HtmlResponse;
56 import org.lastaflute.web.response.StreamResponse;
57 import org.lastaflute.web.ruts.process.ActionRuntime;
58 import org.slf4j.Logger;
59 import org.slf4j.LoggerFactory;
60
61 import com.healthmarketscience.jackcess.RuntimeIOException;
62 import com.orangesignal.csv.CsvConfig;
63 import com.orangesignal.csv.CsvWriter;
64
65
66
67
68 public class AdminBackupAction extends FessAdminAction {
69
70 private static final Logger logger = LoggerFactory.getLogger(AdminBackupAction.class);
71
72 public static final String CSV_EXTENTION = ".csv";
73
74 private static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
75
76 @Resource
77 private AsyncManager asyncManager;
78
79 @Override
80 protected void setupHtmlData(final ActionRuntime runtime) {
81 super.setupHtmlData(runtime);
82 runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameBackup()));
83 }
84
85 @Execute
86 public HtmlResponse index() {
87 saveToken();
88 return asListHtml();
89 }
90
91 @Execute
92 public HtmlResponse upload(final UploadForm form) {
93 validate(form, messages -> {}, () -> asListHtml());
94 verifyToken(() -> asListHtml());
95 asyncManager.async(() -> {
96 final String fileName = form.bulkFile.getFileName();
97 if (fileName.startsWith("system") && fileName.endsWith(".properties")) {
98 try (final InputStream in = form.bulkFile.getInputStream()) {
99 ComponentUtil.getSystemProperties().load(in);
100 } catch (final IOException e) {
101 logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e);
102 }
103 } else {
104 try (CurlResponse response =
105 Curl.post(ResourceUtil.getElasticsearchHttpUrl() + "/_bulk").header("Content-Type", "application/json")
106 .onConnect((req, con) -> {
107 con.setDoOutput(true);
108 try (InputStream in = form.bulkFile.getInputStream(); OutputStream out = con.getOutputStream()) {
109 CopyUtil.copy(in, out);
110 } catch (IOException e) {
111 throw new IORuntimeException(e);
112 }
113 }).execute()) {
114 if (logger.isDebugEnabled()) {
115 logger.debug("Bulk Response:\n" + response.getContentAsString());
116 }
117 systemHelper.reloadConfiguration();
118 } catch (final Exception e) {
119 logger.warn("Failed to process bulk file: " + form.bulkFile.getFileName(), e);
120 }
121 }
122 });
123 saveInfo(messages -> messages.addSuccessBulkProcessStarted(GLOBAL));
124 return redirect(getClass());
125 }
126
127 @Execute
128 public ActionResponse download(final String id) {
129 if (stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
130 if (id.equals("system.properties")) {
131 return asStream(id).contentTypeOctetStream().stream(out -> {
132 try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
133 ComponentUtil.getSystemProperties().store(baos, id);
134 try (final InputStream in = new ByteArrayInputStream(baos.toByteArray())) {
135 out.write(in);
136 }
137 }
138 });
139 } else if (id.endsWith(CSV_EXTENTION)) {
140 final String name = id.substring(0, id.length() - CSV_EXTENTION.length());
141 if ("search_log".equals(name)) {
142 return writeCsvResponse(id, getSearchLogCsvWriteCall());
143 } else if ("user_info".equals(name)) {
144 return writeCsvResponse(id, getUserInfoCsvWriteCall());
145 } else if ("click_log".equals(name)) {
146 return writeCsvResponse(id, getClickLogCsvWriteCall());
147 } else if ("favorite_log".equals(name)) {
148 return writeCsvResponse(id, getFavoriteLogCsvWriteCall());
149 }
150 } else {
151 final String index;
152 final String filename;
153 if (id.endsWith(".bulk")) {
154 index = id.substring(0, id.length() - 5);
155 filename = id;
156 } else {
157 index = id;
158 filename = id + ".bulk";
159 }
160 return asStream(filename).contentTypeOctetStream().stream(
161 out -> {
162 try (CurlResponse response =
163 Curl.get(ResourceUtil.getElasticsearchHttpUrl() + "/" + index + "/_data")
164 .header("Content-Type", "application/json").param("format", "json").execute()) {
165 out.write(response.getContentAsStream());
166 }
167 });
168 }
169 }
170 throwValidationError(messages -> messages.addErrorsCouldNotFindBackupIndex(GLOBAL), () -> {
171 return asListHtml();
172 });
173 return redirect(getClass());
174 }
175
176 private StreamResponse writeCsvResponse(final String id, final Consumer<CsvWriter> writeCall) {
177 return asStream(id)
178 .contentTypeOctetStream()
179 .header("Pragma", "no-cache")
180 .header("Cache-Control", "no-cache")
181 .header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT")
182 .stream(out -> {
183 final CsvConfig cfg = new CsvConfig(',', '"', '"');
184 cfg.setEscapeDisabled(false);
185 cfg.setQuoteDisabled(false);
186 try (final CsvWriter writer =
187 new CsvWriter(new BufferedWriter(new OutputStreamWriter(out.stream(), fessConfig.getCsvFileEncoding())), cfg)) {
188 writeCall.accept(writer);
189 writer.flush();
190 } catch (final Exception e) {
191 logger.warn("Failed to write " + id + " to response.", e);
192 }
193 });
194 }
195
196 public static Consumer<CsvWriter> getSearchLogCsvWriteCall() {
197 return writer -> {
198 final SearchLogBhv bhv = ComponentUtil.getComponent(SearchLogBhv.class);
199 bhv.selectCursor(cb -> {
200 cb.query().matchAll();
201 cb.query().addOrderBy_RequestedAt_Asc();
202 }, entity -> {
203 final List<String> list = new ArrayList<>();
204 addToList(entity.getQueryId(), list);
205 addToList(entity.getUserInfoId(), list);
206 addToList(entity.getUserSessionId(), list);
207 addToList(entity.getUser(), list);
208 addToList(entity.getSearchWord(), list);
209 addToList(entity.getHitCount(), list);
210 addToList(entity.getQueryPageSize(), list);
211 addToList(entity.getQueryOffset(), list);
212 addToList(entity.getReferer(), list);
213 addToList(entity.getLanguages(), list);
214 addToList(entity.getRoles(), list);
215 addToList(entity.getUserAgent(), list);
216 addToList(entity.getClientIp(), list);
217 addToList(entity.getAccessType(), list);
218 addToList(entity.getQueryTime(), list);
219 addToList(entity.getResponseTime(), list);
220 addToList(entity.getRequestedAt(), list);
221 entity.getSearchFieldLogList().stream().forEach(e -> {
222 addToList(e.getFirst(), list);
223 addToList(e.getSecond(), list);
224 });
225 try {
226 writer.writeValues(list);
227 } catch (final IOException e) {
228 throw new RuntimeIOException(e);
229 }
230 });
231 };
232 }
233
234 public static Consumer<CsvWriter> getUserInfoCsvWriteCall() {
235 return writer -> {
236 final UserInfoBhv bhv = ComponentUtil.getComponent(UserInfoBhv.class);
237 bhv.selectCursor(cb -> {
238 cb.query().matchAll();
239 cb.query().addOrderBy_CreatedAt_Asc();
240 }, entity -> {
241 final List<String> list = new ArrayList<>();
242 addToList(entity.getCreatedAt(), list);
243 addToList(entity.getUpdatedAt(), list);
244 try {
245 writer.writeValues(list);
246 } catch (final IOException e) {
247 throw new RuntimeIOException(e);
248 }
249 });
250 };
251 }
252
253 public static Consumer<CsvWriter> getFavoriteLogCsvWriteCall() {
254 return writer -> {
255 final FavoriteLogBhv bhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
256 bhv.selectCursor(cb -> {
257 cb.query().matchAll();
258 cb.query().addOrderBy_CreatedAt_Asc();
259 }, entity -> {
260 final List<String> list = new ArrayList<>();
261 addToList(entity.getQueryId(), list);
262 addToList(entity.getUserInfoId(), list);
263 addToList(entity.getDocId(), list);
264 addToList(entity.getUrl(), list);
265 addToList(entity.getCreatedAt(), list);
266 try {
267 writer.writeValues(list);
268 } catch (final IOException e) {
269 throw new RuntimeIOException(e);
270 }
271 });
272 };
273 }
274
275 public static Consumer<CsvWriter> getClickLogCsvWriteCall() {
276 return writer -> {
277 final ClickLogBhv bhv = ComponentUtil.getComponent(ClickLogBhv.class);
278 bhv.selectCursor(cb -> {
279 cb.query().matchAll();
280 cb.query().addOrderBy_RequestedAt_Asc();
281 }, entity -> {
282 final List<String> list = new ArrayList<>();
283 addToList(entity.getQueryId(), list);
284 addToList(entity.getUserSessionId(), list);
285 addToList(entity.getDocId(), list);
286 addToList(entity.getUrl(), list);
287 addToList(entity.getOrder(), list);
288 addToList(entity.getQueryRequestedAt(), list);
289 addToList(entity.getRequestedAt(), list);
290 try {
291 writer.writeValues(list);
292 } catch (final IOException e) {
293 throw new RuntimeIOException(e);
294 }
295 });
296 };
297 }
298
299 private static void addToList(final Object value, final List<String> list) {
300 if (value == null) {
301 list.add(StringUtil.EMPTY);
302 } else if (value instanceof LocalDateTime) {
303 list.add(((LocalDateTime) value).format(ISO_8601_FORMATTER));
304 } else if (value instanceof String[]) {
305 String.join(",", (String[]) value);
306 } else {
307 list.add(value.toString());
308 }
309 }
310
311 static public List<Map<String, String>> getBackupItems() {
312 final FessConfig fessConfig = ComponentUtil.getFessConfig();
313 return stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.map(name -> {
314 final Map<String, String> map = new HashMap<>();
315 map.put("id", name);
316 map.put("name", name);
317 return map;
318 }).collect(Collectors.toList()));
319 }
320
321 private HtmlResponse asListHtml() {
322 return asHtml(path_AdminBackup_AdminBackupJsp).useForm(UploadForm.class).renderWith(
323 data -> RenderDataUtil.register(data, "backupItems", getBackupItems()));
324 }
325
326 }