1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.admin.maintenance;
17
18 import java.io.BufferedWriter;
19 import java.io.IOException;
20 import java.io.OutputStreamWriter;
21 import java.net.InetAddress;
22 import java.nio.file.Files;
23 import java.nio.file.Path;
24 import java.nio.file.Paths;
25 import java.text.SimpleDateFormat;
26 import java.util.Arrays;
27 import java.util.Date;
28 import java.util.Properties;
29 import java.util.stream.Stream;
30 import java.util.zip.ZipEntry;
31 import java.util.zip.ZipOutputStream;
32
33 import javax.annotation.Resource;
34
35 import org.apache.commons.text.StringEscapeUtils;
36 import org.apache.logging.log4j.LogManager;
37 import org.apache.logging.log4j.Logger;
38 import org.codelibs.core.exception.IORuntimeException;
39 import org.codelibs.core.io.CopyUtil;
40 import org.codelibs.core.lang.StringUtil;
41 import org.codelibs.curl.CurlResponse;
42 import org.codelibs.fesen.action.ActionListener;
43 import org.codelibs.fess.Constants;
44 import org.codelibs.fess.annotation.Secured;
45 import org.codelibs.fess.app.web.base.FessAdminAction;
46 import org.codelibs.fess.es.client.SearchEngineClient;
47 import org.codelibs.fess.mylasta.direction.FessConfig.SimpleImpl;
48 import org.codelibs.fess.util.ComponentUtil;
49 import org.codelibs.fess.util.SearchEngineUtil;
50 import org.lastaflute.web.Execute;
51 import org.lastaflute.web.response.ActionResponse;
52 import org.lastaflute.web.response.HtmlResponse;
53 import org.lastaflute.web.ruts.process.ActionRuntime;
54
55 public class AdminMaintenanceAction extends FessAdminAction {
56
57 public static final String ROLE = "admin-maintenance";
58
59
60
61
62 private static final Logger logger = LogManager.getLogger(AdminMaintenanceAction.class);
63
64 private static final String[] ES_CAT_NAMES =
65 new String[] { "aliases", "allocation", "count", "fielddata", "health", "indices", "master", "nodeattrs", "nodes",
66 "pending_tasks", "plugins", "recovery", "repositories", "thread_pool", "shards", "segments", "snapshots", "templates" };
67
68
69
70
71
72 @Resource
73 protected SearchEngineClient searchEngineClient;
74
75
76
77
78 @Override
79 protected void setupHtmlData(final ActionRuntime runtime) {
80 super.setupHtmlData(runtime);
81 runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameMaintenance()));
82 }
83
84 @Override
85 protected String getActionRole() {
86 return ROLE;
87 }
88
89
90
91
92
93 @Execute
94 @Secured({ ROLE, ROLE + VIEW })
95 public HtmlResponse index() {
96 saveToken();
97 return asIndexHtml();
98 }
99
100 private HtmlResponse asIndexHtml() {
101 return asHtml(path_AdminMaintenance_AdminMaintenanceJsp).useForm(ActionForm.class, op -> op.setup(f -> {
102 f.replaceAliases = Constants.ON;
103 f.resetDictionaries = null;
104 }));
105 }
106
107 @Execute
108 @Secured({ ROLE })
109 public HtmlResponse reindexOnly(final ActionForm form) {
110 validate(form, messages -> {}, this::asIndexHtml);
111 verifyToken(this::asIndexHtml);
112 if (startReindex(isCheckboxEnabled(form.replaceAliases), isCheckboxEnabled(form.resetDictionaries), form.numberOfShardsForDoc,
113 form.autoExpandReplicasForDoc)) {
114 saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
115 }
116 return redirect(getClass());
117 }
118
119 @Execute
120 @Secured({ ROLE })
121 public HtmlResponse reloadDocIndex(final ActionForm form) {
122 validate(form, messages -> {}, this::asIndexHtml);
123 verifyToken(this::asIndexHtml);
124 final String docIndex = fessConfig.getIndexDocumentUpdateIndex();
125 searchEngineClient.admin().indices().prepareClose(docIndex).execute(ActionListener.wrap(res -> {
126 logger.info("Close {}", docIndex);
127 searchEngineClient.admin().indices().prepareOpen(docIndex).execute(
128 ActionListener.wrap(res2 -> logger.info("Open {}", docIndex), e -> logger.warn("Failed to open {}", docIndex, e)));
129 }, e -> logger.warn("Failed to close {}", docIndex, e)));
130 saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
131 return redirect(getClass());
132 }
133
134 @Execute
135 @Secured({ ROLE })
136 public HtmlResponse clearCrawlerIndex(final ActionForm form) {
137 validate(form, messages -> {}, this::asIndexHtml);
138 verifyToken(this::asIndexHtml);
139 searchEngineClient.admin().indices().prepareDelete(
140 fessConfig.getIndexDocumentCrawlerIndex() + ".queue",
141 fessConfig.getIndexDocumentCrawlerIndex() + ".data",
142 fessConfig.getIndexDocumentCrawlerIndex() + ".filter")
143 .execute(ActionListener.wrap(res -> logger.info("Deleted .crawler indices."),
144 e -> logger.warn("Failed to delete .crawler.* indices.", e)));
145 saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
146 return redirect(getClass());
147 }
148
149 @Execute
150 @Secured({ ROLE, ROLE + VIEW })
151 public ActionResponse downloadLogs(final ActionForm form) {
152 validate(form, messages -> {}, this::asIndexHtml);
153 verifyTokenKeep(this::asIndexHtml);
154
155 final String diagnosticId = "log" + new SimpleDateFormat("yyyyMMddHHmm").format(ComponentUtil.getSystemHelper().getCurrentTime());
156 return asStream(diagnosticId + ".zip").contentTypeOctetStream().stream(out -> {
157 try (ZipOutputStream zos = new ZipOutputStream(out.stream())) {
158 writeLogFiles(zos, diagnosticId);
159 writeSystemProperties(zos, diagnosticId);
160 writeFessBasicConfig(zos, diagnosticId);
161 writeFessConfig(zos, diagnosticId);
162 writeFesenCat(zos, diagnosticId);
163 writeFesenJson(zos, diagnosticId);
164 }
165 });
166 }
167
168 protected void writeFesenJson(final ZipOutputStream zos, final String id) {
169 writeElastisearchJsonApi(zos, id, "cluster", "health");
170 writeElastisearchJsonApi(zos, id, "cluster", "state");
171 writeElastisearchJsonApi(zos, id, "cluster", "stats");
172 writeElastisearchJsonApi(zos, id, "cluster", "pending_tasks");
173 writeElastisearchJsonApi(zos, id, "nodes", "stats");
174 writeElastisearchJsonApi(zos, id, "nodes", "_all");
175 writeElastisearchJsonApi(zos, id, "nodes", "usage");
176 writeElastisearchJsonApi(zos, id, "remote", "info");
177 writeElastisearchJsonApi(zos, id, "tasks", "");
178 writeElastisearchJsonApi(zos, id, "nodes", "hot_threads");
179 }
180
181 protected void writeElastisearchJsonApi(final ZipOutputStream zos, final String id, final String v1, final String v2) {
182 final ZipEntry entry = new ZipEntry(id + "/es_" + v1 + "_" + v2 + ".json");
183 try {
184 zos.putNextEntry(entry);
185 try (CurlResponse response = ComponentUtil.getCurlHelper().get("/_" + v1 + "/" + v2).execute()) {
186 CopyUtil.copy(response.getContentAsStream(), zos);
187 }
188 } catch (final Exception e) {
189 logger.warn("Failed to access /_{}/{}", v1, v2, e);
190 }
191 }
192
193 protected void writeFesenCat(final ZipOutputStream zos, final String id) {
194 Arrays.stream(ES_CAT_NAMES).forEach(name -> {
195 final ZipEntry entry = new ZipEntry(id + "/es_cat_" + name + ".txt");
196 try {
197 zos.putNextEntry(entry);
198 try (CurlResponse response = ComponentUtil.getCurlHelper().get("/_cat/" + name).param("v", "").execute()) {
199 CopyUtil.copy(response.getContentAsStream(), zos);
200 }
201 } catch (final Exception e) {
202 logger.warn("Failed to access /_cat/{}", name, e);
203 }
204 });
205 }
206
207 protected void writeFessConfig(final ZipOutputStream zos, final String id) {
208 if (fessConfig instanceof SimpleImpl) {
209 final Properties prop = new Properties();
210 ((SimpleImpl) fessConfig).keySet().stream().forEach(k -> prop.setProperty(k, fessConfig.get(k)));
211
212 final ZipEntry entry = new ZipEntry(id + "/fess_config.properties");
213 try {
214 zos.putNextEntry(entry);
215 prop.store(zos, getHostInfo());
216 } catch (final IOException e) {
217 logger.warn("Failed to access fess_config.properties.", e);
218 }
219 }
220 }
221
222 protected void writeFessBasicConfig(final ZipOutputStream zos, final String id) {
223 final String index = ".fess_basic_config";
224 final ZipEntry entry = new ZipEntry(id + "/fess_basic_config.bulk");
225 try {
226 zos.putNextEntry(entry);
227 SearchEngineUtil.scroll(index, hit -> {
228 final String data = "{\"index\":{\"_index\":\"" + index + "\",\"_id\":\"" + StringEscapeUtils.escapeJson(hit.getId())
229 + "\"}}\n" + hit.getSourceAsString() + "\n";
230 try {
231 zos.write(data.getBytes(Constants.CHARSET_UTF_8));
232 } catch (final IOException e) {
233 logger.warn("Failed to access /{}/{}.", index, hit.getId(), e);
234 }
235 return true;
236 });
237 } catch (final IOException e) {
238 logger.warn("Failed to access /{}.", index, e);
239 }
240 }
241
242 protected void writeSystemProperties(final ZipOutputStream zos, final String id) {
243 final ZipEntry entry = new ZipEntry(id + "/system.properties");
244 try {
245 zos.putNextEntry(entry);
246 ComponentUtil.getSystemProperties().store(zos, getHostInfo());
247 } catch (final IOException e) {
248 logger.warn("Failed to access system.properties.", e);
249 }
250 }
251
252 protected void writeLogFiles(final ZipOutputStream zos, final String id) {
253 final String logFilePath = systemHelper.getLogFilePath();
254 if (StringUtil.isNotBlank(logFilePath)) {
255 final Path logDirPath = Paths.get(logFilePath);
256 try (Stream<Path> stream = Files.list(logDirPath)) {
257 stream.filter(entry -> isLogFilename(entry.getFileName().toString())).forEach(filePath -> {
258 final ZipEntry entry = new ZipEntry(id + "/" + filePath.getFileName().toString());
259 try {
260 zos.putNextEntry(entry);
261 final long len = Files.copy(filePath, zos);
262 if (logger.isDebugEnabled()) {
263 logger.debug("{}: {}", filePath.getFileName(), len);
264 }
265 } catch (final IOException e) {
266 logger.warn("Failed to access {}", filePath, e);
267 }
268 });
269 } catch (final Exception e) {
270 logger.warn("Failed to access log files.", e);
271 }
272 }
273 }
274
275 protected String getHostInfo() {
276 final StringBuilder buf = new StringBuilder();
277 try {
278 final InetAddress ia = InetAddress.getLocalHost();
279 final String hostname = ia.getHostName();
280 if (StringUtil.isNotBlank(hostname)) {
281 buf.append(hostname);
282 }
283 final String ip = ia.getHostAddress();
284 if (StringUtil.isNotBlank(ip)) {
285 if (buf.length() > 0) {
286 buf.append(" : ");
287 }
288 buf.append(ip);
289 }
290 } catch (final Exception e) {
291
292 }
293 return buf.toString();
294 }
295
296 protected boolean isLogFilename(final String name) {
297 return name.endsWith(".log") || name.endsWith(".log.gz");
298 }
299
300 protected boolean startReindex(final boolean replaceAliases, final boolean resetDictionaries, final String numberOfShards,
301 final String autoExpandReplicas) {
302 final String docIndex = "fess";
303 final String fromIndex = fessConfig.getIndexDocumentUpdateIndex();
304 final String toIndex = docIndex + "." + new SimpleDateFormat("yyyyMMddHHmm").format(new Date());
305 if (searchEngineClient.createIndex(docIndex, toIndex, numberOfShards, autoExpandReplicas, resetDictionaries)) {
306 searchEngineClient.admin().cluster().prepareHealth(toIndex).setWaitForYellowStatus().execute(ActionListener.wrap(response -> {
307 searchEngineClient.addMapping(docIndex, "doc", toIndex);
308 if (searchEngineClient.reindex(fromIndex, toIndex, replaceAliases)
309 && (replaceAliases && !searchEngineClient.updateAlias(toIndex))) {
310 logger.warn("Failed to update aliases for {} and {}", fromIndex, toIndex);
311 }
312 }, e -> logger.warn("Failed to reindex from {} to {}", fromIndex, toIndex, e)));
313 return true;
314 }
315 saveError(messages -> messages.addErrorsFailedToReindex(GLOBAL, fromIndex, toIndex));
316 return false;
317 }
318
319 }