1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.admin.wizard;
17
18 import java.io.UnsupportedEncodingException;
19 import java.net.URLEncoder;
20 import java.util.List;
21
22 import javax.annotation.Resource;
23
24 import org.apache.commons.lang3.StringUtils;
25 import org.apache.logging.log4j.LogManager;
26 import org.apache.logging.log4j.Logger;
27 import org.codelibs.core.lang.StringUtil;
28 import org.codelibs.core.misc.DynamicProperties;
29 import org.codelibs.fess.Constants;
30 import org.codelibs.fess.annotation.Secured;
31 import org.codelibs.fess.app.service.FileConfigService;
32 import org.codelibs.fess.app.service.ScheduledJobService;
33 import org.codelibs.fess.app.service.WebConfigService;
34 import org.codelibs.fess.app.web.base.FessAdminAction;
35 import org.codelibs.fess.crawler.util.CharUtil;
36 import org.codelibs.fess.es.config.exentity.FileConfig;
37 import org.codelibs.fess.es.config.exentity.ScheduledJob;
38 import org.codelibs.fess.es.config.exentity.WebConfig;
39 import org.codelibs.fess.helper.ProcessHelper;
40 import org.codelibs.fess.util.ComponentUtil;
41 import org.lastaflute.job.JobManager;
42 import org.lastaflute.job.key.LaJobUnique;
43 import org.lastaflute.web.Execute;
44 import org.lastaflute.web.response.HtmlResponse;
45 import org.lastaflute.web.ruts.process.ActionRuntime;
46
47 public class AdminWizardAction extends FessAdminAction {
48
49 public static final String ROLE = "admin-wizard";
50
51
52
53
54 private static final Logger logger = LogManager.getLogger(AdminWizardAction.class);
55
56
57
58
59 @Resource
60 protected DynamicProperties systemProperties;
61
62 @Resource
63 protected WebConfigService webConfigService;
64
65 @Resource
66 protected FileConfigService fileConfigService;
67
68 @Resource
69 protected ProcessHelper processHelper;
70
71 @Resource
72 protected ScheduledJobService scheduledJobService;
73
74
75
76
77 @Override
78 protected void setupHtmlData(final ActionRuntime runtime) {
79 super.setupHtmlData(runtime);
80 runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameWizard()));
81 }
82
83 @Override
84 protected String getActionRole() {
85 return ROLE;
86 }
87
88
89
90
91
92 @Execute
93 @Secured({ ROLE, ROLE + VIEW })
94 public HtmlResponse index() {
95 return asIndexHtml();
96 }
97
98 private HtmlResponse asIndexHtml() {
99 return asHtml(path_AdminWizard_AdminWizardJsp).useForm(IndexForm.class);
100 }
101
102 @Execute
103 @Secured({ ROLE })
104 public HtmlResponse crawlingConfigForm() {
105 saveToken();
106 return asHtml(path_AdminWizard_AdminWizardConfigJsp).useForm(CrawlingConfigForm.class);
107 }
108
109 @Execute
110 @Secured({ ROLE })
111 public HtmlResponse crawlingConfig(final CrawlingConfigForm form) {
112 validate(form, messages -> {}, () -> asHtml(path_AdminWizard_AdminWizardConfigJsp));
113 verifyTokenKeep(this::asIndexHtml);
114 final String name = crawlingConfigInternal(form);
115 saveInfo(messages -> messages.addSuccessCreateCrawlingConfigAtWizard(GLOBAL, name));
116 return redirectWith(getClass(), moreUrl("crawlingConfigForm"));
117 }
118
119 @Execute
120 @Secured({ ROLE })
121 public HtmlResponse crawlingConfigNext(final CrawlingConfigForm form) {
122 validate(form, messages -> {}, () -> asHtml(path_AdminWizard_AdminWizardConfigJsp));
123 verifyToken(this::asIndexHtml);
124 final String name = crawlingConfigInternal(form);
125 saveInfo(messages -> messages.addSuccessCreateCrawlingConfigAtWizard(GLOBAL, name));
126 return redirectWith(getClass(), moreUrl("startCrawlingForm"));
127 }
128
129 protected String crawlingConfigInternal(final CrawlingConfigForm form) {
130
131 String configName = form.crawlingConfigName;
132 String configPath = form.crawlingConfigPath.trim();
133 if (StringUtil.isBlank(configName)) {
134 configName = StringUtils.abbreviate(configPath, 30);
135 }
136
137
138 final StringBuilder buf = new StringBuilder(1000);
139 for (int i = 0; i < configPath.length(); i++) {
140 final char c = configPath.charAt(i);
141 if (c == '\\') {
142 buf.append('/');
143 } else if (c == ' ') {
144 buf.append("%20");
145 } else if (CharUtil.isUrlChar(c)) {
146 buf.append(c);
147 } else {
148 try {
149 buf.append(URLEncoder.encode(String.valueOf(c), Constants.UTF_8));
150 } catch (final UnsupportedEncodingException e) {}
151 }
152 }
153 configPath = convertCrawlingPath(buf.toString());
154
155 final String username = systemHelper.getUsername();
156 final long now = systemHelper.getCurrentTimeAsLong();
157
158 try {
159 if (isWebCrawlingPath(configPath)) {
160
161 final WebConfig wConfig = new WebConfig();
162 wConfig.setAvailable(Constants.T);
163 wConfig.setBoost(1.0f);
164 wConfig.setCreatedBy(username);
165 wConfig.setCreatedTime(now);
166 if (form.depth != null) {
167 wConfig.setDepth(form.depth);
168 }
169 wConfig.setExcludedDocUrls(getDefaultString("default.config.web.excludedDocUrls", StringUtil.EMPTY));
170 wConfig.setExcludedUrls(getDefaultString("default.config.web.excludedUrls", StringUtil.EMPTY));
171 wConfig.setIncludedDocUrls(getDefaultString("default.config.web.includedDocUrls", StringUtil.EMPTY));
172 wConfig.setIncludedUrls(getDefaultString("default.config.web.includedUrls", StringUtil.EMPTY));
173 wConfig.setIntervalTime(getDefaultInteger("default.config.web.intervalTime", Constants.DEFAULT_INTERVAL_TIME_FOR_WEB));
174 if (form.maxAccessCount != null) {
175 wConfig.setMaxAccessCount(form.maxAccessCount);
176 }
177 wConfig.setName(configName);
178 wConfig.setNumOfThread(getDefaultInteger("default.config.web.numOfThread", Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB));
179 wConfig.setSortOrder(getDefaultInteger("default.config.web.sortOrder", 1));
180 wConfig.setUpdatedBy(username);
181 wConfig.setUpdatedTime(now);
182 wConfig.setUrls(configPath);
183 wConfig.setUserAgent(getDefaultString("default.config.web.userAgent", fessConfig.getUserAgentName()));
184 wConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());
185
186 webConfigService.store(wConfig);
187
188 } else {
189
190 final FileConfig fConfig = new FileConfig();
191 fConfig.setAvailable(Constants.T);
192 fConfig.setBoost(1.0f);
193 fConfig.setCreatedBy(username);
194 fConfig.setCreatedTime(now);
195 if (form.depth != null) {
196 fConfig.setDepth(form.depth);
197 }
198 fConfig.setExcludedDocPaths(getDefaultString("default.config.file.excludedDocPaths", StringUtil.EMPTY));
199 fConfig.setExcludedPaths(getDefaultString("default.config.file.excludedPaths", StringUtil.EMPTY));
200 fConfig.setIncludedDocPaths(getDefaultString("default.config.file.includedDocPaths", StringUtil.EMPTY));
201 fConfig.setIncludedPaths(getDefaultString("default.config.file.includedPaths", StringUtil.EMPTY));
202 fConfig.setIntervalTime(getDefaultInteger("default.config.file.intervalTime", Constants.DEFAULT_INTERVAL_TIME_FOR_FS));
203 if (form.maxAccessCount != null) {
204 fConfig.setMaxAccessCount(form.maxAccessCount);
205 }
206 fConfig.setName(configName);
207 fConfig.setNumOfThread(getDefaultInteger("default.config.file.numOfThread", Constants.DEFAULT_NUM_OF_THREAD_FOR_FS));
208 fConfig.setSortOrder(getDefaultInteger("default.config.file.sortOrder", 1));
209 fConfig.setUpdatedBy(username);
210 fConfig.setUpdatedTime(now);
211 fConfig.setPaths(configPath);
212 fConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());
213
214 fileConfigService.store(fConfig);
215 }
216 return configName;
217 } catch (final Exception e) {
218 logger.error("Failed to create crawling config: {}", form.crawlingConfigPath, e);
219 throwValidationError(messages -> messages.addErrorsFailedToCreateCrawlingConfigAtWizard(GLOBAL),
220 () -> asHtml(path_AdminWizard_AdminWizardConfigJsp));
221 return null;
222 }
223 }
224
225 protected Integer getDefaultInteger(final String key, final Integer defaultValue) {
226 final String value = systemProperties.getProperty(key);
227 if (value != null) {
228 try {
229 return Integer.parseInt(value);
230 } catch (final NumberFormatException e) {}
231 }
232 return defaultValue;
233 }
234
235 protected Long getDefaultLong(final String key, final Long defaultValue) {
236 final String value = systemProperties.getProperty(key);
237 if (value != null) {
238 try {
239 return Long.parseLong(value);
240 } catch (final NumberFormatException e) {}
241 }
242 return defaultValue;
243 }
244
245 protected String getDefaultString(final String key, final String defaultValue) {
246 final String value = systemProperties.getProperty(key);
247 if (value != null) {
248 return value;
249 }
250 return defaultValue;
251 }
252
253 protected boolean isWebCrawlingPath(final String path) {
254 if (path.startsWith("http:") || path.startsWith("https:")) {
255 return true;
256 }
257
258 return false;
259 }
260
261 protected String convertCrawlingPath(final String path) {
262 if (path.startsWith("http:") || path.startsWith("https:") || path.startsWith("smb:") || path.startsWith("smb1:")
263 || path.startsWith("ftp:") || path.startsWith("storage:")) {
264 return path;
265 }
266
267 if (path.startsWith("www.")) {
268 return "http://" + path;
269 }
270
271 if (path.startsWith("//")) {
272 return "file://" + path;
273 }
274 if (path.startsWith("/")) {
275 return "file:" + path;
276 }
277 if (!path.startsWith("file:")) {
278 return "file:/" + path.replace('\\', '/');
279 }
280 return path;
281 }
282
283 @Execute
284 @Secured({ ROLE })
285 public HtmlResponse startCrawlingForm() {
286 saveToken();
287 return asHtml(path_AdminWizard_AdminWizardStartJsp).useForm(StartCrawlingForm.class);
288 }
289
290 @Execute
291 @Secured({ ROLE })
292 public HtmlResponse startCrawling(final StartCrawlingForm form) {
293 verifyToken(this::asIndexHtml);
294 if (!processHelper.isProcessRunning()) {
295 final List<ScheduledJob> scheduledJobList = scheduledJobService.getCrawlerJobList();
296 final JobManager jobManager = ComponentUtil.getJobManager();
297 for (final ScheduledJob scheduledJob : scheduledJobList) {
298 jobManager.findJobByUniqueOf(LaJobUnique.of(scheduledJob.getId())).ifPresent(job -> {
299 job.launchNow();
300 });
301 }
302 saveInfo(messages -> messages.addSuccessStartCrawlProcess(GLOBAL));
303 } else {
304 saveError(messages -> messages.addErrorsFailedToStartCrawlProcess(GLOBAL));
305 }
306 return redirect(AdminWizardAction.class);
307 }
308 }