1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.admin.wizard;
17
18 import java.io.UnsupportedEncodingException;
19 import java.net.URLEncoder;
20 import java.util.List;
21
22 import javax.annotation.Resource;
23
24 import org.apache.commons.lang3.StringUtils;
25 import org.codelibs.core.lang.StringUtil;
26 import org.codelibs.core.misc.DynamicProperties;
27 import org.codelibs.fess.Constants;
28 import org.codelibs.fess.app.service.FileConfigService;
29 import org.codelibs.fess.app.service.ScheduledJobService;
30 import org.codelibs.fess.app.service.WebConfigService;
31 import org.codelibs.fess.app.web.base.FessAdminAction;
32 import org.codelibs.fess.crawler.util.CharUtil;
33 import org.codelibs.fess.es.config.exentity.FileConfig;
34 import org.codelibs.fess.es.config.exentity.ScheduledJob;
35 import org.codelibs.fess.es.config.exentity.WebConfig;
36 import org.codelibs.fess.helper.ProcessHelper;
37 import org.codelibs.fess.util.ComponentUtil;
38 import org.lastaflute.job.JobManager;
39 import org.lastaflute.job.key.LaJobUnique;
40 import org.lastaflute.web.Execute;
41 import org.lastaflute.web.response.HtmlResponse;
42 import org.lastaflute.web.ruts.process.ActionRuntime;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45
46 public class AdminWizardAction extends FessAdminAction {
47
48
49
50
51 private static final Logger logger = LoggerFactory.getLogger(AdminWizardAction.class);
52
53
54
55
56 @Resource
57 protected DynamicProperties systemProperties;
58
59 @Resource
60 protected WebConfigService webConfigService;
61
62 @Resource
63 protected FileConfigService fileConfigService;
64
65 @Resource
66 protected ProcessHelper processHelper;
67
68 @Resource
69 protected ScheduledJobService scheduledJobService;
70
71
72
73
74 @Override
75 protected void setupHtmlData(final ActionRuntime runtime) {
76 super.setupHtmlData(runtime);
77 runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameWizard()));
78 }
79
80
81
82
83
84 @Execute
85 public HtmlResponse index() {
86 return asIndexHtml();
87 }
88
89 private HtmlResponse asIndexHtml() {
90 return asHtml(path_AdminWizard_AdminWizardJsp).useForm(IndexForm.class);
91 }
92
93 @Execute
94 public HtmlResponse crawlingConfigForm() {
95 saveToken();
96 return asHtml(path_AdminWizard_AdminWizardConfigJsp).useForm(CrawlingConfigForm.class);
97 }
98
99 @Execute
100 public HtmlResponse crawlingConfig(final CrawlingConfigForm form) {
101 validate(form, messages -> {}, () -> {
102 return asHtml(path_AdminWizard_AdminWizardConfigJsp);
103 });
104 verifyTokenKeep(() -> asIndexHtml());
105 final String name = crawlingConfigInternal(form);
106 saveInfo(messages -> messages.addSuccessCreateCrawlingConfigAtWizard(GLOBAL, name));
107 return redirectWith(getClass(), moreUrl("crawlingConfigForm"));
108 }
109
110 @Execute
111 public HtmlResponse crawlingConfigNext(final CrawlingConfigForm form) {
112 validate(form, messages -> {}, () -> {
113 return asHtml(path_AdminWizard_AdminWizardConfigJsp);
114 });
115 verifyToken(() -> asIndexHtml());
116 final String name = crawlingConfigInternal(form);
117 saveInfo(messages -> messages.addSuccessCreateCrawlingConfigAtWizard(GLOBAL, name));
118 return redirectWith(getClass(), moreUrl("startCrawlingForm"));
119 }
120
121 protected String crawlingConfigInternal(final CrawlingConfigForm form) {
122
123 String configName = form.crawlingConfigName;
124 String configPath = form.crawlingConfigPath.trim();
125 if (StringUtil.isBlank(configName)) {
126 configName = StringUtils.abbreviate(configPath, 30);
127 }
128
129
130 final StringBuilder buf = new StringBuilder(1000);
131 for (int i = 0; i < configPath.length(); i++) {
132 final char c = configPath.charAt(i);
133 if (c == '\\') {
134 buf.append('/');
135 } else if (c == ' ') {
136 buf.append("%20");
137 } else if (CharUtil.isUrlChar(c)) {
138 buf.append(c);
139 } else {
140 try {
141 buf.append(URLEncoder.encode(String.valueOf(c), Constants.UTF_8));
142 } catch (final UnsupportedEncodingException e) {}
143 }
144 }
145 configPath = convertCrawlingPath(buf.toString());
146
147 final String username = systemHelper.getUsername();
148 final long now = systemHelper.getCurrentTimeAsLong();
149
150 try {
151 if (isWebCrawlingPath(configPath)) {
152
153 final WebConfig wConfig = new WebConfig();
154 wConfig.setAvailable(Constants.T);
155 wConfig.setBoost(1.0f);
156 wConfig.setCreatedBy(username);
157 wConfig.setCreatedTime(now);
158 if (form.depth != null) {
159 wConfig.setDepth(form.depth);
160 }
161 wConfig.setExcludedDocUrls(getDefaultString("default.config.web.excludedDocUrls", StringUtil.EMPTY));
162 wConfig.setExcludedUrls(getDefaultString("default.config.web.excludedUrls", StringUtil.EMPTY));
163 wConfig.setIncludedDocUrls(getDefaultString("default.config.web.includedDocUrls", StringUtil.EMPTY));
164 wConfig.setIncludedUrls(getDefaultString("default.config.web.includedUrls", StringUtil.EMPTY));
165 wConfig.setIntervalTime(getDefaultInteger("default.config.web.intervalTime", Constants.DEFAULT_INTERVAL_TIME_FOR_WEB));
166 if (form.maxAccessCount != null) {
167 wConfig.setMaxAccessCount(form.maxAccessCount);
168 }
169 wConfig.setName(configName);
170 wConfig.setNumOfThread(getDefaultInteger("default.config.web.numOfThread", Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB));
171 wConfig.setSortOrder(getDefaultInteger("default.config.web.sortOrder", 1));
172 wConfig.setUpdatedBy(username);
173 wConfig.setUpdatedTime(now);
174 wConfig.setUrls(configPath);
175 wConfig.setUserAgent(getDefaultString("default.config.web.userAgent", ComponentUtil.getUserAgentName()));
176 wConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());
177
178 webConfigService.store(wConfig);
179
180 } else {
181
182 final FileConfig fConfig = new FileConfig();
183 fConfig.setAvailable(Constants.T);
184 fConfig.setBoost(1.0f);
185 fConfig.setCreatedBy(username);
186 fConfig.setCreatedTime(now);
187 if (form.depth != null) {
188 fConfig.setDepth(form.depth);
189 }
190 fConfig.setExcludedDocPaths(getDefaultString("default.config.file.excludedDocPaths", StringUtil.EMPTY));
191 fConfig.setExcludedPaths(getDefaultString("default.config.file.excludedPaths", StringUtil.EMPTY));
192 fConfig.setIncludedDocPaths(getDefaultString("default.config.file.includedDocPaths", StringUtil.EMPTY));
193 fConfig.setIncludedPaths(getDefaultString("default.config.file.includedPaths", StringUtil.EMPTY));
194 fConfig.setIntervalTime(getDefaultInteger("default.config.file.intervalTime", Constants.DEFAULT_INTERVAL_TIME_FOR_FS));
195 if (form.maxAccessCount != null) {
196 fConfig.setMaxAccessCount(form.maxAccessCount);
197 }
198 fConfig.setName(configName);
199 fConfig.setNumOfThread(getDefaultInteger("default.config.file.numOfThread", Constants.DEFAULT_NUM_OF_THREAD_FOR_FS));
200 fConfig.setSortOrder(getDefaultInteger("default.config.file.sortOrder", 1));
201 fConfig.setUpdatedBy(username);
202 fConfig.setUpdatedTime(now);
203 fConfig.setPaths(configPath);
204 fConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());
205
206 fileConfigService.store(fConfig);
207 }
208 return configName;
209 } catch (final Exception e) {
210 logger.error("Failed to create crawling config: " + form.crawlingConfigPath, e);
211 throwValidationError(messages -> messages.addErrorsFailedToCreateCrawlingConfigAtWizard(GLOBAL), () -> {
212 return asHtml(path_AdminWizard_AdminWizardConfigJsp);
213 });
214 return null;
215 }
216 }
217
218 protected Integer getDefaultInteger(final String key, final Integer defaultValue) {
219 final String value = systemProperties.getProperty(key);
220 if (value != null) {
221 try {
222 return Integer.parseInt(value);
223 } catch (final NumberFormatException e) {}
224 }
225 return defaultValue;
226 }
227
228 protected Long getDefaultLong(final String key, final Long defaultValue) {
229 final String value = systemProperties.getProperty(key);
230 if (value != null) {
231 try {
232 return Long.parseLong(value);
233 } catch (final NumberFormatException e) {}
234 }
235 return defaultValue;
236 }
237
238 protected String getDefaultString(final String key, final String defaultValue) {
239 final String value = systemProperties.getProperty(key);
240 if (value != null) {
241 return value;
242 }
243 return defaultValue;
244 }
245
246 protected boolean isWebCrawlingPath(final String path) {
247 if (path.startsWith("http:") || path.startsWith("https:")) {
248 return true;
249 }
250
251 return false;
252 }
253
254 protected String convertCrawlingPath(final String path) {
255 if (path.startsWith("http:") || path.startsWith("https:") || path.startsWith("smb:") || path.startsWith("ftp:")) {
256 return path;
257 }
258
259 if (path.startsWith("www.")) {
260 return "http://" + path;
261 }
262
263 if (path.startsWith("//")) {
264 return "file://" + path;
265 } else if (path.startsWith("/")) {
266 return "file:" + path;
267 } else if (!path.startsWith("file:")) {
268 return "file:/" + path.replace('\\', '/');
269 }
270 return path;
271 }
272
273 @Execute
274 public HtmlResponse startCrawlingForm() {
275 saveToken();
276 return asHtml(path_AdminWizard_AdminWizardStartJsp).useForm(StartCrawlingForm.class);
277 }
278
279 @Execute
280 public HtmlResponse startCrawling(final StartCrawlingForm form) {
281 verifyToken(() -> asIndexHtml());
282 if (!processHelper.isProcessRunning()) {
283 final List<ScheduledJob> scheduledJobList = scheduledJobService.getCrawlerJobList();
284 final JobManager jobManager = ComponentUtil.getJobManager();
285 for (final ScheduledJob scheduledJob : scheduledJobList) {
286 jobManager.findJobByUniqueOf(LaJobUnique.of(scheduledJob.getId())).ifPresent(job -> {
287 job.launchNow();
288 });
289 }
290 saveInfo(messages -> messages.addSuccessStartCrawlProcess(GLOBAL));
291 } else {
292 saveError(messages -> messages.addErrorsFailedToStartCrawlProcess(GLOBAL));
293 }
294 return redirect(AdminWizardAction.class);
295 }
296 }