1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.thumbnail.impl;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.File;
21 import java.util.ArrayList;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.function.BiPredicate;
26 import java.util.function.Predicate;
27
28 import org.apache.logging.log4j.LogManager;
29 import org.apache.logging.log4j.Logger;
30 import org.codelibs.core.lang.StringUtil;
31 import org.codelibs.core.misc.Tuple3;
32 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
33 import org.codelibs.fess.crawler.client.CrawlerClient;
34 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
35 import org.codelibs.fess.crawler.entity.ResponseData;
36 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
37 import org.codelibs.fess.exception.ThumbnailGenerationException;
38 import org.codelibs.fess.helper.CrawlingConfigHelper;
39 import org.codelibs.fess.helper.IndexingHelper;
40 import org.codelibs.fess.mylasta.direction.FessConfig;
41 import org.codelibs.fess.opensearch.client.SearchEngineClient;
42 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig;
43 import org.codelibs.fess.thumbnail.ThumbnailGenerator;
44 import org.codelibs.fess.util.ComponentUtil;
45 import org.codelibs.fess.util.DocumentUtil;
46
47
48
49
50
51 public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
52 private static final Logger logger = LogManager.getLogger(BaseThumbnailGenerator.class);
53
54
55 protected final Map<String, String> conditionMap = new HashMap<>();
56
57
58 protected int directoryNameLength = 5;
59
60
61 protected List<String> generatorList;
62
63
64 protected Map<String, String> filePathMap = new HashMap<>();
65
66
67 protected String name;
68
69
70 protected int maxRedirectCount = 10;
71
72
73 protected Boolean available = null;
74
75
76
77
78 public void register() {
79 ComponentUtil.getThumbnailManager().add(this);
80 }
81
82
83
84
85 public BaseThumbnailGenerator() {
86
87 }
88
89
90
91
92
93
94 public void addCondition(final String key, final String regex) {
95 final String value = conditionMap.get(key);
96 if (StringUtil.isBlank(value)) {
97 conditionMap.put(key, regex);
98 } else {
99 conditionMap.put(key, value + "|" + regex);
100 }
101 }
102
103 @Override
104 public boolean isTarget(final Map<String, Object> docMap) {
105 final String thumbnailFieldName = ComponentUtil.getFessConfig().getIndexFieldThumbnail();
106 if (logger.isDebugEnabled()) {
107 logger.debug("[{}] thumbnail: {}", name, docMap.get(thumbnailFieldName));
108 }
109 if (!docMap.containsKey(thumbnailFieldName)) {
110 return false;
111 }
112 for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
113 if (docMap.get(entry.getKey()) instanceof final String value && value.matches(entry.getValue())) {
114 if (logger.isDebugEnabled()) {
115 logger.debug("[{}] match {}:{}", entry.getKey(), name, value);
116 }
117 return true;
118 }
119 }
120 return false;
121 }
122
123 @Override
124 public boolean isAvailable() {
125 if (available != null) {
126 return available;
127 }
128 if (generatorList != null && !generatorList.isEmpty()) {
129 String path = System.getenv("PATH");
130 if (path == null) {
131 path = System.getenv("Path");
132 }
133 if (path == null) {
134 path = System.getenv("path");
135 }
136 final List<String> pathList = new ArrayList<>();
137 pathList.add("/usr/share/fess/bin");
138 if (path != null) {
139 stream(path.split(File.pathSeparator)).of(stream -> stream.map(String::trim).forEach(s -> pathList.add(s)));
140 }
141 if (logger.isDebugEnabled()) {
142 logger.debug("search paths: {}", pathList);
143 }
144 available = generatorList.stream().map(s -> {
145 if (s.startsWith("${path}")) {
146 for (final String p : pathList) {
147 final File f = new File(s.replace("${path}", p));
148 if (f.exists()) {
149 final String filePath = f.getAbsolutePath();
150 filePathMap.put(s, filePath);
151 if (logger.isDebugEnabled()) {
152 logger.debug("generator path: {}", filePath);
153 }
154 return filePath;
155 }
156 }
157 }
158 if (logger.isDebugEnabled()) {
159 logger.debug("generator path: {}", s);
160 }
161 return s;
162 }).allMatch(s -> {
163 final boolean found = new File(s).isFile();
164 if (found && logger.isDebugEnabled()) {
165 logger.debug("Generator command found: {}", s);
166 }
167 return found;
168 });
169 } else {
170 available = true;
171 }
172 return available;
173 }
174
175 @Override
176 public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
177 final FessConfig fessConfig = ComponentUtil.getFessConfig();
178 final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
179 final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
180 if (logger.isDebugEnabled()) {
181 logger.debug("Create thumbnail task: {}", task);
182 }
183 return task;
184 }
185
186
187
188
189
190 public void setDirectoryNameLength(final int directoryNameLength) {
191 this.directoryNameLength = directoryNameLength;
192 }
193
194
195
196
197
198
199 protected String expandPath(final String value) {
200 if (value != null && filePathMap.containsKey(value)) {
201 return filePathMap.get(value);
202 }
203 return value;
204 }
205
206
207
208
209
210
211 protected void updateThumbnailField(final String thumbnailId, final String value) {
212
213 final FessConfig fessConfig = ComponentUtil.getFessConfig();
214 try {
215 ComponentUtil.getIndexingHelper()
216 .updateDocument(ComponentUtil.getSearchEngineClient(), thumbnailId, fessConfig.getIndexFieldThumbnail(), value);
217 } catch (final Exception e) {
218 logger.warn("Failed to update thumbnail field at {}", thumbnailId, e);
219 }
220 }
221
222
223
224
225
226
227
228 protected boolean process(final String id, final BiPredicate<String, String> consumer) {
229 final FessConfig fessConfig = ComponentUtil.getFessConfig();
230 final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
231 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
232 try {
233 final Map<String, Object> doc = indexingHelper.getDocument(searchEngineClient, id,
234 new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
235 if (doc == null) {
236 throw new ThumbnailGenerationException("Document is not found: " + id);
237 }
238 final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
239 if (StringUtil.isBlank(url)) {
240 throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
241 }
242 final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
243 if (configId == null || configId.length() < 2) {
244 throw new ThumbnailGenerationException("Invalid configId: " + configId);
245 }
246 return consumer.test(configId, url);
247 } catch (final ThumbnailGenerationException e) {
248 if (e.getCause() == null) {
249 logger.debug(e.getMessage());
250 } else {
251 logger.warn("Failed to process thumbnail: id={}", id, e);
252 }
253 } catch (final Exception e) {
254 logger.warn("Failed to process thumbnail: id={}", id, e);
255 }
256 return false;
257 }
258
259
260
261
262
263
264
265 protected boolean process(final String id, final Predicate<ResponseData> consumer) {
266 return process(id, (configId, url) -> {
267 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
268 final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
269 if (config == null) {
270 throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
271 }
272
273 if (logger.isInfoEnabled()) {
274 logger.info("Generating Thumbnail: {}", url);
275 }
276
277 final CrawlerClientFactory crawlerClientFactory =
278 config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
279 final CrawlerClient client = crawlerClientFactory.getClient(url);
280 if (client == null) {
281 throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
282 }
283 String u = url;
284 for (int i = 0; i < maxRedirectCount; i++) {
285 try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
286 if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
287 u = responseData.getRedirectLocation();
288 continue;
289 }
290 if (StringUtil.isBlank(responseData.getUrl())) {
291 throw new ThumbnailGenerationException(
292 "Failed to process a thumbnail content: " + url + " (Response URL is empty)");
293 }
294 return consumer.test(responseData);
295 } catch (final CrawlingAccessException e) {
296 if (logger.isDebugEnabled()) {
297 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
298 }
299 throw new ThumbnailGenerationException(e.getMessage());
300 } catch (final Exception e) {
301 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
302 }
303 }
304 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
305 });
306 }
307
308
309
310
311
312 public void setGeneratorList(final List<String> generatorList) {
313 this.generatorList = generatorList;
314 }
315
316 @Override
317 public String getName() {
318 return name;
319 }
320
321
322
323
324
325 public void setName(final String name) {
326 this.name = name;
327 }
328
329
330
331
332
333 public void setMaxRedirectCount(final int maxRedirectCount) {
334 this.maxRedirectCount = maxRedirectCount;
335 }
336
337 }