1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.thumbnail;
17
18 import java.io.File;
19 import java.io.IOException;
20 import java.nio.file.FileVisitResult;
21 import java.nio.file.FileVisitor;
22 import java.nio.file.Files;
23 import java.nio.file.Path;
24 import java.nio.file.attribute.BasicFileAttributes;
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.concurrent.BlockingQueue;
30 import java.util.concurrent.LinkedBlockingQueue;
31 import java.util.concurrent.TimeUnit;
32
33 import javax.annotation.PostConstruct;
34 import javax.annotation.PreDestroy;
35
36 import org.codelibs.core.lang.StringUtil;
37 import org.codelibs.core.misc.Tuple3;
38 import org.codelibs.fess.Constants;
39 import org.codelibs.fess.es.client.FessEsClient;
40 import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
41 import org.codelibs.fess.es.config.exentity.ThumbnailQueue;
42 import org.codelibs.fess.exception.FessSystemException;
43 import org.codelibs.fess.exception.JobProcessingException;
44 import org.codelibs.fess.helper.SystemHelper;
45 import org.codelibs.fess.mylasta.direction.FessConfig;
46 import org.codelibs.fess.util.ComponentUtil;
47 import org.codelibs.fess.util.DocumentUtil;
48 import org.codelibs.fess.util.ResourceUtil;
49 import org.elasticsearch.index.query.QueryBuilders;
50 import org.slf4j.Logger;
51 import org.slf4j.LoggerFactory;
52
53 import com.google.common.collect.Lists;
54
55 public class ThumbnailManager {
56 private static final String FESS_THUMBNAIL_PATH = "fess.thumbnail.path";
57
58 private static final String FESS_VAR_PATH = "fess.var.path";
59
60 private static final String NOIMAGE_FILE_SUFFIX = ".txt";
61
62 protected static final String THUMBNAILS_DIR_NAME = "thumbnails";
63
64 private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
65
66 protected File baseDir;
67
68 private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
69
70 private BlockingQueue<Tuple3<String, String, String>> thumbnailTaskQueue;
71
72 private volatile boolean generating;
73
74 private Thread thumbnailQueueThread;
75
76 protected int thumbnailPathCacheSize = 10;
77
78 protected String imageExtention = "png";
79
80 protected int splitSize = 5;
81
82 protected int thumbnailTaskQueueSize = 10000;
83
84 protected int thumbnailTaskBulkSize = 100;
85
86 protected long thumbnailTaskQueueTimeout = 10 * 1000L;
87
88 protected long noImageExpired = 24 * 60 * 60 * 1000L;
89
90 @PostConstruct
91 public void init() {
92 final String thumbnailPath = System.getProperty(FESS_THUMBNAIL_PATH);
93 if (thumbnailPath != null) {
94 baseDir = new File(thumbnailPath);
95 } else {
96 final String varPath = System.getProperty(FESS_VAR_PATH);
97 if (varPath != null) {
98 baseDir = new File(varPath, THUMBNAILS_DIR_NAME);
99 } else {
100 baseDir = ResourceUtil.getThumbnailPath().toFile();
101 }
102 }
103 if (baseDir.mkdirs()) {
104 logger.info("Created: " + baseDir.getAbsolutePath());
105 }
106 if (!baseDir.isDirectory()) {
107 throw new FessSystemException("Not found: " + baseDir.getAbsolutePath());
108 }
109
110 if (logger.isDebugEnabled()) {
111 logger.debug("Thumbnail Directory: " + baseDir.getAbsolutePath());
112 }
113
114 thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
115 generating = true;
116 thumbnailQueueThread = new Thread((Runnable) () -> {
117 final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
118 while (generating) {
119 try {
120 final Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
121 if (task == null) {
122 if (!taskList.isEmpty()) {
123 storeQueue(taskList);
124 }
125 } else if (!taskList.contains(task)) {
126 taskList.add(task);
127 if (taskList.size() > thumbnailTaskBulkSize) {
128 storeQueue(taskList);
129 }
130 }
131 } catch (final InterruptedException e) {
132 if (logger.isDebugEnabled()) {
133 logger.debug("Interupted task.", e);
134 }
135 } catch (final Exception e) {
136 if (generating) {
137 logger.warn("Failed to generate thumbnail.", e);
138 }
139 }
140 }
141 if (!taskList.isEmpty()) {
142 storeQueue(taskList);
143 }
144 }, "ThumbnailGenerator");
145 thumbnailQueueThread.start();
146 }
147
148 @PreDestroy
149 public void destroy() {
150 generating = false;
151 thumbnailQueueThread.interrupt();
152 try {
153 thumbnailQueueThread.join(10000);
154 } catch (final InterruptedException e) {
155 logger.warn("Thumbnail thread is timeouted.", e);
156 }
157 generatorList.forEach(g -> {
158 try {
159 g.destroy();
160 } catch (final Exception e) {
161 logger.warn("Failed to stop thumbnail generator.", e);
162 }
163 });
164 }
165
166 public String getThumbnailPathOption() {
167 return "-D" + FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
168 }
169
170 protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
171 final FessConfig fessConfig = ComponentUtil.getFessConfig();
172 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
173 final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray();
174 final List<ThumbnailQueue> list = new ArrayList<>();
175 taskList.stream().filter(entity -> entity != null).forEach(task -> {
176 for (final String target : targets) {
177 final ThumbnailQueue entity = new ThumbnailQueue();
178 entity.setGenerator(task.getValue1());
179 entity.setThumbnailId(task.getValue2());
180 entity.setPath(task.getValue3());
181 entity.setTarget(target);
182 entity.setCreatedBy(Constants.SYSTEM_USER);
183 entity.setCreatedTime(systemHelper.getCurrentTimeAsLong());
184 list.add(entity);
185 }
186 });
187 taskList.clear();
188 if (logger.isDebugEnabled()) {
189 logger.debug("Storing " + list.size() + " thumbnail tasks.");
190 }
191 final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
192 thumbnailQueueBhv.batchInsert(list);
193 }
194
195 public int generate() {
196 final FessConfig fessConfig = ComponentUtil.getFessConfig();
197 final List<String> idList = new ArrayList<>();
198 final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
199 thumbnailQueueBhv.selectList(cb -> {
200 if (StringUtil.isBlank(fessConfig.getSchedulerTargetName())) {
201 cb.query().setTarget_Equal(Constants.DEFAULT_JOB_TARGET);
202 } else {
203 cb.query().setTarget_InScope(Lists.newArrayList(Constants.DEFAULT_JOB_TARGET, fessConfig.getSchedulerTargetName()));
204 }
205 cb.query().addOrderBy_CreatedTime_Asc();
206 cb.fetchFirst(fessConfig.getPageThumbnailQueueMaxFetchSizeAsInteger());
207 }).forEach(entity -> {
208 if (logger.isDebugEnabled()) {
209 logger.debug("Generating thumbnail: " + entity);
210 }
211 idList.add(entity.getId());
212 final String generatorName = entity.getGenerator();
213 try {
214 final ThumbnailGenerator generator = ComponentUtil.getComponent(generatorName);
215 final File outputFile = new File(baseDir, entity.getPath());
216 final File noImageFile = new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX);
217 if (!noImageFile.isFile() || System.currentTimeMillis() - noImageFile.lastModified() > noImageExpired) {
218 if (noImageFile.isFile() && !noImageFile.delete()) {
219 logger.warn("Failed to delete " + noImageFile.getAbsolutePath());
220 }
221 if (!generator.generate(entity.getThumbnailId(), outputFile)) {
222 new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX).setLastModified(System.currentTimeMillis());
223 } else {
224 final long interval = fessConfig.getThumbnailGeneratorIntervalAsInteger().longValue();
225 if (interval > 0) {
226 Thread.sleep(interval);
227 }
228 }
229 } else if (logger.isDebugEnabled()) {
230 logger.debug("No image file exists: " + noImageFile.getAbsolutePath());
231 }
232 } catch (final Exception e) {
233 logger.warn("Failed to create thumbnail for " + entity, e);
234 }
235 });
236 if (!idList.isEmpty()) {
237 thumbnailQueueBhv.queryDelete(cb -> {
238 cb.query().setId_InScope(idList);
239 });
240 thumbnailQueueBhv.refresh();
241 }
242 return idList.size();
243 }
244
245 public boolean offer(final Map<String, Object> docMap) {
246 for (final ThumbnailGenerator generator : generatorList) {
247 if (generator.isTarget(docMap)) {
248 final String path = getImageFilename(docMap);
249 final Tuple3<String, String, String> task = generator.createTask(path, docMap);
250 if (task != null) {
251 if (logger.isDebugEnabled()) {
252 logger.debug("Add thumbnail task: " + task);
253 }
254 if (!thumbnailTaskQueue.offer(task)) {
255 logger.warn("Failed to add thumbnail task: " + task);
256 }
257 return true;
258 }
259 return false;
260 }
261 }
262 if (logger.isDebugEnabled()) {
263 logger.debug("Thumbnail generator is not found: " + (docMap != null ? docMap.get("url") : docMap));
264 }
265 return false;
266 }
267
268 protected String getImageFilename(final Map<String, Object> docMap) {
269 final StringBuilder buf = new StringBuilder(50);
270 final FessConfig fessConfig = ComponentUtil.getFessConfig();
271 final String docid = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldDocId(), String.class);
272 for (int i = 0; i < docid.length(); i++) {
273 if (i > 0 && i % splitSize == 0) {
274 buf.append('/');
275 }
276 buf.append(docid.charAt(i));
277 }
278 buf.append('.').append(imageExtention);
279 return buf.toString();
280 }
281
282 public File getThumbnailFile(final Map<String, Object> docMap) {
283 final String thumbnailPath = getImageFilename(docMap);
284 if (StringUtil.isNotBlank(thumbnailPath)) {
285 final File file = new File(baseDir, thumbnailPath);
286 if (file.isFile()) {
287 return file;
288 }
289 }
290 return null;
291 }
292
293 public void add(final ThumbnailGenerator generator) {
294 if (generator.isAvailable()) {
295 generatorList.add(generator);
296 }
297 }
298
299 public long purge(final long expiry) {
300 if (!baseDir.exists()) {
301 return 0;
302 }
303 try {
304 final FilePurgeVisitor visitor = new FilePurgeVisitor(baseDir.toPath(), imageExtention, expiry);
305 Files.walkFileTree(baseDir.toPath(), visitor);
306 return visitor.getCount();
307 } catch (final Exception e) {
308 throw new JobProcessingException(e);
309 }
310 }
311
312 protected static class FilePurgeVisitor implements FileVisitor<Path> {
313
314 protected final long expiry;
315
316 protected long count;
317
318 protected final int maxPurgeSize;
319
320 protected final List<Path> deletedFileList = new ArrayList<>();
321
322 protected final Path basePath;
323
324 protected final String imageExtention;
325
326 protected final FessEsClient fessEsClient;
327
328 protected final FessConfig fessConfig;
329
330 FilePurgeVisitor(final Path basePath, final String imageExtention, final long expiry) {
331 this.basePath = basePath;
332 this.imageExtention = imageExtention;
333 this.expiry = expiry;
334 this.fessConfig = ComponentUtil.getFessConfig();
335 this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger();
336 this.fessEsClient = ComponentUtil.getFessEsClient();
337 }
338
339 protected void deleteFiles() {
340 final Map<String, Path> deleteFileMap = new HashMap<>();
341 for (final Path path : deletedFileList) {
342 final String docId = getDocId(path);
343 if (StringUtil.isBlank(docId) || deleteFileMap.containsKey(docId)) {
344 deleteFile(path);
345 } else {
346 deleteFileMap.put(docId, path);
347 }
348 }
349 deletedFileList.clear();
350
351 if (!deleteFileMap.isEmpty()) {
352 final String docIdField = fessConfig.getIndexFieldDocId();
353 fessEsClient.getDocumentList(
354 fessConfig.getIndexDocumentSearchIndex(),
355 fessConfig.getIndexDocumentType(),
356 searchRequestBuilder -> {
357 searchRequestBuilder.setQuery(QueryBuilders.termsQuery(docIdField,
358 deleteFileMap.keySet().toArray(new String[deleteFileMap.size()])));
359 searchRequestBuilder.setFetchSource(new String[] { docIdField }, StringUtil.EMPTY_STRINGS);
360 return true;
361 }).forEach(m -> {
362 final Object docId = m.get(docIdField);
363 if (docId != null) {
364 deleteFileMap.remove(docId);
365 if (logger.isDebugEnabled()) {
366 logger.debug("Keep thumbnail: " + docId);
367 }
368 }
369 });
370 ;
371 deleteFileMap.values().forEach(v -> deleteFile(v));
372 count += deleteFileMap.size();
373 }
374 }
375
376 protected void deleteFile(final Path path) {
377 try {
378 Files.delete(path);
379 if (logger.isDebugEnabled()) {
380 logger.debug("Delete " + path);
381 }
382 } catch (final IOException e) {
383 logger.warn("Failed to delete " + path, e);
384 }
385 }
386
387 protected String getDocId(final Path file) {
388 final String s = file.toUri().toString();
389 final String b = basePath.toUri().toString();
390 final String id = s.replace(b, StringUtil.EMPTY).replace("." + imageExtention, StringUtil.EMPTY).replace("/", StringUtil.EMPTY);
391 if (logger.isDebugEnabled()) {
392 logger.debug("Base: " + b + " File: " + s + " DocId: " + id);
393 }
394 return id;
395 }
396
397 public long getCount() {
398 if (!deletedFileList.isEmpty()) {
399 deleteFiles();
400 }
401 return count;
402 }
403
404 @Override
405 public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
406 return FileVisitResult.CONTINUE;
407 }
408
409 @Override
410 public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
411 if (System.currentTimeMillis() - Files.getLastModifiedTime(file).toMillis() > expiry) {
412 deletedFileList.add(file);
413 if (deletedFileList.size() > maxPurgeSize) {
414 deleteFiles();
415 }
416 }
417 return FileVisitResult.CONTINUE;
418 }
419
420 @Override
421 public FileVisitResult visitFileFailed(final Path file, final IOException e) throws IOException {
422 if (e != null) {
423 logger.warn("I/O exception on " + file, e);
424 }
425 return FileVisitResult.CONTINUE;
426 }
427
428 @Override
429 public FileVisitResult postVisitDirectory(final Path dir, final IOException e) throws IOException {
430 if (e != null) {
431 logger.warn("I/O exception on " + dir, e);
432 }
433 if (dir.toFile().list().length == 0 && !dir.toFile().getName().equals(THUMBNAILS_DIR_NAME)) {
434 Files.delete(dir);
435 }
436 return FileVisitResult.CONTINUE;
437 }
438
439 }
440
441 public void setThumbnailPathCacheSize(final int thumbnailPathCacheSize) {
442 this.thumbnailPathCacheSize = thumbnailPathCacheSize;
443 }
444
445 public void setImageExtention(final String imageExtention) {
446 this.imageExtention = imageExtention;
447 }
448
449 public void setSplitSize(final int splitSize) {
450 this.splitSize = splitSize;
451 }
452
453 public void setThumbnailTaskQueueSize(final int thumbnailTaskQueueSize) {
454 this.thumbnailTaskQueueSize = thumbnailTaskQueueSize;
455 }
456
457 public void setNoImageExpired(final long noImageExpired) {
458 this.noImageExpired = noImageExpired;
459 }
460
461 }