View Javadoc
1   /*
2    * Copyright 2012-2019 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.thumbnail;
17  
18  import java.io.File;
19  import java.io.IOException;
20  import java.nio.file.FileAlreadyExistsException;
21  import java.nio.file.FileVisitResult;
22  import java.nio.file.FileVisitor;
23  import java.nio.file.Files;
24  import java.nio.file.Path;
25  import java.nio.file.attribute.BasicFileAttributes;
26  import java.util.ArrayList;
27  import java.util.HashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.concurrent.BlockingQueue;
31  import java.util.concurrent.ExecutorService;
32  import java.util.concurrent.LinkedBlockingQueue;
33  import java.util.concurrent.TimeUnit;
34  import java.util.stream.Stream;
35  
36  import javax.annotation.PostConstruct;
37  import javax.annotation.PreDestroy;
38  
39  import org.codelibs.core.lang.StringUtil;
40  import org.codelibs.core.misc.Tuple3;
41  import org.codelibs.fess.Constants;
42  import org.codelibs.fess.es.client.FessEsClient;
43  import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
44  import org.codelibs.fess.es.config.exentity.ThumbnailQueue;
45  import org.codelibs.fess.exception.FessSystemException;
46  import org.codelibs.fess.exception.JobProcessingException;
47  import org.codelibs.fess.helper.SystemHelper;
48  import org.codelibs.fess.mylasta.direction.FessConfig;
49  import org.codelibs.fess.util.ComponentUtil;
50  import org.codelibs.fess.util.DocumentUtil;
51  import org.codelibs.fess.util.ResourceUtil;
52  import org.elasticsearch.index.query.QueryBuilders;
53  import org.slf4j.Logger;
54  import org.slf4j.LoggerFactory;
55  
56  import com.google.common.collect.Lists;
57  
58  public class ThumbnailManager {
59      private static final String NOIMAGE_FILE_SUFFIX = ".txt";
60  
61      protected static final String THUMBNAILS_DIR_NAME = "thumbnails";
62  
63      private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
64  
65      protected File baseDir;
66  
67      private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
68  
69      private BlockingQueue<Tuple3<String, String, String>> thumbnailTaskQueue;
70  
71      private volatile boolean generating;
72  
73      private Thread thumbnailQueueThread;
74  
75      protected int thumbnailPathCacheSize = 10;
76  
77      protected String imageExtention = "png";
78  
79      protected int splitSize = 3;
80  
81      protected int thumbnailTaskQueueSize = 10000;
82  
83      protected int thumbnailTaskBulkSize = 100;
84  
85      protected long thumbnailTaskQueueTimeout = 10 * 1000L;
86  
87      protected long noImageExpired = 24 * 60 * 60 * 1000L; // 24 hours
88  
89      @PostConstruct
90      public void init() {
91          if (logger.isDebugEnabled()) {
92              logger.debug("Initialize " + this.getClass().getSimpleName());
93          }
94          final String thumbnailPath = System.getProperty(Constants.FESS_THUMBNAIL_PATH);
95          if (thumbnailPath != null) {
96              baseDir = new File(thumbnailPath);
97          } else {
98              final String varPath = System.getProperty(Constants.FESS_VAR_PATH);
99              if (varPath != null) {
100                 baseDir = new File(varPath, THUMBNAILS_DIR_NAME);
101             } else {
102                 baseDir = ResourceUtil.getThumbnailPath().toFile();
103             }
104         }
105         if (baseDir.mkdirs()) {
106             logger.info("Created: " + baseDir.getAbsolutePath());
107         }
108         if (!baseDir.isDirectory()) {
109             throw new FessSystemException("Not found: " + baseDir.getAbsolutePath());
110         }
111 
112         if (logger.isDebugEnabled()) {
113             logger.debug("Thumbnail Directory: " + baseDir.getAbsolutePath());
114         }
115 
116         thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
117         generating = !Constants.TRUE.equalsIgnoreCase(System.getProperty("fess.thumbnail.process"));
118         thumbnailQueueThread = new Thread((Runnable) () -> {
119             final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
120             while (generating) {
121                 try {
122                     final Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
123                     if (task == null) {
124                         if (!taskList.isEmpty()) {
125                             storeQueue(taskList);
126                         }
127                     } else if (!taskList.contains(task)) {
128                         taskList.add(task);
129                         if (taskList.size() > thumbnailTaskBulkSize) {
130                             storeQueue(taskList);
131                         }
132                     }
133                 } catch (final InterruptedException e) {
134                     if (logger.isDebugEnabled()) {
135                         logger.debug("Interupted task.", e);
136                     }
137                 } catch (final Exception e) {
138                     if (generating) {
139                         logger.warn("Failed to generate thumbnail.", e);
140                     }
141                 }
142             }
143             if (!taskList.isEmpty()) {
144                 storeQueue(taskList);
145             }
146         }, "ThumbnailGenerator");
147         thumbnailQueueThread.start();
148     }
149 
150     @PreDestroy
151     public void destroy() {
152         generating = false;
153         thumbnailQueueThread.interrupt();
154         try {
155             thumbnailQueueThread.join(10000);
156         } catch (final InterruptedException e) {
157             logger.warn("Thumbnail thread is timeouted.", e);
158         }
159         generatorList.forEach(g -> {
160             try {
161                 g.destroy();
162             } catch (final Exception e) {
163                 logger.warn("Failed to stop thumbnail generator.", e);
164             }
165         });
166     }
167 
168     public String getThumbnailPathOption() {
169         return "-D" + Constants.FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
170     }
171 
172     protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
173         final FessConfig fessConfig = ComponentUtil.getFessConfig();
174         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
175         final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray();
176         final List<ThumbnailQueue> list = new ArrayList<>();
177         taskList.stream().filter(entity -> entity != null).forEach(task -> {
178             for (final String target : targets) {
179                 final ThumbnailQueue/ThumbnailQueue.html#ThumbnailQueue">ThumbnailQueue entity = new ThumbnailQueue();
180                 entity.setGenerator(task.getValue1());
181                 entity.setThumbnailId(task.getValue2());
182                 entity.setPath(task.getValue3());
183                 entity.setTarget(target);
184                 entity.setCreatedBy(Constants.SYSTEM_USER);
185                 entity.setCreatedTime(systemHelper.getCurrentTimeAsLong());
186                 list.add(entity);
187             }
188         });
189         taskList.clear();
190         if (logger.isDebugEnabled()) {
191             logger.debug("Storing " + list.size() + " thumbnail tasks.");
192         }
193         final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
194         thumbnailQueueBhv.batchInsert(list);
195     }
196 
197     public int generate(final ExecutorService executorService, final boolean cleanup) {
198         final FessConfig fessConfig = ComponentUtil.getFessConfig();
199         final List<String> idList = new ArrayList<>();
200         final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
201         thumbnailQueueBhv.selectList(cb -> {
202             if (StringUtil.isBlank(fessConfig.getSchedulerTargetName())) {
203                 cb.query().setTarget_Equal(Constants.DEFAULT_JOB_TARGET);
204             } else {
205                 cb.query().setTarget_InScope(Lists.newArrayList(Constants.DEFAULT_JOB_TARGET, fessConfig.getSchedulerTargetName()));
206             }
207             cb.query().addOrderBy_CreatedTime_Asc();
208             cb.fetchFirst(fessConfig.getPageThumbnailQueueMaxFetchSizeAsInteger());
209         }).stream().map(entity -> {
210             idList.add(entity.getId());
211             if (cleanup) {
212                 if (logger.isDebugEnabled()) {
213                     logger.debug("Removing thumbnail queue: " + entity);
214                 }
215                 return null;
216             } else {
217                 return executorService.submit(() -> process(fessConfig, entity));
218             }
219         }).filter(f -> f != null).forEach(f -> {
220             try {
221                 f.get();
222             } catch (final Exception e) {
223                 logger.warn("Failed to process a thumbnail generation.", e);
224             }
225         });
226 
227         if (!idList.isEmpty()) {
228             thumbnailQueueBhv.queryDelete(cb -> {
229                 cb.query().setId_InScope(idList);
230             });
231             thumbnailQueueBhv.refresh();
232         }
233         return idList.size();
234     }
235 
236     protected void process(final FessConfig fessConfig, final ThumbnailQueue entity) {
237         if (logger.isDebugEnabled()) {
238             logger.debug("Processing thumbnail: " + entity);
239         }
240         final String generatorName = entity.getGenerator();
241         try {
242             final File outputFile = new File(baseDir, entity.getPath());
243             final File noImageFile = new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX);
244             if (!noImageFile.isFile() || System.currentTimeMillis() - noImageFile.lastModified() > noImageExpired) {
245                 if (noImageFile.isFile() && !noImageFile.delete()) {
246                     logger.warn("Failed to delete " + noImageFile.getAbsolutePath());
247                 }
248                 final ThumbnailGenerator generator = ComponentUtil.getComponent(generatorName);
249                 if (generator.isAvailable()) {
250                     if (!generator.generate(entity.getThumbnailId(), outputFile)) {
251                         new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX).setLastModified(System.currentTimeMillis());
252                     } else {
253                         final long interval = fessConfig.getThumbnailGeneratorIntervalAsInteger().longValue();
254                         if (interval > 0) {
255                             Thread.sleep(interval);
256                         }
257                     }
258                 } else {
259                     logger.warn(generatorName + " is not available.");
260                 }
261             } else if (logger.isDebugEnabled()) {
262                 logger.debug("No image file exists: " + noImageFile.getAbsolutePath());
263             }
264         } catch (final Exception e) {
265             logger.warn("Failed to create thumbnail for " + entity, e);
266         }
267     }
268 
269     public boolean offer(final Map<String, Object> docMap) {
270         for (final ThumbnailGenerator generator : generatorList) {
271             if (generator.isTarget(docMap)) {
272                 final String path = getImageFilename(docMap);
273                 final Tuple3<String, String, String> task = generator.createTask(path, docMap);
274                 if (task != null) {
275                     if (logger.isDebugEnabled()) {
276                         logger.debug("Add thumbnail task: " + task);
277                     }
278                     if (!thumbnailTaskQueue.offer(task)) {
279                         logger.warn("Failed to add thumbnail task: " + task);
280                     }
281                     return true;
282                 }
283                 return false;
284             }
285         }
286         if (logger.isDebugEnabled()) {
287             logger.debug("Thumbnail generator is not found: " + (docMap != null ? docMap.get("url") : docMap));
288         }
289         return false;
290     }
291 
292     protected String getImageFilename(final Map<String, Object> docMap) {
293         final FessConfig fessConfig = ComponentUtil.getFessConfig();
294         final String docid = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldDocId(), String.class);
295         return getImageFilename(docid);
296     }
297 
298     protected String getImageFilename(final String docid) {
299         final StringBuilder buf = new StringBuilder(50);
300         for (int i = 0; i < docid.length(); i++) {
301             if (i > 0 && i % splitSize == 0) {
302                 buf.append('/');
303             }
304             buf.append(docid.charAt(i));
305         }
306         buf.append('.').append(imageExtention);
307         return buf.toString();
308     }
309 
310     public File getThumbnailFile(final Map<String, Object> docMap) {
311         final String thumbnailPath = getImageFilename(docMap);
312         if (StringUtil.isNotBlank(thumbnailPath)) {
313             final File file = new File(baseDir, thumbnailPath);
314             if (file.isFile()) {
315                 return file;
316             }
317         }
318         return null;
319     }
320 
321     public void add(final ThumbnailGenerator generator) {
322         if (logger.isDebugEnabled()) {
323             logger.debug(generator.getName() + " is available.");
324         }
325         if (generator.isAvailable()) {
326             generatorList.add(generator);
327         }
328     }
329 
330     public long purge(final long expiry) {
331         if (!baseDir.exists()) {
332             return 0;
333         }
334         try {
335             final FilePurgeVisitor visitor = new FilePurgeVisitor(baseDir.toPath(), imageExtention, expiry);
336             Files.walkFileTree(baseDir.toPath(), visitor);
337             return visitor.getCount();
338         } catch (final Exception e) {
339             throw new JobProcessingException(e);
340         }
341     }
342 
343     protected static class FilePurgeVisitor implements FileVisitor<Path> {
344 
345         protected final long expiry;
346 
347         protected long count;
348 
349         protected final int maxPurgeSize;
350 
351         protected final List<Path> deletedFileList = new ArrayList<>();
352 
353         protected final Path basePath;
354 
355         protected final String imageExtention;
356 
357         protected final FessEsClient fessEsClient;
358 
359         protected final FessConfig fessConfig;
360 
361         FilePurgeVisitor(final Path basePath, final String imageExtention, final long expiry) {
362             this.basePath = basePath;
363             this.imageExtention = imageExtention;
364             this.expiry = expiry;
365             this.fessConfig = ComponentUtil.getFessConfig();
366             this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger();
367             this.fessEsClient = ComponentUtil.getFessEsClient();
368         }
369 
370         protected void deleteFiles() {
371             final Map<String, Path> deleteFileMap = new HashMap<>();
372             for (final Path path : deletedFileList) {
373                 final String docId = getDocId(path);
374                 if (StringUtil.isBlank(docId) || deleteFileMap.containsKey(docId)) {
375                     deleteFile(path);
376                 } else {
377                     deleteFileMap.put(docId, path);
378                 }
379             }
380             deletedFileList.clear();
381 
382             if (!deleteFileMap.isEmpty()) {
383                 final String docIdField = fessConfig.getIndexFieldDocId();
384                 fessEsClient.getDocumentList(
385                         fessConfig.getIndexDocumentSearchIndex(),
386                         searchRequestBuilder -> {
387                             searchRequestBuilder.setQuery(QueryBuilders.termsQuery(docIdField,
388                                     deleteFileMap.keySet().toArray(new String[deleteFileMap.size()])));
389                             searchRequestBuilder.setFetchSource(new String[] { docIdField }, StringUtil.EMPTY_STRINGS);
390                             return true;
391                         }).forEach(m -> {
392                     final Object docId = m.get(docIdField);
393                     if (docId != null) {
394                         deleteFileMap.remove(docId);
395                         if (logger.isDebugEnabled()) {
396                             logger.debug("Keep thumbnail: " + docId);
397                         }
398                     }
399                 });
400 
401                 deleteFileMap.values().forEach(v -> deleteFile(v));
402                 count += deleteFileMap.size();
403             }
404         }
405 
406         protected void deleteFile(final Path path) {
407             try {
408                 Files.delete(path);
409                 if (logger.isDebugEnabled()) {
410                     logger.debug("Delete " + path);
411                 }
412 
413                 Path parent = path.getParent();
414                 while (deleteEmptyDirectory(parent)) {
415                     parent = parent.getParent();
416                 }
417             } catch (final IOException e) {
418                 logger.warn("Failed to delete " + path, e);
419             }
420         }
421 
422         protected String getDocId(final Path file) {
423             final String s = file.toUri().toString();
424             final String b = basePath.toUri().toString();
425             final String id = s.replace(b, StringUtil.EMPTY).replace("." + imageExtention, StringUtil.EMPTY).replace("/", StringUtil.EMPTY);
426             if (logger.isDebugEnabled()) {
427                 logger.debug("Base: " + b + " File: " + s + " DocId: " + id);
428             }
429             return id;
430         }
431 
432         public long getCount() {
433             if (!deletedFileList.isEmpty()) {
434                 deleteFiles();
435             }
436             return count;
437         }
438 
439         @Override
440         public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
441             return FileVisitResult.CONTINUE;
442         }
443 
444         @Override
445         public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
446             if (System.currentTimeMillis() - Files.getLastModifiedTime(file).toMillis() > expiry) {
447                 deletedFileList.add(file);
448                 if (deletedFileList.size() > maxPurgeSize) {
449                     deleteFiles();
450                 }
451             }
452             return FileVisitResult.CONTINUE;
453         }
454 
455         @Override
456         public FileVisitResult visitFileFailed(final Path file, final IOException e) throws IOException {
457             if (e != null) {
458                 logger.warn("I/O exception on " + file, e);
459             }
460             return FileVisitResult.CONTINUE;
461         }
462 
463         @Override
464         public FileVisitResult postVisitDirectory(final Path dir, final IOException e) throws IOException {
465             if (e != null) {
466                 logger.warn("I/O exception on " + dir, e);
467             }
468             deleteEmptyDirectory(dir);
469             return FileVisitResult.CONTINUE;
470         }
471 
472         private boolean deleteEmptyDirectory(final Path dir) throws IOException {
473             if (dir == null) {
474                 return false;
475             }
476             final File directory = dir.toFile();
477             if (directory.list() != null && directory.list().length == 0 && !THUMBNAILS_DIR_NAME.equals(directory.getName())) {
478                 Files.delete(dir);
479                 if (logger.isDebugEnabled()) {
480                     logger.debug("Delete " + dir);
481                 }
482                 return true;
483             }
484             return false;
485         }
486 
487     }
488 
489     public void migrate() {
490         new Thread(() -> {
491             final Path basePath = baseDir.toPath();
492             final String suffix = "." + imageExtention;
493             try (Stream<Path> paths = Files.walk(basePath)) {
494                 paths.filter(path -> path.toFile().getName().endsWith(imageExtention)).forEach(path -> {
495                     final Path subPath = basePath.relativize(path);
496                     final String docId = subPath.toString().replace("/", StringUtil.EMPTY).replace(suffix, StringUtil.EMPTY);
497                     final String filename = getImageFilename(docId);
498                     final Path newPath = basePath.resolve(filename);
499                     if (!path.equals(newPath)) {
500                         try {
501                             try {
502                                 Files.createDirectories(newPath.getParent());
503                             } catch (final FileAlreadyExistsException e) {
504                                 // ignore
505                     }
506                     Files.move(path, newPath);
507                     logger.info("Move " + path + " to " + newPath);
508                 } catch (final IOException e) {
509                     logger.warn("Failed to move " + path, e);
510                 }
511             }
512         }       );
513             } catch (final IOException e) {
514                 logger.warn("Failed to migrate thumbnail images.", e);
515             }
516         }, "ThumbnailMigrator").start();
517     }
518 
519     public void setThumbnailPathCacheSize(final int thumbnailPathCacheSize) {
520         this.thumbnailPathCacheSize = thumbnailPathCacheSize;
521     }
522 
523     public void setImageExtention(final String imageExtention) {
524         this.imageExtention = imageExtention;
525     }
526 
527     public void setSplitSize(final int splitSize) {
528         this.splitSize = splitSize;
529     }
530 
531     public void setThumbnailTaskQueueSize(final int thumbnailTaskQueueSize) {
532         this.thumbnailTaskQueueSize = thumbnailTaskQueueSize;
533     }
534 
535     public void setNoImageExpired(final long noImageExpired) {
536         this.noImageExpired = noImageExpired;
537     }
538 
539 }