View Javadoc
1   /*
2    * Copyright 2012-2019 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.thumbnail;
17  
18  import java.io.File;
19  import java.io.IOException;
20  import java.nio.file.FileAlreadyExistsException;
21  import java.nio.file.FileVisitResult;
22  import java.nio.file.FileVisitor;
23  import java.nio.file.Files;
24  import java.nio.file.Path;
25  import java.nio.file.attribute.BasicFileAttributes;
26  import java.util.ArrayList;
27  import java.util.HashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.concurrent.BlockingQueue;
31  import java.util.concurrent.ExecutorService;
32  import java.util.concurrent.LinkedBlockingQueue;
33  import java.util.concurrent.TimeUnit;
34  import java.util.stream.Stream;
35  
36  import javax.annotation.PostConstruct;
37  import javax.annotation.PreDestroy;
38  
39  import org.codelibs.core.lang.StringUtil;
40  import org.codelibs.core.misc.Tuple3;
41  import org.codelibs.fess.Constants;
42  import org.codelibs.fess.es.client.FessEsClient;
43  import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
44  import org.codelibs.fess.es.config.exentity.ThumbnailQueue;
45  import org.codelibs.fess.exception.FessSystemException;
46  import org.codelibs.fess.exception.JobProcessingException;
47  import org.codelibs.fess.helper.SystemHelper;
48  import org.codelibs.fess.mylasta.direction.FessConfig;
49  import org.codelibs.fess.util.ComponentUtil;
50  import org.codelibs.fess.util.DocumentUtil;
51  import org.codelibs.fess.util.ResourceUtil;
52  import org.elasticsearch.index.query.QueryBuilders;
53  import org.slf4j.Logger;
54  import org.slf4j.LoggerFactory;
55  
56  import com.google.common.collect.Lists;
57  
58  public class ThumbnailManager {
59      private static final String NOIMAGE_FILE_SUFFIX = ".txt";
60  
61      protected static final String THUMBNAILS_DIR_NAME = "thumbnails";
62  
63      private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
64  
65      protected File baseDir;
66  
67      private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
68  
69      private BlockingQueue<Tuple3<String, String, String>> thumbnailTaskQueue;
70  
71      private volatile boolean generating;
72  
73      private Thread thumbnailQueueThread;
74  
75      protected int thumbnailPathCacheSize = 10;
76  
77      protected String imageExtention = "png";
78  
79      protected int splitSize = 3;
80  
81      protected int thumbnailTaskQueueSize = 10000;
82  
83      protected int thumbnailTaskBulkSize = 100;
84  
85      protected long thumbnailTaskQueueTimeout = 10 * 1000L;
86  
87      protected long noImageExpired = 24 * 60 * 60 * 1000L; // 24 hours
88  
89      @PostConstruct
90      public void init() {
91          final String thumbnailPath = System.getProperty(Constants.FESS_THUMBNAIL_PATH);
92          if (thumbnailPath != null) {
93              baseDir = new File(thumbnailPath);
94          } else {
95              final String varPath = System.getProperty(Constants.FESS_VAR_PATH);
96              if (varPath != null) {
97                  baseDir = new File(varPath, THUMBNAILS_DIR_NAME);
98              } else {
99                  baseDir = ResourceUtil.getThumbnailPath().toFile();
100             }
101         }
102         if (baseDir.mkdirs()) {
103             logger.info("Created: " + baseDir.getAbsolutePath());
104         }
105         if (!baseDir.isDirectory()) {
106             throw new FessSystemException("Not found: " + baseDir.getAbsolutePath());
107         }
108 
109         if (logger.isDebugEnabled()) {
110             logger.debug("Thumbnail Directory: " + baseDir.getAbsolutePath());
111         }
112 
113         thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
114         generating = !Constants.TRUE.equalsIgnoreCase(System.getProperty("fess.thumbnail.process"));
115         thumbnailQueueThread = new Thread((Runnable) () -> {
116             final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
117             while (generating) {
118                 try {
119                     final Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
120                     if (task == null) {
121                         if (!taskList.isEmpty()) {
122                             storeQueue(taskList);
123                         }
124                     } else if (!taskList.contains(task)) {
125                         taskList.add(task);
126                         if (taskList.size() > thumbnailTaskBulkSize) {
127                             storeQueue(taskList);
128                         }
129                     }
130                 } catch (final InterruptedException e) {
131                     if (logger.isDebugEnabled()) {
132                         logger.debug("Interupted task.", e);
133                     }
134                 } catch (final Exception e) {
135                     if (generating) {
136                         logger.warn("Failed to generate thumbnail.", e);
137                     }
138                 }
139             }
140             if (!taskList.isEmpty()) {
141                 storeQueue(taskList);
142             }
143         }, "ThumbnailGenerator");
144         thumbnailQueueThread.start();
145     }
146 
147     @PreDestroy
148     public void destroy() {
149         generating = false;
150         thumbnailQueueThread.interrupt();
151         try {
152             thumbnailQueueThread.join(10000);
153         } catch (final InterruptedException e) {
154             logger.warn("Thumbnail thread is timeouted.", e);
155         }
156         generatorList.forEach(g -> {
157             try {
158                 g.destroy();
159             } catch (final Exception e) {
160                 logger.warn("Failed to stop thumbnail generator.", e);
161             }
162         });
163     }
164 
165     public String getThumbnailPathOption() {
166         return "-D" + Constants.FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
167     }
168 
169     protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
170         final FessConfig fessConfig = ComponentUtil.getFessConfig();
171         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
172         final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray();
173         final List<ThumbnailQueue> list = new ArrayList<>();
174         taskList.stream().filter(entity -> entity != null).forEach(task -> {
175             for (final String target : targets) {
176                 final ThumbnailQueue/ThumbnailQueue.html#ThumbnailQueue">ThumbnailQueue entity = new ThumbnailQueue();
177                 entity.setGenerator(task.getValue1());
178                 entity.setThumbnailId(task.getValue2());
179                 entity.setPath(task.getValue3());
180                 entity.setTarget(target);
181                 entity.setCreatedBy(Constants.SYSTEM_USER);
182                 entity.setCreatedTime(systemHelper.getCurrentTimeAsLong());
183                 list.add(entity);
184             }
185         });
186         taskList.clear();
187         if (logger.isDebugEnabled()) {
188             logger.debug("Storing " + list.size() + " thumbnail tasks.");
189         }
190         final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
191         thumbnailQueueBhv.batchInsert(list);
192     }
193 
194     public int generate(final ExecutorService executorService, final boolean cleanup) {
195         final FessConfig fessConfig = ComponentUtil.getFessConfig();
196         final List<String> idList = new ArrayList<>();
197         final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
198         thumbnailQueueBhv.selectList(cb -> {
199             if (StringUtil.isBlank(fessConfig.getSchedulerTargetName())) {
200                 cb.query().setTarget_Equal(Constants.DEFAULT_JOB_TARGET);
201             } else {
202                 cb.query().setTarget_InScope(Lists.newArrayList(Constants.DEFAULT_JOB_TARGET, fessConfig.getSchedulerTargetName()));
203             }
204             cb.query().addOrderBy_CreatedTime_Asc();
205             cb.fetchFirst(fessConfig.getPageThumbnailQueueMaxFetchSizeAsInteger());
206         }).stream().map(entity -> {
207             idList.add(entity.getId());
208             if (cleanup) {
209                 if (logger.isDebugEnabled()) {
210                     logger.debug("Removing thumbnail queue: " + entity);
211                 }
212                 return null;
213             } else {
214                 return executorService.submit(() -> process(fessConfig, entity));
215             }
216         }).filter(f -> f != null).forEach(f -> {
217             try {
218                 f.get();
219             } catch (final Exception e) {
220                 logger.warn("Failed to process a thumbnail generation.", e);
221             }
222         });
223 
224         if (!idList.isEmpty()) {
225             thumbnailQueueBhv.queryDelete(cb -> {
226                 cb.query().setId_InScope(idList);
227             });
228             thumbnailQueueBhv.refresh();
229         }
230         return idList.size();
231     }
232 
233     protected void process(final FessConfig fessConfig, final ThumbnailQueue entity) {
234         if (logger.isDebugEnabled()) {
235             logger.debug("Processing thumbnail: " + entity);
236         }
237         final String generatorName = entity.getGenerator();
238         try {
239             final File outputFile = new File(baseDir, entity.getPath());
240             final File noImageFile = new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX);
241             if (!noImageFile.isFile() || System.currentTimeMillis() - noImageFile.lastModified() > noImageExpired) {
242                 if (noImageFile.isFile() && !noImageFile.delete()) {
243                     logger.warn("Failed to delete " + noImageFile.getAbsolutePath());
244                 }
245                 final ThumbnailGenerator generator = ComponentUtil.getComponent(generatorName);
246                 if (generator.isAvailable()) {
247                     if (!generator.generate(entity.getThumbnailId(), outputFile)) {
248                         new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX).setLastModified(System.currentTimeMillis());
249                     } else {
250                         final long interval = fessConfig.getThumbnailGeneratorIntervalAsInteger().longValue();
251                         if (interval > 0) {
252                             Thread.sleep(interval);
253                         }
254                     }
255                 } else {
256                     logger.warn(generatorName + " is not available.");
257                 }
258             } else if (logger.isDebugEnabled()) {
259                 logger.debug("No image file exists: " + noImageFile.getAbsolutePath());
260             }
261         } catch (final Exception e) {
262             logger.warn("Failed to create thumbnail for " + entity, e);
263         }
264     }
265 
266     public boolean offer(final Map<String, Object> docMap) {
267         for (final ThumbnailGenerator generator : generatorList) {
268             if (generator.isTarget(docMap)) {
269                 final String path = getImageFilename(docMap);
270                 final Tuple3<String, String, String> task = generator.createTask(path, docMap);
271                 if (task != null) {
272                     if (logger.isDebugEnabled()) {
273                         logger.debug("Add thumbnail task: " + task);
274                     }
275                     if (!thumbnailTaskQueue.offer(task)) {
276                         logger.warn("Failed to add thumbnail task: " + task);
277                     }
278                     return true;
279                 }
280                 return false;
281             }
282         }
283         if (logger.isDebugEnabled()) {
284             logger.debug("Thumbnail generator is not found: " + (docMap != null ? docMap.get("url") : docMap));
285         }
286         return false;
287     }
288 
289     protected String getImageFilename(final Map<String, Object> docMap) {
290         final FessConfig fessConfig = ComponentUtil.getFessConfig();
291         final String docid = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldDocId(), String.class);
292         return getImageFilename(docid);
293     }
294 
295     protected String getImageFilename(final String docid) {
296         final StringBuilder buf = new StringBuilder(50);
297         for (int i = 0; i < docid.length(); i++) {
298             if (i > 0 && i % splitSize == 0) {
299                 buf.append('/');
300             }
301             buf.append(docid.charAt(i));
302         }
303         buf.append('.').append(imageExtention);
304         return buf.toString();
305     }
306 
307     public File getThumbnailFile(final Map<String, Object> docMap) {
308         final String thumbnailPath = getImageFilename(docMap);
309         if (StringUtil.isNotBlank(thumbnailPath)) {
310             final File file = new File(baseDir, thumbnailPath);
311             if (file.isFile()) {
312                 return file;
313             }
314         }
315         return null;
316     }
317 
318     public void add(final ThumbnailGenerator generator) {
319         if (logger.isDebugEnabled()) {
320             logger.debug(generator.getName() + " is available.");
321         }
322         if (generator.isAvailable()) {
323             generatorList.add(generator);
324         }
325     }
326 
327     public long purge(final long expiry) {
328         if (!baseDir.exists()) {
329             return 0;
330         }
331         try {
332             final FilePurgeVisitor visitor = new FilePurgeVisitor(baseDir.toPath(), imageExtention, expiry);
333             Files.walkFileTree(baseDir.toPath(), visitor);
334             return visitor.getCount();
335         } catch (final Exception e) {
336             throw new JobProcessingException(e);
337         }
338     }
339 
340     protected static class FilePurgeVisitor implements FileVisitor<Path> {
341 
342         protected final long expiry;
343 
344         protected long count;
345 
346         protected final int maxPurgeSize;
347 
348         protected final List<Path> deletedFileList = new ArrayList<>();
349 
350         protected final Path basePath;
351 
352         protected final String imageExtention;
353 
354         protected final FessEsClient fessEsClient;
355 
356         protected final FessConfig fessConfig;
357 
358         FilePurgeVisitor(final Path basePath, final String imageExtention, final long expiry) {
359             this.basePath = basePath;
360             this.imageExtention = imageExtention;
361             this.expiry = expiry;
362             this.fessConfig = ComponentUtil.getFessConfig();
363             this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger();
364             this.fessEsClient = ComponentUtil.getFessEsClient();
365         }
366 
367         protected void deleteFiles() {
368             final Map<String, Path> deleteFileMap = new HashMap<>();
369             for (final Path path : deletedFileList) {
370                 final String docId = getDocId(path);
371                 if (StringUtil.isBlank(docId) || deleteFileMap.containsKey(docId)) {
372                     deleteFile(path);
373                 } else {
374                     deleteFileMap.put(docId, path);
375                 }
376             }
377             deletedFileList.clear();
378 
379             if (!deleteFileMap.isEmpty()) {
380                 final String docIdField = fessConfig.getIndexFieldDocId();
381                 fessEsClient.getDocumentList(
382                         fessConfig.getIndexDocumentSearchIndex(),
383                         searchRequestBuilder -> {
384                             searchRequestBuilder.setQuery(QueryBuilders.termsQuery(docIdField,
385                                     deleteFileMap.keySet().toArray(new String[deleteFileMap.size()])));
386                             searchRequestBuilder.setFetchSource(new String[] { docIdField }, StringUtil.EMPTY_STRINGS);
387                             return true;
388                         }).forEach(m -> {
389                     final Object docId = m.get(docIdField);
390                     if (docId != null) {
391                         deleteFileMap.remove(docId);
392                         if (logger.isDebugEnabled()) {
393                             logger.debug("Keep thumbnail: " + docId);
394                         }
395                     }
396                 });
397 
398                 deleteFileMap.values().forEach(v -> deleteFile(v));
399                 count += deleteFileMap.size();
400             }
401         }
402 
403         protected void deleteFile(final Path path) {
404             try {
405                 Files.delete(path);
406                 if (logger.isDebugEnabled()) {
407                     logger.debug("Delete " + path);
408                 }
409 
410                 Path parent = path.getParent();
411                 while (deleteEmptyDirectory(parent)) {
412                     parent = parent.getParent();
413                 }
414             } catch (final IOException e) {
415                 logger.warn("Failed to delete " + path, e);
416             }
417         }
418 
419         protected String getDocId(final Path file) {
420             final String s = file.toUri().toString();
421             final String b = basePath.toUri().toString();
422             final String id = s.replace(b, StringUtil.EMPTY).replace("." + imageExtention, StringUtil.EMPTY).replace("/", StringUtil.EMPTY);
423             if (logger.isDebugEnabled()) {
424                 logger.debug("Base: " + b + " File: " + s + " DocId: " + id);
425             }
426             return id;
427         }
428 
429         public long getCount() {
430             if (!deletedFileList.isEmpty()) {
431                 deleteFiles();
432             }
433             return count;
434         }
435 
436         @Override
437         public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
438             return FileVisitResult.CONTINUE;
439         }
440 
441         @Override
442         public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
443             if (System.currentTimeMillis() - Files.getLastModifiedTime(file).toMillis() > expiry) {
444                 deletedFileList.add(file);
445                 if (deletedFileList.size() > maxPurgeSize) {
446                     deleteFiles();
447                 }
448             }
449             return FileVisitResult.CONTINUE;
450         }
451 
452         @Override
453         public FileVisitResult visitFileFailed(final Path file, final IOException e) throws IOException {
454             if (e != null) {
455                 logger.warn("I/O exception on " + file, e);
456             }
457             return FileVisitResult.CONTINUE;
458         }
459 
460         @Override
461         public FileVisitResult postVisitDirectory(final Path dir, final IOException e) throws IOException {
462             if (e != null) {
463                 logger.warn("I/O exception on " + dir, e);
464             }
465             deleteEmptyDirectory(dir);
466             return FileVisitResult.CONTINUE;
467         }
468 
469         private boolean deleteEmptyDirectory(final Path dir) throws IOException {
470             if (dir == null) {
471                 return false;
472             }
473             final File directory = dir.toFile();
474             if (directory.list() != null && directory.list().length == 0 && !THUMBNAILS_DIR_NAME.equals(directory.getName())) {
475                 Files.delete(dir);
476                 if (logger.isDebugEnabled()) {
477                     logger.debug("Delete " + dir);
478                 }
479                 return true;
480             }
481             return false;
482         }
483 
484     }
485 
486     public void migrate() {
487         new Thread(() -> {
488             final Path basePath = baseDir.toPath();
489             final String suffix = "." + imageExtention;
490             try (Stream<Path> paths = Files.walk(basePath)) {
491                 paths.filter(path -> path.toFile().getName().endsWith(imageExtention)).forEach(path -> {
492                     final Path subPath = basePath.relativize(path);
493                     final String docId = subPath.toString().replace("/", StringUtil.EMPTY).replace(suffix, StringUtil.EMPTY);
494                     final String filename = getImageFilename(docId);
495                     final Path newPath = basePath.resolve(filename);
496                     if (!path.equals(newPath)) {
497                         try {
498                             try {
499                                 Files.createDirectories(newPath.getParent());
500                             } catch (final FileAlreadyExistsException e) {
501                                 // ignore
502                     }
503                     Files.move(path, newPath);
504                     logger.info("Move " + path + " to " + newPath);
505                 } catch (final IOException e) {
506                     logger.warn("Failed to move " + path, e);
507                 }
508             }
509         }       );
510             } catch (final IOException e) {
511                 logger.warn("Failed to migrate thumbnail images.", e);
512             }
513         }, "ThumbnailMigrator").start();
514     }
515 
516     public void setThumbnailPathCacheSize(final int thumbnailPathCacheSize) {
517         this.thumbnailPathCacheSize = thumbnailPathCacheSize;
518     }
519 
520     public void setImageExtention(final String imageExtention) {
521         this.imageExtention = imageExtention;
522     }
523 
524     public void setSplitSize(final int splitSize) {
525         this.splitSize = splitSize;
526     }
527 
528     public void setThumbnailTaskQueueSize(final int thumbnailTaskQueueSize) {
529         this.thumbnailTaskQueueSize = thumbnailTaskQueueSize;
530     }
531 
532     public void setNoImageExpired(final long noImageExpired) {
533         this.noImageExpired = noImageExpired;
534     }
535 
536 }