View Javadoc
1   /*
2    * Copyright 2012-2019 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.thumbnail.impl;
17  
18  import static org.codelibs.core.stream.StreamUtil.stream;
19  
20  import java.io.File;
21  import java.util.ArrayList;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.function.BiPredicate;
26  import java.util.function.Predicate;
27  
28  import org.codelibs.core.lang.StringUtil;
29  import org.codelibs.core.misc.Tuple3;
30  import org.codelibs.fess.crawler.builder.RequestDataBuilder;
31  import org.codelibs.fess.crawler.client.CrawlerClient;
32  import org.codelibs.fess.crawler.client.CrawlerClientFactory;
33  import org.codelibs.fess.crawler.entity.ResponseData;
34  import org.codelibs.fess.crawler.exception.CrawlingAccessException;
35  import org.codelibs.fess.es.client.FessEsClient;
36  import org.codelibs.fess.es.config.exentity.CrawlingConfig;
37  import org.codelibs.fess.exception.ThumbnailGenerationException;
38  import org.codelibs.fess.helper.CrawlingConfigHelper;
39  import org.codelibs.fess.helper.IndexingHelper;
40  import org.codelibs.fess.mylasta.direction.FessConfig;
41  import org.codelibs.fess.thumbnail.ThumbnailGenerator;
42  import org.codelibs.fess.util.ComponentUtil;
43  import org.codelibs.fess.util.DocumentUtil;
44  import org.slf4j.Logger;
45  import org.slf4j.LoggerFactory;
46  
47  public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
48      private static final Logger logger = LoggerFactory.getLogger(BaseThumbnailGenerator.class);
49  
50      protected final Map<String, String> conditionMap = new HashMap<>();
51  
52      protected int directoryNameLength = 5;
53  
54      protected List<String> generatorList;
55  
56      protected Map<String, String> filePathMap = new HashMap<>();
57  
58      protected String name;
59  
60      protected int maxRedirectCount = 10;
61  
62      protected Boolean available = null;
63  
64      public void addCondition(final String key, final String regex) {
65          final String value = conditionMap.get(key);
66          if (StringUtil.isBlank(value)) {
67              conditionMap.put(key, regex);
68          } else {
69              conditionMap.put(key, value + "|" + regex);
70          }
71      }
72  
73      @Override
74      public boolean isTarget(final Map<String, Object> docMap) {
75          if (!docMap.containsKey(ComponentUtil.getFessConfig().getIndexFieldThumbnail())) {
76              return false;
77          }
78          for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
79              final Object value = docMap.get(entry.getKey());
80              if (value instanceof String && ((String) value).matches(entry.getValue())) {
81                  return true;
82              }
83          }
84          return false;
85      }
86  
87      @Override
88      public boolean isAvailable() {
89          if (available != null) {
90              return available;
91          }
92          if (generatorList != null && !generatorList.isEmpty()) {
93              String path = System.getenv("PATH");
94              if (path == null) {
95                  path = System.getenv("Path");
96              }
97              if (path == null) {
98                  path = System.getenv("path");
99              }
100             final List<String> pathList = new ArrayList<>();
101             pathList.add("/usr/share/fess/bin");
102             if (path != null) {
103                 stream(path.split(File.pathSeparator)).of(stream -> stream.map(s -> s.trim()).forEach(s -> pathList.add(s)));
104             }
105             available = generatorList.stream().map(s -> {
106                 if (s.startsWith("${path}")) {
107                     for (final String p : pathList) {
108                         final File f = new File(s.replace("${path}", p));
109                         if (f.exists()) {
110                             final String filePath = f.getAbsolutePath();
111                             filePathMap.put(s, filePath);
112                             return filePath;
113                         }
114                     }
115                 }
116                 return s;
117             }).allMatch(s -> new File(s).isFile());
118         } else {
119             available = true;
120         }
121         return available;
122     }
123 
124     @Override
125     public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
126         final FessConfig fessConfig = ComponentUtil.getFessConfig();
127         final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
128         final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
129         if (logger.isDebugEnabled()) {
130             logger.debug("Create thumbnail task: " + task);
131         }
132         return task;
133     }
134 
135     public void setDirectoryNameLength(final int directoryNameLength) {
136         this.directoryNameLength = directoryNameLength;
137     }
138 
139     protected String expandPath(final String value) {
140         if (value != null && filePathMap.containsKey(value)) {
141             return filePathMap.get(value);
142         }
143         return value;
144     }
145 
146     protected void updateThumbnailField(final String thumbnailId, final String value) {
147         // TODO bulk
148         final FessConfig fessConfig = ComponentUtil.getFessConfig();
149         try {
150             ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getFessEsClient(), thumbnailId,
151                     fessConfig.getIndexFieldThumbnail(), value);
152         } catch (final Exception e) {
153             logger.warn("Failed to update thumbnail field at " + thumbnailId, e);
154         }
155     }
156 
157     protected boolean process(final String id, final BiPredicate<String, String> consumer) {
158         final FessConfig fessConfig = ComponentUtil.getFessConfig();
159         final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
160         final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
161         try {
162             final Map<String, Object> doc =
163                     indexingHelper.getDocument(fessEsClient, id,
164                             new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
165             if (doc == null) {
166                 throw new ThumbnailGenerationException("Document is not found: " + id);
167             }
168             final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
169             if (StringUtil.isBlank(url)) {
170                 throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
171             }
172             final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
173             if (configId == null || configId.length() < 2) {
174                 throw new ThumbnailGenerationException("Invalid configId: " + configId);
175             }
176             return consumer.test(configId, url);
177         } catch (final ThumbnailGenerationException e) {
178             if (e.getCause() == null) {
179                 logger.debug(e.getMessage());
180             } else {
181                 logger.warn("Failed to process " + id, e);
182             }
183         } catch (final Exception e) {
184             logger.warn("Failed to process " + id, e);
185         }
186         return false;
187     }
188 
189     protected boolean process(final String id, final Predicate<ResponseData> consumer) {
190         return process(id,
191                 (configId, url) -> {
192                     final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
193                     final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
194                     if (config == null) {
195                         throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
196                     }
197 
198                     if (logger.isInfoEnabled()) {
199                         logger.info("Generating Thumbnail: " + url);
200                     }
201 
202                     final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
203                     config.initializeClientFactory(crawlerClientFactory);
204                     final CrawlerClient client = crawlerClientFactory.getClient(url);
205                     if (client == null) {
206                         throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
207                     }
208                     String u = url;
209                     for (int i = 0; i < maxRedirectCount; i++) {
210                         try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
211                             if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
212                                 u = responseData.getRedirectLocation();
213                                 continue;
214                             }
215                             if (StringUtil.isBlank(responseData.getUrl())) {
216                                 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url
217                                         + " (Response URL is empty)");
218                             }
219                             return consumer.test(responseData);
220                         } catch (final CrawlingAccessException e) {
221                             if (logger.isDebugEnabled()) {
222                                 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
223                             } else {
224                                 throw new ThumbnailGenerationException(e.getMessage());
225                             }
226                         } catch (final Exception e) {
227                             throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
228                         }
229                     }
230                     throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
231                 });
232     }
233 
234     public void setGeneratorList(final List<String> generatorList) {
235         this.generatorList = generatorList;
236     }
237 
238     @Override
239     public String getName() {
240         return name;
241     }
242 
243     public void setName(final String name) {
244         this.name = name;
245     }
246 
247     public void setMaxRedirectCount(final int maxRedirectCount) {
248         this.maxRedirectCount = maxRedirectCount;
249     }
250 
251 }