1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.thumbnail.impl;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.File;
21 import java.util.ArrayList;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.function.BiPredicate;
26 import java.util.function.Predicate;
27
28 import org.apache.logging.log4j.LogManager;
29 import org.apache.logging.log4j.Logger;
30 import org.codelibs.core.lang.StringUtil;
31 import org.codelibs.core.misc.Tuple3;
32 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
33 import org.codelibs.fess.crawler.client.CrawlerClient;
34 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
35 import org.codelibs.fess.crawler.entity.ResponseData;
36 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
37 import org.codelibs.fess.es.client.SearchEngineClient;
38 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
39 import org.codelibs.fess.exception.ThumbnailGenerationException;
40 import org.codelibs.fess.helper.CrawlingConfigHelper;
41 import org.codelibs.fess.helper.IndexingHelper;
42 import org.codelibs.fess.mylasta.direction.FessConfig;
43 import org.codelibs.fess.thumbnail.ThumbnailGenerator;
44 import org.codelibs.fess.util.ComponentUtil;
45 import org.codelibs.fess.util.DocumentUtil;
46
47 public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
48 private static final Logger logger = LogManager.getLogger(BaseThumbnailGenerator.class);
49
50 protected final Map<String, String> conditionMap = new HashMap<>();
51
52 protected int directoryNameLength = 5;
53
54 protected List<String> generatorList;
55
56 protected Map<String, String> filePathMap = new HashMap<>();
57
58 protected String name;
59
60 protected int maxRedirectCount = 10;
61
62 protected Boolean available = null;
63
64 public void register() {
65 ComponentUtil.getThumbnailManager().add(this);
66 }
67
68 public void addCondition(final String key, final String regex) {
69 final String value = conditionMap.get(key);
70 if (StringUtil.isBlank(value)) {
71 conditionMap.put(key, regex);
72 } else {
73 conditionMap.put(key, value + "|" + regex);
74 }
75 }
76
77 @Override
78 public boolean isTarget(final Map<String, Object> docMap) {
79 if (!docMap.containsKey(ComponentUtil.getFessConfig().getIndexFieldThumbnail())) {
80 return false;
81 }
82 for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
83 final Object value = docMap.get(entry.getKey());
84 if (value instanceof String && ((String) value).matches(entry.getValue())) {
85 return true;
86 }
87 }
88 return false;
89 }
90
91 @Override
92 public boolean isAvailable() {
93 if (available != null) {
94 return available;
95 }
96 if (generatorList != null && !generatorList.isEmpty()) {
97 String path = System.getenv("PATH");
98 if (path == null) {
99 path = System.getenv("Path");
100 }
101 if (path == null) {
102 path = System.getenv("path");
103 }
104 final List<String> pathList = new ArrayList<>();
105 pathList.add("/usr/share/fess/bin");
106 if (path != null) {
107 stream(path.split(File.pathSeparator)).of(stream -> stream.map(String::trim).forEach(s -> pathList.add(s)));
108 }
109 available = generatorList.stream().map(s -> {
110 if (s.startsWith("${path}")) {
111 for (final String p : pathList) {
112 final File f = new File(s.replace("${path}", p));
113 if (f.exists()) {
114 final String filePath = f.getAbsolutePath();
115 filePathMap.put(s, filePath);
116 return filePath;
117 }
118 }
119 }
120 return s;
121 }).allMatch(s -> new File(s).isFile());
122 } else {
123 available = true;
124 }
125 return available;
126 }
127
128 @Override
129 public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
130 final FessConfig fessConfig = ComponentUtil.getFessConfig();
131 final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
132 final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
133 if (logger.isDebugEnabled()) {
134 logger.debug("Create thumbnail task: {}", task);
135 }
136 return task;
137 }
138
139 public void setDirectoryNameLength(final int directoryNameLength) {
140 this.directoryNameLength = directoryNameLength;
141 }
142
143 protected String expandPath(final String value) {
144 if (value != null && filePathMap.containsKey(value)) {
145 return filePathMap.get(value);
146 }
147 return value;
148 }
149
150 protected void updateThumbnailField(final String thumbnailId, final String value) {
151
152 final FessConfig fessConfig = ComponentUtil.getFessConfig();
153 try {
154 ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getSearchEngineClient(), thumbnailId,
155 fessConfig.getIndexFieldThumbnail(), value);
156 } catch (final Exception e) {
157 logger.warn("Failed to update thumbnail field at {}", thumbnailId, e);
158 }
159 }
160
161 protected boolean process(final String id, final BiPredicate<String, String> consumer) {
162 final FessConfig fessConfig = ComponentUtil.getFessConfig();
163 final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
164 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
165 try {
166 final Map<String, Object> doc = indexingHelper.getDocument(searchEngineClient, id,
167 new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
168 if (doc == null) {
169 throw new ThumbnailGenerationException("Document is not found: " + id);
170 }
171 final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
172 if (StringUtil.isBlank(url)) {
173 throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
174 }
175 final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
176 if (configId == null || configId.length() < 2) {
177 throw new ThumbnailGenerationException("Invalid configId: " + configId);
178 }
179 return consumer.test(configId, url);
180 } catch (final ThumbnailGenerationException e) {
181 if (e.getCause() == null) {
182 logger.debug(e.getMessage());
183 } else {
184 logger.warn("Failed to process {}", id, e);
185 }
186 } catch (final Exception e) {
187 logger.warn("Failed to process {}", id, e);
188 }
189 return false;
190 }
191
192 protected boolean process(final String id, final Predicate<ResponseData> consumer) {
193 return process(id, (configId, url) -> {
194 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
195 final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
196 if (config == null) {
197 throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
198 }
199
200 if (logger.isInfoEnabled()) {
201 logger.info("Generating Thumbnail: {}", url);
202 }
203
204 final CrawlerClientFactory crawlerClientFactory =
205 config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
206 final CrawlerClient client = crawlerClientFactory.getClient(url);
207 if (client == null) {
208 throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
209 }
210 String u = url;
211 for (int i = 0; i < maxRedirectCount; i++) {
212 try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
213 if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
214 u = responseData.getRedirectLocation();
215 continue;
216 }
217 if (StringUtil.isBlank(responseData.getUrl())) {
218 throw new ThumbnailGenerationException(
219 "Failed to process a thumbnail content: " + url + " (Response URL is empty)");
220 }
221 return consumer.test(responseData);
222 } catch (final CrawlingAccessException e) {
223 if (logger.isDebugEnabled()) {
224 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
225 }
226 throw new ThumbnailGenerationException(e.getMessage());
227 } catch (final Exception e) {
228 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
229 }
230 }
231 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
232 });
233 }
234
235 public void setGeneratorList(final List<String> generatorList) {
236 this.generatorList = generatorList;
237 }
238
239 @Override
240 public String getName() {
241 return name;
242 }
243
244 public void setName(final String name) {
245 this.name = name;
246 }
247
248 public void setMaxRedirectCount(final int maxRedirectCount) {
249 this.maxRedirectCount = maxRedirectCount;
250 }
251
252 }