1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.thumbnail.impl;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.File;
21 import java.util.ArrayList;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.function.BiPredicate;
26 import java.util.function.Predicate;
27
28 import org.codelibs.core.lang.StringUtil;
29 import org.codelibs.core.misc.Tuple3;
30 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
31 import org.codelibs.fess.crawler.client.CrawlerClient;
32 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
33 import org.codelibs.fess.crawler.entity.ResponseData;
34 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
35 import org.codelibs.fess.es.client.FessEsClient;
36 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
37 import org.codelibs.fess.exception.ThumbnailGenerationException;
38 import org.codelibs.fess.helper.CrawlingConfigHelper;
39 import org.codelibs.fess.helper.IndexingHelper;
40 import org.codelibs.fess.mylasta.direction.FessConfig;
41 import org.codelibs.fess.thumbnail.ThumbnailGenerator;
42 import org.codelibs.fess.util.ComponentUtil;
43 import org.codelibs.fess.util.DocumentUtil;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47 public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
48 private static final Logger logger = LoggerFactory.getLogger(BaseThumbnailGenerator.class);
49
50 protected final Map<String, String> conditionMap = new HashMap<>();
51
52 protected int directoryNameLength = 5;
53
54 protected List<String> generatorList;
55
56 protected Map<String, String> filePathMap = new HashMap<>();
57
58 protected String name;
59
60 protected int maxRedirectCount = 10;
61
62 public void addCondition(final String key, final String regex) {
63 final String value = conditionMap.get(key);
64 if (StringUtil.isBlank(value)) {
65 conditionMap.put(key, regex);
66 } else {
67 conditionMap.put(key, value + "|" + regex);
68 }
69 }
70
71 @Override
72 public boolean isTarget(final Map<String, Object> docMap) {
73 if (!docMap.containsKey(ComponentUtil.getFessConfig().getIndexFieldThumbnail())) {
74 return false;
75 }
76 for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
77 final Object value = docMap.get(entry.getKey());
78 if (value instanceof String && ((String) value).matches(entry.getValue())) {
79 return true;
80 }
81 }
82 return false;
83 }
84
85 @Override
86 public boolean isAvailable() {
87 if (generatorList != null && !generatorList.isEmpty()) {
88 String path = System.getenv("PATH");
89 if (path == null) {
90 path = System.getenv("Path");
91 }
92 if (path == null) {
93 path = System.getenv("path");
94 }
95 final List<String> pathList = new ArrayList<>();
96 pathList.add("/usr/share/fess/bin");
97 if (path != null) {
98 stream(path.split(File.pathSeparator)).of(stream -> stream.map(s -> s.trim()).forEach(s -> pathList.add(s)));
99 }
100 return generatorList.stream().map(s -> {
101 if (s.startsWith("${path}")) {
102 for (final String p : pathList) {
103 final File f = new File(s.replace("${path}", p));
104 if (f.exists()) {
105 final String filePath = f.getAbsolutePath();
106 filePathMap.put(s, filePath);
107 return filePath;
108 }
109 }
110 }
111 return s;
112 }).allMatch(s -> new File(s).isFile());
113 }
114 return true;
115 }
116
117 @Override
118 public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
119 final FessConfig fessConfig = ComponentUtil.getFessConfig();
120 final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
121 final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
122 if (logger.isDebugEnabled()) {
123 logger.debug("Create thumbnail task: " + task);
124 }
125 return task;
126 }
127
128 public void setDirectoryNameLength(final int directoryNameLength) {
129 this.directoryNameLength = directoryNameLength;
130 }
131
132 protected String expandPath(final String value) {
133 if (value != null && filePathMap.containsKey(value)) {
134 return filePathMap.get(value);
135 }
136 return value;
137 }
138
139 protected void updateThumbnailField(final String thumbnailId, final String value) {
140
141 final FessConfig fessConfig = ComponentUtil.getFessConfig();
142 try {
143 ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getFessEsClient(), thumbnailId,
144 fessConfig.getIndexFieldThumbnail(), value);
145 } catch (final Exception e) {
146 logger.warn("Failed to update thumbnail field at " + thumbnailId, e);
147 }
148 }
149
150 protected boolean process(final String id, final BiPredicate<String, String> consumer) {
151 final FessConfig fessConfig = ComponentUtil.getFessConfig();
152 final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
153 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
154 try {
155 final Map<String, Object> doc =
156 indexingHelper.getDocument(fessEsClient, id,
157 new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
158 if (doc == null) {
159 throw new ThumbnailGenerationException("Document is not found: " + id);
160 }
161 final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
162 if (StringUtil.isBlank(url)) {
163 throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
164 }
165 final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
166 if (configId == null || configId.length() < 2) {
167 throw new ThumbnailGenerationException("Invalid configId: " + configId);
168 }
169 return consumer.test(configId, url);
170 } catch (final ThumbnailGenerationException e) {
171 if (e.getCause() == null) {
172 logger.debug(e.getMessage());
173 } else {
174 logger.warn("Failed to process " + id, e);
175 }
176 } catch (final Exception e) {
177 logger.warn("Failed to process " + id, e);
178 }
179 return false;
180 }
181
182 protected boolean process(final String id, final Predicate<ResponseData> consumer) {
183 return process(id,
184 (configId, url) -> {
185 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
186 final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
187 if (config == null) {
188 throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
189 }
190 final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
191 config.initializeClientFactory(crawlerClientFactory);
192 final CrawlerClient client = crawlerClientFactory.getClient(url);
193 if (client == null) {
194 throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
195 }
196 String u = url;
197 for (int i = 0; i < maxRedirectCount; i++) {
198 try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
199 if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
200 u = responseData.getRedirectLocation();
201 continue;
202 }
203 if (StringUtil.isBlank(responseData.getUrl())) {
204 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url
205 + " (Response URL is empty)");
206 }
207 return consumer.test(responseData);
208 } catch (final CrawlingAccessException e) {
209 if (logger.isDebugEnabled()) {
210 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
211 } else {
212 throw new ThumbnailGenerationException(e.getMessage());
213 }
214 } catch (final Exception e) {
215 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
216 }
217 }
218 throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
219 });
220 }
221
222 public void setGeneratorList(final List<String> generatorList) {
223 this.generatorList = generatorList;
224 }
225
226 @Override
227 public String getName() {
228 return name;
229 }
230
231 public void setName(final String name) {
232 this.name = name;
233 }
234
235 public void setMaxRedirectCount(final int maxRedirectCount) {
236 this.maxRedirectCount = maxRedirectCount;
237 }
238
239 }