1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.ds.impl;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedReader;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.InputStreamReader;
24 import java.util.ArrayList;
25 import java.util.HashMap;
26 import java.util.LinkedHashMap;
27 import java.util.List;
28 import java.util.Locale;
29 import java.util.Map;
30 import java.util.regex.Pattern;
31
32 import org.apache.commons.io.IOUtils;
33 import org.apache.commons.text.StringEscapeUtils;
34 import org.codelibs.core.lang.StringUtil;
35 import org.codelibs.fess.Constants;
36 import org.codelibs.fess.app.service.FailureUrlService;
37 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
38 import org.codelibs.fess.crawler.exception.MultipleCrawlingAccessException;
39 import org.codelibs.fess.ds.IndexUpdateCallback;
40 import org.codelibs.fess.es.config.exentity.DataConfig;
41 import org.codelibs.fess.exception.DataStoreCrawlingException;
42 import org.codelibs.fess.exception.DataStoreException;
43 import org.codelibs.fess.util.ComponentUtil;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47 import com.orangesignal.csv.CsvConfig;
48 import com.orangesignal.csv.CsvReader;
49
50 public class CsvDataStoreImpl extends AbstractDataStoreImpl {
51 private static final Logger logger = LoggerFactory.getLogger(CsvDataStoreImpl.class);
52
53 protected static final String ESCAPE_CHARACTER_PARAM = "escapeCharacter";
54
55 protected static final String QUOTE_CHARACTER_PARAM = "quoteCharacter";
56
57 protected static final String SEPARATOR_CHARACTER_PARAM = "separatorCharacter";
58
59 protected static final String SKIP_LINES_PARAM = "skipLines";
60
61 protected static final String IGNORE_LINE_PATTERNS_PARAM = "ignoreLinePatterns";
62
63 protected static final String IGNORE_EMPTY_LINES_PARAM = "ignoreEmptyLines";
64
65 protected static final String IGNORE_TRAILING_WHITESPACES_PARAM = "ignoreTrailingWhitespaces";
66
67 protected static final String IGNORE_LEADING_WHITESPACES_PARAM = "ignoreLeadingWhitespaces";
68
69 protected static final String NULL_STRING_PARAM = "nullString";
70
71 protected static final String BREAK_STRING_PARAM = "breakString";
72
73 protected static final String ESCAPE_DISABLED_PARAM = "escapeDisabled";
74
75 protected static final String QUOTE_DISABLED_PARAM = "quoteDisabled";
76
77 protected static final String CSV_FILE_ENCODING_PARAM = "fileEncoding";
78
79 protected static final String CSV_FILES_PARAM = "files";
80
81 protected static final String CSV_DIRS_PARAM = "directories";
82
83 protected static final String CELL_PREFIX = "cell";
84
85 public String[] csvFileSuffixs = new String[] { ".csv", ".tsv" };
86
87 protected List<File> getCsvFileList(final Map<String, String> paramMap) {
88 String value = paramMap.get(CSV_FILES_PARAM);
89 final List<File> fileList = new ArrayList<>();
90 if (StringUtil.isBlank(value)) {
91 value = paramMap.get(CSV_DIRS_PARAM);
92 if (StringUtil.isBlank(value)) {
93 throw new DataStoreException(CSV_FILES_PARAM + " and " + CSV_DIRS_PARAM + " are blank.");
94 }
95 logger.info(CSV_DIRS_PARAM + "=" + value);
96 final String[] values = value.split(",");
97 for (final String path : values) {
98 final File dir = new File(path);
99 if (dir.isDirectory()) {
100 stream(dir.listFiles()).of(
101 stream -> stream.filter(f -> isCsvFile(f.getParentFile(), f.getName()))
102 .sorted((f1, f2) -> (int) (f1.lastModified() - f2.lastModified())).forEach(f -> fileList.add(f)));
103 } else {
104 logger.warn(path + " is not a directory.");
105 }
106 }
107 } else {
108 logger.info(CSV_FILES_PARAM + "=" + value);
109 final String[] values = value.split(",");
110 for (final String path : values) {
111 final File file = new File(path);
112 if (file.isFile() && isCsvFile(file.getParentFile(), file.getName())) {
113 fileList.add(file);
114 } else {
115 logger.warn(path + " is not found.");
116 }
117 }
118 }
119 if (fileList.isEmpty() && logger.isDebugEnabled()) {
120 logger.debug("No csv files in " + value);
121 }
122 return fileList;
123 }
124
125 protected boolean isCsvFile(final File parentFile, final String filename) {
126 final String name = filename.toLowerCase(Locale.ROOT);
127 for (final String suffix : csvFileSuffixs) {
128 if (name.endsWith(suffix)) {
129 return true;
130 }
131 }
132 return false;
133 }
134
135 protected String getCsvFileEncoding(final Map<String, String> paramMap) {
136 final String value = paramMap.get(CSV_FILE_ENCODING_PARAM);
137 if (StringUtil.isBlank(value)) {
138 return Constants.UTF_8;
139 }
140 return value;
141 }
142
143 protected boolean hasHeaderLine(final Map<String, String> paramMap) {
144 final String value = paramMap.get("hasHeaderLine");
145 if (StringUtil.isBlank(value)) {
146 return false;
147 }
148 try {
149 return Boolean.parseBoolean(value);
150 } catch (final Exception e) {
151 return false;
152 }
153 }
154
155 @Override
156 protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
157 final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
158
159 final long readInterval = getReadInterval(paramMap);
160
161 final List<File> csvFileList = getCsvFileList(paramMap);
162 if (csvFileList.isEmpty()) {
163 logger.warn("No CSV file.");
164 return;
165 }
166
167 final String csvFileEncoding = getCsvFileEncoding(paramMap);
168 final boolean hasHeaderLine = hasHeaderLine(paramMap);
169 final CsvConfig csvConfig = buildCsvConfig(paramMap);
170
171 for (final File csvFile : csvFileList) {
172 processCsv(dataConfig, callback, paramMap, scriptMap, defaultDataMap, csvConfig, csvFile, readInterval, csvFileEncoding,
173 hasHeaderLine);
174 }
175 }
176
177 protected void processCsv(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
178 final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap, final CsvConfig csvConfig, final File csvFile,
179 final long readInterval, final String csvFileEncoding, final boolean hasHeaderLine) {
180 logger.info("Loading " + csvFile.getAbsolutePath());
181 CsvReader csvReader = null;
182 try {
183 csvReader = new CsvReader(new BufferedReader(new InputStreamReader(new FileInputStream(csvFile), csvFileEncoding)), csvConfig);
184 List<String> headerList = null;
185 if (hasHeaderLine) {
186 headerList = csvReader.readValues();
187 }
188 List<String> list;
189 boolean loop = true;
190 while ((list = csvReader.readValues()) != null && loop && alive) {
191 final Map<String, Object> dataMap = new HashMap<>();
192 dataMap.putAll(defaultDataMap);
193 final Map<String, Object> resultMap = new LinkedHashMap<>();
194 resultMap.putAll(paramMap);
195 resultMap.put("csvfile", csvFile.getAbsolutePath());
196 resultMap.put("csvfilename", csvFile.getName());
197 resultMap.put("crawlingConfig", dataConfig);
198 boolean foundValues = false;
199 for (int i = 0; i < list.size(); i++) {
200 String key = null;
201 String value = list.get(i);
202 if (value == null) {
203 value = StringUtil.EMPTY;
204 }
205 if (StringUtil.isNotBlank(value)) {
206 foundValues = true;
207 }
208 if (headerList != null && headerList.size() > i) {
209 key = headerList.get(i);
210 if (StringUtil.isNotBlank(key)) {
211 resultMap.put(key, value);
212 }
213 }
214 key = CELL_PREFIX + Integer.toString(i + 1);
215 resultMap.put(key, value);
216 }
217 if (!foundValues) {
218 logger.debug("No data in line: {}", resultMap);
219 continue;
220 }
221
222 if (logger.isDebugEnabled()) {
223 for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {
224 logger.debug(entry.getKey() + "=" + entry.getValue());
225 }
226 }
227
228 final Map<String, Object> crawlingContext = new HashMap<>();
229 crawlingContext.put("doc", dataMap);
230 resultMap.put("crawlingContext", crawlingContext);
231 for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
232 final Object convertValue = convertValue(entry.getValue(), resultMap);
233 if (convertValue != null) {
234 dataMap.put(entry.getKey(), convertValue);
235 }
236 }
237
238 if (logger.isDebugEnabled()) {
239 for (final Map.Entry<String, Object> entry : dataMap.entrySet()) {
240 logger.debug(entry.getKey() + "=" + entry.getValue());
241 }
242 }
243
244 try {
245 callback.store(paramMap, dataMap);
246 } catch (final CrawlingAccessException e) {
247 logger.warn("Crawling Access Exception at : " + dataMap, e);
248
249 Throwable target = e;
250 if (target instanceof MultipleCrawlingAccessException) {
251 final Throwable[] causes = ((MultipleCrawlingAccessException) target).getCauses();
252 if (causes.length > 0) {
253 target = causes[causes.length - 1];
254 }
255 }
256
257 String errorName;
258 final Throwable cause = target.getCause();
259 if (cause != null) {
260 errorName = cause.getClass().getCanonicalName();
261 } else {
262 errorName = target.getClass().getCanonicalName();
263 }
264
265 String url;
266 if (target instanceof DataStoreCrawlingException) {
267 final DataStoreCrawlingException dce = (DataStoreCrawlingException) target;
268 url = dce.getUrl();
269 if (dce.aborted()) {
270 loop = false;
271 }
272 } else {
273 url = csvFile.getAbsolutePath() + ":" + csvReader.getLineNumber();
274 }
275 final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
276 failureUrlService.store(dataConfig, errorName, url, target);
277 } catch (final Throwable t) {
278 logger.warn("Crawling Access Exception at : " + dataMap, t);
279 final String url = csvFile.getAbsolutePath() + ":" + csvReader.getLineNumber();
280 final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
281 failureUrlService.store(dataConfig, t.getClass().getCanonicalName(), url, t);
282 }
283
284 if (readInterval > 0) {
285 sleep(readInterval);
286 }
287 }
288 } catch (final Exception e) {
289 throw new DataStoreException("Failed to crawl data when reading csv file.", e);
290 } finally {
291 IOUtils.closeQuietly(csvReader);
292 }
293 }
294
295 protected CsvConfig buildCsvConfig(final Map<String, String> paramMap) {
296 final CsvConfig csvConfig = new CsvConfig();
297
298 if (paramMap.containsKey(SEPARATOR_CHARACTER_PARAM)) {
299 final String value = paramMap.get(SEPARATOR_CHARACTER_PARAM);
300 if (StringUtil.isNotBlank(value)) {
301 try {
302 csvConfig.setSeparator(StringEscapeUtils.unescapeJava(value).charAt(0));
303 } catch (final Exception e) {
304 logger.warn("Failed to load " + SEPARATOR_CHARACTER_PARAM, e);
305 }
306 }
307 }
308
309 if (paramMap.containsKey(QUOTE_CHARACTER_PARAM)) {
310 final String value = paramMap.get(QUOTE_CHARACTER_PARAM);
311 if (StringUtil.isNotBlank(value)) {
312 try {
313 csvConfig.setQuote(value.charAt(0));
314 } catch (final Exception e) {
315 logger.warn("Failed to load " + QUOTE_CHARACTER_PARAM, e);
316 }
317 }
318 }
319
320 if (paramMap.containsKey(ESCAPE_CHARACTER_PARAM)) {
321 final String value = paramMap.get(ESCAPE_CHARACTER_PARAM);
322 if (StringUtil.isNotBlank(value)) {
323 try {
324 csvConfig.setEscape(value.charAt(0));
325 } catch (final Exception e) {
326 logger.warn("Failed to load " + ESCAPE_CHARACTER_PARAM, e);
327 }
328 }
329 }
330
331 if (paramMap.containsKey(QUOTE_DISABLED_PARAM)) {
332 final String value = paramMap.get(QUOTE_DISABLED_PARAM);
333 if (StringUtil.isNotBlank(value)) {
334 try {
335
336 csvConfig.setQuoteDisabled(Boolean.parseBoolean(value));
337 } catch (final Exception e) {
338 logger.warn("Failed to load " + QUOTE_DISABLED_PARAM, e);
339 }
340 }
341 }
342
343 if (paramMap.containsKey(ESCAPE_DISABLED_PARAM)) {
344 final String value = paramMap.get(ESCAPE_DISABLED_PARAM);
345 if (StringUtil.isNotBlank(value)) {
346 try {
347
348 csvConfig.setEscapeDisabled(Boolean.parseBoolean(value));
349 } catch (final Exception e) {
350 logger.warn("Failed to load " + ESCAPE_DISABLED_PARAM, e);
351 }
352 }
353 }
354
355 if (paramMap.containsKey(BREAK_STRING_PARAM)) {
356 final String value = paramMap.get(BREAK_STRING_PARAM);
357 if (StringUtil.isNotBlank(value)) {
358
359 csvConfig.setBreakString(value);
360 }
361 }
362
363 if (paramMap.containsKey(NULL_STRING_PARAM)) {
364 final String value = paramMap.get(NULL_STRING_PARAM);
365 if (StringUtil.isNotBlank(value)) {
366
367 csvConfig.setNullString(value);
368 }
369 }
370
371 if (paramMap.containsKey(IGNORE_LEADING_WHITESPACES_PARAM)) {
372 final String value = paramMap.get(IGNORE_LEADING_WHITESPACES_PARAM);
373 if (StringUtil.isNotBlank(value)) {
374 try {
375
376 csvConfig.setIgnoreLeadingWhitespaces(Boolean.parseBoolean(value));
377 } catch (final Exception e) {
378 logger.warn("Failed to load " + IGNORE_LEADING_WHITESPACES_PARAM, e);
379 }
380 }
381 }
382
383 if (paramMap.containsKey(IGNORE_TRAILING_WHITESPACES_PARAM)) {
384 final String value = paramMap.get(IGNORE_TRAILING_WHITESPACES_PARAM);
385 if (StringUtil.isNotBlank(value)) {
386 try {
387
388 csvConfig.setIgnoreTrailingWhitespaces(Boolean.parseBoolean(value));
389 } catch (final Exception e) {
390 logger.warn("Failed to load " + IGNORE_TRAILING_WHITESPACES_PARAM, e);
391 }
392 }
393 }
394
395 if (paramMap.containsKey(IGNORE_EMPTY_LINES_PARAM)) {
396 final String value = paramMap.get(IGNORE_EMPTY_LINES_PARAM);
397 if (StringUtil.isNotBlank(value)) {
398 try {
399
400 csvConfig.setIgnoreEmptyLines(Boolean.parseBoolean(value));
401 } catch (final Exception e) {
402 logger.warn("Failed to load " + IGNORE_EMPTY_LINES_PARAM, e);
403 }
404 }
405 }
406
407 if (paramMap.containsKey(IGNORE_LINE_PATTERNS_PARAM)) {
408 final String value = paramMap.get(IGNORE_LINE_PATTERNS_PARAM);
409 if (StringUtil.isNotBlank(value)) {
410
411 csvConfig.setIgnoreLinePatterns(Pattern.compile(value));
412 }
413 }
414
415 if (paramMap.containsKey(SKIP_LINES_PARAM)) {
416 final String value = paramMap.get(SKIP_LINES_PARAM);
417 if (StringUtil.isNotBlank(value)) {
418 try {
419
420 csvConfig.setSkipLines(Integer.parseInt(value));
421 } catch (final Exception e) {
422 logger.warn("Failed to load " + SKIP_LINES_PARAM, e);
423 }
424 }
425 }
426
427 return csvConfig;
428 }
429 }