1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.ds.impl;
17
18 import java.sql.Connection;
19 import java.sql.DriverManager;
20 import java.sql.ResultSet;
21 import java.sql.ResultSetMetaData;
22 import java.sql.SQLException;
23 import java.sql.Statement;
24 import java.util.Collection;
25 import java.util.HashMap;
26 import java.util.Map;
27 import java.util.Set;
28
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.fess.app.service.FailureUrlService;
31 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
32 import org.codelibs.fess.crawler.exception.MultipleCrawlingAccessException;
33 import org.codelibs.fess.ds.IndexUpdateCallback;
34 import org.codelibs.fess.es.config.exentity.DataConfig;
35 import org.codelibs.fess.exception.DataStoreCrawlingException;
36 import org.codelibs.fess.exception.DataStoreException;
37 import org.codelibs.fess.exception.FessSystemException;
38 import org.codelibs.fess.util.ComponentUtil;
39 import org.slf4j.Logger;
40 import org.slf4j.LoggerFactory;
41
42 public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
43 private static final Logger logger = LoggerFactory.getLogger(DatabaseDataStoreImpl.class);
44
45 private static final String SQL_PARAM = "sql";
46
47 private static final String URL_PARAM = "url";
48
49 private static final String PASSWORD_PARAM = "password";
50
51 private static final String USERNAME_PARAM = "username";
52
53 private static final String DRIVER_PARAM = "driver";
54
55 protected String getDriverClass(final Map<String, String> paramMap) {
56 final String driverName = paramMap.get(DRIVER_PARAM);
57 if (StringUtil.isBlank(driverName)) {
58 throw new DataStoreException("JDBC driver is null");
59 }
60 return driverName;
61 }
62
63 protected String getUsername(final Map<String, String> paramMap) {
64 return paramMap.get(USERNAME_PARAM);
65 }
66
67 protected String getPassword(final Map<String, String> paramMap) {
68 return paramMap.get(PASSWORD_PARAM);
69 }
70
71 protected String getUrl(final Map<String, String> paramMap) {
72 return paramMap.get(URL_PARAM);
73 }
74
75 protected String getSql(final Map<String, String> paramMap) {
76 final String sql = paramMap.get(SQL_PARAM);
77 if (StringUtil.isBlank(sql)) {
78 throw new DataStoreException("sql is null");
79 }
80 return sql;
81 }
82
83 @Override
84 protected void storeData(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> paramMap,
85 final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
86
87 final long readInterval = getReadInterval(paramMap);
88
89 Connection con = null;
90 Statement stmt = null;
91 ResultSet rs = null;
92 try {
93 Class.forName(getDriverClass(paramMap));
94
95 final String jdbcUrl = getUrl(paramMap);
96 final String username = getUsername(paramMap);
97 final String password = getPassword(paramMap);
98 if (StringUtil.isNotEmpty(username)) {
99 con = DriverManager.getConnection(jdbcUrl, username, password);
100 } else {
101 con = DriverManager.getConnection(jdbcUrl);
102 }
103
104 final String sql = getSql(paramMap);
105 stmt = con.createStatement();
106 rs = stmt.executeQuery(sql);
107 boolean loop = true;
108 while (rs.next() && loop && alive) {
109 final Map<String, Object> dataMap = new HashMap<>();
110 dataMap.putAll(defaultDataMap);
111 final Map<String, Object> crawlingContext = new HashMap<>();
112 crawlingContext.put("doc", dataMap);
113 for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
114 final Object convertValue =
115 convertValue(entry.getValue(), new ResultSetParamMap(config, crawlingContext, rs, paramMap));
116 if (convertValue != null) {
117 dataMap.put(entry.getKey(), convertValue);
118 }
119 }
120
121 try {
122 callback.store(paramMap, dataMap);
123 } catch (final CrawlingAccessException e) {
124 logger.warn("Crawling Access Exception at : " + dataMap, e);
125
126 Throwable target = e;
127 if (target instanceof MultipleCrawlingAccessException) {
128 final Throwable[] causes = ((MultipleCrawlingAccessException) target).getCauses();
129 if (causes.length > 0) {
130 target = causes[causes.length - 1];
131 }
132 }
133
134 String errorName;
135 final Throwable cause = target.getCause();
136 if (cause != null) {
137 errorName = cause.getClass().getCanonicalName();
138 } else {
139 errorName = target.getClass().getCanonicalName();
140 }
141
142 String url;
143 if (target instanceof DataStoreCrawlingException) {
144 final DataStoreCrawlingException dce = (DataStoreCrawlingException) target;
145 url = dce.getUrl();
146 if (dce.aborted()) {
147 loop = false;
148 }
149 } else {
150 url = sql + ":" + rs.getRow();
151 }
152 final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
153 failureUrlService.store(config, errorName, url, target);
154 } catch (final Throwable t) {
155 logger.warn("Crawling Access Exception at : " + dataMap, t);
156 final String url = sql + ":" + rs.getRow();
157 final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
158 failureUrlService.store(config, t.getClass().getCanonicalName(), url, t);
159 }
160
161 if (readInterval > 0) {
162 sleep(readInterval);
163 }
164 }
165 } catch (final Exception e) {
166 throw new DataStoreException("Failed to crawl data in DB.", e);
167 } finally {
168 try {
169 if (rs != null) {
170 rs.close();
171 }
172 } catch (final SQLException e) {
173 logger.warn("Failed to close a result set.", e);
174 } finally {
175 try {
176 if (stmt != null) {
177 stmt.close();
178 }
179 } catch (final SQLException e) {
180 logger.warn("Failed to close a statement.", e);
181 } finally {
182 try {
183 if (con != null) {
184 con.close();
185 }
186 } catch (final SQLException e) {
187 logger.warn("Failed to close a db connection.", e);
188 }
189 }
190 }
191
192 }
193 }
194
195 protected static class ResultSetParamMap implements Map<String, Object> {
196 private final Map<String, Object> paramMap = new HashMap<>();
197
198 public ResultSetParamMap(final DataConfig config, final Map<String, Object> crawlingContext, final ResultSet resultSet,
199 final Map<String, String> paramMap) {
200 this.paramMap.putAll(paramMap);
201 this.paramMap.put("crawlingConfig", config);
202 this.paramMap.put("crawlingContext", crawlingContext);
203
204 try {
205 final ResultSetMetaData metaData = resultSet.getMetaData();
206 final int columnCount = metaData.getColumnCount();
207 for (int i = 0; i < columnCount; i++) {
208 try {
209 final String label = metaData.getColumnLabel(i + 1);
210 final String value = resultSet.getString(i + 1);
211 this.paramMap.put(label, value);
212 } catch (final SQLException e) {
213 logger.warn("Failed to parse data in a result set. The column is " + (i + 1) + ".", e);
214 }
215 }
216 } catch (final Exception e) {
217 throw new FessSystemException("Failed to access meta data.", e);
218 }
219
220 }
221
222 @Override
223 public void clear() {
224 paramMap.clear();
225 }
226
227 @Override
228 public boolean containsKey(final Object key) {
229 return paramMap.containsKey(key);
230 }
231
232 @Override
233 public boolean containsValue(final Object value) {
234 return paramMap.containsValue(value);
235 }
236
237 @Override
238 public Set<java.util.Map.Entry<String, Object>> entrySet() {
239 return paramMap.entrySet();
240 }
241
242 @Override
243 public Object get(final Object key) {
244 return paramMap.get(key);
245 }
246
247 @Override
248 public boolean isEmpty() {
249 return paramMap.isEmpty();
250 }
251
252 @Override
253 public Set<String> keySet() {
254 return paramMap.keySet();
255 }
256
257 @Override
258 public Object put(final String key, final Object value) {
259 return paramMap.put(key, value);
260 }
261
262 @Override
263 public void putAll(final Map<? extends String, ? extends Object> m) {
264 paramMap.putAll(m);
265 }
266
267 @Override
268 public Object remove(final Object key) {
269 return paramMap.remove(key);
270 }
271
272 @Override
273 public int size() {
274 return paramMap.size();
275 }
276
277 @Override
278 public Collection<Object> values() {
279 return paramMap.values();
280 }
281
282 }
283
284 }