1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.ds.impl;
17
18 import java.io.InputStream;
19 import java.net.URI;
20 import java.net.URISyntaxException;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.function.Consumer;
27 import java.util.stream.Collectors;
28
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.elasticsearch.runner.net.Curl;
31 import org.codelibs.elasticsearch.runner.net.CurlResponse;
32 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
33 import org.codelibs.fess.crawler.client.http.HcHttpClient;
34 import org.codelibs.fess.crawler.client.http.RequestHeader;
35 import org.codelibs.fess.ds.IndexUpdateCallback;
36 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
37 import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
38 import org.codelibs.fess.es.config.exentity.DataConfig;
39 import org.codelibs.fess.helper.SystemHelper;
40 import org.codelibs.fess.mylasta.direction.FessConfig;
41 import org.codelibs.fess.util.ComponentUtil;
42 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
43 import org.elasticsearch.common.xcontent.json.JsonXContent;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47 import com.fasterxml.jackson.core.type.TypeReference;
48 import com.fasterxml.jackson.databind.ObjectMapper;
49
50
51
52
53 public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
54 private static final Logger logger = LoggerFactory.getLogger(GitBucketDataStoreImpl.class);
55
56 private static final int MAX_DEPTH = 20;
57
58 protected static final String TOKEN_PARAM = "token";
59 protected static final String GITBUCKET_URL_PARAM = "url";
60 protected static final String PRIVATE_REPOSITORY_PARAM = "is_private";
61 protected static final String COLLABORATORS_PARAM = "collaborators";
62
63 @Override
64 protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
65 final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
66
67 final String rootURL = getRootURL(paramMap);
68 final String authToken = getAuthToken(paramMap);
69 final long readInterval = getReadInterval(paramMap);
70
71
72 if (rootURL.isEmpty() || authToken.isEmpty()) {
73 logger.warn("parameter \"" + TOKEN_PARAM + "\" and \"" + GITBUCKET_URL_PARAM + "\" are required");
74 return;
75 }
76
77
78 final List<Map<String, Object>> repositoryList = getRepositoryList(rootURL, authToken);
79 if (repositoryList.isEmpty()) {
80 logger.warn("Token is invalid or no Repository");
81 return;
82 }
83
84
85 final Map<String, String> pluginInfo = getFessPluginInfo(rootURL, authToken);
86 final String sourceLabel = pluginInfo.get("source_label");
87 final String issueLabel = pluginInfo.get("issue_label");
88 final String wikiLabel = pluginInfo.get("wiki_label");
89
90 final CrawlingConfig crawlingConfig = new CrawlingConfigWrapper(dataConfig) {
91 @Override
92 public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
93 final Map<String, Object> paramMap = super.initializeClientFactory(crawlerClientFactory);
94 final List<RequestHeader> headerList = new ArrayList<>();
95 final RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY);
96 if (headers != null) {
97 for (final RequestHeader header : headers) {
98 headerList.add(header);
99 }
100 }
101 headerList.add(new RequestHeader("Authorization", "token " + authToken));
102 headerList.add(new RequestHeader("Accept", "application/vnd.github.v3.raw"));
103 paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, headerList.toArray(new RequestHeader[headerList.size()]));
104 return paramMap;
105 }
106 };
107
108
109 for (final Map<String, Object> repository : repositoryList) {
110 try {
111 final String owner = (String) repository.get("owner");
112 final String name = (String) repository.get("name");
113
114 final String branch = (String) repository.getOrDefault("branch", "master");
115 final int issueCount = (int) repository.get("issue_count");
116 final int pullCount = (int) repository.get("pull_count");
117 final List<String> roleList = createRoleList(owner, repository);
118
119
120 if (StringUtil.isNotEmpty(branch)) {
121 final String refStr = getGitRef(rootURL, authToken, owner, name, branch);
122 logger.info("Crawl " + owner + "/" + name);
123
124 crawlFileContents(
125 rootURL,
126 authToken,
127 owner,
128 name,
129 refStr,
130 StringUtil.EMPTY,
131 0,
132 readInterval,
133 path -> {
134 storeFileContent(rootURL, authToken, sourceLabel, owner, name, refStr, roleList, path, crawlingConfig,
135 callback, paramMap, scriptMap, defaultDataMap);
136 if (readInterval > 0) {
137 sleep(readInterval);
138 }
139 });
140 }
141
142 logger.info("Crawl issues in " + owner + "/" + name);
143
144 for (int issueId = 1; issueId <= issueCount + pullCount; issueId++) {
145 storeIssueById(rootURL, authToken, issueLabel, owner, name, new Integer(issueId), roleList, crawlingConfig, callback,
146 paramMap, scriptMap, defaultDataMap);
147
148 if (readInterval > 0) {
149 sleep(readInterval);
150 }
151 }
152
153 logger.info("Crawl Wiki in " + owner + "/" + name);
154
155 storeWikiContents(rootURL, authToken, wikiLabel, owner, name, roleList, crawlingConfig, callback, paramMap, scriptMap,
156 defaultDataMap, readInterval);
157
158 } catch (final Exception e) {
159 logger.warn("Failed to access to " + repository, e);
160 }
161 }
162
163 }
164
165 protected String getRootURL(final Map<String, String> paramMap) {
166 if (paramMap.containsKey(GITBUCKET_URL_PARAM)) {
167 final String url = paramMap.get(GITBUCKET_URL_PARAM);
168 if (!url.endsWith("/")) {
169 return url + "/";
170 }
171 return url;
172 }
173 return StringUtil.EMPTY;
174 }
175
176 protected String getAuthToken(final Map<String, String> paramMap) {
177 if (paramMap.containsKey(TOKEN_PARAM)) {
178 return paramMap.get(TOKEN_PARAM);
179 }
180 return StringUtil.EMPTY;
181 }
182
183 protected Map<String, String> getFessPluginInfo(final String rootURL, final String authToken) {
184 final FessConfig fessConfig = ComponentUtil.getFessConfig();
185 final String url = rootURL + "api/v3/fess/info";
186 try (CurlResponse curlResponse =
187 Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
188 @SuppressWarnings({ "rawtypes", "unchecked" })
189 final Map<String, String> map = (Map) curlResponse.getContentAsMap();
190 assert (map.containsKey("version"));
191 assert (map.containsKey("source_label") && map.containsKey("wiki_label") && map.containsKey("issue_label"));
192 return map;
193
194 } catch (final Exception e) {
195 logger.warn("Failed to access to " + url, e);
196 return Collections.emptyMap();
197 }
198 }
199
200 protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
201 final FessConfig fessConfig = ComponentUtil.getFessConfig();
202 final String url = rootURL + "api/v3/fess/repos";
203 int totalCount = -1;
204 final List<Map<String, Object>> repoList = new ArrayList<>();
205
206 do {
207 final String urlWithOffset = url + "?offset=" + repoList.size();
208
209 try (CurlResponse curlResponse =
210 Curl.get(urlWithOffset).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
211 final Map<String, Object> map = curlResponse.getContentAsMap();
212
213 assert (map.containsKey("total_count"));
214 assert (map.containsKey("response_count"));
215 assert (map.containsKey("repositories"));
216
217 totalCount = (int) map.get("total_count");
218 final int responseCount = (int) map.get("response_count");
219 if (responseCount == 0) {
220 break;
221 }
222
223 @SuppressWarnings("unchecked")
224 final List<Map<String, Object>> repos = (ArrayList<Map<String, Object>>) map.get("repositories");
225 repoList.addAll(repos);
226 } catch (final Exception e) {
227 logger.warn("Failed to access to " + urlWithOffset, e);
228 break;
229 }
230 } while (repoList.size() < totalCount);
231
232 logger.info("There exist " + repoList.size() + " repositories");
233 return repoList;
234 }
235
236 protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
237 final FessConfig fessConfig = ComponentUtil.getFessConfig();
238 final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch, null);
239
240 try (CurlResponse curlResponse =
241 Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
242 final Map<String, Object> map = curlResponse.getContentAsMap();
243 assert (map.containsKey("object"));
244 @SuppressWarnings("unchecked")
245 final Map<String, String> objmap = (Map<String, String>) map.get("object");
246 assert (objmap.containsKey("sha"));
247 return objmap.get("sha");
248 } catch (final Exception e) {
249 logger.warn("Failed to access to " + url, e);
250 return branch;
251 }
252 }
253
254 private List<String> createRoleList(final String owner, final Map<String, Object> repository) {
255 Boolean isPrivate = true;
256 if (repository.containsKey(PRIVATE_REPOSITORY_PARAM)) {
257 isPrivate = (Boolean) repository.get(PRIVATE_REPOSITORY_PARAM);
258 }
259 if (!isPrivate) {
260 return Collections.singletonList("Rguest");
261 }
262
263 @SuppressWarnings("unchecked")
264 final List<String> collaboratorList = (List<String>) repository.get(COLLABORATORS_PARAM);
265 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
266 collaboratorList.add(owner);
267 return collaboratorList.stream().map(user -> systemHelper.getSearchRoleByUser(user)).collect(Collectors.toList());
268 }
269
270 private List<Object> parseList(final InputStream is) {
271 try {
272 return JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, is).list();
273 } catch (final Exception e) {
274 logger.warn("Failed to parse a list.", e);
275 return Collections.emptyList();
276 }
277 }
278
279 private void storeFileContent(final String rootURL, final String authToken, final String sourceLabel, final String owner,
280 final String name, final String refStr, final List<String> roleList, final String path, final CrawlingConfig crawlingConfig,
281 final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
282 final Map<String, Object> defaultDataMap) {
283 final String apiUrl = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, null);
284 final String viewUrl = encode(rootURL, owner + "/" + name + "/blob/" + refStr + "/" + path, null);
285
286 if (logger.isInfoEnabled()) {
287 logger.info("Get a content from " + apiUrl);
288 }
289 final Map<String, Object> dataMap = new HashMap<>();
290 dataMap.putAll(defaultDataMap);
291 dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"),
292 apiUrl + "?ref=" + refStr + "&large_file=true"));
293
294 dataMap.put("url", viewUrl);
295 dataMap.put("role", roleList);
296 dataMap.put("label", Collections.singletonList(sourceLabel));
297
298
299
300 callback.store(paramMap, dataMap);
301
302 return;
303 }
304
305 private void storeIssueById(final String rootURL, final String authToken, final String issueLabel, final String owner,
306 final String name, final Integer issueId, final List<String> roleList, final CrawlingConfig crawlingConfig,
307 final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
308 final Map<String, Object> defaultDataMap) {
309 final FessConfig fessConfig = ComponentUtil.getFessConfig();
310
311 final String issueUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/issues/" + issueId.toString();
312 final String viewUrl = rootURL + owner + "/" + name + "/issues/" + issueId.toString();
313
314 if (logger.isInfoEnabled()) {
315 logger.info("Get a content from " + issueUrl);
316 }
317
318 final Map<String, Object> dataMap = new HashMap<>();
319 String contentStr = "";
320 dataMap.putAll(defaultDataMap);
321
322
323
324 try (CurlResponse curlResponse =
325 Curl.get(issueUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
326 final Map<String, Object> map = curlResponse.getContentAsMap();
327 dataMap.put("title", map.getOrDefault("title", ""));
328 contentStr = (String) map.getOrDefault("body", "");
329 } catch (final Exception e) {
330 logger.warn("Failed to access to " + issueUrl, e);
331 }
332
333 final String commentsStr = String.join("\n", getIssueComments(issueUrl, authToken));
334 contentStr += "\n" + commentsStr;
335
336 dataMap.put("content", contentStr);
337 dataMap.put("url", viewUrl);
338 dataMap.put("role", roleList);
339 dataMap.put("label", Collections.singletonList(issueLabel));
340
341
342
343 callback.store(paramMap, dataMap);
344
345 return;
346 }
347
348 private List<String> getIssueComments(final String issueUrl, final String authToken) {
349 final FessConfig fessConfig = ComponentUtil.getFessConfig();
350 final String commentsUrl = issueUrl + "/comments";
351 final List<String> commentList = new ArrayList<>();
352
353 try (CurlResponse curlResponse =
354 Curl.get(commentsUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
355 final String commentsJson = curlResponse.getContentAsString();
356 final List<Map<String, Object>> comments =
357 new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
358 });
359
360 for (final Map<String, Object> comment : comments) {
361 if (comment.containsKey("body")) {
362 commentList.add((String) comment.get("body"));
363 }
364 }
365 } catch (final Exception e) {
366 logger.warn("Failed to access to " + issueUrl, e);
367 }
368
369 return commentList;
370 }
371
372 @SuppressWarnings("unchecked")
373 private void storeWikiContents(final String rootURL, final String authToken, final String wikiLabel, final String owner,
374 final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
375 final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap,
376 final long readInterval) {
377 final FessConfig fessConfig = ComponentUtil.getFessConfig();
378 final String wikiUrl = rootURL + "api/v3/fess/" + owner + "/" + name + "/wiki";
379
380 List<String> pageList = Collections.emptyList();
381
382
383 try (CurlResponse curlResponse =
384 Curl.get(wikiUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
385 final Map<String, Object> map = curlResponse.getContentAsMap();
386 pageList = (List<String>) map.get("pages");
387 } catch (final Exception e) {
388 logger.warn("Failed to access to " + wikiUrl, e);
389 }
390
391 for (final String page : pageList) {
392
393 final String pageUrl = wikiUrl + "/contents/" + page + ".md";
394 final String viewUrl = rootURL + owner + "/" + name + "/wiki/" + page;
395
396 if (logger.isInfoEnabled()) {
397 logger.info("Get a content from " + pageUrl);
398 }
399
400 final Map<String, Object> dataMap = new HashMap<>();
401 dataMap.putAll(defaultDataMap);
402 dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), pageUrl));
403
404 dataMap.put("url", viewUrl);
405 dataMap.put("role", roleList);
406 dataMap.put("label", Collections.singletonList(wikiLabel));
407
408
409
410 callback.store(paramMap, dataMap);
411 logger.info("Stored " + pageUrl);
412
413 if (readInterval > 0) {
414 sleep(readInterval);
415 }
416 }
417
418 }
419
420 protected void crawlFileContents(final String rootURL, final String authToken, final String owner, final String name,
421 final String refStr, final String path, final int depth, final long readInterval, final Consumer<String> consumer) {
422
423 if (MAX_DEPTH <= depth) {
424 return;
425 }
426
427 final FessConfig fessConfig = ComponentUtil.getFessConfig();
428 final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, "ref=" + refStr);
429
430 try (CurlResponse curlResponse =
431 Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
432 final InputStream iStream = curlResponse.getContentAsStream();
433 final List<Object> fileList = parseList(iStream);
434
435 for (int i = 0; i < fileList.size(); ++i) {
436 @SuppressWarnings("unchecked")
437 final Map<String, String> file = (Map<String, String>) fileList.get(i);
438 final String newPath = path.isEmpty() ? file.get("name") : path + "/" + file.get("name");
439 switch (file.get("type")) {
440 case "file":
441 consumer.accept(newPath);
442 break;
443 case "dir":
444 if (readInterval > 0) {
445 sleep(readInterval);
446 }
447 crawlFileContents(rootURL, authToken, owner, name, refStr, newPath, depth + 1, readInterval, consumer);
448 break;
449 }
450 }
451 } catch (final Exception e) {
452 logger.warn("Failed to access to " + url, e);
453 }
454 }
455
456 private String encode(final String rootURL, final String path, final String query) {
457 try {
458 final URI rootURI = new URI(rootURL);
459 final URI uri =
460 new URI(rootURI.getScheme(), rootURI.getUserInfo(), rootURI.getHost(), rootURI.getPort(), rootURI.getPath() + path,
461 query, null);
462 return uri.toASCIIString();
463 } catch (final URISyntaxException e) {
464 logger.warn("Failed to parse " + rootURL + path + "?" + query, e);
465 if (StringUtil.isEmpty(query)) {
466 return rootURL + path;
467 }
468 return rootURL + path + "?" + query;
469 }
470 }
471 }