1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.io.ByteArrayInputStream;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.net.Proxy;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Comparator;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.concurrent.TimeUnit;
35 import java.util.regex.Matcher;
36 import java.util.regex.Pattern;
37 import java.util.stream.Collectors;
38
39 import javax.xml.XMLConstants;
40 import javax.xml.parsers.DocumentBuilder;
41 import javax.xml.parsers.DocumentBuilderFactory;
42
43 import org.apache.commons.lang3.StringUtils;
44 import org.apache.logging.log4j.LogManager;
45 import org.apache.logging.log4j.Logger;
46 import org.codelibs.core.io.CopyUtil;
47 import org.codelibs.core.lang.StringUtil;
48 import org.codelibs.curl.Curl;
49 import org.codelibs.curl.CurlRequest;
50 import org.codelibs.curl.CurlResponse;
51 import org.codelibs.fess.crawler.Constants;
52 import org.codelibs.fess.exception.PluginException;
53 import org.codelibs.fess.util.ComponentUtil;
54 import org.codelibs.fess.util.ResourceUtil;
55 import org.lastaflute.di.exception.IORuntimeException;
56 import org.w3c.dom.Document;
57 import org.w3c.dom.Node;
58 import org.w3c.dom.NodeList;
59 import org.xml.sax.SAXException;
60
61 import com.fasterxml.jackson.databind.ObjectMapper;
62 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
63 import com.google.common.cache.CacheBuilder;
64 import com.google.common.cache.CacheLoader;
65 import com.google.common.cache.LoadingCache;
66
67 public class PluginHelper {
68 private static final Logger logger = LogManager.getLogger(PluginHelper.class);
69
70 protected LoadingCache<ArtifactType, Artifact[]> availableArtifacts = CacheBuilder.newBuilder().maximumSize(10)
71 .expireAfterWrite(5, TimeUnit.MINUTES).build(new CacheLoader<ArtifactType, Artifact[]>() {
72 @Override
73 public Artifact[] load(final ArtifactType key) {
74 final List<Artifact> list = new ArrayList<>();
75 for (final String url : getRepositories()) {
76 if (url.endsWith(".yaml")) {
77 if (key == ArtifactType.UNKNOWN) {
78 list.addAll(loadArtifactsFromRepository(url));
79 }
80 } else {
81 list.addAll(processRepository(key, url));
82 }
83 }
84 return list.toArray(new Artifact[list.size()]);
85 }
86 });
87
88 public Artifact[] getAvailableArtifacts(final ArtifactType artifactType) {
89 try {
90 return availableArtifacts.get(artifactType);
91 } catch (final Exception e) {
92 throw new PluginException("Failed to access " + artifactType, e);
93 }
94 }
95
96 protected String[] getRepositories() {
97 return split(ComponentUtil.getFessConfig().getPluginRepositories(), ",")
98 .get(stream -> stream.map(String::trim).toArray(n -> new String[n]));
99 }
100
101 protected List<Artifact> loadArtifactsFromRepository(final String url) {
102 final String content = getRepositoryContent(url);
103 final ObjectMapper objectMapper = new YAMLMapper();
104 try {
105 @SuppressWarnings("unchecked")
106 final List<Map<?, ?>> result = objectMapper.readValue(content, List.class);
107 if (result != null) {
108 return result.stream().map(o -> new Artifact((String) o.get("name"), (String) o.get("version"), (String) o.get("url")))
109 .collect(Collectors.toList());
110 }
111 return Collections.emptyList();
112 } catch (final Exception e) {
113 throw new PluginException("Failed to access " + url, e);
114 }
115 }
116
117 protected List<Artifact> processRepository(final ArtifactType artifactType, final String url) {
118 final List<Artifact> list = new ArrayList<>();
119 final String repoContent = getRepositoryContent(url);
120 final Matcher matcher = Pattern.compile("href=\"[^\"]*(" + artifactType.getId() + "[a-zA-Z0-9\\-]+)/?\"").matcher(repoContent);
121 while (matcher.find()) {
122 final String name = matcher.group(1);
123 final String pluginUrl = url + (url.endsWith("/") ? name + "/" : "/" + name + "/");
124 final String pluginMetaContent = getRepositoryContent(pluginUrl + "maven-metadata.xml");
125 try (final InputStream is = new ByteArrayInputStream(pluginMetaContent.getBytes(Constants.UTF_8_CHARSET))) {
126 final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
127 factory.setFeature(Constants.FEATURE_SECURE_PROCESSING, true);
128 factory.setFeature(Constants.FEATURE_EXTERNAL_GENERAL_ENTITIES, false);
129 factory.setFeature(Constants.FEATURE_EXTERNAL_PARAMETER_ENTITIES, false);
130 factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, StringUtil.EMPTY);
131 factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, StringUtil.EMPTY);
132 final DocumentBuilder builder = factory.newDocumentBuilder();
133 final Document document = builder.parse(is);
134 final NodeList nodeList = document.getElementsByTagName("version");
135 for (int i = 0; i < nodeList.getLength(); i++) {
136 final String version = nodeList.item(i).getTextContent();
137 if (isTargetPluginVersion(version)) {
138 if (version.endsWith("SNAPSHOT")) {
139 final String snapshotVersion = getSnapshotActualVersion(builder, pluginUrl, version);
140 if (StringUtil.isNotBlank(snapshotVersion)) {
141 final String actualVersion = version.replace("SNAPSHOT", snapshotVersion);
142 list.add(
143 new Artifact(name, actualVersion, pluginUrl + version + "/" + name + "-" + actualVersion + ".jar"));
144 } else if (logger.isDebugEnabled()) {
145 logger.debug("Snapshot name is not found: {}/{}", name, version);
146 }
147 } else {
148 list.add(new Artifact(name, version, pluginUrl + version + "/" + name + "-" + version + ".jar"));
149 }
150 } else if (logger.isDebugEnabled()) {
151 logger.debug("{}:{} is ignored.", name, version);
152 }
153 }
154 } catch (final Exception e) {
155 logger.warn("Failed to parse {}maven-metadata.xml.", pluginUrl, e);
156 }
157 }
158 return list;
159 }
160
161 protected boolean isTargetPluginVersion(final String version) {
162 return ComponentUtil.getFessConfig().isTargetPluginVersion(version);
163 }
164
165 protected String getSnapshotActualVersion(final DocumentBuilder builder, final String pluginUrl, final String version)
166 throws SAXException, IOException {
167 String timestamp = null;
168 String buildNumber = null;
169 final String versionMetaContent = getRepositoryContent(pluginUrl + version + "/maven-metadata.xml");
170 try (final InputStream is = new ByteArrayInputStream(versionMetaContent.getBytes(Constants.UTF_8_CHARSET))) {
171 final Document doc = builder.parse(is);
172 final NodeList snapshotNodeList = doc.getElementsByTagName("snapshot");
173 if (snapshotNodeList.getLength() > 0) {
174 final NodeList nodeList = snapshotNodeList.item(0).getChildNodes();
175 for (int i = 0; i < nodeList.getLength(); i++) {
176 final Node node = nodeList.item(i);
177 if ("timestamp".equalsIgnoreCase(node.getNodeName())) {
178 timestamp = node.getTextContent();
179 } else if ("buildNumber".equalsIgnoreCase(node.getNodeName())) {
180 buildNumber = node.getTextContent();
181 }
182 }
183 }
184 }
185 if (StringUtil.isNotBlank(timestamp) && StringUtil.isNotBlank(buildNumber)) {
186 return timestamp + "-" + buildNumber;
187 }
188 return null;
189 }
190
191 protected String getRepositoryContent(final String url) {
192 if (logger.isDebugEnabled()) {
193 logger.debug("Loading {}", url);
194 }
195 try (final CurlResponse response = createCurlRequest(url).execute()) {
196 return response.getContentAsString();
197 } catch (final IOException e) {
198 throw new IORuntimeException(e);
199 }
200 }
201
202 public Artifact[] getInstalledArtifacts(final ArtifactType artifactType) {
203 if (artifactType == ArtifactType.UNKNOWN) {
204 final File[] jarFiles = ResourceUtil.getPluginJarFiles((d, n) -> {
205 for (final ArtifactType type : ArtifactType.values()) {
206 if (n.startsWith(type.getId())) {
207 return false;
208 }
209 }
210 return true;
211 });
212 final List<Artifact> list = new ArrayList<>(jarFiles.length);
213 for (final File file : jarFiles) {
214 list.add(getArtifactFromFileName(artifactType, file.getName()));
215 }
216 list.sort(Comparator.comparing(Artifact::getName));
217 return list.toArray(new Artifact[list.size()]);
218 }
219
220 final File[] jarFiles = ResourceUtil.getPluginJarFiles(artifactType.getId());
221 final List<Artifact> list = new ArrayList<>(jarFiles.length);
222 for (final File file : jarFiles) {
223 list.add(getArtifactFromFileName(artifactType, file.getName()));
224 }
225 list.sort(Comparator.comparing(Artifact::getName));
226 return list.toArray(new Artifact[list.size()]);
227 }
228
229 protected Artifact getArtifactFromFileName(final ArtifactType artifactType, final String filename) {
230 return getArtifactFromFileName(artifactType, filename, null);
231 }
232
233 public Artifact getArtifactFromFileName(final ArtifactType artifactType, final String filename, final String url) {
234 final String baseName = StringUtils.removeEndIgnoreCase(filename, ".jar");
235 final List<String> nameList = new ArrayList<>();
236 final List<String> versionList = new ArrayList<>();
237 boolean isName = true;
238 for (final String value : baseName.split("-")) {
239 if (isName && value.length() > 0 && value.charAt(0) >= '0' && value.charAt(0) <= '9') {
240 isName = false;
241 }
242 if (isName) {
243 nameList.add(value);
244 } else {
245 versionList.add(value);
246 }
247 }
248 return new Artifact(nameList.stream().collect(Collectors.joining("-")), versionList.stream().collect(Collectors.joining("-")), url);
249 }
250
251 public void installArtifact(final Artifact artifact) {
252 switch (artifact.getType()) {
253 case THEME:
254 install(artifact);
255 ComponentUtil.getThemeHelper().install(artifact);
256 break;
257 default:
258 install(artifact);
259 break;
260 }
261 }
262
263 protected void install(final Artifact artifact) {
264 final String fileName = artifact.getFileName();
265 final String url = artifact.getUrl();
266 if (StringUtil.isBlank(url)) {
267 throw new PluginException("url is blank: " + artifact.getName());
268 }
269 if (url.startsWith("http:") || url.startsWith("https:")) {
270 try (final CurlResponse response = createCurlRequest(url).execute()) {
271 if (response.getHttpStatusCode() != 200) {
272 throw new PluginException("HTTP Status " + response.getHttpStatusCode() + " : failed to get the artifact from " + url);
273 }
274 try (final InputStream in = response.getContentAsStream()) {
275 CopyUtil.copy(in, ResourceUtil.getPluginPath(fileName).toFile());
276 }
277 } catch (final Exception e) {
278 throw new PluginException("Failed to install the artifact " + artifact.getName(), e);
279 }
280 } else {
281 try (final InputStream in = new FileInputStream(url)) {
282 CopyUtil.copy(in, ResourceUtil.getPluginPath(fileName).toFile());
283 } catch (final Exception e) {
284 throw new PluginException("Failed to install the artifact " + artifact.getName(), e);
285 }
286 }
287 }
288
289 protected CurlRequest createCurlRequest(final String url) {
290 final CurlRequest request = Curl.get(url);
291 final Proxy proxy = ComponentUtil.getFessConfig().getHttpProxy();
292 if (proxy != null && !Proxy.NO_PROXY.equals(proxy)) {
293 request.proxy(proxy);
294 }
295 return request;
296 }
297
298 public void deleteInstalledArtifact(final Artifact artifact) {
299 final String fileName = artifact.getFileName();
300 final Path jarPath = Paths.get(ResourceUtil.getPluginPath().toString(), fileName);
301 if (!Files.exists(jarPath)) {
302 throw new PluginException(fileName + " does not exist.");
303 }
304
305 switch (artifact.getType()) {
306 case THEME:
307 ComponentUtil.getThemeHelper().uninstall(artifact);
308 uninstall(fileName, jarPath);
309 break;
310 default:
311 uninstall(fileName, jarPath);
312 break;
313 }
314
315 }
316
317 protected void uninstall(final String fileName, final Path jarPath) {
318 try {
319 Files.delete(jarPath);
320 } catch (final IOException e) {
321 throw new PluginException("Failed to delete the artifact " + fileName, e);
322 }
323 }
324
325 public Artifact getArtifact(final String name, final String version) {
326 if (StringUtil.isBlank(name) || StringUtil.isBlank(version)) {
327 return null;
328 }
329 for (final Artifact artifact : getAvailableArtifacts(ArtifactType.getType(name))) {
330 if (name.equals(artifact.getName()) && version.equals(artifact.getVersion())) {
331 return artifact;
332 }
333 }
334 return null;
335 }
336
337 public static class Artifact {
338 protected final String name;
339 protected final String version;
340 protected final String url;
341
342 public Artifact(final String name, final String version, final String url) {
343 this.name = name;
344 this.version = version;
345 this.url = url;
346 }
347
348 public Artifact(final String name, final String version) {
349 this(name, version, null);
350 }
351
352 public String getName() {
353 return name;
354 }
355
356 public String getVersion() {
357 return version;
358 }
359
360 public String getFileName() {
361 return name + "-" + version + ".jar";
362 }
363
364 public String getUrl() {
365 return url;
366 }
367
368 public ArtifactType getType() {
369 return ArtifactType.getType(name);
370 }
371
372 @Override
373 public String toString() {
374 return name + ":" + version;
375 }
376 }
377
378 public enum ArtifactType {
379 DATA_STORE("fess-ds"), THEME("fess-theme"), INGEST("fess-ingest"), SCRIPT("fess-script"), WEBAPP("fess-webapp"), UNKNOWN("jar");
380
381 private final String id;
382
383 ArtifactType(final String id) {
384 this.id = id;
385 }
386
387 public String getId() {
388 return id;
389 }
390
391 public static ArtifactType getType(final String name) {
392 if (name.startsWith(DATA_STORE.getId())) {
393 return DATA_STORE;
394 }
395 if (name.startsWith(THEME.getId())) {
396 return THEME;
397 }
398 if (name.startsWith(INGEST.getId())) {
399 return INGEST;
400 }
401 if (name.startsWith(SCRIPT.getId())) {
402 return SCRIPT;
403 }
404 if (name.startsWith(WEBAPP.getId())) {
405 return WEBAPP;
406 }
407 return UNKNOWN;
408 }
409 }
410
411 }