1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.web.admin.storage;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.io.InputStream;
21 import java.net.URLDecoder;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27
28 import org.apache.logging.log4j.LogManager;
29 import org.apache.logging.log4j.Logger;
30 import org.codelibs.core.lang.StringUtil;
31 import org.codelibs.fess.annotation.Secured;
32 import org.codelibs.fess.app.web.base.FessAdminAction;
33 import org.codelibs.fess.crawler.Constants;
34 import org.codelibs.fess.exception.StorageException;
35 import org.codelibs.fess.mylasta.direction.FessConfig;
36 import org.codelibs.fess.util.ComponentUtil;
37 import org.codelibs.fess.util.RenderDataUtil;
38 import org.dbflute.optional.OptionalThing;
39 import org.lastaflute.web.Execute;
40 import org.lastaflute.web.response.ActionResponse;
41 import org.lastaflute.web.response.HtmlResponse;
42 import org.lastaflute.web.response.StreamResponse;
43 import org.lastaflute.web.ruts.multipart.MultipartFormFile;
44 import org.lastaflute.web.ruts.process.ActionRuntime;
45 import org.lastaflute.web.servlet.request.stream.WrittenStreamOut;
46
47 import io.minio.GetObjectArgs;
48 import io.minio.ListObjectsArgs;
49 import io.minio.MakeBucketArgs;
50 import io.minio.MinioClient;
51 import io.minio.PutObjectArgs;
52 import io.minio.RemoveObjectArgs;
53 import io.minio.Result;
54 import io.minio.errors.ErrorResponseException;
55 import io.minio.messages.Item;
56
57
58
59
60 public class AdminStorageAction extends FessAdminAction {
61
62 public static final String ROLE = "admin-storage";
63
64 private static final Logger logger = LogManager.getLogger(AdminStorageAction.class);
65
66 @Override
67 protected void setupHtmlData(final ActionRuntime runtime) {
68 super.setupHtmlData(runtime);
69 runtime.registerData("helpLink", systemHelper.getHelpLink(fessConfig.getOnlineHelpNameStorage()));
70 }
71
72 @Override
73 protected String getActionRole() {
74 return ROLE;
75 }
76
77 @Execute
78 @Secured({ ROLE, ROLE + VIEW })
79 public HtmlResponse index() {
80 saveToken();
81 return asListHtml(StringUtil.EMPTY);
82 }
83
84 @Execute
85 @Secured({ ROLE, ROLE + VIEW })
86 public ActionResponse list(final OptionalThing<String> id) {
87 saveToken();
88 if (id.isPresent() && id.get() != null) {
89 return asListHtml(decodePath(id.get()));
90 }
91 return redirect(getClass());
92 }
93
94 @Execute
95 @Secured({ ROLE })
96 public HtmlResponse upload(final ItemForm form) {
97 validate(form, messages -> {}, () -> asListHtml(form.path));
98 if (form.uploadFile == null) {
99 throwValidationError(messages -> messages.addErrorsStorageNoUploadFile(GLOBAL), () -> asListHtml(form.path));
100 }
101 verifyToken(() -> asListHtml(form.path));
102 try {
103 uploadObject(getObjectName(form.path, form.uploadFile.getFileName()), form.uploadFile);
104 } catch (final StorageException e) {
105 if (logger.isDebugEnabled()) {
106 logger.debug("Failed to upload {}", form.uploadFile.getFileName(), e);
107 }
108 throwValidationError(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, form.uploadFile.getFileName()),
109 () -> asListHtml(encodeId(form.path)));
110
111 }
112 saveInfo(messages -> messages.addSuccessUploadFileToStorage(GLOBAL, form.uploadFile.getFileName()));
113 return redirectWith(getClass(), moreUrl("list/" + encodeId(form.path)));
114 }
115
116 @Execute
117 @Secured({ ROLE, ROLE + VIEW })
118 public ActionResponse download(final String id) {
119 final String[] values = decodeId(id);
120 if (StringUtil.isEmpty(values[1])) {
121 throwValidationError(messages -> messages.addErrorsStorageFileNotFound(GLOBAL), () -> asListHtml(encodeId(values[0])));
122 }
123 final StreamResponse response = new StreamResponse(StringUtil.EMPTY);
124 final String name = values[1];
125 final String encodedName = URLEncoder.encode(name, Constants.UTF_8_CHARSET).replace("+", "%20");
126 response.header("Content-Disposition", "attachment; filename=\"" + name + "\"; filename*=utf-8''" + encodedName);
127 response.header("Pragma", "no-cache");
128 response.header("Cache-Control", "no-cache");
129 response.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT");
130 response.contentTypeOctetStream();
131 return response.stream(out -> {
132 try {
133 downloadObject(getObjectName(values[0], values[1]), out);
134 } catch (final StorageException e) {
135 if (logger.isDebugEnabled()) {
136 logger.debug("Failed to download {}", values[1], e);
137 }
138 throwValidationError(messages -> messages.addErrorsStorageFileDownloadFailure(GLOBAL, values[1]),
139 () -> asListHtml(encodeId(values[0])));
140 }
141 });
142 }
143
144 @Execute
145 @Secured({ ROLE })
146 public HtmlResponse delete(final String id) {
147 final String[] values = decodeId(id);
148 if (StringUtil.isEmpty(values[1])) {
149 throwValidationError(messages -> messages.addErrorsStorageFileNotFound(GLOBAL), () -> asListHtml(encodeId(values[0])));
150 }
151 final String objectName = getObjectName(values[0], values[1]);
152 try {
153 deleteObject(objectName);
154 } catch (final StorageException e) {
155 logger.debug("Failed to delete {}", values[1], e);
156 throwValidationError(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, values[1]),
157 () -> asListHtml(encodeId(values[0])));
158 }
159 saveInfo(messages -> messages.addSuccessDeleteFile(GLOBAL, values[1]));
160 return redirectWith(getClass(), moreUrl("list/" + encodeId(values[0])));
161 }
162
163 @Execute
164 @Secured({ ROLE })
165 public HtmlResponse createDir(final ItemForm form) {
166 validate(form, messages -> {}, () -> asListHtml(form.path));
167 if (StringUtil.isBlank(form.name)) {
168 throwValidationError(messages -> messages.addErrorsStorageDirectoryNameIsInvalid(GLOBAL), () -> asListHtml(form.path));
169 }
170 return redirectWith(getClass(), moreUrl("list/" + encodeId(getObjectName(form.path, form.name))));
171 }
172
173 public static void uploadObject(final String objectName, final MultipartFormFile uploadFile) {
174 try (final InputStream in = uploadFile.getInputStream()) {
175 final FessConfig fessConfig = ComponentUtil.getFessConfig();
176 final MinioClient minioClient = createClient(fessConfig);
177 final PutObjectArgs args = PutObjectArgs.builder().bucket(fessConfig.getStorageBucket()).object(objectName)
178 .stream(in, uploadFile.getFileSize(), -1).contentType("application/octet-stream").build();
179 minioClient.putObject(args);
180 } catch (final Exception e) {
181 throw new StorageException("Failed to upload " + objectName, e);
182 }
183 }
184
185 public static void downloadObject(final String objectName, final WrittenStreamOut out) {
186 final FessConfig fessConfig = ComponentUtil.getFessConfig();
187 final GetObjectArgs args = GetObjectArgs.builder().bucket(fessConfig.getStorageBucket()).object(objectName).build();
188 try (InputStream in = createClient(fessConfig).getObject(args)) {
189 out.write(in);
190 } catch (final Exception e) {
191 throw new StorageException("Failed to download " + objectName, e);
192 }
193 }
194
195 public static void deleteObject(final String objectName) {
196 try {
197 final FessConfig fessConfig = ComponentUtil.getFessConfig();
198 final MinioClient minioClient = createClient(fessConfig);
199 final RemoveObjectArgs args = RemoveObjectArgs.builder().bucket(fessConfig.getStorageBucket()).object(objectName).build();
200 minioClient.removeObject(args);
201 } catch (final Exception e) {
202 throw new StorageException("Failed to delete " + objectName, e);
203 }
204 }
205
206 protected static MinioClient createClient(final FessConfig fessConfig) {
207 try {
208 return MinioClient.builder().endpoint(fessConfig.getStorageEndpoint())
209 .credentials(fessConfig.getStorageAccessKey(), fessConfig.getStorageSecretKey()).build();
210 } catch (final Exception e) {
211 throw new StorageException("Failed to create MinioClient: " + fessConfig.getStorageEndpoint(), e);
212 }
213 }
214
215 public static List<Map<String, Object>> getFileItems(final String prefix) {
216 final FessConfig fessConfig = ComponentUtil.getFessConfig();
217 final ArrayList<Map<String, Object>> list = new ArrayList<>();
218 try {
219 final MinioClient minioClient = createClient(fessConfig);
220 final ListObjectsArgs args = ListObjectsArgs.builder().bucket(fessConfig.getStorageBucket())
221 .prefix(prefix != null && prefix.length() > 0 ? prefix + "/" : prefix).recursive(false).includeUserMetadata(false)
222 .useApiVersion1(false).build();
223 for (final Result<Item> result : minioClient.listObjects(args)) {
224 final Map<String, Object> map = new HashMap<>();
225 final Item item = result.get();
226 final String objectName = item.objectName();
227 map.put("id", encodeId(objectName));
228 map.put("name", getName(objectName));
229 map.put("hashCode", item.hashCode());
230 map.put("size", item.size());
231 map.put("directory", item.isDir());
232 if (!item.isDir()) {
233 map.put("lastModified", item.lastModified());
234 }
235 list.add(map);
236 if (list.size() > fessConfig.getStorageMaxItemsInPageAsInteger()) {
237 break;
238 }
239 }
240 } catch (final ErrorResponseException e) {
241 final String code = e.errorResponse().code();
242 if ("NoSuchBucket".equals(code)) {
243 final MinioClient minioClient = createClient(fessConfig);
244 try {
245 final MakeBucketArgs args = MakeBucketArgs.builder().bucket(fessConfig.getStorageBucket()).build();
246 minioClient.makeBucket(args);
247 logger.info("Created bucket: {}", fessConfig.getStorageBucket());
248 } catch (final Exception e1) {
249 logger.warn("Failed to create bucket: {}", fessConfig.getStorageBucket(), e1);
250 }
251 } else if (logger.isDebugEnabled()) {
252 logger.debug("Failed to access {}", fessConfig.getStorageEndpoint(), e);
253 }
254 } catch (final Exception e) {
255 if (logger.isDebugEnabled()) {
256 logger.debug("Failed to access {}", fessConfig.getStorageEndpoint(), e);
257 }
258 }
259 return list;
260 }
261
262 private static String getName(final String objectName) {
263 final String[] values = objectName.split("/");
264 if (values.length == 0) {
265 return StringUtil.EMPTY;
266 }
267 return values[values.length - 1];
268 }
269
270 public static String decodePath(final String id) {
271 final String[] values = decodeId(id);
272 if (StringUtil.isEmpty(values[0]) && StringUtil.isEmpty(values[1])) {
273 return StringUtil.EMPTY;
274 }
275 if (StringUtil.isEmpty(values[0])) {
276 return values[1];
277 }
278 return values[0] + "/" + values[1];
279 }
280
281 public static String[] decodeId(final String id) {
282 final String value = urlDecode(urlDecode(id));
283 final String[] values = split(value, "/").get(stream -> stream.filter(StringUtil::isNotEmpty).toArray(n -> new String[n]));
284 if (values.length == 0) {
285
286 return new String[] { StringUtil.EMPTY, StringUtil.EMPTY };
287 }
288 if (values.length == 1) {
289 return new String[] { StringUtil.EMPTY, values[0] };
290 }
291 final StringBuilder buf = new StringBuilder();
292 for (int i = 0; i < values.length - 1; i++) {
293 if (buf.length() > 0) {
294 buf.append('/');
295 }
296 buf.append(values[i]);
297 }
298 return new String[] { buf.toString(), values[values.length - 1] };
299 }
300
301 protected static String createParentId(final String prefix) {
302 if (prefix == null) {
303 return StringUtil.EMPTY;
304 }
305 final String[] values = prefix.split("/");
306 if (values.length > 1) {
307 final StringBuilder buf = new StringBuilder();
308 for (int i = 0; i < values.length - 1; i++) {
309 if (buf.length() > 0) {
310 buf.append('/');
311 }
312 buf.append(values[i]);
313 }
314 return urlEncode(buf.toString());
315 }
316 return StringUtil.EMPTY;
317 }
318
319 protected static List<Map<String, String>> createPathItems(final String prefix) {
320 final List<Map<String, String>> list = new ArrayList<>();
321 final StringBuilder buf = new StringBuilder();
322 split(prefix, "/").of(stream -> stream.filter(StringUtil::isNotEmpty).forEach(s -> {
323 if (buf.length() > 0) {
324 buf.append('/');
325 }
326 buf.append(s);
327 final Map<String, String> map = new HashMap<>();
328 map.put("id", urlEncode(buf.toString()));
329 map.put("name", s);
330 list.add(map);
331 }));
332 return list;
333 }
334
335 protected static String getPathPrefix(final String path) {
336 return StringUtil.isEmpty(path) ? StringUtil.EMPTY : path + "/";
337 }
338
339 public static String getObjectName(final String path, final String name) {
340 return getPathPrefix(path) + name;
341 }
342
343 protected static String urlEncode(final String str) {
344 if (str == null) {
345 return StringUtil.EMPTY;
346 }
347 return URLEncoder.encode(str, Constants.UTF_8_CHARSET);
348 }
349
350 protected static String urlDecode(final String str) {
351 if (str == null) {
352 return StringUtil.EMPTY;
353 }
354 return URLDecoder.decode(str, Constants.UTF_8_CHARSET);
355 }
356
357 protected static String encodeId(final String objectName) {
358 return urlEncode(urlEncode(objectName));
359 }
360
361 private HtmlResponse asListHtml(final String prefix) {
362 return asHtml(path_AdminStorage_AdminStorageJsp).useForm(ItemForm.class).renderWith(data -> {
363 RenderDataUtil.register(data, "endpoint", fessConfig.getStorageEndpoint());
364 RenderDataUtil.register(data, "bucket", fessConfig.getStorageBucket());
365 RenderDataUtil.register(data, "path", prefix);
366 RenderDataUtil.register(data, "pathItems", createPathItems(prefix));
367 RenderDataUtil.register(data, "parentId", createParentId(prefix));
368 RenderDataUtil.register(data, "fileItems", getFileItems(prefix));
369 });
370 }
371
372 }