1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.storage;
17
18 import java.io.InputStream;
19 import java.io.OutputStream;
20 import java.net.URI;
21 import java.time.ZonedDateTime;
22 import java.util.ArrayList;
23 import java.util.Base64;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.stream.Collectors;
27
28 import org.apache.logging.log4j.LogManager;
29 import org.apache.logging.log4j.Logger;
30 import org.codelibs.core.lang.StringUtil;
31 import org.codelibs.fess.crawler.Constants;
32 import org.codelibs.fess.exception.StorageException;
33
34 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
35 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
36 import software.amazon.awssdk.core.sync.RequestBody;
37 import software.amazon.awssdk.regions.Region;
38 import software.amazon.awssdk.services.s3.S3Client;
39 import software.amazon.awssdk.services.s3.S3ClientBuilder;
40 import software.amazon.awssdk.services.s3.model.CommonPrefix;
41 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
42 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
43 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
44 import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
45 import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse;
46 import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
47 import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
48 import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
49 import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
50 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
51 import software.amazon.awssdk.services.s3.model.PutObjectTaggingRequest;
52 import software.amazon.awssdk.services.s3.model.S3Object;
53 import software.amazon.awssdk.services.s3.model.Tag;
54 import software.amazon.awssdk.services.s3.model.Tagging;
55
56
57
58
59
60 public class S3StorageClient implements StorageClient {
61
62 private static final Logger logger = LogManager.getLogger(S3StorageClient.class);
63
64 private final S3Client s3Client;
65 private final String bucket;
66
67
68
69
70
71
72
73
74
75
76 public S3StorageClient(final String endpoint, final String accessKey, final String secretKey, final String bucket,
77 final String region) {
78 this.bucket = bucket;
79
80 final AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
81 final S3ClientBuilder builder =
82 S3Client.builder().credentialsProvider(StaticCredentialsProvider.create(credentials)).region(Region.of(getRegion(region)));
83
84
85 if (StringUtil.isNotBlank(endpoint)) {
86 builder.endpointOverride(URI.create(endpoint)).forcePathStyle(true);
87 }
88
89 this.s3Client = builder.build();
90 }
91
92 private String getRegion(final String region) {
93 return StringUtil.isNotBlank(region) ? region : "us-east-1";
94 }
95
96 @Override
97 public void uploadObject(final String objectName, final InputStream inputStream, final long size, final String contentType) {
98 try {
99 final PutObjectRequest request =
100 PutObjectRequest.builder().bucket(bucket).key(objectName).contentType(contentType).contentLength(size).build();
101 s3Client.putObject(request, RequestBody.fromInputStream(inputStream, size));
102 } catch (final Exception e) {
103 throw new StorageException("Failed to upload " + objectName, e);
104 }
105 }
106
107 @Override
108 public void downloadObject(final String objectName, final OutputStream outputStream) {
109 try {
110 final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(objectName).build();
111 try (InputStream in = s3Client.getObject(request)) {
112 in.transferTo(outputStream);
113 }
114 } catch (final Exception e) {
115 throw new StorageException("Failed to download " + objectName, e);
116 }
117 }
118
119 @Override
120 public void deleteObject(final String objectName) {
121 try {
122 final DeleteObjectRequest request = DeleteObjectRequest.builder().bucket(bucket).key(objectName).build();
123 s3Client.deleteObject(request);
124 } catch (final Exception e) {
125 throw new StorageException("Failed to delete " + objectName, e);
126 }
127 }
128
129 @Override
130 public List<StorageItem> listObjects(final String prefix, final int maxItems) {
131 final List<StorageItem> items = new ArrayList<>();
132
133 try {
134 final ListObjectsV2Request.Builder requestBuilder =
135 ListObjectsV2Request.builder().bucket(bucket).delimiter("/").maxKeys(maxItems);
136
137 if (StringUtil.isNotBlank(prefix)) {
138 final String normalizedPrefix = prefix.endsWith("/") ? prefix : prefix + "/";
139 requestBuilder.prefix(normalizedPrefix);
140 }
141
142 final ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
143
144
145 for (final CommonPrefix commonPrefix : response.commonPrefixes()) {
146 final String dirName = getName(commonPrefix.prefix());
147 if (StringUtil.isNotBlank(dirName)) {
148 items.add(new StorageItem(dirName, prefix, true, 0, null, encodeId(commonPrefix.prefix())));
149 }
150 }
151
152
153 for (final S3Object s3Object : response.contents()) {
154 final String objectKey = s3Object.key();
155
156 if (!objectKey.endsWith("/")) {
157 final String fileName = getName(objectKey);
158 final ZonedDateTime lastModified =
159 s3Object.lastModified() != null ? s3Object.lastModified().atZone(java.time.ZoneId.systemDefault()) : null;
160 items.add(new StorageItem(fileName, prefix, false, s3Object.size(), lastModified, encodeId(objectKey)));
161 }
162 }
163 } catch (final NoSuchBucketException e) {
164 logger.info("Bucket does not exist: {}", bucket);
165 } catch (final Exception e) {
166 if (logger.isDebugEnabled()) {
167 logger.debug("Failed to list objects in {}", bucket, e);
168 }
169 }
170
171 return items;
172 }
173
174 @Override
175 public Map<String, String> getObjectTags(final String objectName) {
176 try {
177 final GetObjectTaggingRequest request = GetObjectTaggingRequest.builder().bucket(bucket).key(objectName).build();
178 final GetObjectTaggingResponse response = s3Client.getObjectTagging(request);
179 return response.tagSet().stream().collect(Collectors.toMap(Tag::key, Tag::value));
180 } catch (final Exception e) {
181 throw new StorageException("Failed to get tags from " + objectName, e);
182 }
183 }
184
185 @Override
186 public void setObjectTags(final String objectName, final Map<String, String> tags) {
187 try {
188 final List<Tag> tagList = tags.entrySet()
189 .stream()
190 .map(e -> Tag.builder().key(e.getKey()).value(e.getValue()).build())
191 .collect(Collectors.toList());
192
193 final PutObjectTaggingRequest request = PutObjectTaggingRequest.builder()
194 .bucket(bucket)
195 .key(objectName)
196 .tagging(Tagging.builder().tagSet(tagList).build())
197 .build();
198 s3Client.putObjectTagging(request);
199 } catch (final Exception e) {
200 throw new StorageException("Failed to update tags for " + objectName, e);
201 }
202 }
203
204 @Override
205 public void ensureBucketExists() {
206 try {
207 final HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
208 s3Client.headBucket(request);
209 } catch (final NoSuchBucketException e) {
210 try {
211 final CreateBucketRequest createRequest = CreateBucketRequest.builder().bucket(bucket).build();
212 s3Client.createBucket(createRequest);
213 logger.info("Created storage bucket: {}", bucket);
214 } catch (final Exception e1) {
215 logger.warn("Failed to create storage bucket: {}", bucket, e1);
216 }
217 } catch (final Exception e) {
218 if (logger.isDebugEnabled()) {
219 logger.debug("Failed to check bucket: {}", bucket, e);
220 }
221 }
222 }
223
224 @Override
225 public boolean isAvailable() {
226 try {
227 final HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
228 s3Client.headBucket(request);
229 return true;
230 } catch (final Exception e) {
231 return false;
232 }
233 }
234
235 @Override
236 public void close() {
237 if (s3Client != null) {
238 s3Client.close();
239 }
240 }
241
242
243
244
245
246
247
248 private String getName(final String objectName) {
249 if (StringUtil.isBlank(objectName)) {
250 return StringUtil.EMPTY;
251 }
252
253 String name = objectName;
254 if (name.endsWith("/")) {
255 name = name.substring(0, name.length() - 1);
256 }
257 final String[] values = name.split("/");
258 if (values.length == 0) {
259 return StringUtil.EMPTY;
260 }
261 return values[values.length - 1];
262 }
263
264
265
266
267
268
269
270 private String encodeId(final String objectName) {
271 if (objectName == null) {
272 return StringUtil.EMPTY;
273 }
274 return new String(Base64.getUrlEncoder().encode(objectName.getBytes(Constants.UTF_8_CHARSET)), Constants.UTF_8_CHARSET);
275 }
276 }