View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.storage;
17  
18  import java.io.InputStream;
19  import java.io.OutputStream;
20  import java.net.URI;
21  import java.time.ZonedDateTime;
22  import java.util.ArrayList;
23  import java.util.Base64;
24  import java.util.List;
25  import java.util.Map;
26  import java.util.stream.Collectors;
27  
28  import org.apache.logging.log4j.LogManager;
29  import org.apache.logging.log4j.Logger;
30  import org.codelibs.core.lang.StringUtil;
31  import org.codelibs.fess.crawler.Constants;
32  import org.codelibs.fess.exception.StorageException;
33  
34  import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
35  import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
36  import software.amazon.awssdk.core.sync.RequestBody;
37  import software.amazon.awssdk.regions.Region;
38  import software.amazon.awssdk.services.s3.S3Client;
39  import software.amazon.awssdk.services.s3.S3ClientBuilder;
40  import software.amazon.awssdk.services.s3.model.CommonPrefix;
41  import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
42  import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
43  import software.amazon.awssdk.services.s3.model.GetObjectRequest;
44  import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
45  import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse;
46  import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
47  import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
48  import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
49  import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
50  import software.amazon.awssdk.services.s3.model.PutObjectRequest;
51  import software.amazon.awssdk.services.s3.model.PutObjectTaggingRequest;
52  import software.amazon.awssdk.services.s3.model.S3Object;
53  import software.amazon.awssdk.services.s3.model.Tag;
54  import software.amazon.awssdk.services.s3.model.Tagging;
55  
56  /**
57   * S3-compatible storage client implementation using AWS SDK v2.
58   * Supports Amazon S3, MinIO, and other S3-compatible storage systems.
59   */
60  public class S3StorageClient implements StorageClient {
61  
62      private static final Logger logger = LogManager.getLogger(S3StorageClient.class);
63  
64      private final S3Client s3Client;
65      private final String bucket;
66  
67      /**
68       * Creates a new S3StorageClient instance.
69       *
70       * @param endpoint the S3 endpoint URL (null for AWS default)
71       * @param accessKey the AWS access key
72       * @param secretKey the AWS secret key
73       * @param bucket the bucket name
74       * @param region the AWS region
75       */
76      public S3StorageClient(final String endpoint, final String accessKey, final String secretKey, final String bucket,
77              final String region) {
78          this.bucket = bucket;
79  
80          final AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
81          final S3ClientBuilder builder =
82                  S3Client.builder().credentialsProvider(StaticCredentialsProvider.create(credentials)).region(Region.of(getRegion(region)));
83  
84          // For non-AWS endpoints (MinIO, etc.), set custom endpoint with path-style access
85          if (StringUtil.isNotBlank(endpoint)) {
86              builder.endpointOverride(URI.create(endpoint)).forcePathStyle(true);
87          }
88  
89          this.s3Client = builder.build();
90      }
91  
92      private String getRegion(final String region) {
93          return StringUtil.isNotBlank(region) ? region : "us-east-1";
94      }
95  
96      @Override
97      public void uploadObject(final String objectName, final InputStream inputStream, final long size, final String contentType) {
98          try {
99              final PutObjectRequest request =
100                     PutObjectRequest.builder().bucket(bucket).key(objectName).contentType(contentType).contentLength(size).build();
101             s3Client.putObject(request, RequestBody.fromInputStream(inputStream, size));
102         } catch (final Exception e) {
103             throw new StorageException("Failed to upload " + objectName, e);
104         }
105     }
106 
107     @Override
108     public void downloadObject(final String objectName, final OutputStream outputStream) {
109         try {
110             final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(objectName).build();
111             try (InputStream in = s3Client.getObject(request)) {
112                 in.transferTo(outputStream);
113             }
114         } catch (final Exception e) {
115             throw new StorageException("Failed to download " + objectName, e);
116         }
117     }
118 
119     @Override
120     public void deleteObject(final String objectName) {
121         try {
122             final DeleteObjectRequest request = DeleteObjectRequest.builder().bucket(bucket).key(objectName).build();
123             s3Client.deleteObject(request);
124         } catch (final Exception e) {
125             throw new StorageException("Failed to delete " + objectName, e);
126         }
127     }
128 
129     @Override
130     public List<StorageItem> listObjects(final String prefix, final int maxItems) {
131         final List<StorageItem> items = new ArrayList<>();
132 
133         try {
134             final ListObjectsV2Request.Builder requestBuilder =
135                     ListObjectsV2Request.builder().bucket(bucket).delimiter("/").maxKeys(maxItems);
136 
137             if (StringUtil.isNotBlank(prefix)) {
138                 final String normalizedPrefix = prefix.endsWith("/") ? prefix : prefix + "/";
139                 requestBuilder.prefix(normalizedPrefix);
140             }
141 
142             final ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
143 
144             // Process common prefixes (directories)
145             for (final CommonPrefix commonPrefix : response.commonPrefixes()) {
146                 final String dirName = getName(commonPrefix.prefix());
147                 if (StringUtil.isNotBlank(dirName)) {
148                     items.add(new StorageItem(dirName, prefix, true, 0, null, encodeId(commonPrefix.prefix())));
149                 }
150             }
151 
152             // Process objects (files)
153             for (final S3Object s3Object : response.contents()) {
154                 final String objectKey = s3Object.key();
155                 // Skip directory markers (objects ending with /)
156                 if (!objectKey.endsWith("/")) {
157                     final String fileName = getName(objectKey);
158                     final ZonedDateTime lastModified =
159                             s3Object.lastModified() != null ? s3Object.lastModified().atZone(java.time.ZoneId.systemDefault()) : null;
160                     items.add(new StorageItem(fileName, prefix, false, s3Object.size(), lastModified, encodeId(objectKey)));
161                 }
162             }
163         } catch (final NoSuchBucketException e) {
164             logger.info("Bucket does not exist: {}", bucket);
165         } catch (final Exception e) {
166             if (logger.isDebugEnabled()) {
167                 logger.debug("Failed to list objects in {}", bucket, e);
168             }
169         }
170 
171         return items;
172     }
173 
174     @Override
175     public Map<String, String> getObjectTags(final String objectName) {
176         try {
177             final GetObjectTaggingRequest request = GetObjectTaggingRequest.builder().bucket(bucket).key(objectName).build();
178             final GetObjectTaggingResponse response = s3Client.getObjectTagging(request);
179             return response.tagSet().stream().collect(Collectors.toMap(Tag::key, Tag::value));
180         } catch (final Exception e) {
181             throw new StorageException("Failed to get tags from " + objectName, e);
182         }
183     }
184 
185     @Override
186     public void setObjectTags(final String objectName, final Map<String, String> tags) {
187         try {
188             final List<Tag> tagList = tags.entrySet()
189                     .stream()
190                     .map(e -> Tag.builder().key(e.getKey()).value(e.getValue()).build())
191                     .collect(Collectors.toList());
192 
193             final PutObjectTaggingRequest request = PutObjectTaggingRequest.builder()
194                     .bucket(bucket)
195                     .key(objectName)
196                     .tagging(Tagging.builder().tagSet(tagList).build())
197                     .build();
198             s3Client.putObjectTagging(request);
199         } catch (final Exception e) {
200             throw new StorageException("Failed to update tags for " + objectName, e);
201         }
202     }
203 
204     @Override
205     public void ensureBucketExists() {
206         try {
207             final HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
208             s3Client.headBucket(request);
209         } catch (final NoSuchBucketException e) {
210             try {
211                 final CreateBucketRequest createRequest = CreateBucketRequest.builder().bucket(bucket).build();
212                 s3Client.createBucket(createRequest);
213                 logger.info("Created storage bucket: {}", bucket);
214             } catch (final Exception e1) {
215                 logger.warn("Failed to create storage bucket: {}", bucket, e1);
216             }
217         } catch (final Exception e) {
218             if (logger.isDebugEnabled()) {
219                 logger.debug("Failed to check bucket: {}", bucket, e);
220             }
221         }
222     }
223 
224     @Override
225     public boolean isAvailable() {
226         try {
227             final HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
228             s3Client.headBucket(request);
229             return true;
230         } catch (final Exception e) {
231             return false;
232         }
233     }
234 
235     @Override
236     public void close() {
237         if (s3Client != null) {
238             s3Client.close();
239         }
240     }
241 
242     /**
243      * Extracts the file/directory name from a full object path.
244      *
245      * @param objectName the full object path
246      * @return the name portion of the path
247      */
248     private String getName(final String objectName) {
249         if (StringUtil.isBlank(objectName)) {
250             return StringUtil.EMPTY;
251         }
252         // Remove trailing slash if present
253         String name = objectName;
254         if (name.endsWith("/")) {
255             name = name.substring(0, name.length() - 1);
256         }
257         final String[] values = name.split("/");
258         if (values.length == 0) {
259             return StringUtil.EMPTY;
260         }
261         return values[values.length - 1];
262     }
263 
264     /**
265      * Encodes an object name to a URL-safe base64 string.
266      *
267      * @param objectName the object name to encode
268      * @return base64 encoded string
269      */
270     private String encodeId(final String objectName) {
271         if (objectName == null) {
272             return StringUtil.EMPTY;
273         }
274         return new String(Base64.getUrlEncoder().encode(objectName.getBytes(Constants.UTF_8_CHARSET)), Constants.UTF_8_CHARSET);
275     }
276 }