View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.storage;
17  
18  import java.io.FileInputStream;
19  import java.io.IOException;
20  import java.io.InputStream;
21  import java.io.OutputStream;
22  import java.time.ZonedDateTime;
23  import java.util.ArrayList;
24  import java.util.Base64;
25  import java.util.Collections;
26  import java.util.HashMap;
27  import java.util.List;
28  import java.util.Map;
29  
30  import org.apache.logging.log4j.LogManager;
31  import org.apache.logging.log4j.Logger;
32  import org.codelibs.core.lang.StringUtil;
33  import org.codelibs.fess.crawler.Constants;
34  import org.codelibs.fess.exception.StorageException;
35  
36  import com.google.api.gax.paging.Page;
37  import com.google.auth.oauth2.GoogleCredentials;
38  import com.google.cloud.NoCredentials;
39  import com.google.cloud.storage.Blob;
40  import com.google.cloud.storage.BlobId;
41  import com.google.cloud.storage.BlobInfo;
42  import com.google.cloud.storage.Bucket;
43  import com.google.cloud.storage.BucketInfo;
44  import com.google.cloud.storage.Storage;
45  import com.google.cloud.storage.Storage.BlobListOption;
46  import com.google.cloud.storage.StorageOptions;
47  
48  /**
49   * Google Cloud Storage client implementation.
50   */
51  public class GcsStorageClient implements StorageClient {
52  
53      private static final Logger logger = LogManager.getLogger(GcsStorageClient.class);
54  
55      private final Storage storage;
56      private final String bucket;
57  
58      /**
59       * Creates a new GcsStorageClient instance.
60       *
61       * @param projectId the GCS project ID
62       * @param bucket the bucket name
63       * @param endpoint the custom endpoint URL (optional, for fake-gcs-server etc.)
64       * @param credentialsPath the path to the credentials JSON file (optional)
65       */
66      public GcsStorageClient(final String projectId, final String bucket, final String endpoint, final String credentialsPath) {
67          this.bucket = bucket;
68  
69          final StorageOptions.Builder builder = StorageOptions.newBuilder();
70  
71          if (StringUtil.isNotBlank(projectId)) {
72              builder.setProjectId(projectId);
73          }
74  
75          if (StringUtil.isNotBlank(endpoint)) {
76              // For fake-gcs-server or custom endpoint
77              builder.setHost(endpoint);
78              builder.setCredentials(NoCredentials.getInstance());
79              if (logger.isDebugEnabled()) {
80                  logger.debug("Using custom GCS endpoint: {}", endpoint);
81              }
82          } else {
83              // Production: use credentials file or default credentials
84              if (StringUtil.isNotBlank(credentialsPath)) {
85                  try (FileInputStream fis = new FileInputStream(credentialsPath)) {
86                      final GoogleCredentials credentials = GoogleCredentials.fromStream(fis);
87                      builder.setCredentials(credentials);
88                  } catch (final IOException e) {
89                      throw new StorageException("Failed to load GCS credentials from " + credentialsPath, e);
90                  }
91              }
92              // If no credentials path, uses default credentials (GOOGLE_APPLICATION_CREDENTIALS env var)
93          }
94  
95          this.storage = builder.build().getService();
96      }
97  
98      @Override
99      public void uploadObject(final String objectName, final InputStream inputStream, final long size, final String contentType) {
100         try {
101             final BlobId blobId = BlobId.of(bucket, objectName);
102             final BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(contentType).build();
103             storage.createFrom(blobInfo, inputStream);
104         } catch (final Exception e) {
105             throw new StorageException("Failed to upload " + objectName, e);
106         }
107     }
108 
109     @Override
110     public void downloadObject(final String objectName, final OutputStream outputStream) {
111         try {
112             final Blob blob = storage.get(BlobId.of(bucket, objectName));
113             if (blob == null) {
114                 throw new StorageException("Object not found: " + objectName);
115             }
116             blob.downloadTo(outputStream);
117         } catch (final StorageException e) {
118             throw e;
119         } catch (final Exception e) {
120             throw new StorageException("Failed to download " + objectName, e);
121         }
122     }
123 
124     @Override
125     public void deleteObject(final String objectName) {
126         try {
127             final boolean deleted = storage.delete(BlobId.of(bucket, objectName));
128             if (!deleted && logger.isDebugEnabled()) {
129                 logger.debug("Object may not exist: {}", objectName);
130             }
131         } catch (final Exception e) {
132             throw new StorageException("Failed to delete " + objectName, e);
133         }
134     }
135 
136     @Override
137     public List<StorageItem> listObjects(final String prefix, final int maxItems) {
138         final List<StorageItem> items = new ArrayList<>();
139         final List<StorageItem> fileItems = new ArrayList<>();
140 
141         try {
142             final String searchPrefix = StringUtil.isNotBlank(prefix) ? (prefix.endsWith("/") ? prefix : prefix + "/") : "";
143 
144             final Page<Blob> blobs = storage.list(bucket, BlobListOption.prefix(searchPrefix), BlobListOption.currentDirectory(),
145                     BlobListOption.pageSize(maxItems));
146 
147             for (final Blob blob : blobs.iterateAll()) {
148                 final String blobName = blob.getName();
149 
150                 // Skip the prefix itself
151                 if (blobName.equals(searchPrefix)) {
152                     continue;
153                 }
154 
155                 final boolean isDirectory = blobName.endsWith("/");
156                 final String name = getName(blobName);
157 
158                 if (StringUtil.isBlank(name)) {
159                     continue;
160                 }
161 
162                 final ZonedDateTime lastModified =
163                         blob.getUpdateTimeOffsetDateTime() != null ? blob.getUpdateTimeOffsetDateTime().toZonedDateTime() : null;
164 
165                 final StorageItem item = new StorageItem(name, prefix, isDirectory, isDirectory ? 0 : blob.getSize(),
166                         isDirectory ? null : lastModified, encodeId(blobName));
167 
168                 if (isDirectory) {
169                     items.add(item);
170                 } else {
171                     fileItems.add(item);
172                 }
173 
174                 if (items.size() + fileItems.size() >= maxItems) {
175                     break;
176                 }
177             }
178         } catch (final Exception e) {
179             if (logger.isDebugEnabled()) {
180                 logger.debug("Failed to list objects in {}", bucket, e);
181             }
182         }
183 
184         items.addAll(fileItems);
185         return items;
186     }
187 
188     @Override
189     public Map<String, String> getObjectTags(final String objectName) {
190         try {
191             final Blob blob = storage.get(BlobId.of(bucket, objectName));
192             if (blob == null) {
193                 return Collections.emptyMap();
194             }
195             // GCS uses metadata instead of tags
196             final Map<String, String> metadata = blob.getMetadata();
197             return metadata != null ? new HashMap<>(metadata) : Collections.emptyMap();
198         } catch (final Exception e) {
199             throw new StorageException("Failed to get tags from " + objectName, e);
200         }
201     }
202 
203     @Override
204     public void setObjectTags(final String objectName, final Map<String, String> tags) {
205         try {
206             final Blob blob = storage.get(BlobId.of(bucket, objectName));
207             if (blob != null) {
208                 // GCS uses metadata instead of tags
209                 blob.toBuilder().setMetadata(tags).build().update();
210             } else {
211                 throw new StorageException("Object not found: " + objectName);
212             }
213         } catch (final StorageException e) {
214             throw e;
215         } catch (final Exception e) {
216             throw new StorageException("Failed to update tags for " + objectName, e);
217         }
218     }
219 
220     @Override
221     public void ensureBucketExists() {
222         try {
223             final Bucket existingBucket = storage.get(bucket);
224             if (existingBucket == null) {
225                 storage.create(BucketInfo.newBuilder(bucket).build());
226                 logger.info("Created storage bucket: {}", bucket);
227             }
228         } catch (final Exception e) {
229             logger.warn("Failed to ensure bucket exists: {}", bucket, e);
230         }
231     }
232 
233     @Override
234     public boolean isAvailable() {
235         try {
236             return storage.get(bucket) != null;
237         } catch (final Exception e) {
238             return false;
239         }
240     }
241 
242     @Override
243     public void close() {
244         // GCS Storage client doesn't require explicit close
245         // but we can try to close it if needed
246         try {
247             storage.close();
248         } catch (final Exception e) {
249             if (logger.isDebugEnabled()) {
250                 logger.debug("Failed to close GCS storage client", e);
251             }
252         }
253     }
254 
255     /**
256      * Extracts the file/directory name from a full object path.
257      *
258      * @param objectName the full object path
259      * @return the name portion of the path
260      */
261     private String getName(final String objectName) {
262         if (StringUtil.isBlank(objectName)) {
263             return StringUtil.EMPTY;
264         }
265         // Remove trailing slash if present
266         String name = objectName;
267         if (name.endsWith("/")) {
268             name = name.substring(0, name.length() - 1);
269         }
270         final String[] values = name.split("/");
271         if (values.length == 0) {
272             return StringUtil.EMPTY;
273         }
274         return values[values.length - 1];
275     }
276 
277     /**
278      * Encodes an object name to a URL-safe base64 string.
279      *
280      * @param objectName the object name to encode
281      * @return base64 encoded string
282      */
283     private String encodeId(final String objectName) {
284         if (objectName == null) {
285             return StringUtil.EMPTY;
286         }
287         return new String(Base64.getUrlEncoder().encode(objectName.getBytes(Constants.UTF_8_CHARSET)), Constants.UTF_8_CHARSET);
288     }
289 }