1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.storage;
17
18 import java.io.FileInputStream;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.io.OutputStream;
22 import java.time.ZonedDateTime;
23 import java.util.ArrayList;
24 import java.util.Base64;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.codelibs.core.lang.StringUtil;
33 import org.codelibs.fess.crawler.Constants;
34 import org.codelibs.fess.exception.StorageException;
35
36 import com.google.api.gax.paging.Page;
37 import com.google.auth.oauth2.GoogleCredentials;
38 import com.google.cloud.NoCredentials;
39 import com.google.cloud.storage.Blob;
40 import com.google.cloud.storage.BlobId;
41 import com.google.cloud.storage.BlobInfo;
42 import com.google.cloud.storage.Bucket;
43 import com.google.cloud.storage.BucketInfo;
44 import com.google.cloud.storage.Storage;
45 import com.google.cloud.storage.Storage.BlobListOption;
46 import com.google.cloud.storage.StorageOptions;
47
48
49
50
51 public class GcsStorageClient implements StorageClient {
52
53 private static final Logger logger = LogManager.getLogger(GcsStorageClient.class);
54
55 private final Storage storage;
56 private final String bucket;
57
58
59
60
61
62
63
64
65
66 public GcsStorageClient(final String projectId, final String bucket, final String endpoint, final String credentialsPath) {
67 this.bucket = bucket;
68
69 final StorageOptions.Builder builder = StorageOptions.newBuilder();
70
71 if (StringUtil.isNotBlank(projectId)) {
72 builder.setProjectId(projectId);
73 }
74
75 if (StringUtil.isNotBlank(endpoint)) {
76
77 builder.setHost(endpoint);
78 builder.setCredentials(NoCredentials.getInstance());
79 if (logger.isDebugEnabled()) {
80 logger.debug("Using custom GCS endpoint: {}", endpoint);
81 }
82 } else {
83
84 if (StringUtil.isNotBlank(credentialsPath)) {
85 try (FileInputStream fis = new FileInputStream(credentialsPath)) {
86 final GoogleCredentials credentials = GoogleCredentials.fromStream(fis);
87 builder.setCredentials(credentials);
88 } catch (final IOException e) {
89 throw new StorageException("Failed to load GCS credentials from " + credentialsPath, e);
90 }
91 }
92
93 }
94
95 this.storage = builder.build().getService();
96 }
97
98 @Override
99 public void uploadObject(final String objectName, final InputStream inputStream, final long size, final String contentType) {
100 try {
101 final BlobId blobId = BlobId.of(bucket, objectName);
102 final BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(contentType).build();
103 storage.createFrom(blobInfo, inputStream);
104 } catch (final Exception e) {
105 throw new StorageException("Failed to upload " + objectName, e);
106 }
107 }
108
109 @Override
110 public void downloadObject(final String objectName, final OutputStream outputStream) {
111 try {
112 final Blob blob = storage.get(BlobId.of(bucket, objectName));
113 if (blob == null) {
114 throw new StorageException("Object not found: " + objectName);
115 }
116 blob.downloadTo(outputStream);
117 } catch (final StorageException e) {
118 throw e;
119 } catch (final Exception e) {
120 throw new StorageException("Failed to download " + objectName, e);
121 }
122 }
123
124 @Override
125 public void deleteObject(final String objectName) {
126 try {
127 final boolean deleted = storage.delete(BlobId.of(bucket, objectName));
128 if (!deleted && logger.isDebugEnabled()) {
129 logger.debug("Object may not exist: {}", objectName);
130 }
131 } catch (final Exception e) {
132 throw new StorageException("Failed to delete " + objectName, e);
133 }
134 }
135
136 @Override
137 public List<StorageItem> listObjects(final String prefix, final int maxItems) {
138 final List<StorageItem> items = new ArrayList<>();
139 final List<StorageItem> fileItems = new ArrayList<>();
140
141 try {
142 final String searchPrefix = StringUtil.isNotBlank(prefix) ? (prefix.endsWith("/") ? prefix : prefix + "/") : "";
143
144 final Page<Blob> blobs = storage.list(bucket, BlobListOption.prefix(searchPrefix), BlobListOption.currentDirectory(),
145 BlobListOption.pageSize(maxItems));
146
147 for (final Blob blob : blobs.iterateAll()) {
148 final String blobName = blob.getName();
149
150
151 if (blobName.equals(searchPrefix)) {
152 continue;
153 }
154
155 final boolean isDirectory = blobName.endsWith("/");
156 final String name = getName(blobName);
157
158 if (StringUtil.isBlank(name)) {
159 continue;
160 }
161
162 final ZonedDateTime lastModified =
163 blob.getUpdateTimeOffsetDateTime() != null ? blob.getUpdateTimeOffsetDateTime().toZonedDateTime() : null;
164
165 final StorageItem item = new StorageItem(name, prefix, isDirectory, isDirectory ? 0 : blob.getSize(),
166 isDirectory ? null : lastModified, encodeId(blobName));
167
168 if (isDirectory) {
169 items.add(item);
170 } else {
171 fileItems.add(item);
172 }
173
174 if (items.size() + fileItems.size() >= maxItems) {
175 break;
176 }
177 }
178 } catch (final Exception e) {
179 if (logger.isDebugEnabled()) {
180 logger.debug("Failed to list objects in {}", bucket, e);
181 }
182 }
183
184 items.addAll(fileItems);
185 return items;
186 }
187
188 @Override
189 public Map<String, String> getObjectTags(final String objectName) {
190 try {
191 final Blob blob = storage.get(BlobId.of(bucket, objectName));
192 if (blob == null) {
193 return Collections.emptyMap();
194 }
195
196 final Map<String, String> metadata = blob.getMetadata();
197 return metadata != null ? new HashMap<>(metadata) : Collections.emptyMap();
198 } catch (final Exception e) {
199 throw new StorageException("Failed to get tags from " + objectName, e);
200 }
201 }
202
203 @Override
204 public void setObjectTags(final String objectName, final Map<String, String> tags) {
205 try {
206 final Blob blob = storage.get(BlobId.of(bucket, objectName));
207 if (blob != null) {
208
209 blob.toBuilder().setMetadata(tags).build().update();
210 } else {
211 throw new StorageException("Object not found: " + objectName);
212 }
213 } catch (final StorageException e) {
214 throw e;
215 } catch (final Exception e) {
216 throw new StorageException("Failed to update tags for " + objectName, e);
217 }
218 }
219
220 @Override
221 public void ensureBucketExists() {
222 try {
223 final Bucket existingBucket = storage.get(bucket);
224 if (existingBucket == null) {
225 storage.create(BucketInfo.newBuilder(bucket).build());
226 logger.info("Created storage bucket: {}", bucket);
227 }
228 } catch (final Exception e) {
229 logger.warn("Failed to ensure bucket exists: {}", bucket, e);
230 }
231 }
232
233 @Override
234 public boolean isAvailable() {
235 try {
236 return storage.get(bucket) != null;
237 } catch (final Exception e) {
238 return false;
239 }
240 }
241
242 @Override
243 public void close() {
244
245
246 try {
247 storage.close();
248 } catch (final Exception e) {
249 if (logger.isDebugEnabled()) {
250 logger.debug("Failed to close GCS storage client", e);
251 }
252 }
253 }
254
255
256
257
258
259
260
261 private String getName(final String objectName) {
262 if (StringUtil.isBlank(objectName)) {
263 return StringUtil.EMPTY;
264 }
265
266 String name = objectName;
267 if (name.endsWith("/")) {
268 name = name.substring(0, name.length() - 1);
269 }
270 final String[] values = name.split("/");
271 if (values.length == 0) {
272 return StringUtil.EMPTY;
273 }
274 return values[values.length - 1];
275 }
276
277
278
279
280
281
282
283 private String encodeId(final String objectName) {
284 if (objectName == null) {
285 return StringUtil.EMPTY;
286 }
287 return new String(Base64.getUrlEncoder().encode(objectName.getBytes(Constants.UTF_8_CHARSET)), Constants.UTF_8_CHARSET);
288 }
289 }