1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.dict.stopwords;
17
18 import java.io.BufferedReader;
19 import java.io.BufferedWriter;
20 import java.io.Closeable;
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.InputStreamReader;
26 import java.io.OutputStreamWriter;
27 import java.io.Writer;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.Date;
31 import java.util.List;
32
33 import org.codelibs.core.io.CloseableUtil;
34 import org.codelibs.core.lang.StringUtil;
35 import org.codelibs.curl.CurlResponse;
36 import org.codelibs.fess.Constants;
37 import org.codelibs.fess.dict.DictionaryException;
38 import org.codelibs.fess.dict.DictionaryFile;
39 import org.codelibs.fess.util.ComponentUtil;
40 import org.dbflute.optional.OptionalEntity;
41
42
43
44
45
46
47
48
49
50
51 public class StopwordsFile extends DictionaryFile<StopwordsItem> {
52 private static final String STOPWORDS = "stopwords";
53
54
55 List<StopwordsItem> stopwordsItemList;
56
57
58
59
60
61
62
63
64 public StopwordsFile(final String id, final String path, final Date timestamp) {
65 super(id, path, timestamp);
66 }
67
68 @Override
69 public String getType() {
70 return STOPWORDS;
71 }
72
73 @Override
74 public String getPath() {
75 return path;
76 }
77
78 @Override
79 public synchronized OptionalEntity<StopwordsItem> get(final long id) {
80 if (stopwordsItemList == null) {
81 reload(null);
82 }
83
84 for (final StopwordsItem StopwordsItem : stopwordsItemList) {
85 if (id == StopwordsItem.getId()) {
86 return OptionalEntity.of(StopwordsItem);
87 }
88 }
89 return OptionalEntity.empty();
90 }
91
92 @Override
93 public synchronized PagingList<StopwordsItem> selectList(final int offset, final int size) {
94 if (stopwordsItemList == null) {
95 reload(null);
96 }
97
98 if (offset >= stopwordsItemList.size() || offset < 0) {
99 return new PagingList<>(Collections.<StopwordsItem> emptyList(), offset, size, stopwordsItemList.size());
100 }
101
102 int toIndex = offset + size;
103 if (toIndex > stopwordsItemList.size()) {
104 toIndex = stopwordsItemList.size();
105 }
106
107 return new PagingList<>(stopwordsItemList.subList(offset, toIndex), offset, size, stopwordsItemList.size());
108 }
109
110 @Override
111 public synchronized void insert(final StopwordsItem item) {
112 try (StopwordsUpdater updater = new StopwordsUpdater(item)) {
113 reload(updater);
114 }
115 }
116
117 @Override
118 public synchronized void update(final StopwordsItem item) {
119 try (StopwordsUpdater updater = new StopwordsUpdater(item)) {
120 reload(updater);
121 }
122 }
123
124 @Override
125 public synchronized void delete(final StopwordsItem item) {
126 final StopwordsItem stopwordsItem = item;
127 stopwordsItem.setNewInput(StringUtil.EMPTY);
128 try (StopwordsUpdater updater = new StopwordsUpdater(item)) {
129 reload(updater);
130 }
131 }
132
133
134
135
136
137
138
139 protected void reload(final StopwordsUpdater updater) {
140 try (CurlResponse curlResponse = dictionaryManager.getContentResponse(this)) {
141 reload(updater, curlResponse.getContentAsStream());
142 } catch (final IOException e) {
143 throw new DictionaryException("Failed to parse " + path, e);
144 }
145 }
146
147
148
149
150
151
152
153
154 protected void reload(final StopwordsUpdater updater, final InputStream in) {
155 final List<StopwordsItem> itemList = new ArrayList<>();
156 try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
157 long id = 0;
158 String line = null;
159 while ((line = reader.readLine()) != null) {
160 if (line.length() == 0 || line.charAt(0) == '#') {
161 if (updater != null) {
162 updater.write(line);
163 }
164 continue;
165 }
166
167 final String inputStrings = line;
168 final String input = unescape(inputStrings);
169
170 if (input.length() > 0) {
171 id++;
172 final StopwordsItem item = new StopwordsItem(id, input);
173 if (updater != null) {
174 final StopwordsItem newItem = updater.write(item);
175 if (newItem != null) {
176 itemList.add(newItem);
177 } else {
178 id--;
179 }
180 } else {
181 itemList.add(item);
182 }
183 }
184 }
185 if (updater != null) {
186 final StopwordsItem item = updater.commit();
187 if (item != null) {
188 itemList.add(item);
189 }
190 }
191 stopwordsItemList = itemList;
192 } catch (final IOException e) {
193 throw new DictionaryException("Failed to parse " + path, e);
194 }
195 }
196
197 private String unescape(final String s) {
198 if (s.indexOf('\\') >= 0) {
199 final StringBuilder sb = new StringBuilder();
200 for (int i = 0; i < s.length(); i++) {
201 final char ch = s.charAt(i);
202 if (ch == '\\' && i < s.length() - 1) {
203 i++;
204 sb.append(s.charAt(i));
205 } else {
206 sb.append(ch);
207 }
208 }
209 return sb.toString();
210 }
211 return s;
212 }
213
214
215
216
217
218
219 public String getSimpleName() {
220 return new File(path).getName();
221 }
222
223
224
225
226
227
228
229 public synchronized void update(final InputStream in) throws IOException {
230 try (StopwordsUpdater updater = new StopwordsUpdater(null)) {
231 reload(updater, in);
232 }
233 }
234
235 @Override
236 public String toString() {
237 return "StopwordsFile [path=" + path + ", stopwordsItemList=" + stopwordsItemList + ", id=" + id + "]";
238 }
239
240
241
242
243
244
245 protected class StopwordsUpdater implements Closeable {
246
247
248 protected boolean isCommit = false;
249
250
251 protected File newFile;
252
253
254 protected Writer writer;
255
256
257 protected StopwordsItem item;
258
259
260
261
262
263
264
265 protected StopwordsUpdater(final StopwordsItem newItem) {
266 FileOutputStream fos = null;
267 try {
268 newFile = ComponentUtil.getSystemHelper().createTempFile(STOPWORDS, ".txt");
269 fos = new FileOutputStream(newFile);
270 writer = new BufferedWriter(new OutputStreamWriter(fos, Constants.UTF_8));
271 fos = null;
272 } catch (final Exception e) {
273 if (fos != null) {
274 try {
275 fos.close();
276 } catch (final IOException ioe) {
277
278 }
279 }
280 if (newFile != null) {
281 newFile.delete();
282 }
283 throw new DictionaryException("Failed to write a userDict file.", e);
284 }
285 item = newItem;
286 }
287
288
289
290
291
292
293
294
295
296 public StopwordsItem write(final StopwordsItem oldItem) {
297 try {
298 if (item == null || item.getId() != oldItem.getId() || !item.isUpdated()) {
299 writer.write(oldItem.toLineString());
300 writer.write(Constants.LINE_SEPARATOR);
301 return oldItem;
302 }
303 if (!item.equals(oldItem)) {
304 throw new DictionaryException("Stopwords file was updated: old=" + oldItem + " : new=" + item);
305 }
306 try {
307 if (!item.isDeleted()) {
308
309 writer.write(item.toLineString());
310 writer.write(Constants.LINE_SEPARATOR);
311 return new StopwordsItem(item.getId(), item.getNewInput());
312 }
313 return null;
314 } finally {
315 item.setNewInput(null);
316 }
317 } catch (final IOException e) {
318 throw new DictionaryException("Failed to write: " + oldItem + " -> " + item, e);
319 }
320 }
321
322
323
324
325
326
327
328 public void write(final String line) {
329 try {
330 writer.write(line);
331 writer.write(Constants.LINE_SEPARATOR);
332 } catch (final IOException e) {
333 throw new DictionaryException("Failed to write: " + line, e);
334 }
335 }
336
337
338
339
340
341
342
343
344 public StopwordsItem commit() {
345 isCommit = true;
346 if (item != null && item.isUpdated()) {
347 try {
348 writer.write(item.toLineString());
349 writer.write(Constants.LINE_SEPARATOR);
350 return item;
351 } catch (final IOException e) {
352 throw new DictionaryException("Failed to write: " + item, e);
353 }
354 }
355 return null;
356 }
357
358 @Override
359 public void close() {
360 try {
361 writer.flush();
362 } catch (final IOException e) {
363
364 }
365 CloseableUtil.closeQuietly(writer);
366
367 if (isCommit) {
368 try {
369 dictionaryManager.store(StopwordsFile.this, newFile);
370 } finally {
371 newFile.delete();
372 }
373 } else {
374 newFile.delete();
375 }
376 }
377 }
378 }