View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.dict.stopwords;
17  
18  import java.io.BufferedReader;
19  import java.io.BufferedWriter;
20  import java.io.Closeable;
21  import java.io.File;
22  import java.io.FileOutputStream;
23  import java.io.IOException;
24  import java.io.InputStream;
25  import java.io.InputStreamReader;
26  import java.io.OutputStreamWriter;
27  import java.io.Writer;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.Date;
31  import java.util.List;
32  
33  import org.codelibs.core.io.CloseableUtil;
34  import org.codelibs.core.lang.StringUtil;
35  import org.codelibs.curl.CurlResponse;
36  import org.codelibs.fess.Constants;
37  import org.codelibs.fess.dict.DictionaryException;
38  import org.codelibs.fess.dict.DictionaryFile;
39  import org.codelibs.fess.util.ComponentUtil;
40  import org.dbflute.optional.OptionalEntity;
41  
42  /**
43   * Manages a dictionary file for stopwords.
44   * This class handles reading, parsing, and updating files that contain
45   * a list of stopwords. Each line in the file represents a single stopword.
46   *
47   * The class provides methods for retrieving, adding, updating, and
48   * deleting stopword items, as well as reloading the dictionary
49   * from its source file.
50   */
51  public class StopwordsFile extends DictionaryFile<StopwordsItem> {
52      private static final String STOPWORDS = "stopwords";
53  
54      /** The list of stopword items loaded from the dictionary file. */
55      List<StopwordsItem> stopwordsItemList;
56  
57      /**
58       * Constructs a new stopwords file.
59       *
60       * @param id        The unique identifier for this dictionary file.
61       * @param path      The path to the dictionary file.
62       * @param timestamp The last modified timestamp of the file.
63       */
64      public StopwordsFile(final String id, final String path, final Date timestamp) {
65          super(id, path, timestamp);
66      }
67  
68      @Override
69      public String getType() {
70          return STOPWORDS;
71      }
72  
73      @Override
74      public String getPath() {
75          return path;
76      }
77  
78      @Override
79      public synchronized OptionalEntity<StopwordsItem> get(final long id) {
80          if (stopwordsItemList == null) {
81              reload(null);
82          }
83  
84          for (final StopwordsItem StopwordsItem : stopwordsItemList) {
85              if (id == StopwordsItem.getId()) {
86                  return OptionalEntity.of(StopwordsItem);
87              }
88          }
89          return OptionalEntity.empty();
90      }
91  
92      @Override
93      public synchronized PagingList<StopwordsItem> selectList(final int offset, final int size) {
94          if (stopwordsItemList == null) {
95              reload(null);
96          }
97  
98          if (offset >= stopwordsItemList.size() || offset < 0) {
99              return new PagingList<>(Collections.<StopwordsItem> emptyList(), offset, size, stopwordsItemList.size());
100         }
101 
102         int toIndex = offset + size;
103         if (toIndex > stopwordsItemList.size()) {
104             toIndex = stopwordsItemList.size();
105         }
106 
107         return new PagingList<>(stopwordsItemList.subList(offset, toIndex), offset, size, stopwordsItemList.size());
108     }
109 
110     @Override
111     public synchronized void insert(final StopwordsItem item) {
112         try (StopwordsUpdater updater = new StopwordsUpdater(item)) {
113             reload(updater);
114         }
115     }
116 
117     @Override
118     public synchronized void update(final StopwordsItem item) {
119         try (StopwordsUpdater updater = new StopwordsUpdater(item)) {
120             reload(updater);
121         }
122     }
123 
124     @Override
125     public synchronized void delete(final StopwordsItem item) {
126         final StopwordsItem stopwordsItem = item;
127         stopwordsItem.setNewInput(StringUtil.EMPTY);
128         try (StopwordsUpdater updater = new StopwordsUpdater(item)) {
129             reload(updater);
130         }
131     }
132 
133     /**
134      * Reloads the stopwords dictionary from its source file.
135      *
136      * @param updater An optional updater to apply changes during reload.
137      * @throws DictionaryException if the dictionary file cannot be read.
138      */
139     protected void reload(final StopwordsUpdater updater) {
140         try (CurlResponse curlResponse = dictionaryManager.getContentResponse(this)) {
141             reload(updater, curlResponse.getContentAsStream());
142         } catch (final IOException e) {
143             throw new DictionaryException("Failed to parse " + path, e);
144         }
145     }
146 
147     /**
148      * Reloads the stopwords dictionary from an input stream.
149      *
150      * @param updater An optional updater to apply changes.
151      * @param in      The input stream to read the dictionary from.
152      * @throws DictionaryException if the input stream cannot be parsed.
153      */
154     protected void reload(final StopwordsUpdater updater, final InputStream in) {
155         final List<StopwordsItem> itemList = new ArrayList<>();
156         try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
157             long id = 0;
158             String line = null;
159             while ((line = reader.readLine()) != null) {
160                 if (line.length() == 0 || line.charAt(0) == '#') {
161                     if (updater != null) {
162                         updater.write(line);
163                     }
164                     continue; // ignore empty lines and comments
165                 }
166 
167                 final String inputStrings = line;
168                 final String input = unescape(inputStrings);
169 
170                 if (input.length() > 0) {
171                     id++;
172                     final StopwordsItem item = new StopwordsItem(id, input);
173                     if (updater != null) {
174                         final StopwordsItem newItem = updater.write(item);
175                         if (newItem != null) {
176                             itemList.add(newItem);
177                         } else {
178                             id--;
179                         }
180                     } else {
181                         itemList.add(item);
182                     }
183                 }
184             }
185             if (updater != null) {
186                 final StopwordsItem item = updater.commit();
187                 if (item != null) {
188                     itemList.add(item);
189                 }
190             }
191             stopwordsItemList = itemList;
192         } catch (final IOException e) {
193             throw new DictionaryException("Failed to parse " + path, e);
194         }
195     }
196 
197     private String unescape(final String s) {
198         if (s.indexOf('\\') >= 0) {
199             final StringBuilder sb = new StringBuilder();
200             for (int i = 0; i < s.length(); i++) {
201                 final char ch = s.charAt(i);
202                 if (ch == '\\' && i < s.length() - 1) {
203                     i++;
204                     sb.append(s.charAt(i));
205                 } else {
206                     sb.append(ch);
207                 }
208             }
209             return sb.toString();
210         }
211         return s;
212     }
213 
214     /**
215      * Returns the simple name of the dictionary file.
216      *
217      * @return The file name without the path.
218      */
219     public String getSimpleName() {
220         return new File(path).getName();
221     }
222 
223     /**
224      * Updates the dictionary file with content from an input stream.
225      *
226      * @param in The input stream containing the new dictionary content.
227      * @throws IOException if an I/O error occurs.
228      */
229     public synchronized void update(final InputStream in) throws IOException {
230         try (StopwordsUpdater updater = new StopwordsUpdater(null)) {
231             reload(updater, in);
232         }
233     }
234 
235     @Override
236     public String toString() {
237         return "StopwordsFile [path=" + path + ", stopwordsItemList=" + stopwordsItemList + ", id=" + id + "]";
238     }
239 
240     /**
241      * An inner class for updating the stopwords file.
242      * This class handles the process of writing changes to a temporary file
243      * and then replacing the original file upon successful commit.
244      */
245     protected class StopwordsUpdater implements Closeable {
246 
247         /** A flag indicating whether the changes have been committed. */
248         protected boolean isCommit = false;
249 
250         /** The temporary file to write changes to. */
251         protected File newFile;
252 
253         /** The writer for the temporary file. */
254         protected Writer writer;
255 
256         /** The stopword item being added or updated. */
257         protected StopwordsItem item;
258 
259         /**
260          * Constructs a new updater for a stopword item.
261          *
262          * @param newItem The item to be added or updated.
263          * @throws DictionaryException if the temporary file cannot be created.
264          */
265         protected StopwordsUpdater(final StopwordsItem newItem) {
266             FileOutputStream fos = null;
267             try {
268                 newFile = ComponentUtil.getSystemHelper().createTempFile(STOPWORDS, ".txt");
269                 fos = new FileOutputStream(newFile);
270                 writer = new BufferedWriter(new OutputStreamWriter(fos, Constants.UTF_8));
271                 fos = null; // Successfully wrapped, no need to close explicitly
272             } catch (final Exception e) {
273                 if (fos != null) {
274                     try {
275                         fos.close();
276                     } catch (final IOException ioe) {
277                         // Ignore close exception
278                     }
279                 }
280                 if (newFile != null) {
281                     newFile.delete();
282                 }
283                 throw new DictionaryException("Failed to write a userDict file.", e);
284             }
285             item = newItem;
286         }
287 
288         /**
289          * Writes a stopword item to the temporary file.
290          * If the item is being updated, it writes the new version.
291          *
292          * @param oldItem The original item from the dictionary.
293          * @return The written item, or null if the item was deleted.
294          * @throws DictionaryException if the file was updated concurrently.
295          */
296         public StopwordsItem write(final StopwordsItem oldItem) {
297             try {
298                 if (item == null || item.getId() != oldItem.getId() || !item.isUpdated()) {
299                     writer.write(oldItem.toLineString());
300                     writer.write(Constants.LINE_SEPARATOR);
301                     return oldItem;
302                 }
303                 if (!item.equals(oldItem)) {
304                     throw new DictionaryException("Stopwords file was updated: old=" + oldItem + " : new=" + item);
305                 }
306                 try {
307                     if (!item.isDeleted()) {
308                         // update
309                         writer.write(item.toLineString());
310                         writer.write(Constants.LINE_SEPARATOR);
311                         return new StopwordsItem(item.getId(), item.getNewInput());
312                     }
313                     return null;
314                 } finally {
315                     item.setNewInput(null);
316                 }
317             } catch (final IOException e) {
318                 throw new DictionaryException("Failed to write: " + oldItem + " -> " + item, e);
319             }
320         }
321 
322         /**
323          * Writes a raw line to the temporary file.
324          *
325          * @param line The line to write.
326          * @throws DictionaryException if an I/O error occurs.
327          */
328         public void write(final String line) {
329             try {
330                 writer.write(line);
331                 writer.write(Constants.LINE_SEPARATOR);
332             } catch (final IOException e) {
333                 throw new DictionaryException("Failed to write: " + line, e);
334             }
335         }
336 
337         /**
338          * Commits the changes to the dictionary file.
339          * If there is a pending new item, it is written to the file.
340          *
341          * @return The committed item, or null if no item was committed.
342          * @throws DictionaryException if an I/O error occurs.
343          */
344         public StopwordsItem commit() {
345             isCommit = true;
346             if (item != null && item.isUpdated()) {
347                 try {
348                     writer.write(item.toLineString());
349                     writer.write(Constants.LINE_SEPARATOR);
350                     return item;
351                 } catch (final IOException e) {
352                     throw new DictionaryException("Failed to write: " + item, e);
353                 }
354             }
355             return null;
356         }
357 
358         @Override
359         public void close() {
360             try {
361                 writer.flush();
362             } catch (final IOException e) {
363                 // ignore
364             }
365             CloseableUtil.closeQuietly(writer);
366 
367             if (isCommit) {
368                 try {
369                     dictionaryManager.store(StopwordsFile.this, newFile);
370                 } finally {
371                     newFile.delete();
372                 }
373             } else {
374                 newFile.delete();
375             }
376         }
377     }
378 }