View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.dict.stemmeroverride;
17  
18  import java.io.BufferedReader;
19  import java.io.BufferedWriter;
20  import java.io.Closeable;
21  import java.io.File;
22  import java.io.FileOutputStream;
23  import java.io.IOException;
24  import java.io.InputStream;
25  import java.io.InputStreamReader;
26  import java.io.OutputStreamWriter;
27  import java.io.Writer;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.Date;
31  import java.util.List;
32  import java.util.regex.Matcher;
33  import java.util.regex.Pattern;
34  
35  import org.apache.logging.log4j.LogManager;
36  import org.apache.logging.log4j.Logger;
37  import org.codelibs.core.io.CloseableUtil;
38  import org.codelibs.core.lang.StringUtil;
39  import org.codelibs.curl.CurlResponse;
40  import org.codelibs.fess.Constants;
41  import org.codelibs.fess.dict.DictionaryException;
42  import org.codelibs.fess.dict.DictionaryFile;
43  import org.codelibs.fess.util.ComponentUtil;
44  import org.dbflute.optional.OptionalEntity;
45  
46  /**
47   * Manages a dictionary file for stemmer overrides.
48   * This class handles reading, parsing, and updating files that contain
49   * stemmer override rules, where each rule maps an input word to an
50   * output stem. The file format is expected to be `input => output`.
51   *
52   * The class provides methods for retrieving, adding, updating, and
53   * deleting stemmer override items, as well as reloading the dictionary
54   * from its source file.
55   */
56  public class StemmerOverrideFile extends DictionaryFile<StemmerOverrideItem> {
57      private static final Logger logger = LogManager.getLogger(StemmerOverrideFile.class);
58  
59      private static final String STEMMER_OVERRIDE = "stemmeroverride";
60  
61      /** The list of stemmer override items loaded from the dictionary file. */
62      List<StemmerOverrideItem> stemmerOverrideItemList;
63  
64      /**
65       * Constructs a new stemmer override file.
66       *
67       * @param id        The unique identifier for this dictionary file.
68       * @param path      The path to the dictionary file.
69       * @param timestamp The last modified timestamp of the file.
70       */
71      public StemmerOverrideFile(final String id, final String path, final Date timestamp) {
72          super(id, path, timestamp);
73      }
74  
75      @Override
76      public String getType() {
77          return STEMMER_OVERRIDE;
78      }
79  
80      @Override
81      public String getPath() {
82          return path;
83      }
84  
85      @Override
86      public synchronized OptionalEntity<StemmerOverrideItem> get(final long id) {
87          if (stemmerOverrideItemList == null) {
88              reload(null);
89          }
90  
91          for (final StemmerOverrideItem stemmerOverrideItem : stemmerOverrideItemList) {
92              if (id == stemmerOverrideItem.getId()) {
93                  return OptionalEntity.of(stemmerOverrideItem);
94              }
95          }
96          return OptionalEntity.empty();
97      }
98  
99      @Override
100     public synchronized PagingList<StemmerOverrideItem> selectList(final int offset, final int size) {
101         if (stemmerOverrideItemList == null) {
102             reload(null);
103         }
104 
105         if (offset >= stemmerOverrideItemList.size() || offset < 0) {
106             return new PagingList<>(Collections.<StemmerOverrideItem> emptyList(), offset, size, stemmerOverrideItemList.size());
107         }
108 
109         int toIndex = offset + size;
110         if (toIndex > stemmerOverrideItemList.size()) {
111             toIndex = stemmerOverrideItemList.size();
112         }
113 
114         return new PagingList<>(stemmerOverrideItemList.subList(offset, toIndex), offset, size, stemmerOverrideItemList.size());
115     }
116 
117     @Override
118     public synchronized void insert(final StemmerOverrideItem item) {
119         try (StemmerOverrideUpdater updater = new StemmerOverrideUpdater(item)) {
120             reload(updater);
121         }
122     }
123 
124     @Override
125     public synchronized void update(final StemmerOverrideItem item) {
126         try (StemmerOverrideUpdater updater = new StemmerOverrideUpdater(item)) {
127             reload(updater);
128         }
129     }
130 
131     @Override
132     public synchronized void delete(final StemmerOverrideItem item) {
133         final StemmerOverrideItem stemmerOverrideItem = item;
134         stemmerOverrideItem.setNewInput(StringUtil.EMPTY);
135         stemmerOverrideItem.setNewOutput(StringUtil.EMPTY);
136         try (StemmerOverrideUpdater updater = new StemmerOverrideUpdater(item)) {
137             reload(updater);
138         }
139     }
140 
141     /**
142      * Reloads the stemmer override dictionary from its source file.
143      *
144      * @param updater An optional updater to apply changes during reload.
145      * @throws DictionaryException if the dictionary file cannot be read.
146      */
147     protected void reload(final StemmerOverrideUpdater updater) {
148         try (CurlResponse curlResponse = dictionaryManager.getContentResponse(this)) {
149             reload(updater, curlResponse.getContentAsStream());
150         } catch (final IOException e) {
151             throw new DictionaryException("Failed to parse " + path, e);
152         }
153     }
154 
155     /**
156      * Reloads the stemmer override dictionary from an input stream.
157      *
158      * @param updater An optional updater to apply changes.
159      * @param in      The input stream to read the dictionary from.
160      * @throws DictionaryException if the input stream cannot be parsed.
161      */
162     protected void reload(final StemmerOverrideUpdater updater, final InputStream in) {
163         final Pattern parsePattern = Pattern.compile("(.*?)\\s*+=>\\s*+(.*?)\\s*+$");
164         final List<StemmerOverrideItem> itemList = new ArrayList<>();
165         try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
166             long id = 0;
167             String line = null;
168             while ((line = reader.readLine()) != null) {
169                 // Remove comments
170                 final String replacedLine = line.replaceAll("#.*$", StringUtil.EMPTY).trim();
171 
172                 // Skip empty lines or comment lines
173                 if (replacedLine.length() == 0) {
174                     if (updater != null) {
175                         updater.write(line);
176                     }
177                     continue;
178                 }
179 
180                 final Matcher m = parsePattern.matcher(replacedLine);
181 
182                 if (!m.find()) {
183                     logger.warn("Failed to parse stemmer override: line={}, path={}", line, path);
184                     if (updater != null) {
185                         updater.write("# " + line);
186                     }
187                     continue;
188                 }
189 
190                 final String input = m.group(1).trim();
191                 final String output = m.group(2).trim();
192 
193                 if (input == null || output == null) {
194                     logger.warn("Failed to parse stemmer override: line={}, path={}", line, path);
195                     if (updater != null) {
196                         updater.write("# " + line);
197                     }
198                     continue;
199                 }
200 
201                 id++;
202                 final StemmerOverrideItem item = new StemmerOverrideItem(id, input, output);
203 
204                 if (updater != null) {
205                     final StemmerOverrideItem newItem = updater.write(item);
206                     if (newItem != null) {
207                         itemList.add(newItem);
208                     } else {
209                         id--;
210                     }
211                 } else {
212                     itemList.add(item);
213                 }
214             }
215             if (updater != null) {
216                 final StemmerOverrideItem item = updater.commit();
217                 if (item != null) {
218                     itemList.add(item);
219                 }
220             }
221             stemmerOverrideItemList = itemList;
222         } catch (final IOException e) {
223             throw new DictionaryException("Failed to parse " + path, e);
224         }
225     }
226 
227     /**
228      * Returns the simple name of the dictionary file.
229      *
230      * @return The file name without the path.
231      */
232     public String getSimpleName() {
233         return new File(path).getName();
234     }
235 
236     /**
237      * Updates the dictionary file with content from an input stream.
238      *
239      * @param in The input stream containing the new dictionary content.
240      * @throws IOException if an I/O error occurs.
241      */
242     public synchronized void update(final InputStream in) throws IOException {
243         try (StemmerOverrideUpdater updater = new StemmerOverrideUpdater(null)) {
244             reload(updater, in);
245         }
246     }
247 
248     @Override
249     public String toString() {
250         return "StemmerOverrideFile [path=" + path + ", stemmerOverrideItemList=" + stemmerOverrideItemList + ", id=" + id + "]";
251     }
252 
253     /**
254      * An inner class for updating the stemmer override file.
255      * This class handles the process of writing changes to a temporary file
256      * and then replacing the original file upon successful commit.
257      */
258     protected class StemmerOverrideUpdater implements Closeable {
259 
260         /** A flag indicating whether the changes have been committed. */
261         protected boolean isCommit = false;
262 
263         /** The temporary file to write changes to. */
264         protected File newFile;
265 
266         /** The writer for the temporary file. */
267         protected Writer writer;
268 
269         /** The stemmer override item being added or updated. */
270         protected StemmerOverrideItem item;
271 
272         /**
273          * Constructs a new updater for a stemmer override item.
274          *
275          * @param newItem The item to be added or updated.
276          * @throws DictionaryException if the temporary file cannot be created.
277          */
278         protected StemmerOverrideUpdater(final StemmerOverrideItem newItem) {
279             FileOutputStream fos = null;
280             try {
281                 newFile = ComponentUtil.getSystemHelper().createTempFile(STEMMER_OVERRIDE, ".txt");
282                 fos = new FileOutputStream(newFile);
283                 writer = new BufferedWriter(new OutputStreamWriter(fos, Constants.UTF_8));
284                 fos = null; // Successfully wrapped, no need to close explicitly
285             } catch (final Exception e) {
286                 if (fos != null) {
287                     try {
288                         fos.close();
289                     } catch (final IOException ioe) {
290                         // Ignore close exception
291                     }
292                 }
293                 if (newFile != null) {
294                     newFile.delete();
295                 }
296                 throw new DictionaryException("Failed to write a userDict file.", e);
297             }
298             item = newItem;
299         }
300 
301         /**
302          * Writes a stemmer override item to the temporary file.
303          * If the item is being updated, it writes the new version.
304          *
305          * @param oldItem The original item from the dictionary.
306          * @return The written item, or null if the item was deleted.
307          * @throws DictionaryException if the file was updated concurrently.
308          */
309         public StemmerOverrideItem write(final StemmerOverrideItem oldItem) {
310             try {
311                 if (item == null || item.getId() != oldItem.getId() || !item.isUpdated()) {
312                     writer.write(oldItem.toLineString());
313                     writer.write(Constants.LINE_SEPARATOR);
314                     return oldItem;
315                 }
316                 if (!item.equals(oldItem)) {
317                     throw new DictionaryException("StemmerOverride file was updated: old=" + oldItem + " : new=" + item);
318                 }
319                 try {
320                     if (!item.isDeleted()) {
321                         // update
322                         writer.write(item.toLineString());
323                         writer.write(Constants.LINE_SEPARATOR);
324                         return new StemmerOverrideItem(item.getId(), item.getNewInput(), item.getNewOutput());
325                     }
326                     return null;
327                 } finally {
328                     item.setNewInput(null);
329                     item.setNewOutput(null);
330                 }
331             } catch (final IOException e) {
332                 throw new DictionaryException("Failed to write: " + oldItem + " -> " + item, e);
333             }
334         }
335 
336         /**
337          * Writes a raw line to the temporary file.
338          *
339          * @param line The line to write.
340          * @throws DictionaryException if an I/O error occurs.
341          */
342         public void write(final String line) {
343             try {
344                 writer.write(line);
345                 writer.write(Constants.LINE_SEPARATOR);
346             } catch (final IOException e) {
347                 throw new DictionaryException("Failed to write: " + line, e);
348             }
349         }
350 
351         /**
352          * Commits the changes to the dictionary file.
353          * If there is a pending new item, it is written to the file.
354          *
355          * @return The committed item, or null if no item was committed.
356          * @throws DictionaryException if an I/O error occurs.
357          */
358         public StemmerOverrideItem commit() {
359             isCommit = true;
360             if (item != null && item.isUpdated()) {
361                 try {
362                     writer.write(item.toLineString());
363                     writer.write(Constants.LINE_SEPARATOR);
364                     return item;
365                 } catch (final IOException e) {
366                     throw new DictionaryException("Failed to write: " + item, e);
367                 }
368             }
369             return null;
370         }
371 
372         @Override
373         public void close() {
374             try {
375                 writer.flush();
376             } catch (final IOException e) {
377                 // ignore
378             }
379             CloseableUtil.closeQuietly(writer);
380 
381             if (isCommit) {
382                 try {
383                     dictionaryManager.store(StemmerOverrideFile.this, newFile);
384                 } finally {
385                     newFile.delete();
386                 }
387             } else {
388                 newFile.delete();
389             }
390         }
391     }
392 
393 }