View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.dict.kuromoji;
17  
18  import java.io.BufferedReader;
19  import java.io.BufferedWriter;
20  import java.io.Closeable;
21  import java.io.File;
22  import java.io.FileOutputStream;
23  import java.io.IOException;
24  import java.io.InputStream;
25  import java.io.InputStreamReader;
26  import java.io.OutputStreamWriter;
27  import java.io.Writer;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.Date;
31  import java.util.List;
32  
33  import org.codelibs.core.io.CloseableUtil;
34  import org.codelibs.core.lang.StringUtil;
35  import org.codelibs.curl.CurlResponse;
36  import org.codelibs.fess.Constants;
37  import org.codelibs.fess.dict.DictionaryException;
38  import org.codelibs.fess.dict.DictionaryFile;
39  import org.codelibs.fess.util.ComponentUtil;
40  import org.codelibs.fess.util.KuromojiCSVUtil;
41  import org.dbflute.optional.OptionalEntity;
42  
43  /**
44   * A dictionary file for Kuromoji.
45   */
46  public class KuromojiFile extends DictionaryFile<KuromojiItem> {
47      private static final String KUROMOJI = "kuromoji";
48  
49      List<KuromojiItem> kuromojiItemList;
50  
51      /**
52       * Constructs a new Kuromoji file.
53       *
54       * @param id The ID of the dictionary file.
55       * @param path The path of the dictionary file.
56       * @param timestamp The timestamp of the dictionary file.
57       */
58      public KuromojiFile(final String id, final String path, final Date timestamp) {
59          super(id, path, timestamp);
60      }
61  
62      @Override
63      public String getType() {
64          return KUROMOJI;
65      }
66  
67      @Override
68      public String getPath() {
69          return path;
70      }
71  
72      @Override
73      public synchronized OptionalEntity<KuromojiItem> get(final long id) {
74          if (kuromojiItemList == null) {
75              reload(null);
76          }
77  
78          for (final KuromojiItem kuromojiItem : kuromojiItemList) {
79              if (id == kuromojiItem.getId()) {
80                  return OptionalEntity.of(kuromojiItem);
81              }
82          }
83          return OptionalEntity.empty();
84      }
85  
86      @Override
87      public synchronized PagingList<KuromojiItem> selectList(final int offset, final int size) {
88          if (kuromojiItemList == null) {
89              reload(null);
90          }
91  
92          if (offset >= kuromojiItemList.size() || offset < 0) {
93              return new PagingList<>(Collections.<KuromojiItem> emptyList(), offset, size, kuromojiItemList.size());
94          }
95  
96          int toIndex = offset + size;
97          if (toIndex > kuromojiItemList.size()) {
98              toIndex = kuromojiItemList.size();
99          }
100 
101         return new PagingList<>(kuromojiItemList.subList(offset, toIndex), offset, size, kuromojiItemList.size());
102     }
103 
104     @Override
105     public synchronized void insert(final KuromojiItem item) {
106         try (KuromojiUpdater updater = new KuromojiUpdater(item)) {
107             reload(updater);
108         }
109     }
110 
111     @Override
112     public synchronized void update(final KuromojiItem item) {
113         try (KuromojiUpdater updater = new KuromojiUpdater(item)) {
114             reload(updater);
115         }
116     }
117 
118     @Override
119     public synchronized void delete(final KuromojiItem item) {
120         final KuromojiItem kuromojiItem = item;
121         kuromojiItem.setNewToken(StringUtil.EMPTY);
122         try (KuromojiUpdater updater = new KuromojiUpdater(item)) {
123             reload(updater);
124         }
125     }
126 
127     /**
128      * Reloads the dictionary file.
129      *
130      * @param updater The updater.
131      */
132     protected void reload(final KuromojiUpdater updater) {
133         try (CurlResponse curlResponse = dictionaryManager.getContentResponse(this)) {
134             reload(updater, curlResponse.getContentAsStream());
135         } catch (final IOException e) {
136             throw new DictionaryException("Failed to parse " + path, e);
137         }
138     }
139 
140     /**
141      * Reloads the dictionary file.
142      *
143      * @param updater The updater.
144      * @param in The input stream.
145      */
146     protected void reload(final KuromojiUpdater updater, final InputStream in) {
147         final List<KuromojiItem> itemList = new ArrayList<>();
148         try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
149             long id = 0;
150             String line = null;
151             while ((line = reader.readLine()) != null) {
152                 // Remove comments
153                 final String replacedLine = line.replaceAll("#.*$", StringUtil.EMPTY).trim();
154 
155                 // Skip empty lines or comment lines
156                 if (replacedLine.length() == 0) {
157                     if (updater != null) {
158                         updater.write(line);
159                     }
160                     continue;
161                 }
162 
163                 final String[] values = KuromojiCSVUtil.parse(replacedLine);
164                 String token = null;
165                 String segmentation = null;
166                 String reading = null;
167                 String pos = null;
168                 switch (values.length) {
169                 case 4:
170                     pos = values[3];
171                 case 3:
172                     reading = values[2];
173                 case 2:
174                     segmentation = values[1];
175                 case 1:
176                     token = values[0];
177                 default:
178                     break;
179                 }
180 
181                 id++;
182                 final KuromojiItem item = new KuromojiItem(id, token, segmentation, reading, pos);
183                 if (updater != null) {
184                     final KuromojiItem newItem = updater.write(item);
185                     if (newItem != null) {
186                         itemList.add(newItem);
187                     } else {
188                         id--;
189                     }
190                 } else {
191                     itemList.add(item);
192                 }
193             }
194             if (updater != null) {
195                 final KuromojiItem item = updater.commit();
196                 if (item != null) {
197                     itemList.add(item);
198                 }
199             }
200             kuromojiItemList = itemList;
201         } catch (final IOException e) {
202             throw new DictionaryException("Failed to parse " + path, e);
203         }
204     }
205 
206     /**
207      * Returns the simple name of the file.
208      *
209      * @return The simple name of the file.
210      */
211     public String getSimpleName() {
212         return new File(path).getName();
213     }
214 
215     /**
216      * Updates the dictionary file with the given input stream.
217      *
218      * @param in The input stream.
219      * @throws IOException If an I/O error occurs.
220      */
221     public synchronized void update(final InputStream in) throws IOException {
222         try (KuromojiUpdater updater = new KuromojiUpdater(null)) {
223             reload(updater, in);
224         }
225     }
226 
227     @Override
228     public String toString() {
229         return "KuromojiFile [path=" + path + ", kuromojiItemList=" + kuromojiItemList + ", id=" + id + "]";
230     }
231 
232     /**
233      * An updater for Kuromoji files.
234      */
235     protected class KuromojiUpdater implements Closeable {
236 
237         /** True if the changes have been committed. */
238         protected boolean isCommit = false;
239 
240         /** The new file. */
241         protected File newFile;
242 
243         /** The writer for the new file. */
244         protected Writer writer;
245 
246         /** The item to be added or updated. */
247         protected KuromojiItem item;
248 
249         /**
250          * Constructs a new Kuromoji updater.
251          *
252          * @param newItem The new item to be added or updated.
253          */
254         protected KuromojiUpdater(final KuromojiItem newItem) {
255             FileOutputStream fos = null;
256             try {
257                 newFile = ComponentUtil.getSystemHelper().createTempFile(KUROMOJI, ".txt");
258                 fos = new FileOutputStream(newFile);
259                 writer = new BufferedWriter(new OutputStreamWriter(fos, Constants.UTF_8));
260                 fos = null; // Successfully wrapped, no need to close explicitly
261             } catch (final Exception e) {
262                 if (fos != null) {
263                     try {
264                         fos.close();
265                     } catch (final IOException ioe) {
266                         // Ignore close exception
267                     }
268                 }
269                 if (newFile != null) {
270                     newFile.delete();
271                 }
272                 throw new DictionaryException("Failed to write a userDict file.", e);
273             }
274             item = newItem;
275         }
276 
277         /**
278          * Writes the old item to the new file.
279          *
280          * @param oldItem The old item.
281          * @return The new item if it was updated, otherwise the old item.
282          */
283         public KuromojiItem write(final KuromojiItem oldItem) {
284             try {
285                 if (item == null || item.getId() != oldItem.getId() || !item.isUpdated()) {
286                     writer.write(oldItem.toLineString());
287                     writer.write(Constants.LINE_SEPARATOR);
288                     return oldItem;
289                 }
290                 if (!item.equals(oldItem)) {
291                     throw new DictionaryException("Kuromoji file was updated: old=" + oldItem + " : new=" + item);
292                 }
293                 try {
294                     if (!item.isDeleted()) {
295                         // update
296                         writer.write(item.toLineString());
297                         writer.write(Constants.LINE_SEPARATOR);
298                         return new KuromojiItem(item.getId(), item.getNewToken(), item.getNewSegmentation(), item.getNewReading(),
299                                 item.getNewPos());
300                     }
301                     return null;
302                 } finally {
303                     item.setNewToken(null);
304                 }
305             } catch (final IOException e) {
306                 throw new DictionaryException("Failed to write: " + oldItem + " -> " + item, e);
307             }
308         }
309 
310         /**
311          * Writes a line to the new file.
312          *
313          * @param line The line to write.
314          */
315         public void write(final String line) {
316             try {
317                 writer.write(line);
318                 writer.write(Constants.LINE_SEPARATOR);
319             } catch (final IOException e) {
320                 throw new DictionaryException("Failed to write: " + line, e);
321             }
322         }
323 
324         /**
325          * Commits the changes.
326          *
327          * @return The new item if it was committed, otherwise null.
328          */
329         public KuromojiItem commit() {
330             isCommit = true;
331             if (item != null && item.isUpdated()) {
332                 try {
333                     writer.write(item.toLineString());
334                     writer.write(Constants.LINE_SEPARATOR);
335                     return item;
336                 } catch (final IOException e) {
337                     throw new DictionaryException("Failed to write: " + item, e);
338                 }
339             }
340             return null;
341         }
342 
343         @Override
344         public void close() {
345             try {
346                 writer.flush();
347             } catch (final IOException e) {
348                 // ignore
349             }
350             CloseableUtil.closeQuietly(writer);
351 
352             if (isCommit) {
353                 try {
354                     dictionaryManager.store(KuromojiFile.this, newFile);
355                 } finally {
356                     newFile.delete();
357                 }
358             } else {
359                 newFile.delete();
360             }
361         }
362     }
363 
364 }