View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.app.service;
17  
18  import java.io.IOException;
19  import java.io.PrintWriter;
20  import java.io.StringWriter;
21  import java.util.List;
22  
23  import org.apache.logging.log4j.LogManager;
24  import org.apache.logging.log4j.Logger;
25  import org.codelibs.core.beans.util.BeanUtil;
26  import org.codelibs.core.lang.StringUtil;
27  import org.codelibs.fess.Constants;
28  import org.codelibs.fess.app.pager.FailureUrlPager;
29  import org.codelibs.fess.exception.ContainerNotAvailableException;
30  import org.codelibs.fess.helper.SystemHelper;
31  import org.codelibs.fess.mylasta.direction.FessConfig;
32  import org.codelibs.fess.opensearch.config.cbean.FailureUrlCB;
33  import org.codelibs.fess.opensearch.config.exbhv.FailureUrlBhv;
34  import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig;
35  import org.codelibs.fess.opensearch.config.exentity.FailureUrl;
36  import org.codelibs.fess.util.ComponentUtil;
37  import org.dbflute.cbean.result.PagingResultBean;
38  import org.dbflute.optional.OptionalEntity;
39  
40  import jakarta.annotation.Resource;
41  
42  /**
43   * Service class for managing failure URLs that occur during web crawling.
44   * Provides functionality to store, retrieve, and manage failed crawling attempts
45   * with their associated error information.
46   */
47  public class FailureUrlService {
48  
49      /** Logger instance for this class */
50      private static final Logger logger = LogManager.getLogger(FailureUrlService.class);
51  
52      /**
53       * Default constructor.
54       */
55      public FailureUrlService() {
56          // Default constructor
57      }
58  
59      /** Behavior class for FailureUrl entity operations */
60      @Resource
61      protected FailureUrlBhv failureUrlBhv;
62  
63      /** Configuration settings for Fess */
64      @Resource
65      protected FessConfig fessConfig;
66  
67      /**
68       * Retrieves a paginated list of failure URLs based on the provided pager criteria.
69       *
70       * @param failureUrlPager the pager containing search criteria and pagination settings
71       * @return a list of FailureUrl entities matching the criteria
72       */
73      public List<FailureUrl> getFailureUrlList(final FailureUrlPager failureUrlPager) {
74  
75          final PagingResultBean<FailureUrl> failureUrlList = failureUrlBhv.selectPage(cb -> {
76              cb.paging(failureUrlPager.getPageSize(), failureUrlPager.getCurrentPageNumber());
77              setupListCondition(cb, failureUrlPager);
78          });
79  
80          // update pager
81          BeanUtil.copyBeanToBean(failureUrlList, failureUrlPager, option -> option.include(Constants.PAGER_CONVERSION_RULE));
82          failureUrlPager.setPageNumberList(failureUrlList.pageRange(op -> {
83              op.rangeSize(fessConfig.getPagingPageRangeSizeAsInteger());
84          }).createPageNumberList());
85  
86          return failureUrlList;
87      }
88  
89      /**
90       * Retrieves a specific failure URL by its ID.
91       *
92       * @param id the unique identifier of the failure URL
93       * @return an OptionalEntity containing the FailureUrl if found, empty otherwise
94       */
95      public OptionalEntity<FailureUrl> getFailureUrl(final String id) {
96          return failureUrlBhv.selectByPK(id);
97      }
98  
99      /**
100      * Stores or updates a failure URL entity in the data store.
101      *
102      * @param failureUrl the FailureUrl entity to store or update
103      */
104     public void store(final FailureUrl failureUrl) {
105 
106         failureUrlBhv.insertOrUpdate(failureUrl, op -> {
107             op.setRefreshPolicy(Constants.TRUE);
108         });
109 
110     }
111 
112     /**
113      * Deletes a failure URL entity from the data store.
114      *
115      * @param failureUrl the FailureUrl entity to delete
116      */
117     public void delete(final FailureUrl failureUrl) {
118 
119         failureUrlBhv.delete(failureUrl, op -> {
120             op.setRefreshPolicy(Constants.TRUE);
121         });
122 
123     }
124 
125     /**
126      * Sets up the condition builder for listing failure URLs with pagination and filtering.
127      *
128      * @param cb the condition builder to configure
129      * @param failureUrlPager the pager containing filter criteria
130      */
131     protected void setupListCondition(final FailureUrlCB cb, final FailureUrlPager failureUrlPager) {
132         if (failureUrlPager.id != null) {
133             cb.query().docMeta().setId_Equal(failureUrlPager.id);
134         }
135         // TODO Long, Integer, String supported only.
136 
137         // setup condition
138         cb.query().addOrderBy_LastAccessTime_Desc();
139 
140         buildSearchCondition(failureUrlPager, cb);
141     }
142 
143     /**
144      * Deletes all failure URLs that match the criteria specified in the pager.
145      *
146      * @param failureUrlPager the pager containing deletion criteria
147      */
148     public void deleteAll(final FailureUrlPager failureUrlPager) {
149         failureUrlBhv.queryDelete(cb -> {
150             buildSearchCondition(failureUrlPager, cb);
151         });
152     }
153 
154     /**
155      * Builds search conditions for failure URL queries based on pager criteria.
156      *
157      * @param failureUrlPager the pager containing search criteria
158      * @param cb the condition builder to configure with search conditions
159      */
160     private void buildSearchCondition(final FailureUrlPager failureUrlPager, final FailureUrlCB cb) {
161         // search
162         if (StringUtil.isNotBlank(failureUrlPager.url)) {
163             cb.query().setUrl_Wildcard(failureUrlPager.url);
164         }
165 
166         if (StringUtil.isNotBlank(failureUrlPager.errorCountMax)) {
167             cb.query().setErrorCount_LessEqual(Integer.parseInt(failureUrlPager.errorCountMax));
168         }
169         if (StringUtil.isNotBlank(failureUrlPager.errorCountMin)) {
170             cb.query().setErrorCount_GreaterEqual(Integer.parseInt(failureUrlPager.errorCountMin));
171         }
172 
173         if (StringUtil.isNotBlank(failureUrlPager.errorName)) {
174             cb.query().setErrorName_Wildcard(failureUrlPager.errorName);
175         }
176 
177     }
178 
179     /**
180      * Deletes all failure URLs associated with a specific configuration ID.
181      *
182      * @param configId the configuration ID to delete failure URLs for
183      */
184     public void deleteByConfigId(final String configId) {
185         failureUrlBhv.queryDelete(cb -> {
186             cb.query().setConfigId_Equal(configId);
187         });
188     }
189 
190     /**
191      * Stores a new failure URL or updates an existing one with error information.
192      * Creates a new failure URL entry or increments the error count for an existing URL.
193      *
194      * @param crawlingConfig the crawling configuration associated with the failure
195      * @param errorName the name/type of the error that occurred
196      * @param url the URL that failed to be crawled
197      * @param e the exception that caused the failure
198      * @return the stored or updated FailureUrl entity, or null if the exception should be ignored
199      */
200     public FailureUrl store(final CrawlingConfig crawlingConfig, final String errorName, final String url, final Throwable e) {
201         if (e instanceof ContainerNotAvailableException) {
202             return null;
203         }
204 
205         final FailureUrlBhv bhv = ComponentUtil.getComponent(FailureUrlBhv.class);
206         final FailureUrl failureUrl = bhv.selectEntity(cb -> {
207             cb.query().setUrl_Equal(url);
208             if (crawlingConfig != null) {
209                 cb.query().setConfigId_Equal(crawlingConfig.getConfigId());
210             }
211         }).map(entity -> {
212             entity.setErrorCount(entity.getErrorCount() + 1);
213             return entity;
214         }).orElseGet(() -> {
215             final FailureUrl entity = new FailureUrl();
216             entity.setErrorCount(1);
217             entity.setUrl(url);
218             if (crawlingConfig != null) {
219                 entity.setConfigId(crawlingConfig.getConfigId());
220             }
221             return entity;
222         });
223 
224         failureUrl.setErrorName(errorName);
225         failureUrl.setErrorLog(getStackTrace(e));
226         failureUrl.setLastAccessTime(ComponentUtil.getSystemHelper().getCurrentTimeAsLong());
227         failureUrl.setThreadName(Thread.currentThread().getName());
228 
229         bhv.insertOrUpdate(failureUrl, op -> {
230             op.setRefreshPolicy(Constants.TRUE);
231         });
232         return failureUrl;
233     }
234 
235     /**
236      * Extracts and returns the stack trace from a throwable as a string.
237      * The stack trace is abbreviated if it exceeds the configured maximum length.
238      *
239      * @param t the throwable to extract the stack trace from
240      * @return the stack trace as a string, or empty string if extraction fails
241      */
242     private String getStackTrace(final Throwable t) {
243         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
244         try (final StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw)) {
245             t.printStackTrace(pw);
246             pw.flush();
247             return systemHelper.abbreviateLongText(sw.toString());
248         } catch (final IOException e) {
249             logger.warn("Failed to print the stack trace {}", t.getMessage(), e);
250         }
251         return StringUtil.EMPTY;
252     }
253 }