161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(crawler) Reduce log spam in HttpFetcherImpl

Browse Source
This commit is contained in:
Viktor Lofgren 2023-07-29 19:16:53 +02:00
parent 7611b7900d
commit d2b6b2044c
1 changed files with 5 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java
View File

@ -12,7 +12,6 @@ import nu.marginalia.crawling.model.CrawlerDocumentStatus;
import nu.marginalia.crawling.model.ContentType;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.bigstring.BigString;
import nu.marginalia.crawl.retreival.logic.ContentTypeLogic;
import nu.marginalia.crawl.retreival.logic.ContentTypeParser;
import okhttp3.*;
@ -25,9 +24,7 @@ import javax.net.ssl.SSLException;
import javax.net.ssl.X509TrustManager;
import java.io.EOFException;
import java.io.IOException;
import java.net.SocketTimeoutException;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.net.*;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.StandardCharsets;
@ -200,16 +197,16 @@ public class HttpFetcherImpl implements HttpFetcher {
catch (SocketTimeoutException ex) {
return createTimeoutErrorRsp(url, ex);
}
catch (IllegalCharsetNameException | SSLException | EOFException ex) {
catch (UnknownHostException ex) {
return createUnknownHostError(url, ex);
}
catch (SocketException | ProtocolException | IllegalCharsetNameException | SSLException | EOFException ex) {
// This is a bit of a grab-bag of errors that crop up
// IllegalCharsetName is egg on our face,
// but SSLException and EOFException are probably the server's fault
return createHardErrorRsp(url, ex);
}
catch (UnknownHostException ex) {
return createUnknownHostError(url, ex);
}
catch (Exception ex) {
logger.error("Error during fetching", ex);
return createHardErrorRsp(url, ex);