161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(warc) Fix NPE in WarcRecorder

Browse Source
This commit is contained in:
Viktor Lofgren 2023-12-25 00:54:38 +01:00
parent 1755b646b8
commit c5aab7e8db
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View File

@ -289,10 +289,12 @@ public class CrawlerRetreiver implements AutoCloseable {
new ContentType(doc.contentType, "UTF-8"),
doc.documentBody);
var parsed = Jsoup.parse(doc.documentBody);
if (doc.documentBody != null) {
var parsed = Jsoup.parse(doc.documentBody);
crawlFrontier.enqueueLinksFromDocument(top, parsed);
crawlFrontier.addVisited(top);
crawlFrontier.enqueueLinksFromDocument(top, parsed);
crawlFrontier.addVisited(top);
}
}
else if (fetchedDoc instanceof HttpFetchResult.ResultException ex) {
errorCount ++;