diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/retreival/CrawlerRetreiver.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/retreival/CrawlerRetreiver.java index 0c6105e7..a8793a36 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/retreival/CrawlerRetreiver.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/crawling/retreival/CrawlerRetreiver.java @@ -168,10 +168,11 @@ public class CrawlerRetreiver { var d = doc.get(); crawledDomainWriter.accept(d); - try { - visited.add(new EdgeUrl(d.url)); + if (d.url != null) { + try { + visited.add(new EdgeUrl(d.url)); + } catch (URISyntaxException ex) {} } - catch (URISyntaxException ex) {} }