From 75d87c73d1accfc0477eba66581dd51445110270 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sat, 30 Dec 2023 13:56:44 +0100 Subject: [PATCH] (crawler) Disable Java's infinite DNS caching --- .../src/main/java/nu/marginalia/crawl/CrawlerMain.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java index edbf219f..4b97200b 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.security.Security; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; @@ -100,11 +101,15 @@ public class CrawlerMain { } public static void main(String... args) throws Exception { + if (!AbortMonitor.getInstance().isAlive()) { System.err.println("Remove abort file first"); return; } + // Prevent Java from caching DNS lookups forever (filling up the system RAM as a result) + Security.setProperty("networkaddress.cache.ttl" , "3600"); + // This must run *early* System.setProperty("http.agent", WmsaHome.getUserAgent().uaString());