161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(crawler) Disable Java's infinite DNS caching

Browse Source
This commit is contained in:
Viktor Lofgren 2023-12-30 13:56:44 +01:00
parent 0fe44c9bf2
commit 75d87c73d1
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -41,6 +41,7 @@ import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.security.Security;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.*; import java.util.*;
import java.util.concurrent.*; import java.util.concurrent.*;
@ -100,11 +101,15 @@ public class CrawlerMain {
} }
public static void main(String... args) throws Exception { public static void main(String... args) throws Exception {
if (!AbortMonitor.getInstance().isAlive()) { if (!AbortMonitor.getInstance().isAlive()) {
System.err.println("Remove abort file first"); System.err.println("Remove abort file first");
return; return;
} }
// Prevent Java from caching DNS lookups forever (filling up the system RAM as a result)
Security.setProperty("networkaddress.cache.ttl" , "3600");
// This must run *early* // This must run *early*
System.setProperty("http.agent", WmsaHome.getUserAgent().uaString()); System.setProperty("http.agent", WmsaHome.getUserAgent().uaString());