161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(crawler) Clean up stale warc files

Browse Source 
We should probably have an option to keep them, but not by default!
This commit is contained in:
Viktor Lofgren 2023-12-25 15:07:36 +01:00
parent 88551043cd
commit 25d086c4e1
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -254,13 +254,14 @@ public class CrawlerMain {
logger.info("Fetched {}", domain); logger.info("Fetched {}", domain);
} catch (Exception e) { } catch (Exception e) {
logger.error("Error fetching domain " + domain, e); logger.error("Error fetching domain " + domain, e);
Files.deleteIfExists(newWarcFile);
Files.deleteIfExists(tempFile);
} }
finally { finally {
// We don't need to double-count these; it's also kept int he workLog // We don't need to double-count these; it's also kept int he workLog
processingIds.remove(domain); processingIds.remove(domain);
Thread.currentThread().setName("[idle]"); Thread.currentThread().setName("[idle]");
Files.deleteIfExists(newWarcFile);
Files.deleteIfExists(tempFile);
} }
} }