(crawler) Be a bit more conservative about pulling etags and so on if the previous fetch wasn't OK

2023-12-25 00:55:05 +01:00 · 2023-12-25 00:55:05 +01:00 · 84563b0d46
commit 84563b0d46
parent c5aab7e8db
1 changed files with 3 additions and 0 deletions
--- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java
+++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java
@ -49,6 +49,9 @@ public record DocumentWithReference(
        if (null == doc)
            return ContentTags.empty();

+        if (doc.documentBody == null || doc.httpStatus != 200)
+            return ContentTags.empty();
+
        String lastmod = doc.getLastModified();
        String etag = doc.getEtag();