161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(converter) Swallow errors in parquet data stream

Browse Source
This commit is contained in:
Viktor Lofgren 2023-12-28 19:45:35 +01:00
parent ff7d1a250e
commit bcecc93e39
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/ParquetSerializableCrawlDataStream.java
View File

@ -46,7 +46,13 @@ public class ParquetSerializableCrawlDataStream implements AutoCloseable, Serial
createDomainRecord(nextRecord);
wroteDomainRecord = true;
}
createDocumentRecord(nextRecord);
try {
createDocumentRecord(nextRecord);
}
catch (Exception ex) {
logger.error("Failed to create document record", ex);
}
}
return !nextQ.isEmpty();
}