161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(converter) Add upper 128KB limit to how much HTML we'll parse

Browse Source
This commit is contained in:
Viktor Lofgren 2024-01-03 23:14:03 +01:00
parent f7560cb1d8
commit 60361f88ed
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
View File

@ -107,6 +107,10 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
throw new DisqualifiedException(DisqualificationReason.LANGUAGE);
}
if (documentBody.length() > 128_000) { // 128kb
documentBody = documentBody.substring(0, 128_000);
}
Document doc = Jsoup.parse(documentBody);
if (!metaRobotsTag.allowIndexingByMetaTag(doc)) {