161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(summary) Reduce the chance of expensive operations

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-16 15:48:34 +02:00
parent 46d761f34f
commit db0216936e
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/SummarizingDOMFilter.java
View File

@ -101,9 +101,15 @@ public class SummarizingDOMFilter implements NodeFilter {
for (var stats : in) {
// text() is expensive, we don't mind sifting through superfluous whitespace
int cnt = stats.score(tn ->
countOccurrencesOfAnyWord(tn.getWholeText(), importantWords)
- countOccurrencesOfAnyWord(tn.getWholeText(), badWords));
int cnt = stats.score(tn -> {
String wholeText = tn.getWholeText();
if (wholeText.length() > 128)
return 0;
return countOccurrencesOfAnyWord(wholeText, importantWords)
- countOccurrencesOfAnyWord(wholeText, badWords);
});
if (cnt > 0) {
ret.put(stats, -cnt);