161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Tweaks for search result relevance

Browse Source
This commit is contained in:
vlofgren 2022-08-29 18:01:07 +02:00
parent 3f2854a5e9
commit 813399401e
40 changed files with 509 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
marginalia_nu/src/main/java/nu/marginalia/util/DenseBitMap.java
View File

@ -1,6 +1,10 @@
package nu.marginalia.util;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
public class DenseBitMap {
public static final long MAX_CAPACITY_2GB_16BN_ITEMS=(1L<<34)-8;
@ -15,6 +19,31 @@ public class DenseBitMap {
this.buffer = ByteBuffer.allocateDirect((int)((cardinality / 8) + (misaligned ? 1 : 0)));
}
public static DenseBitMap loadFromFile(Path file) throws IOException {
long size = Files.size(file);
var dbm = new DenseBitMap(size/8);
try (var bc = Files.newByteChannel(file)) {
while (dbm.buffer.position() < dbm.buffer.capacity()) {
bc.read(dbm.buffer);
}
}
dbm.buffer.clear();
return dbm;
}
public void writeToFile(Path file) throws IOException {
try (var bc = Files.newByteChannel(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
while (buffer.position() < buffer.capacity()) {
bc.write(buffer);
}
}
buffer.clear();
}
public boolean get(long pos) {
return (buffer.get((int)(pos >>> 3)) & ((byte)1 << (int)(pos & 7))) != 0;
}

4
marginalia_nu/src/main/java/nu/marginalia/util/language/DocumentDebugger.java
View File

@ -8,7 +8,7 @@ import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.language.processing.model.DocumentSentence;
import nu.marginalia.util.language.processing.model.WordRep;
import nu.marginalia.util.language.processing.model.tag.WordSeparator;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.jsoup.nodes.Document;
import java.io.FileNotFoundException;
@ -30,7 +30,7 @@ public class DocumentDebugger {
Path tempDir;
public DocumentDebugger(LanguageModels lm) throws IOException {
se = new SentenceExtractor(lm);
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
ke = new KeywordExtractor();
kc = new KeywordCounter(dict, ke);

5
marginalia_nu/src/main/java/nu/marginalia/util/language/conf/LanguageModels.java
View File

@ -6,8 +6,9 @@ import java.nio.file.Path;
@AllArgsConstructor
public class LanguageModels {
public final Path ngramDictionary;
public final Path ngramFrequency;
public final Path ngramBloomFilter;
public final Path termFrequencies;
public final Path openNLPSentenceDetectionData;
public final Path posRules;
public final Path posDict;

42
marginalia_nu/src/main/java/nu/marginalia/util/language/processing/DocumentKeywordExtractor.java
View File

@ -4,7 +4,7 @@ import com.google.common.collect.Sets;
import nu.marginalia.util.language.WordPatterns;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import nu.marginalia.util.language.processing.model.WordRep;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.index.model.IndexBlock;
import nu.marginalia.wmsa.edge.model.crawl.EdgePageWordSet;
import nu.marginalia.wmsa.edge.model.crawl.EdgePageWords;
@ -19,29 +19,54 @@ public class DocumentKeywordExtractor {
private final KeywordExtractor keywordExtractor;
private final KeywordCounter tfIdfCounter;
private final NameCounter nameCounter;
private final LongNameCounter longNameCounter;
private final SubjectCounter subjectCounter;
private final NGramDict dict;
private final TermFrequencyDict dict;
private final double docCount;
@Inject
public DocumentKeywordExtractor(NGramDict dict) {
public DocumentKeywordExtractor(TermFrequencyDict dict) {
this.dict = dict;
docCount = dict.docCount();
keywordExtractor = new KeywordExtractor();
tfIdfCounter = new KeywordCounter(dict, keywordExtractor);
nameCounter = new NameCounter(keywordExtractor);
longNameCounter = new LongNameCounter(dict, keywordExtractor);
subjectCounter = new SubjectCounter(keywordExtractor);
}
public EdgePageWordSet extractKeywordsMinimal(DocumentLanguageData documentLanguageData) {
List<WordRep> titleWords = extractTitleWords(documentLanguageData);
KeywordCounter.WordHistogram wordsTfIdf = tfIdfCounter.countHisto(documentLanguageData);
List<WordRep> wordsNamesAll = nameCounter.count(documentLanguageData, 1);
List<WordRep> subjects = subjectCounter.count(documentLanguageData);
List<WordRep> midKeywords = new ArrayList<>(wordsTfIdf.mid());
List<WordRep> topKeywords = new ArrayList<>(wordsTfIdf.top());
Collection<String> artifacts = getArtifacts(documentLanguageData);
return new EdgePageWordSet(
createWords(IndexBlock.Subjects, subjects),
createWords(IndexBlock.Title, titleWords),
createWords(IndexBlock.NamesWords, wordsNamesAll),
createWords(IndexBlock.Tfidf_Top, topKeywords),
createWords(IndexBlock.Tfidf_Middle, midKeywords),
new EdgePageWords(IndexBlock.Artifacts, artifacts)
);
}
public EdgePageWordSet extractKeywords(DocumentLanguageData documentLanguageData) {
List<WordRep> titleWords = extractTitleWords(documentLanguageData);
KeywordCounter.WordHistogram wordsTfIdf = tfIdfCounter.countHisto(documentLanguageData);
List<WordRep> wordsNamesRepeated = nameCounter.count(documentLanguageData, 2);
List<WordRep> wordsNamesAll = nameCounter.count(documentLanguageData, 1);
List<WordRep> subjects = subjectCounter.count(documentLanguageData);
@ -49,12 +74,9 @@ public class DocumentKeywordExtractor {
List<WordRep> midKeywords = new ArrayList<>(wordsTfIdf.mid());
List<WordRep> topKeywords = new ArrayList<>(wordsTfIdf.top());
var wordsToMatchWithTitle = joinWordLists(topKeywords, wordsNamesRepeated, subjects);
Collection<String> artifacts = getArtifacts(documentLanguageData);
var wordSet = new EdgePageWordSet(
createWords(IndexBlock.TitleKeywords, overlappingStems(titleWords, wordsToMatchWithTitle)),
createWords(IndexBlock.Subjects, subjects),
createWords(IndexBlock.Title, titleWords),
createWords(IndexBlock.NamesWords, wordsNamesAll),
@ -121,7 +143,7 @@ public class DocumentKeywordExtractor {
else {
lastSet = counts.entrySet().stream()
.sorted(Comparator.comparing(e -> {
double N = 11820118.; // Number of documents in term freq dictionary
double N = docCount; // Number of documents in term freq dictionary
// Caveat: This is actually the *negated* term score, because the second logarithm has
// its parameter inverted (log(a^b) = b log(a); here b = -1)

10
marginalia_nu/src/main/java/nu/marginalia/util/language/processing/KeywordCounter.java
View File

@ -2,7 +2,7 @@ package nu.marginalia.util.language.processing;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import nu.marginalia.util.language.processing.model.WordRep;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import java.util.HashMap;
import java.util.HashSet;
@ -12,11 +12,13 @@ import java.util.regex.Pattern;
public class KeywordCounter {
private final KeywordExtractor keywordExtractor;
private final NGramDict dict;
private final TermFrequencyDict dict;
private final double docCount;
public KeywordCounter(NGramDict dict, KeywordExtractor keywordExtractor) {
public KeywordCounter(TermFrequencyDict dict, KeywordExtractor keywordExtractor) {
this.dict = dict;
this.keywordExtractor = keywordExtractor;
this.docCount = (double) dict.docCount();
}
public WordHistogram countHisto(DocumentLanguageData dld) {
@ -71,7 +73,7 @@ public class KeywordCounter {
if (freq < 1) {
freq = 10;
}
return (0.1 + 0.9*value/maxValue) * Math.log((1.1+freq)/11820118.);
return (0.1 + 0.9*value/maxValue) * Math.log((1.1+freq)/docCount);
}
public record WordHistogram(Set<WordRep> lower, Set<WordRep> mid, Set<WordRep> top) { }

4
marginalia_nu/src/main/java/nu/marginalia/util/language/processing/KeywordExtractor.java
View File

@ -1,9 +1,9 @@
package nu.marginalia.util.language.processing;
import nu.marginalia.util.language.WordPatterns;
import nu.marginalia.util.language.processing.model.DocumentSentence;
import nu.marginalia.util.language.processing.model.WordSpan;
import nu.marginalia.util.language.processing.model.tag.WordSeparator;
import nu.marginalia.util.language.WordPatterns;
import java.lang.ref.SoftReference;
import java.util.ArrayList;
@ -377,4 +377,6 @@ public class KeywordExtractor {
return (posTag.startsWith("JJ") || posTag.startsWith("R") || posTag.startsWith("VBG"));
}
}

9
marginalia_nu/src/main/java/nu/marginalia/util/language/processing/LongNameCounter.java
View File

@ -3,7 +3,7 @@ package nu.marginalia.util.language.processing;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import nu.marginalia.util.language.processing.model.DocumentSentence;
import nu.marginalia.util.language.processing.model.WordRep;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import java.util.*;
import java.util.regex.Pattern;
@ -11,10 +11,11 @@ import java.util.stream.Collectors;
public class LongNameCounter {
private final KeywordExtractor keywordExtractor;
private final NGramDict dict;
public LongNameCounter(NGramDict dict, KeywordExtractor keywordExtractor) {
private final TermFrequencyDict dict;
private final double docCount;
public LongNameCounter(TermFrequencyDict dict, KeywordExtractor keywordExtractor) {
this.dict = dict;
docCount = (double) dict.docCount();
this.keywordExtractor = keywordExtractor;
}

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java
View File

@ -87,7 +87,7 @@ public class WmsaHome {
final Path home = getHomePath();
return new LanguageModels(
home.resolve("model/ngrams-generous-emstr.bin"),
home.resolve("model/ngrams.bin"),
home.resolve("model/tfreq-new-algo3.bin"),
home.resolve("model/opennlp-sentence.bin"),
home.resolve("model/English.RDR"),

78
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/dict/NGramBloomFilter.java Normal file
View File

@ -0,0 +1,78 @@
package nu.marginalia.wmsa.edge.assistant.dict;
import ca.rmen.porterstemmer.PorterStemmer;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.inject.Inject;
import nu.marginalia.util.DenseBitMap;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.configuration.WmsaHome;
import nu.marginalia.wmsa.edge.index.lexicon.journal.KeywordLexiconJournalFile;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
public class NGramBloomFilter {
private final DenseBitMap bitMap;
private static final PorterStemmer ps = new PorterStemmer();
private static final HashFunction hasher = Hashing.murmur3_128(0);
@Inject
public NGramBloomFilter() throws IOException {
this(WmsaHome.getLanguageModels());
}
public NGramBloomFilter(LanguageModels lm) throws IOException {
this(DenseBitMap.loadFromFile(lm.ngramBloomFilter));
}
public NGramBloomFilter(DenseBitMap bitMap) {
this.bitMap = bitMap;
}
public boolean isKnownNGram(String word) {
long bit = bitForWord(word, bitMap.cardinality);
return bitMap.get(bit);
}
public static void main(String... args) throws IOException {
var filter = convertFromDictionaryFile(new File(args[0]));
filter.bitMap.writeToFile(Path.of(args[1]));
}
public static NGramBloomFilter load(Path file) throws IOException {
return new NGramBloomFilter(DenseBitMap.loadFromFile(file));
}
public static NGramBloomFilter convertFromDictionaryFile(File file) throws IOException {
DenseBitMap bitMap = new DenseBitMap(1024*1024*1024L);
AtomicInteger popCount = new AtomicInteger();
try (var f = new KeywordLexiconJournalFile(file)) {
f.loadFile(data -> {
long bit = bitForWord(new String(data), bitMap.cardinality);
if (!bitMap.set(bit))
popCount.incrementAndGet();
});
}
System.out.println("popcount = " + popCount.get());
return new NGramBloomFilter(bitMap);
}
private static final Pattern underscore = Pattern.compile("_");
private static long bitForWord(String s, long n) {
String[] parts = underscore.split(s);
long hc = 0;
for (String part : parts) {
hc = hc * 31 + hasher.hashString(ps.stemWord(part), StandardCharsets.UTF_8).padToLong();
}
return (hc & 0x7FFF_FFFF_FFFF_FFFFL) % n;
}
}

117
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/dict/NGramDict.java → marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/dict/TermFrequencyDict.java
View File

@ -9,7 +9,6 @@ import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import nu.marginalia.wmsa.configuration.WmsaHome;
import nu.marginalia.wmsa.edge.converting.processor.logic.DomPruner;
import nu.marginalia.wmsa.edge.crawling.CrawlPlanLoader;
import opennlp.tools.langdetect.LanguageDetector;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
@ -21,12 +20,17 @@ import javax.inject.Singleton;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Singleton
public class NGramDict {
public class TermFrequencyDict {
private final TLongIntHashMap wordRates = new TLongIntHashMap(1_000_000, 0.5f, 0, 0);
@ -34,21 +38,22 @@ public class NGramDict {
private static final Pattern separator = Pattern.compile("[_ ]+");
private static final PorterStemmer ps = new PorterStemmer();
private static final long DOC_COUNT_KEY = ~0L;
private static long fileSize(Path p) throws IOException {
return Files.size(p);
}
@Inject
public NGramDict(@Nullable LanguageModels models) {
public TermFrequencyDict(@Nullable LanguageModels models) {
if (models == null) {
return;
}
if (models.ngramFrequency != null) {
if (models.termFrequencies != null) {
try (var frequencyData = new DataInputStream(new BufferedInputStream(new FileInputStream(models.ngramFrequency.toFile())))) {
try (var frequencyData = new DataInputStream(new BufferedInputStream(new FileInputStream(models.termFrequencies.toFile())))) {
wordRates.ensureCapacity((int)(fileSize(models.ngramFrequency)/16));
wordRates.ensureCapacity((int)(fileSize(models.termFrequencies)/16));
for (;;) {
wordRates.put(frequencyData.readLong(), (int) frequencyData.readLong());
@ -56,7 +61,7 @@ public class NGramDict {
} catch (EOFException eof) {
// ok
} catch (IOException e) {
logger.error("IO Exception reading " + models.ngramFrequency, e);
logger.error("IO Exception reading " + models.termFrequencies, e);
}
}
@ -64,60 +69,100 @@ public class NGramDict {
}
public static void main(String... args) throws IOException {
public int docCount() {
int cnt = wordRates.get(DOC_COUNT_KEY);
if (cnt == 0) {
cnt = 11820118; // legacy
}
return cnt;
}
public static void main(String... args) throws IOException, InterruptedException {
if (args.length != 2) {
System.err.println("Expected arguments: plan.yaml out-file");
}
String inFile = args[0];
String outFile = args[1];
var plan = new CrawlPlanLoader().load(Path.of(args[0]));
SentenceExtractor se = new SentenceExtractor(WmsaHome.getLanguageModels());
ThreadLocal<SentenceExtractor> se = ThreadLocal.withInitial(() -> new SentenceExtractor(WmsaHome.getLanguageModels()));
DomPruner pruner = new DomPruner();
LanguageFilter lf = new LanguageFilter();
Map<String, Integer> counts = new HashMap<>(100_000_000);
Set<String> words = new HashSet<>(10_000);
TLongIntHashMap counts = new TLongIntHashMap(100_000_000, 0.7f, -1, -1);
ForkJoinPool fjp = new ForkJoinPool(24);
AtomicInteger docCount = new AtomicInteger();
for (var domain : plan.domainsIterable()) { // leaks file descriptor, is fine
if (domain.doc == null)
continue;
for (var doc : domain.doc) {
if (doc.documentBody == null)
continue;
fjp.execute(() -> {
Document parsed = Jsoup.parse(doc.documentBody);
pruner.prune(parsed, 0.5);
for (var doc : domain.doc) {
if (doc.documentBody == null)
continue;
docCount.incrementAndGet();
DocumentLanguageData dld = se.extractSentences(parsed);
Document parsed = Jsoup.parse(doc.documentBody);
pruner.prune(parsed, 0.5);
if (lf.dictionaryAgreement(dld) < 0.1) {
continue;
}
DocumentLanguageData dld = se.get().extractSentences(parsed);
for (var sent : dld.sentences) {
for (var word : sent) {
words.add(word.stemmed());
if (lf.dictionaryAgreement(dld) < 0.1) {
return;
}
}
for (var word : words) {
counts.merge(word, 1, Integer::sum);
}
Set<String> words = new HashSet<>(10_000);
words.clear();
for (var sent : dld.sentences) {
for (var word : sent) {
words.add(word.stemmed());
}
}
fjp.execute(() -> {
synchronized (counts) {
for (var word : words) {
counts.adjustOrPutValue(longHash(word.getBytes()), 1, 1);
}
}
});
}
});
}
fjp.shutdown();
fjp.awaitTermination(10, TimeUnit.SECONDS);
try (var dos = new DataOutputStream(Files.newOutputStream(Path.of(outFile)))) {
synchronized (counts) {
counts.put(DOC_COUNT_KEY, docCount.get());
counts.forEachEntry((hash, cnt) -> {
try {
dos.writeLong(hash);
dos.writeLong(cnt);
} catch (IOException e) {
throw new RuntimeException(e);
}
return true;
});
}
}
counts.forEach((w,c) -> {
if (c > 3) {
System.out.println(w + ":" + c);
}
});
System.out.println(docCount.get());
//
// counts.forEachEntry((w,c) -> {
// if (c > 3L) {
// System.out.println(w + ":" + c);
// }
// return true;
// });
}

10
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/suggest/Suggestions.java
View File

@ -2,8 +2,8 @@ package nu.marginalia.wmsa.edge.assistant.suggest;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.SpellChecker;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.converting.processor.logic.HtmlFeature;
import org.apache.commons.collections4.trie.PatriciaTrie;
import org.slf4j.Logger;
@ -21,7 +21,7 @@ import java.util.stream.Stream;
public class Suggestions {
private final PatriciaTrie<String> suggestionsTrie;
private final NGramDict nGramDict;
private final TermFrequencyDict termFrequencyDict;
private final SpellChecker spellChecker;
private static final Pattern suggestionPattern = Pattern.compile("^[a-zA-Z0-9]+( [a-zA-Z0-9]+)*$");
@ -31,12 +31,12 @@ public class Suggestions {
@Inject
public Suggestions(@Named("suggestions-file") Path suggestionsFile,
SpellChecker spellChecker,
NGramDict dict
TermFrequencyDict dict
) {
this.spellChecker = spellChecker;
suggestionsTrie = loadSuggestions(suggestionsFile);
nGramDict = dict;
termFrequencyDict = dict;
logger.info("Loaded {} suggestions", suggestionsTrie.size());
}
@ -138,7 +138,7 @@ public class Suggestions {
}
Map<String, Long> scach = new HashMap<>(512);
Function<String, Long> valr = s -> -nGramDict.getTermFreqHash(scach.computeIfAbsent(s, NGramDict::getStringHash));
Function<String, Long> valr = s -> -termFrequencyDict.getTermFreqHash(scach.computeIfAbsent(s, TermFrequencyDict::getStringHash));
return Stream.iterate(start.getKey(), Objects::nonNull, suggestionsTrie::nextKey)
.takeWhile(s -> s.startsWith(prefix))

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/atags/AnchorTextExtractor.java
View File

@ -6,7 +6,7 @@ import lombok.SneakyThrows;
import nu.marginalia.util.DenseBitMap;
import nu.marginalia.util.language.WordPatterns;
import nu.marginalia.wmsa.configuration.WmsaHome;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.converting.processor.logic.LinkParser;
import nu.marginalia.wmsa.edge.model.EdgeUrl;
import org.apache.logging.log4j.util.Strings;
@ -36,7 +36,7 @@ public class AnchorTextExtractor {
// de-duplicating billions of shuffled (url, word) tuples on limited hardware
private final DenseBitMap deduplicateHashBitset = new DenseBitMap(DenseBitMap.MAX_CAPACITY_2GB_16BN_ITEMS);
private final NGramDict ngramDict = new NGramDict(WmsaHome.getLanguageModels());
private final TermFrequencyDict ngramDict = new TermFrequencyDict(WmsaHome.getLanguageModels());
public AnchorTextExtractor(Predicate<String> includeDomainPredicate,
Predicate<EdgeUrl> includeUrlPredicate,

3
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/model/DisqualifiedException.java
View File

@ -17,6 +17,7 @@ public class DisqualifiedException extends Exception {
LANGUAGE,
STATUS,
QUALITY,
ACCEPTABLE_ADS
ACCEPTABLE_ADS,
FORBIDDEN
}
}

29
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/DocumentProcessor.java
View File

@ -86,10 +86,6 @@ public class DocumentProcessor {
if (isAcceptedContentType(crawledDocument)) {
var detailsWords = createDetails(crawledDomain, crawledDocument);
if (detailsWords.details().quality < minDocumentQuality) {
throw new DisqualifiedException(DisqualificationReason.QUALITY);
}
ret.details = detailsWords.details();
ret.words = detailsWords.words();
}
@ -141,11 +137,14 @@ public class DocumentProcessor {
private DetailsWithWords createDetails(CrawledDomain crawledDomain, CrawledDocument crawledDocument)
throws DisqualifiedException, URISyntaxException {
var doc = Jsoup.parse(crawledDocument.documentBody);
Document doc = Jsoup.parse(crawledDocument.documentBody);
if (AcceptableAds.hasAcceptableAdsTag(doc)) {
throw new DisqualifiedException(DisqualificationReason.ACCEPTABLE_ADS);
}
if (doc.select("meta[name=robots]").attr("content").contains("noindex")) {
throw new DisqualifiedException(DisqualificationReason.FORBIDDEN);
}
DomPruner domPruner = new DomPruner();
Document prunedDoc = doc.clone();
@ -160,11 +159,17 @@ public class DocumentProcessor {
ret.length = getLength(doc);
ret.standard = getHtmlStandard(doc);
ret.title = titleExtractor.getTitleAbbreviated(doc, dld, crawledDocument.url);
ret.features = featureExtractor.getFeatures(crawledDomain, doc);
ret.features = featureExtractor.getFeatures(crawledDomain, doc, dld);
ret.quality = documentValuator.getQuality(ret.standard, doc, dld);
ret.hashCode = HashCode.fromString(crawledDocument.documentBodyHash).asLong();
var words = getWords(dld);
EdgePageWordSet words;
if (ret.quality < minDocumentQuality || dld.totalNumWords() < minDocumentLength) {
words = keywordExtractor.extractKeywordsMinimal(dld);
}
else {
words = keywordExtractor.extractKeywords(dld);
}
var url = new EdgeUrl(crawledDocument.url);
addMetaWords(ret, url, crawledDomain, words);
@ -195,7 +200,6 @@ public class DocumentProcessor {
ret.features.stream().map(HtmlFeature::getKeyword).forEach(tagWords::add);
words.append(IndexBlock.Meta, tagWords);
words.append(IndexBlock.Words_1, tagWords);
}
private void getLinks(EdgeUrl baseUrl, ProcessedDocumentDetails ret, Document doc, EdgePageWordSet words) {
@ -255,7 +259,6 @@ public class DocumentProcessor {
private void synthesizeFilenameKeyword(Set<String> fileKeywords, EdgeUrl link) {
Path pFilename = Path.of(link.path.toLowerCase()).getFileName();
if (pFilename == null) return;
@ -273,10 +276,6 @@ public class DocumentProcessor {
}
private void checkDocumentLanguage(DocumentLanguageData dld) throws DisqualifiedException {
if (dld.totalNumWords() < minDocumentLength) {
throw new DisqualifiedException(DisqualificationReason.LENGTH);
}
double languageAgreement = languageFilter.dictionaryAgreement(dld);
if (languageAgreement < 0.1) {
throw new DisqualifiedException(DisqualificationReason.LANGUAGE);
@ -292,10 +291,6 @@ public class DocumentProcessor {
return htmlStandard;
}
private EdgePageWordSet getWords(DocumentLanguageData dld) {
return keywordExtractor.extractKeywords(dld);
}
private String getDescription(Document doc) {
return summaryExtractor.extractSummary(doc);
}

42
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/DomainProcessor.java
View File

@ -6,13 +6,13 @@ import nu.marginalia.wmsa.edge.converting.model.ProcessedDocument;
import nu.marginalia.wmsa.edge.converting.model.ProcessedDomain;
import nu.marginalia.wmsa.edge.crawling.model.CrawledDomain;
import nu.marginalia.wmsa.edge.crawling.model.CrawlerDomainStatus;
import nu.marginalia.wmsa.edge.index.model.IndexBlock;
import nu.marginalia.wmsa.edge.model.EdgeDomain;
import nu.marginalia.wmsa.edge.model.crawl.EdgeDomainIndexingState;
import nu.marginalia.wmsa.edge.model.crawl.EdgeUrlState;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
public class DomainProcessor {
private final DocumentProcessor documentProcessor;
@ -45,6 +45,8 @@ public class DomainProcessor {
ret.documents.add(processedDoc);
}
}
addCommonSiteWords(ret);
}
else {
ret.documents = Collections.emptyList();
@ -60,6 +62,40 @@ public class DomainProcessor {
return ret;
}
private void addCommonSiteWords(ProcessedDomain ret) {
if (ret.documents.size() < 25)
return;
Map<String, Integer> topKeywordCount = new HashMap<>(ret.documents.size()*10);
for (var doc : ret.documents) {
if (doc.words == null)
continue;
for (var word : doc.words.get(IndexBlock.Tfidf_Top).words) {
topKeywordCount.merge(word, -1, Integer::sum);
}
}
if (topKeywordCount.values().stream().mapToInt(i -> i).sum() > -100)
return;
Set<String> topWords = topKeywordCount.entrySet().stream()
.filter(e -> e.getValue() < -10)
.sorted(Map.Entry.comparingByValue()).limit(5)
.map(Map.Entry::getKey)
.collect(Collectors.toSet());
if (!topWords.isEmpty()) {
for (var doc : ret.documents) {
if (doc.words != null) {
doc.words.get(IndexBlock.Site).addAll(topWords);
}
}
}
}
private double getAverageQuality(List<ProcessedDocument> documents) {
int n = 0;
double q = 0.;

169
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/DomPruner.java
View File

@ -4,7 +4,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.NodeVisitor;
import org.jsoup.select.NodeFilter;
import java.util.HashMap;
import java.util.Map;
@ -12,100 +12,103 @@ import java.util.Map;
public class DomPruner {
public void prune(Document document, double pruneThreshold) {
PruningVisitor pruningVisitor = new PruningVisitor();
document.traverse(pruningVisitor);
pruningVisitor.data.forEach((node, data) -> {
if (data.depth <= 1) {
return;
}
if (data.signalNodeSize == 0) node.remove();
else if (data.noiseNodeSize > 0
&& data.signalRate() < pruneThreshold
&& data.treeSize > 3) {
node.remove();
}
});
document.filter(new PruningFilter(pruneThreshold));
}
}
private static class PruningVisitor implements NodeVisitor {
class PruningFilter implements NodeFilter {
private final Map<Node, NodeData> data = new HashMap<>();
private final NodeData dummy = new NodeData(Integer.MAX_VALUE, 1, 0);
private final Map<Node, NodeData> data = new HashMap<>();
private final NodeData dummy = new NodeData(Integer.MAX_VALUE, 1, 0);
private double pruneThreshold;
@Override
public void head(Node node, int depth) {}
@Override
public void tail(Node node, int depth) {
final NodeData dataForNode;
if (node instanceof TextNode tn) {
dataForNode = new NodeData(depth, tn.text().length(), 0);
}
else if (isSignal(node)) {
dataForNode = new NodeData(depth, 0,0);
for (var childNode : node.childNodes()) {
dataForNode.add(data.getOrDefault(childNode, dummy));
}
}
else {
dataForNode = new NodeData(depth, 0,0);
for (var childNode : node.childNodes()) {
dataForNode.addAsNoise(data.getOrDefault(childNode, dummy));
}
}
data.put(node, dataForNode);
}
public boolean isSignal(Node node) {
if (node instanceof Element e) {
if ("a".equalsIgnoreCase(e.tagName()))
return false;
if ("nav".equalsIgnoreCase(e.tagName()))
return false;
if ("footer".equalsIgnoreCase(e.tagName()))
return false;
if ("header".equalsIgnoreCase(e.tagName()))
return false;
}
return true;
}
public PruningFilter(double pruneThreshold) {
this.pruneThreshold = pruneThreshold;
}
private static class NodeData {
int signalNodeSize;
int noiseNodeSize;
int treeSize = 1;
int depth;
@Override
public FilterResult head(Node node, int depth) {
return FilterResult.CONTINUE;
}
private NodeData(int depth, int signalNodeSize, int noiseNodeSize) {
this.depth = depth;
this.signalNodeSize = signalNodeSize;
this.noiseNodeSize = noiseNodeSize;
@Override
public FilterResult tail(Node node, int depth) {
final NodeData dataForNode;
if (node instanceof TextNode tn) {
dataForNode = new NodeData(depth, tn.text().length(), 0);
}
else if (isSignal(node)) {
dataForNode = new NodeData(depth, 0,0);
for (var childNode : node.childNodes()) {
dataForNode.add(data.getOrDefault(childNode, dummy));
}
}
else {
dataForNode = new NodeData(depth, 0,0);
for (var childNode : node.childNodes()) {
dataForNode.addAsNoise(data.getOrDefault(childNode, dummy));
}
}
public void add(NodeData other) {
signalNodeSize += other.signalNodeSize;
noiseNodeSize += other.noiseNodeSize;
treeSize += other.treeSize;
data.put(node, dataForNode);
if (dataForNode.depth <= 1)
return FilterResult.CONTINUE;
if (dataForNode.signalNodeSize == 0)
return FilterResult.REMOVE;
if (dataForNode.noiseNodeSize > 0
&& dataForNode.signalRate() < pruneThreshold
&& dataForNode.treeSize > 3)
return FilterResult.REMOVE;
return FilterResult.CONTINUE;
}
public boolean isSignal(Node node) {
if (node instanceof Element e) {
if ("a".equalsIgnoreCase(e.tagName()))
return false;
if ("nav".equalsIgnoreCase(e.tagName()))
return false;
if ("footer".equalsIgnoreCase(e.tagName()))
return false;
if ("header".equalsIgnoreCase(e.tagName()))
return false;
}
public void addAsNoise(NodeData other) {
noiseNodeSize += other.noiseNodeSize + other.signalNodeSize;
treeSize += other.treeSize;
}
public double signalRate() {
return signalNodeSize / (double)(signalNodeSize + noiseNodeSize);
}
return true;
}
}
class NodeData {
int signalNodeSize;
int noiseNodeSize;
int treeSize = 1;
int depth;
NodeData(int depth, int signalNodeSize, int noiseNodeSize) {
this.depth = depth;
this.signalNodeSize = signalNodeSize;
this.noiseNodeSize = noiseNodeSize;
}
public void add(NodeData other) {
signalNodeSize += other.signalNodeSize;
noiseNodeSize += other.noiseNodeSize;
treeSize += other.treeSize;
}
public void addAsNoise(NodeData other) {
noiseNodeSize += other.noiseNodeSize + other.signalNodeSize;
treeSize += other.treeSize;
}
public double signalRate() {
return signalNodeSize / (double)(signalNodeSize + noiseNodeSize);
}
}

25
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/FeatureExtractor.java
View File

@ -2,7 +2,11 @@ package nu.marginalia.wmsa.edge.converting.processor.logic;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import nu.marginalia.wmsa.edge.converting.processor.logic.topic.AdblockSimulator;
import nu.marginalia.wmsa.edge.converting.processor.logic.topic.RecipeDetector;
import nu.marginalia.wmsa.edge.converting.processor.logic.topic.TextileCraftDetector;
import nu.marginalia.wmsa.edge.converting.processor.logic.topic.WoodworkingDetector;
import nu.marginalia.wmsa.edge.crawling.model.CrawledDomain;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -35,14 +39,20 @@ public class FeatureExtractor {
"d31qbv1cthcecs.cloudfront.net",
"linkedin.com");
private AdblockSimulator adblockSimulator;
private final AdblockSimulator adblockSimulator;
private final RecipeDetector recipeDetector;
private final TextileCraftDetector textileCraftDetector;
private final WoodworkingDetector woodworkingDetector;
@Inject
public FeatureExtractor(AdblockSimulator adblockSimulator) {
public FeatureExtractor(AdblockSimulator adblockSimulator, RecipeDetector recipeDetector, TextileCraftDetector textileCraftDetector, WoodworkingDetector woodworkingDetector) {
this.adblockSimulator = adblockSimulator;
this.recipeDetector = recipeDetector;
this.textileCraftDetector = textileCraftDetector;
this.woodworkingDetector = woodworkingDetector;
}
public Set<HtmlFeature> getFeatures(CrawledDomain domain, Document doc) {
public Set<HtmlFeature> getFeatures(CrawledDomain domain, Document doc, DocumentLanguageData dld) {
final Set<HtmlFeature> features = new HashSet<>();
final Elements scriptTags = doc.getElementsByTag("script");
@ -81,9 +91,14 @@ public class FeatureExtractor {
}
}
if (!domain.cookies.isEmpty()) {
if (!domain.cookies.isEmpty())
features.add(HtmlFeature.COOKIES);
}
if (recipeDetector.testP(dld) > 0.5)
features.add(HtmlFeature.CATEGORY_FOOD);
// these should be mutually exclusive
else if (woodworkingDetector.testP(dld) > 0.3 || textileCraftDetector.testP(dld) > 0.3)
features.add(HtmlFeature.CATEGORY_CRAFTS);
return features;
}

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/HtmlFeature.java
View File

@ -12,6 +12,8 @@ public enum HtmlFeature {
CATEGORY_FOOD("category:food"),
ADVERTISEMENT("special:ads"),
CATEGORY_CRAFTS("category:crafts"),
;
private final String keyword;

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/topic/RecipeDetector.java
View File

@ -1,6 +1,7 @@
package nu.marginalia.wmsa.edge.converting.processor.logic.topic;
import ca.rmen.porterstemmer.PorterStemmer;
import com.google.inject.Inject;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import java.util.HashMap;
@ -14,6 +15,7 @@ public class RecipeDetector {
private final Map<String, Double> termValues = new HashMap<>();
@Inject
public RecipeDetector() {
PorterStemmer ps = new PorterStemmer();

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/topic/TextileCraftDetector.java
View File

@ -1,6 +1,7 @@
package nu.marginalia.wmsa.edge.converting.processor.logic.topic;
import ca.rmen.porterstemmer.PorterStemmer;
import com.google.inject.Inject;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import java.util.HashMap;
@ -14,6 +15,7 @@ public class TextileCraftDetector {
private final Map<String, Double> termValues = new HashMap<>();
@Inject
public TextileCraftDetector() {
PorterStemmer ps = new PorterStemmer();

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/topic/WoodworkingDetector.java
View File

@ -1,6 +1,7 @@
package nu.marginalia.wmsa.edge.converting.processor.logic.topic;
import ca.rmen.porterstemmer.PorterStemmer;
import com.google.inject.Inject;
import nu.marginalia.util.language.processing.model.DocumentLanguageData;
import java.util.HashMap;
@ -14,6 +15,7 @@ public class WoodworkingDetector {
private final Map<String, Double> termValues = new HashMap<>();
@Inject
public WoodworkingDetector() {
PorterStemmer ps = new PorterStemmer();

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/lexicon/journal/KeywordLexiconJournalFile.java
View File

@ -15,7 +15,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
public class KeywordLexiconJournalFile {
public class KeywordLexiconJournalFile implements AutoCloseable {
private final RandomAccessFile journalFileRAF;
private final File journalFile;
private final Logger logger = LoggerFactory.getLogger(getClass());

14
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/model/IndexBlock.java
View File

@ -4,22 +4,24 @@ public enum IndexBlock {
TitleKeywords(0, 0),
Title(1, 1),
Link(2, 1.25),
Link(2, 1.15),
Subjects(3, 0.5),
NamesWords(4, 5),
Subjects(3, 3.0),
NamesWords(4, 3.0),
Artifacts(5, 10),
Meta(6, 7),
Tfidf_Top(7, 2),
Tfidf_Middle(8, 2.5),
Tfidf_Lower(9, 5.0),
Tfidf_Top(7, 0.5),
Tfidf_Middle(8, 1.25),
Tfidf_Lower(9, 1.5),
Words_1(10, 3.0),
Words_2(11, 3.5),
Words_4(12, 4.0),
Words_8(13, 4.5),
Words_16Plus(14, 7.0),
Site(15, 1.2),
;
public final int id;

16
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/reader/SearchIndexReader.java
View File

@ -29,7 +29,6 @@ public class SearchIndexReader implements AutoCloseable {
IndexBlock.Tfidf_Top,
IndexBlock.Tfidf_Middle,
IndexBlock.Tfidf_Lower,
IndexBlock.NamesWords,
IndexBlock.Words_1,
IndexBlock.Words_2,
IndexBlock.Words_4,
@ -62,15 +61,14 @@ public class SearchIndexReader implements AutoCloseable {
queryBuilders = new EnumMap<>(IndexBlock.class);
underspecifiedQueryBuilders = new EnumMap<>(IndexBlock.class);
queryBuilders.put(IndexBlock.Words_1, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleKeywordsIndex, titleIndex, topIndex, words1), words1));
queryBuilders.put(IndexBlock.Words_2, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleKeywordsIndex, titleIndex, topIndex, words2), words1));
queryBuilders.put(IndexBlock.Words_4, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleKeywordsIndex, titleIndex, topIndex, words4), words1));
queryBuilders.put(IndexBlock.Words_8, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleKeywordsIndex, titleIndex, topIndex, words8), words1));
queryBuilders.put(IndexBlock.Words_16Plus, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleKeywordsIndex, titleIndex, topIndex, words1, words2, words4, words8, words16, artifacts), words1));
queryBuilders.put(IndexBlock.Words_1, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleIndex, topIndex, words1), words1));
queryBuilders.put(IndexBlock.Words_2, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleIndex, topIndex, words2), words1));
queryBuilders.put(IndexBlock.Words_4, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleIndex, topIndex, words4), words1));
queryBuilders.put(IndexBlock.Words_8, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleIndex, topIndex, words8), words1));
queryBuilders.put(IndexBlock.Words_16Plus, new IndexQueryBuilder(listOfNonNulls(metaIndex, titleIndex, topIndex, words1, words2, words4, words8, words16, artifacts), words1));
underspecifiedQueryBuilders.put(IndexBlock.TitleKeywords, new IndexQueryBuilder(listOfNonNulls(titleKeywordsIndex, linkIndex, topicIndex, topIndex, midIndex, lowIndex, namesIndex, metaIndex), words1));
underspecifiedQueryBuilders.put(IndexBlock.Title, new IndexQueryBuilder(listOfNonNulls(titleIndex, topicIndex, linkIndex, topicIndex, topIndex, midIndex, lowIndex, namesIndex, metaIndex), words1));
underspecifiedQueryBuilders.put(IndexBlock.Tfidf_Top, new IndexQueryBuilder(listOfNonNulls(topIndex, linkIndex, midIndex, lowIndex, namesIndex, metaIndex), words1));
underspecifiedQueryBuilders.put(IndexBlock.Title, new IndexQueryBuilder(listOfNonNulls(titleIndex, linkIndex, topIndex, namesIndex, topicIndex, metaIndex), words1));
underspecifiedQueryBuilders.put(IndexBlock.Tfidf_Top, new IndexQueryBuilder(listOfNonNulls(linkIndex, namesIndex, topIndex, midIndex, lowIndex, topicIndex, metaIndex), words1));
}
@SafeVarargs

3
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgePageWordSet.java
View File

@ -19,7 +19,8 @@ public class EdgePageWordSet {
public EdgePageWords get(IndexBlock block) {
var words = wordSets.get(block);
if (words == null) {
return new EdgePageWords(block);
words = new EdgePageWords(block);
wordSets.put(block, words);
}
return words;
}

6
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/EnglishDictionary.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.wmsa.edge.search.query;
import com.google.inject.Inject;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -13,11 +13,11 @@ import java.util.stream.Collectors;
public class EnglishDictionary {
private final Set<String> englishWords = new HashSet<>();
private final NGramDict dict;
private final TermFrequencyDict dict;
private final Logger logger = LoggerFactory.getLogger(getClass());
@Inject
public EnglishDictionary(NGramDict dict) {
public EnglishDictionary(TermFrequencyDict dict) {
this.dict = dict;
try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream("dictionary/en-words"),
"Could not load word frequency table");

11
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java
View File

@ -4,7 +4,8 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.util.language.WordPatterns;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.NGramBloomFilter;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.index.model.IndexBlock;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchSpecification;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchSubquery;
@ -22,20 +23,22 @@ import java.util.*;
public class QueryFactory {
private final LanguageModels lm;
private final NGramDict dict;
private final TermFrequencyDict dict;
private final EnglishDictionary englishDictionary;
private final NGramBloomFilter nGramBloomFilter;
private final Logger logger = LoggerFactory.getLogger(getClass());
@Inject
public QueryFactory(LanguageModels lm, NGramDict dict, EnglishDictionary englishDictionary) {
public QueryFactory(LanguageModels lm, TermFrequencyDict dict, EnglishDictionary englishDictionary, NGramBloomFilter nGramBloomFilter) {
this.lm = lm;
this.dict = dict;
this.englishDictionary = englishDictionary;
this.nGramBloomFilter = nGramBloomFilter;
}
public QueryParser getParser() {
return new QueryParser(englishDictionary, new QueryVariants(lm ,dict, englishDictionary));
return new QueryParser(englishDictionary, new QueryVariants(lm ,dict, nGramBloomFilter, englishDictionary));
}
public EdgeSearchQuery createQuery(EdgeUserSearchParameters params) {

29
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryVariants.java
View File

@ -10,7 +10,8 @@ import nu.marginalia.util.language.processing.KeywordExtractor;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.language.processing.model.DocumentSentence;
import nu.marginalia.util.language.processing.model.WordSpan;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.NGramBloomFilter;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import opennlp.tools.stemmer.PorterStemmer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -24,14 +25,18 @@ public class QueryVariants {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final KeywordExtractor keywordExtractor;
private final SentenceExtractor sentenceExtractor;
private final NGramDict dict;
private final TermFrequencyDict dict;
private final PorterStemmer ps = new PorterStemmer();
private final static int MAX_NGRAM_LENGTH = 4;
private final NGramBloomFilter nGramBloomFilter;
private final EnglishDictionary englishDictionary;
@Inject
public QueryVariants(LanguageModels lm, NGramDict dict, EnglishDictionary englishDictionary) {
public QueryVariants(LanguageModels lm,
TermFrequencyDict dict,
NGramBloomFilter nGramBloomFilter,
EnglishDictionary englishDictionary) {
this.nGramBloomFilter = nGramBloomFilter;
this.englishDictionary = englishDictionary;
this.keywordExtractor = new KeywordExtractor();
this.sentenceExtractor = new SentenceExtractor(lm);
@ -154,11 +159,11 @@ public class QueryVariants {
double q = 0;
for (var word : lst) {
String[] parts = underscore.split(word);
StringJoiner combined = new StringJoiner("_");
double qp = 0;
for (String part : parts) {
combined.add(ps.stem(part));
qp += 1./(1+ dict.getTermFreq(part));
}
q += Math.log(1 + dict.getTermFreqStemmed(combined.toString()));
q += 1.0 / qp;
}
ret.add(new QueryVariant(lst, q));
}
@ -215,8 +220,8 @@ public class QueryVariants {
while (wordMatcher.find(ws) && stemmedMatcher.find(ss)) {
ws = wordMatcher.start()+1;
ss = stemmedMatcher.start()+1;
if (dict.getTermFreqStemmed(splitAtNumBoundaryAndStem(span.word, stemmedMatcher.start(), "_")) > 0
|| dict.getTermFreqStemmed(splitAtNumBoundaryAndStem(span.word, stemmedMatcher.start(), "-")) > 0)
if (nGramBloomFilter.isKnownNGram(splitAtNumBoundary(span.word, stemmedMatcher.start(), "_"))
|| nGramBloomFilter.isKnownNGram(splitAtNumBoundary(span.word, stemmedMatcher.start(), "-")))
{
String combined = splitAtNumBoundary(span.word, wordMatcher.start(), "_");
asTokens2.add(combined);
@ -242,7 +247,7 @@ public class QueryVariants {
for (var span : ls) {
var matcher = dashBoundary.matcher(span.word);
if (matcher.find() && dict.getTermFreqStemmed(ps.stem(dashBoundary.matcher(span.word).replaceAll(""))) > 0) {
if (matcher.find() && nGramBloomFilter.isKnownNGram(ps.stem(dashBoundary.matcher(span.word).replaceAll("")))) {
dash = true;
String combined = dashBoundary.matcher(span.word).replaceAll("");
asTokens2.add(combined);
@ -262,10 +267,6 @@ public class QueryVariants {
return in.substring(0, splitPoint+1) + joiner + in.substring(splitPoint+1);
}
private String splitAtNumBoundaryAndStem(String in, int splitPoint, String joiner) {
return ps.stem(in.substring(0, splitPoint+1)) + joiner + ps.stem(in.substring(splitPoint+1));
}
private List<List<Word>> getWordSpans(TreeMap<Integer, List<WordSpan>> byStart, DocumentSentence sentence, List<ArrayList<WordSpan>> livingSpans) {
List<List<Word>> goodSpans = new ArrayList<>();
for (int i = 0; i < sentence.length(); i++) {

6
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultValuator.java
View File

@ -3,7 +3,7 @@ package nu.marginalia.wmsa.edge.search.results;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.util.language.WordPatterns;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.index.model.IndexBlock;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchResultKeywordScore;
@ -12,7 +12,7 @@ import java.util.regex.Pattern;
@Singleton
public class SearchResultValuator {
private final NGramDict dict;
private final TermFrequencyDict dict;
private static final Pattern separator = Pattern.compile("_");
@ -20,7 +20,7 @@ public class SearchResultValuator {
private static final int AVG_LENGTH = 1400;
@Inject
public SearchResultValuator(NGramDict dict) {
public SearchResultValuator(TermFrequencyDict dict) {
this.dict = dict;
}

2
marginalia_nu/src/main/resources/templates/edge/index.hdb
View File

@ -2,7 +2,7 @@
<html>
<head>
<meta charset="UTF-8">
<title>Marginalia Search}</title>
<title>Marginalia Search</title>
<link rel="stylesheet" href="/style-new.css" />
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Marginalia">

4
marginalia_nu/src/test/java/nu/marginalia/util/TestLanguageModels.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.util;
import nu.marginalia.wmsa.configuration.WmsaHome;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.configuration.WmsaHome;
import java.nio.file.Files;
import java.nio.file.Path;
@ -26,7 +26,7 @@ public class TestLanguageModels {
var languageModelsHome = getLanguageModelsPath();
return new LanguageModels(
languageModelsHome.resolve("ngrams-generous-emstr.bin"),
languageModelsHome.resolve("ngrams.bin"),
languageModelsHome.resolve("tfreq-generous-emstr.bin"),
languageModelsHome.resolve("opennlp-sentence.bin"),
languageModelsHome.resolve("English.RDR"),

6
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/assistant/suggest/SuggestionsTest.java
View File

@ -1,9 +1,9 @@
package nu.marginalia.wmsa.edge.assistant.suggest;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.SpellChecker;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.SpellChecker;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@ -17,7 +17,7 @@ class SuggestionsTest {
public static void setUp() {
LanguageModels lm = TestLanguageModels.getLanguageModels();
suggestions = new Suggestions(Path.of("/home/vlofgren/Work/sql-titles-clean"),
new SpellChecker(), new NGramDict(lm));
new SpellChecker(), new TermFrequencyDict(lm));
}
@Test

10
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/crawling/SentenceExtractorTest.java
View File

@ -10,7 +10,7 @@ import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.language.processing.model.WordRep;
import nu.marginalia.util.language.processing.model.WordSpan;
import nu.marginalia.util.language.processing.model.tag.WordSeparator;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.integration.wikipedia.WikipediaReader;
import nu.marginalia.wmsa.edge.model.EdgeDomain;
import org.jsoup.Jsoup;
@ -45,7 +45,7 @@ class SentenceExtractorTest {
System.out.println("Running");
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
SentenceExtractor se = new SentenceExtractor(lm);
KeywordExtractor keywordExtractor = new KeywordExtractor();
@ -85,7 +85,7 @@ class SentenceExtractorTest {
System.out.println("Running");
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
DocumentKeywordExtractor documentKeywordExtractor = new DocumentKeywordExtractor(dict);
@ -110,7 +110,7 @@ class SentenceExtractorTest {
System.out.println("Running");
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
DocumentKeywordExtractor documentKeywordExtractor = new DocumentKeywordExtractor(dict);
@ -154,7 +154,7 @@ class SentenceExtractorTest {
public void testSE() {
var result = newSe.extractSentences(Jsoup.parse(new URL("https://memex.marginalia.nu/log/26-personalized-pagerank.gmi"), 10000));
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
System.out.println(new DocumentKeywordExtractor(dict).extractKeywords(result));

4
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/integration/arxiv/ArxivParserTest.java
View File

@ -1,10 +1,10 @@
package nu.marginalia.wmsa.edge.integration.arxiv;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.DocumentKeywordExtractor;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.integration.arxiv.model.ArxivMetadata;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@ -26,7 +26,7 @@ class ArxivParserTest {
@Test
void extractKeywords() throws IOException {
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
DocumentKeywordExtractor documentKeywordExtractor = new DocumentKeywordExtractor(dict);

6
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/integration/stackoverflow/StackOverflowPostsTest.java
View File

@ -1,11 +1,11 @@
package nu.marginalia.wmsa.edge.integration.stackoverflow;
import nu.marginalia.util.ParallelPipe;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.DocumentKeywordExtractor;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.ParallelPipe;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.integration.model.BasicDocumentData;
import nu.marginalia.wmsa.edge.integration.stackoverflow.model.StackOverflowPost;
import nu.marginalia.wmsa.edge.model.EdgeDomain;
@ -20,7 +20,7 @@ public class StackOverflowPostsTest {
@Test @Disabled("this is stupidly slow")
public void test() throws ParserConfigurationException, SAXException, InterruptedException {
var documentKeywordExtractor = new DocumentKeywordExtractor(new NGramDict(lm));
var documentKeywordExtractor = new DocumentKeywordExtractor(new TermFrequencyDict(lm));
ThreadLocal<StackOverflowPostProcessor> processor = ThreadLocal.withInitial(() -> {
return new StackOverflowPostProcessor(new SentenceExtractor(lm), documentKeywordExtractor);

8
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/integration/wikipedia/WikipediaTest.java
View File

@ -1,13 +1,13 @@
package nu.marginalia.wmsa.edge.integration.wikipedia;
import lombok.SneakyThrows;
import nu.marginalia.util.ParallelPipe;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.DocumentDebugger;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.DocumentKeywordExtractor;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.ParallelPipe;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import nu.marginalia.wmsa.edge.integration.model.BasicDocumentData;
import nu.marginalia.wmsa.edge.integration.wikipedia.model.WikipediaArticle;
import nu.marginalia.wmsa.edge.model.EdgeDomain;
@ -21,7 +21,7 @@ public class WikipediaTest {
@Test @SneakyThrows
public void test() {
var documentKeywordExtractor = new DocumentKeywordExtractor(new NGramDict(lm));
var documentKeywordExtractor = new DocumentKeywordExtractor(new TermFrequencyDict(lm));
ThreadLocal<WikipediaProcessor> processor = ThreadLocal.withInitial(() -> {
return new WikipediaProcessor(new SentenceExtractor(lm), documentKeywordExtractor);
});
@ -48,7 +48,7 @@ public class WikipediaTest {
@Test @SneakyThrows
public void test2() {
var documentKeywordExtractor = new DocumentKeywordExtractor(new NGramDict(lm));
var documentKeywordExtractor = new DocumentKeywordExtractor(new TermFrequencyDict(lm));
var debugger = new DocumentDebugger(lm);
ThreadLocal<WikipediaProcessor> processor = ThreadLocal.withInitial(() -> {

16
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/BodyQueryParserTest.java
View File

@ -1,31 +1,35 @@
package nu.marginalia.wmsa.edge.search.query;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramBloomFilter;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
class BodyQueryParserTest {
private QueryParser parser;
private static NGramDict dict;
private static TermFrequencyDict dict;
private static EnglishDictionary englishDictionary;
private static NGramBloomFilter nGramBloomFilter;
private static final LanguageModels lm = TestLanguageModels.getLanguageModels();
@BeforeAll
public static void init() {
dict = new NGramDict(lm);
public static void init() throws IOException {
dict = new TermFrequencyDict(lm);
nGramBloomFilter = new NGramBloomFilter(lm);
englishDictionary = new EnglishDictionary(dict);
}
@BeforeEach
public void setUp() {
parser = new QueryParser(englishDictionary, new QueryVariants(lm, dict, englishDictionary));
parser = new QueryParser(englishDictionary, new QueryVariants(lm, dict, nGramBloomFilter, englishDictionary));
}
@Test

4
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/EnglishDictionaryTest.java
View File

@ -1,8 +1,8 @@
package nu.marginalia.wmsa.edge.search.query;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.junit.jupiter.api.Test;
class EnglishDictionaryTest {
@ -11,7 +11,7 @@ class EnglishDictionaryTest {
void getWordVariants() {
LanguageModels lm = TestLanguageModels.getLanguageModels();
var dict = new NGramDict(lm);
var dict = new TermFrequencyDict(lm);
new EnglishDictionary(dict).getWordVariants("dos").forEach(System.out::println);
}
}

14
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryParserTest.java
View File

@ -1,25 +1,29 @@
package nu.marginalia.wmsa.edge.search.query;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramBloomFilter;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.stream.Collectors;
class QueryParserTest {
private QueryParser parser;
private static NGramDict dict;
private static TermFrequencyDict dict;
private static EnglishDictionary englishDictionary;
private static NGramBloomFilter nGramBloomFilter;
private static final LanguageModels lm = TestLanguageModels.getLanguageModels();
@BeforeEach
public void setUp() {
dict = new NGramDict(lm);
public void setUp() throws IOException {
dict = new TermFrequencyDict(lm);
nGramBloomFilter = new NGramBloomFilter(lm);
englishDictionary = new EnglishDictionary(dict);
parser = new QueryParser(englishDictionary, new QueryVariants(lm, dict, englishDictionary));
parser = new QueryParser(englishDictionary, new QueryVariants(lm, dict, nGramBloomFilter, englishDictionary));
}
@Test

12
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java
View File

@ -3,23 +3,27 @@ package nu.marginalia.wmsa.edge.search.query;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.wmsa.edge.assistant.dict.NGramBloomFilter;
import nu.marginalia.wmsa.edge.assistant.dict.TermFrequencyDict;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
class QueryVariantsTest {
QueryVariants variants;
QueryParser parser;
SentenceExtractor se;
@BeforeEach
public void setUp() {
public void setUp() throws IOException {
LanguageModels lm = TestLanguageModels.getLanguageModels();
se = new SentenceExtractor(lm);
var dict = new NGramDict(lm);
variants = new QueryVariants(lm, dict, new EnglishDictionary(dict));
var dict = new TermFrequencyDict(lm);
var ngrams = new NGramBloomFilter(lm);
variants = new QueryVariants(lm, dict, ngrams, new EnglishDictionary(dict));
parser = new QueryParser(new EnglishDictionary(dict), variants);
}