161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'Cooking mode' (#81) from master into release

Browse Source 
Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/81
This commit is contained in:
Viktor Lofgren 2022-08-08 18:09:37 +02:00
commit dbb5a4d3bf
9 changed files with 107 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/conversion/SearchIndexPreconverter.java
View File

@ -23,18 +23,6 @@ public class SearchIndexPreconverter {
public record Shard(int bucket, int block) {}
private record ShardOutput(Shard shard, RandomAccessFile raf, FileChannel fc) {
public static ShardOutput fromFile(Shard s, File f) {
try {
var v = new RandomAccessFile(f, "rw");
v.seek(SearchIndexJournalReader.FILE_HEADER_SIZE_BYTES);
return new ShardOutput(s, v, v.getChannel());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
};
@SneakyThrows
@Inject
public SearchIndexPreconverter(File inputFile,
@ -57,7 +45,9 @@ public class SearchIndexPreconverter {
logger.info("{}", indexJournalReader.fileHeader);
ShardOutput[] outputs = outputFiles.entrySet().stream().map(entry -> ShardOutput.fromFile(entry.getKey(), entry.getValue())).toArray(ShardOutput[]::new);
ShardOutput[] outputs = outputFiles.entrySet().stream()
.map(entry -> ShardOutput.fromFile(entry.getKey(), entry.getValue()))
.toArray(ShardOutput[]::new);
var lock = partitioner.getReadLock();
try {
@ -69,18 +59,14 @@ public class SearchIndexPreconverter {
continue;
}
int domainId = entry.domainId();
buffer.clear();
entry.copyToBuffer(buffer);
for (int i = 0; i < outputs.length; i++) {
if (outputs[i].shard.block == entry.header.block().id
&& partitioner.filterUnsafe(domainId, outputs[i].shard.bucket))
{
for (ShardOutput output : outputs) {
if (output.shouldWrite(partitioner, entry)) {
buffer.flip();
while (buffer.position() < buffer.limit())
outputs[i].fc.write(buffer);
output.write(buffer);
}
}
}
@ -90,16 +76,42 @@ public class SearchIndexPreconverter {
}
logger.info("Finalizing preconversion");
for (int i = 0; i < outputs.length; i++) {
long pos = outputs[i].raf.getFilePointer();
outputs[i].raf.seek(0);
outputs[i].raf.writeLong(pos);
outputs[i].raf.writeLong(wordCountOriginal);
outputs[i].fc.force(true);
outputs[i].fc.close();
outputs[i].raf.close();
for (ShardOutput output : outputs) {
output.finish(wordCountOriginal);
}
}
private record ShardOutput(Shard shard, RandomAccessFile raf, FileChannel fc) {
public static ShardOutput fromFile(Shard s, File f) {
try {
var v = new RandomAccessFile(f, "rw");
v.seek(SearchIndexJournalReader.FILE_HEADER_SIZE_BYTES);
return new ShardOutput(s, v, v.getChannel());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public boolean shouldWrite(SearchIndexPartitioner partitioner, SearchIndexJournalReader.JournalEntry entry) {
return shard.block == entry.header.block().id
&& partitioner.filterUnsafe(entry.domainId(), shard.bucket);
}
public void finish(long wordCountOriginal) throws IOException {
long pos = raf.getFilePointer();
raf.seek(0);
raf.writeLong(pos);
raf.writeLong(wordCountOriginal);
fc.force(true);
fc.close();
raf.close();
}
public void write(ByteBuffer buffer) throws IOException {
while (buffer.position() < buffer.limit())
fc.write(buffer);
}
};
}

13
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchProfile.java
View File

@ -1,6 +1,8 @@
package nu.marginalia.wmsa.edge.search;
import nu.marginalia.wmsa.edge.converting.processor.logic.HtmlFeature;
import nu.marginalia.wmsa.edge.index.model.IndexBlock;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchSubquery;
import java.util.Arrays;
import java.util.List;
@ -25,6 +27,9 @@ public enum EdgeSearchProfile {
ACADEMIA("academia",
List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords),
3),
FOOD("food",
List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords),
2, 0),
;
@ -49,7 +54,15 @@ public enum EdgeSearchProfile {
case "default" -> DEFAULT;
case "corpo" -> CORPO;
case "academia" -> ACADEMIA;
case "food" -> FOOD;
default -> YOLO;
};
}
public void addTacitTerms(EdgeSearchSubquery subquery) {
if (this == FOOD) {
subquery.searchTermsInclude.add(HtmlFeature.CATEGORY_FOOD.getKeyword());
}
}
}

7
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/command/SearchJsParameter.java
View File

@ -1,6 +1,9 @@
package nu.marginalia.wmsa.edge.search.command;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchSubquery;
import javax.annotation.Nullable;
import java.util.Arrays;
public enum SearchJsParameter {
DEFAULT("default"),
@ -21,4 +24,8 @@ public enum SearchJsParameter {
return DEFAULT;
}
public void addTacitTerms(EdgeSearchSubquery subquery) {
subquery.searchTermsExclude.addAll(Arrays.asList(implictExcludeSearchTerms));
}
}

8
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/EnglishDictionary.java
View File

@ -46,9 +46,15 @@ public class EnglishDictionary {
var variants = findWordVariants(s);
long freqBaseline = dict.getTermFreq(s);
return variants.stream()
var ret = variants.stream()
.filter(var -> freqBaseline*10 > dict.getTermFreq(var) && freqBaseline/10 < dict.getTermFreq(var)
).collect(Collectors.toList());
if (s.equals("recipe") || s.equals("recipes")) {
ret.add("category:food");
}
return ret;
}

12
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java
View File

@ -54,7 +54,6 @@ public class QueryFactory {
}
private List<EdgeSearchSubquery> reevaluateSubqueries(EdgeSearchQuery processedQuery, EdgeUserSearchParameters params) {
final var jsSetting = params.jsSetting();
final var profile = params.profile();
List<EdgeSearchSubquery> subqueries =
@ -66,10 +65,6 @@ public class QueryFactory {
}
}
subqueries.forEach(sq -> {
sq.searchTermsExclude.addAll(Arrays.asList(jsSetting.implictExcludeSearchTerms));
});
subqueries.sort(Comparator.comparing(sq -> -sq.termSize()*2.3 + sq.block.sortOrder));
return subqueries;
@ -132,7 +127,12 @@ public class QueryFactory {
}
}
subqueries.add(new EdgeSearchSubquery(searchTermsInclude, searchTermsExclude, IndexBlock.TitleKeywords));
EdgeSearchSubquery subquery = new EdgeSearchSubquery(searchTermsInclude, searchTermsExclude, IndexBlock.TitleKeywords);
params.profile().addTacitTerms(subquery);
params.jsSetting().addTacitTerms(subquery);
subqueries.add(subquery);
}

6
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryVariants.java
View File

@ -5,12 +5,12 @@ import com.google.inject.Singleton;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.KeywordExtractor;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.language.processing.model.DocumentSentence;
import nu.marginalia.util.language.processing.model.WordSpan;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import opennlp.tools.stemmer.PorterStemmer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -115,7 +115,6 @@ public class QueryVariants {
for (var span : goodSpans) {
alternativeQueries.addAll(joinTerms(span));
// alternativeQueries.addAll(swapTerms(span));
}
for (var ls : goodSpans) {
@ -134,8 +133,11 @@ public class QueryVariants {
}
}
QueryVariantSet returnValue = new QueryVariantSet();
returnValue.faithful.addAll(evaluateQueries(faithfulQueries));
returnValue.faithful.addAll(evaluateQueries(alternativeQueries));
returnValue.faithful.sort(Comparator.comparing(QueryVariant::getValue));

1
marginalia_nu/src/main/resources/static/edge/index.html
View File

@ -38,6 +38,7 @@
<option value="modern">Blogs and Personal Websites</option>
<option value="academia">Academia, Forums, Big Websites</option>
<option value="yolo" selected>Default Ranking Algorithm</option>
<option value="food">Recipes &#127859;</option>
<option value="corpo">Experimental</option>
</select>
<select name="js" id="js">

1
marginalia_nu/src/main/resources/templates/edge/parts/search-form.hdb
View File

@ -11,6 +11,7 @@
<option {{#eq profile "modern"}}selected{{/eq}} value="modern">Blogs and Personal Websites</option>
<option {{#eq profile "academia"}}selected{{/eq}} value="academia">Academia, Forums, Big Websites</option>
<option {{#eq profile "yolo"}}selected{{/eq}} value="yolo">Default Ranking Algorithm</option>
<option {{#eq profile "food"}}selected{{/eq}} value="food">Recipes &#127859;</option>
<option {{#eq profile "corpo"}}selected{{/eq}} value="corpo">Experimental</option>
</select>
<select name="js" id="js">

52
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java
View File

@ -1,14 +1,12 @@
package nu.marginalia.wmsa.edge.search.query;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
class QueryVariantsTest {
QueryVariants variants;
QueryParser parser;
@ -25,30 +23,36 @@ class QueryVariantsTest {
parser = new QueryParser(new EnglishDictionary(dict), variants);
}
@Test @SuppressWarnings("unchecked")
void getQueryVariants() {
@Test
void getQueryFood() {
System.out.println(se.extractSentence("we are alone"));
testCase("DOS", List.of("DOS"));
testCase("dos", List.of("dos"));
testCase("we are alone", List.of("dos"));
testCase("3D Realms", List.of("dos"));
testCase("I am alone", List.of("dos"));
testCase("plato cave", List.of("dos"));
testCase("The internet is dead", List.of("dos"));
testCase("TRS80", List.of("trs_80"), List.of("trs80"));
testCase("TRS-80", List.of("trs-80"), List.of("trs80"));
testCase("TRS-80", List.of("trs-80"), List.of("trs80"));
testCase("Raspberry Pi 2", List.of("trs-80"), List.of("trs80"));
testCase("Duke Nukem 3D", List.of("trs-80"), List.of("trs80"));
testCase("The Man of Tomorrow", List.of("trs-80"), List.of("trs80"));
testCase("Computer Manual", List.of("trs-80"), List.of("trs80"));
testCase("Knitting", List.of("trs-80"), List.of("trs80"));
testCase("capcom", List.of("trs-80"), List.of("trs80"));
testCase("the man of tomorrow", List.of("trs-80"), List.of("trs80"));
testCase("Omelet recipe");
}
private void testCase(String input, List<String>... expected) {
@Test
void getQueryVariants() {
System.out.println(se.extractSentence("we are alone"));
testCase("DOS");
testCase("dos");
testCase("we are alone");
testCase("3D Realms");
testCase("I am alone");
testCase("plato cave");
testCase("The internet is dead");
testCase("TRS80");
testCase("TRS-80");
testCase("TRS-80");
testCase("Raspberry Pi 2");
testCase("Duke Nukem 3D");
testCase("The Man of Tomorrow");
testCase("Computer Manual");
testCase("Knitting");
testCase("capcom");
testCase("the man of tomorrow");
}
private void testCase(String input) {
var tokens = variants.getQueryVariants(parser.extractBasicTokens(input));
System.out.println(tokens);
}