161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Smarter queries, better logging.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-04-02 12:05:09 +02:00
parent 105d93cd85
commit f7a6ef2179
2 changed files with 32 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java
View File

@ -124,14 +124,13 @@ public class SearchIndex {
}
// Finally consider terms in the full index
queryHeads.add(indexReader.findFullWord(orderedIncludes[0], ReverseIndexEntrySourceBehavior.DO_PREFER));
queryHeads.add(indexReader.findFullWord(orderedIncludes[0], ReverseIndexEntrySourceBehavior.DO_NOT_PREFER));
for (var query : queryHeads) {
if (query == null) {
return Collections.emptyList();
}
for (int orderedInclude : orderedIncludes) {
query = query.alsoFull(orderedInclude);
}

32
code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java
View File

@ -149,24 +149,54 @@ public class IndexQueryService {
continue;
}
logSearchTerms(subquery, searchTerms);
int subqueryCount = 0;
// These queries are different indices for one subquery
List<IndexQuery> queries = params.createIndexQueries(index, searchTerms);
for (var query : queries) {
var resultsForSq = executeQuery(query, params);
logger.info(queryMarker, "{} from {}", resultsForSq.size(), subquery);
logger.info(queryMarker, "{} from {}", resultsForSq.size(), query);
results.addAll(resultsForSq);
subqueryCount += resultsForSq.size();
if (!params.hasTimeLeft()) {
logger.info("Query timed out {}, ({}), -{}",
subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude);
break outer;
}
if (subqueryCount >= 100)
break;
}
}
return results;
}
private void logSearchTerms(SearchSubquery subquery, SearchIndexSearchTerms searchTerms) {
if (!logger.isInfoEnabled(queryMarker)) {
return;
}
var includes = subquery.searchTermsInclude;
var excludes = subquery.searchTermsExclude;
var priority = subquery.searchTermsPriority;
for (int i = 0; i < subquery.searchTermsInclude.size(); i++) {
logger.info(queryMarker, "{} -> {} I", includes.get(i), searchTerms.includes().getInt(i));
}
for (int i = 0; i < subquery.searchTermsExclude.size(); i++) {
logger.info(queryMarker, "{} -> {} E", excludes.get(i), searchTerms.excludes().getInt(i));
}
for (int i = 0; i < subquery.searchTermsPriority.size(); i++) {
logger.info(queryMarker, "{} -> {} p", priority.get(i), searchTerms.priority().getInt(i));
}
}
private TLongArrayList executeQuery(IndexQuery query, SearchParameters params)
{
final TLongArrayList results = new TLongArrayList(params.fetchSize);