161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(index) Clean up

Browse Source
This commit is contained in:
Viktor Lofgren 2024-02-25 11:11:59 +01:00
parent f7f0100174
commit 9429bf5c45
3 changed files with 3 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java
View File

@ -70,7 +70,7 @@ public class SearchResultItem implements Comparable<SearchResultItem> {
@Override
public int compareTo(@NotNull SearchResultItem o) {
// this looks like a bug, but we actually want this in a reversed order
int diff = Double.compare(o.getScore(), getScore());
int diff = Double.compare(getScore(), o.getScore());
if (diff != 0)
return diff;

7
code/index/java/nu/marginalia/index/IndexGrpcService.java
View File

@ -1,6 +1,5 @@
package nu.marginalia.index;
import com.google.common.collect.MinMaxPriorityQueue;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.grpc.stub.StreamObserver;
@ -190,7 +189,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
ResultRankingContext rankingContext = createRankingContext(params.rankingParams, params.subqueries);
logger.info(queryMarker, "{}", params.queryParams);
// logger.info(queryMarker, "{}", params.queryParams);
return new QueryExecution(rankingContext, params.fetchSize)
.run(params);
@ -243,8 +242,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
this.subquery = subquery;
this.parameters = parameters;
logger.info("Starting index task");
remainingIndexTasks.incrementAndGet();
}
@ -254,7 +251,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
subquery,
parameters.queryParams,
parameters.budget,
parameters.fetchSize,
this::drain
);
}
@ -264,7 +260,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
remainingIndexTasks.notifyAll();
}
}
logger.info("Terminating index task");
}
}

60
code/index/java/nu/marginalia/index/index/IndexQueryService.java
View File

@ -38,28 +38,11 @@ public class IndexQueryService {
public void evaluateSubquery(SearchSubquery subquery,
QueryParams queryParams,
IndexSearchBudget timeout,
int fetchSize,
Consumer<CombinedDocIdList> drain)
{
// These queries are various term combinations
if (!timeout.hasTimeLeft()) {
logger.info("Query timed out {}, ({}), -{}",
subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude);
return;
}
logger.info(queryMarker, "{}", subquery);
final SearchTerms searchTerms = new SearchTerms(subquery);
if (searchTerms.isEmpty()) {
logger.info(queryMarker, "empty");
return;
}
final Roaring64Bitmap results = new Roaring64Bitmap();
// logSearchTerms(subquery, searchTerms);
// These queries are different indices for one subquery
List<IndexQuery> queries = index.createQueries(searchTerms, queryParams);
for (var query : queries) {
@ -69,9 +52,7 @@ public class IndexQueryService {
final LongQueryBuffer buffer = new LongQueryBuffer(512);
while (query.hasMore()
&& results.getIntCardinality() < fetchSize * query.fetchSizeMultiplier
&& timeout.hasTimeLeft())
while (query.hasMore() && timeout.hasTimeLeft())
{
buffer.reset();
query.getMoreResults(buffer);
@ -85,8 +66,6 @@ public class IndexQueryService {
results.clear();
}
}
logger.info(queryMarker, "{} from {}", results.getIntCardinality(), query);
}
if (!results.isEmpty()) {
@ -94,41 +73,4 @@ public class IndexQueryService {
}
}
private void logSearchTerms(SearchSubquery subquery, SearchTerms searchTerms) {
// This logging should only be enabled in testing, as it is very verbose
// and contains sensitive information
if (!logger.isInfoEnabled(queryMarker)) {
return;
}
var includes = subquery.searchTermsInclude;
var advice = subquery.searchTermsAdvice;
var excludes = subquery.searchTermsExclude;
var priority = subquery.searchTermsPriority;
for (int i = 0; i < includes.size(); i++) {
logger.info(queryMarker, "{} -> {} I", includes.get(i),
Long.toHexString(searchTerms.includes().getLong(i))
);
}
for (int i = 0; i < advice.size(); i++) {
logger.info(queryMarker, "{} -> {} A", advice.get(i),
Long.toHexString(searchTerms.includes().getLong(includes.size() + i))
);
}
for (int i = 0; i < excludes.size(); i++) {
logger.info(queryMarker, "{} -> {} E", excludes.get(i),
Long.toHexString(searchTerms.excludes().getLong(i))
);
}
for (int i = 0; i < priority.size(); i++) {
logger.info(queryMarker, "{} -> {} P", priority.get(i),
Long.toHexString(searchTerms.priority().getLong(i))
);
}
}
}