161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(index) Index optimization

Browse Source
This commit is contained in:
Viktor Lofgren 2024-02-25 14:01:58 +01:00
parent 3eb0800742
commit 1a51ec2d69
2 changed files with 31 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
code/index/java/nu/marginalia/index/IndexGrpcService.java
View File

@ -14,7 +14,10 @@ import nu.marginalia.api.searchquery.model.results.*;
import nu.marginalia.index.index.IndexQueryService;
import nu.marginalia.index.index.StatefulIndex;
import nu.marginalia.index.model.SearchParameters;
import nu.marginalia.index.model.SearchTerms;
import nu.marginalia.index.model.SearchTermsUtil;
import nu.marginalia.index.query.IndexQuery;
import nu.marginalia.index.query.IndexSearchBudget;
import nu.marginalia.index.results.IndexResultValuatorService;
import nu.marginalia.index.results.model.ids.CombinedDocIdList;
import nu.marginalia.index.searchset.SearchSetsService;
@ -252,8 +255,15 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
/** Execute a search query */
public SearchResultSet run(SearchParameters parameters) throws SQLException, InterruptedException {
for (var subquery : parameters.subqueries) {
workerPool.execute(new IndexLookup(subquery, parameters));
var terms = new SearchTerms(subquery);
if (terms.isEmpty())
continue;
for (var indexQuery : index.createQueries(terms, parameters.queryParams)) {
workerPool.execute(new IndexLookup(indexQuery, parameters.budget));
}
}
for (int i = 0; i < indexValuationThreads; i++) {
@ -283,12 +293,13 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
* resultCandidateQueue, which depending on the state of the valuator threads may
* or may not block*/
class IndexLookup implements Runnable {
private final SearchSubquery subquery;
private final SearchParameters parameters;
private final IndexQuery query;
private final IndexSearchBudget budget;
IndexLookup(SearchSubquery subquery, SearchParameters parameters) {
this.subquery = subquery;
this.parameters = parameters;
IndexLookup(IndexQuery query,
IndexSearchBudget budget) {
this.query = query;
this.budget = budget;
remainingIndexTasks.incrementAndGet();
}
@ -296,9 +307,8 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
public void run() {
try {
indexQueryService.evaluateSubquery(
subquery,
parameters.queryParams,
parameters.budget,
query,
budget,
this::drain
);
}
@ -312,7 +322,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
}
private void drain(CombinedDocIdList resultIds) {
long remainingTime = parameters.budget.timeLeft();
long remainingTime = budget.timeLeft();
try {
if (!resultCandidateQueue.offer(resultIds)) {

34
code/index/java/nu/marginalia/index/index/IndexQueryService.java
View File

@ -16,7 +16,6 @@ import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
import org.slf4j.MarkerFactory;
import java.util.List;
import java.util.function.Consumer;
@Singleton
@ -35,36 +34,25 @@ public class IndexQueryService {
* at different priorty depths until timeout is reached or the results are all visited.
* Then the results are combined.
* */
public void evaluateSubquery(SearchSubquery subquery,
QueryParams queryParams,
public void evaluateSubquery(IndexQuery query,
IndexSearchBudget timeout,
Consumer<CombinedDocIdList> drain)
{
final SearchTerms searchTerms = new SearchTerms(subquery);
final Roaring64Bitmap results = new Roaring64Bitmap();
final LongArrayList results = new LongArrayList(512);
// These queries are different indices for one subquery
List<IndexQuery> queries = index.createQueries(searchTerms, queryParams);
for (var query : queries) {
final LongQueryBuffer buffer = new LongQueryBuffer(512);
if (!timeout.hasTimeLeft())
break;
while (query.hasMore() && timeout.hasTimeLeft())
{
buffer.reset();
query.getMoreResults(buffer);
final LongQueryBuffer buffer = new LongQueryBuffer(512);
results.addElements(0, buffer.data, 0, buffer.end);
while (query.hasMore() && timeout.hasTimeLeft())
{
buffer.reset();
query.getMoreResults(buffer);
for (int i = 0; i < buffer.size(); i++) {
results.add(buffer.data[i]);
}
if (results.getIntCardinality() > 512) {
drain.accept(new CombinedDocIdList(results));
results.clear();
}
if (results.size() < 512) {
drain.accept(new CombinedDocIdList(results));
results.clear();
}
}