Move index to top level of code

code/api/executor-api/build.gradle

@@ -24,7 +24,7 @@ sourceSets {
 
 dependencies {
     implementation project(':code:common:model')
-    implementation project(':code:functions:index:api')
+    implementation project(':code:index:api')
     implementation project(':code:common:config')
     implementation project(':code:common:db')
     implementation project(':code:libraries:message-queue')

code/functions/index/domain-ranking/build.gradle

@@ -1,44 +0,0 @@
-plugins {
-    id 'java'
-
-
-    id "de.undercouch.download" version "5.1.0"
-
-    id 'jvm-test-suite'
-}
-
-java {
-    toolchain {
-        languageVersion.set(JavaLanguageVersion.of(21))
-    }
-}
-
-dependencies {
-    implementation project(':code:common:db')
-    implementation project(':code:common:model')
-    implementation project(':code:common:service')
-    implementation project(':code:functions:domain-links:api')
-
-    implementation 'org.jgrapht:jgrapht-core:1.5.2'
-
-    implementation libs.bundles.slf4j
-    implementation libs.bundles.mariadb
-    implementation libs.guice
-    implementation libs.notnull
-    implementation libs.roaringbitmap
-    implementation libs.trove
-    implementation libs.fastutil
-    implementation libs.hll
-
-    testImplementation project(':code:libraries:array')
-
-    testImplementation libs.commons.lang3
-    testImplementation libs.bundles.slf4j.test
-    testImplementation libs.bundles.junit
-    testImplementation libs.mockito
-
-    testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
-    testImplementation 'org.testcontainers:mariadb:1.17.4'
-    testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
-    testImplementation project(':code:libraries:test-helpers')
-}

code/functions/index/domain-ranking/readme.md

@@ -1,39 +0,0 @@
-# Domain Ranking
-
-Contains domain ranking algorithms.  The domain ranking algorithms are based on 
-the JGraphT library.  
-
-Two principal algorithms are available, the standard PageRank algorithm,
-and personalized pagerank; each are available for two graphs, the link graph
-and a similarity graph where each edge corresponds to the similarity between
-the sets of incident links to two domains, their cosine similarity acting as
-the weight of the links.
-
-With the standard PageRank algorithm, the similarity graph does not produce
-anything useful, but something magical happens when you apply Personalized PageRank 
-to this graph.  It turns into a very good "vibe"-sensitive ranking algorithm.
-
-It's unclear if this is a well known result, but it's a very interesting one
-for creating a ranking algorithm that is focused on a particular segment of the web.
-
-## Central Classes
-
-* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java) - Ranks domains using the 
- PageRank or Personalized PageRank algorithm depending on whether a list of influence domains is provided.
-
-### Data sources
-
-* [LinkGraphSource](src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java) - fetches the link graph
-* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java) - fetches the inverted link graph
-* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java) - fetches the similarity graph from the database 
-
-Note that the similarity graph needs to be precomputed and stored in the database for
-the similarity graph source to be available.
-
-## See Also
-
-* [result-ranking](../result-ranking) - Ranks search results
-
-## Useful Resources
-
-* [The PageRank Citation Ranking: Bringing Order to the Web](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf)

code/functions/index/readme.md

@@ -1,24 +0,0 @@
-# Index
-
-These are components that offer functionality for the [index-service](../../services-core/index-service).
-
-## Indexes
-
-There are two indexes with accompanying tools for constructing them.
-
-* [index-reverse](index-reverse/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. 
-
-* [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. 
-
-These indices rely heavily on the [libraries/btree](../../libraries/btree) and [libraries/array](../../libraries/array) components.
-
-## Algorithms
-
-* [domain-ranking](domain-ranking/) contains domain ranking algorithms.
-* [result-ranking](result-ranking/) contains logic for ranking search results by relevance.
-
-# Libraries
-
-* [index-query](index-query/) contains structures for evaluating search queries.
-* [index-journal](index-journal/) contains tools for writing and reading index data.
-

code/functions/index/result-ranking/build.gradle

@@ -1,31 +0,0 @@
-plugins {
-    id 'java'
-    id "de.undercouch.download" version "5.1.0"
-    id 'jvm-test-suite'
-}
-
-java {
-    toolchain {
-        languageVersion.set(JavaLanguageVersion.of(21))
-    }
-}
-
-dependencies {
-    implementation project(':code:common:model')
-    implementation project(':code:common:service')
-    implementation project(':code:functions:index:api')
-    implementation project(':code:functions:search-query:api')
-
-    implementation libs.bundles.slf4j
-    implementation libs.guice
-    implementation libs.notnull
-    implementation libs.trove
-    implementation libs.fastutil
-
-    testImplementation libs.bundles.slf4j.test
-    testImplementation libs.bundles.junit
-    testImplementation libs.mockito
-    testImplementation project(':code:libraries:term-frequency-dict')
-    testImplementation project(':code:libraries:braille-block-punch-cards')
-}
-

code/functions/index/result-ranking/readme.md

@@ -1,14 +0,0 @@
-# Result Ranking
-
-Contains various heuristics for deciding which search results are important
-with regard to a query. In broad strokes [BM-25](https://nlp.stanford.edu/IR-book/html/htmledition/okapi-bm25-a-non-binary-model-1.html)
-is used, with a number of additional bonuses and penalties to rank the appropriate search
-results higher.
-
-## Central Classes
-
-* [ResultValuator](src/main/java/nu/marginalia/ranking/ResultValuator.java)
-
-## See Also
-
-* [features-index/domain-ranking](../domain-ranking) - Ranks domains

code/functions/search-query/api/build.gradle

@@ -26,7 +26,7 @@ apply from: "$rootProject.projectDir/protobuf.gradle"
 dependencies {
     implementation project(':code:common:model')
     implementation project(':code:common:config')
-    implementation project(':code:functions:index:query')
+    implementation project(':code:index:query')
     implementation project(':code:common:service-discovery')
 
     implementation libs.bundles.slf4j

code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java

@@ -5,7 +5,7 @@ package nu.marginalia.api.searchquery.model.results;
  * @param k  determines the size of the impact of a single term
  * @param b  determines the magnitude of the length normalization
  *
- * @see nu.marginalia.ranking.factors.Bm25Factor
+ * @see nu.marginalia.ranking.results.factors.Bm25Factor
  */
 public record Bm25Parameters(double k, double b) {
 }

code/functions/search-query/build.gradle

@@ -10,7 +10,7 @@ java {
 }
 
 dependencies {
-    implementation project(':code:functions:index:api')
+    implementation project(':code:index:api')
 
     implementation project(':code:common:db')
     implementation project(':code:common:config')
@@ -19,7 +19,7 @@ dependencies {
 
     implementation project(':code:functions:search-query:api')
 
-    implementation project(':code:functions:index:query')
+    implementation project(':code:index:query')
 
     implementation project(':code:libraries:language-processing')
     implementation project(':code:libraries:term-frequency-dict')

code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java

@@ -8,19 +8,14 @@ import lombok.SneakyThrows;
 import nu.marginalia.api.searchquery.*;
 import nu.marginalia.api.searchquery.model.query.QueryParams;
 import nu.marginalia.db.DomainBlacklist;
-import nu.marginalia.functions.index.api.IndexClient;
+import nu.marginalia.index.api.IndexClient;
 import nu.marginalia.functions.searchquery.svc.QueryFactory;
 import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
 import nu.marginalia.model.id.UrlIdCodec;
-import nu.marginalia.service.client.GrpcChannelPoolFactory;
-import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
-import nu.marginalia.service.discovery.property.ServiceKey;
-import nu.marginalia.service.discovery.property.ServicePartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.*;
-import java.util.concurrent.*;
 
 @Singleton
 public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase {

code/index/api/src/main/java/nu/marginalia/index/api/IndexClient.java

@@ -1,4 +1,4 @@
-package nu.marginalia.functions.index.api;
+package nu.marginalia.index.api;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;

code/index/api/src/main/java/nu/marginalia/index/api/IndexMqClient.java

@@ -1,4 +1,4 @@
-package nu.marginalia.functions.index.api;
+package nu.marginalia.index.api;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;

code/index/api/src/main/java/nu/marginalia/index/api/IndexMqEndpoints.java

@@ -1,4 +1,4 @@
-package nu.marginalia.functions.index.api;
+package nu.marginalia.index.api;
 
 public class IndexMqEndpoints {
     public static final String INDEX_IS_BLOCKED = "INDEX-IS-BLOCKED";

code/index/build.gradle

@@ -10,9 +10,12 @@ java {
 }
 
 dependencies {
+    implementation 'org.jgrapht:jgrapht-core:1.5.2'
+
     implementation project(':third-party:commons-codec')
 
-    implementation project(':code:functions:index:api')
+    implementation project(':code:index:api')
+    implementation project(':code:functions:domain-links:api')
 
     implementation project(':code:libraries:array')
     implementation project(':code:libraries:btree')
@@ -26,17 +29,16 @@ dependencies {
 
     implementation project(':code:functions:search-query:api')
 
-    implementation project(':code:functions:index:domain-ranking')
-    implementation project(':code:functions:index:forward-index')
-    implementation project(':code:functions:index:reverse-index')
-    implementation project(':code:functions:index:query')
-    implementation project(':code:functions:index:index-journal')
-    implementation project(':code:functions:index:result-ranking')
+    implementation project(':code:index:forward-index')
+    implementation project(':code:index:reverse-index')
+    implementation project(':code:index:query')
+    implementation project(':code:index:index-journal')
 
 
     implementation libs.bundles.slf4j
 
     implementation libs.prometheus
+    implementation libs.roaringbitmap
     implementation libs.bundles.grpc
     implementation libs.notnull
     implementation libs.guice
@@ -48,6 +50,14 @@ dependencies {
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit
     testImplementation libs.mockito
+    testImplementation libs.commons.lang3
     testImplementation project(':code:common:process')
+    testImplementation project(':code:libraries:array')
 
+    testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
+    testImplementation 'org.testcontainers:mariadb:1.17.4'
+    testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
+    testImplementation project(':code:libraries:term-frequency-dict')
+    testImplementation project(':code:libraries:braille-block-punch-cards')
 }

code/index/forward-index/build.gradle

@@ -13,9 +13,8 @@ java {
 dependencies {
     implementation project(':code:libraries:array')
     implementation project(':code:libraries:btree')
-    implementation project(':code:functions:index:domain-ranking')
-    implementation project(':code:functions:index:query')
-    implementation project(':code:functions:index:index-journal')
+    implementation project(':code:index:query')
+    implementation project(':code:index:index-journal')
     implementation project(':code:common:model')
     implementation project(':code:common:process')
 

code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java

@@ -2,12 +2,12 @@ package nu.marginalia.index.forward;
 
 import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
 import nu.marginalia.array.LongArrayFactory;
+import nu.marginalia.index.domainrankings.DomainRankings;
 import nu.marginalia.index.journal.reader.IndexJournalReader;
 import nu.marginalia.array.LongArray;
 import nu.marginalia.model.id.UrlIdCodec;
 import nu.marginalia.model.idx.DocumentMetadata;
 import nu.marginalia.process.control.ProcessHeartbeat;
-import nu.marginalia.ranking.DomainRankings;
 import org.roaringbitmap.longlong.LongConsumer;
 import org.roaringbitmap.longlong.Roaring64Bitmap;
 import org.slf4j.Logger;

code/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java

@@ -1,13 +1,13 @@
 package nu.marginalia.index.forward;
 
 import lombok.SneakyThrows;
+import nu.marginalia.index.domainrankings.DomainRankings;
 import nu.marginalia.index.journal.model.IndexJournalEntry;
 import nu.marginalia.index.journal.reader.IndexJournalReaderSingleFile;
 import nu.marginalia.index.journal.writer.IndexJournalWriter;
 import nu.marginalia.index.journal.writer.IndexJournalWriterSingleFileImpl;
 import nu.marginalia.model.id.UrlIdCodec;
 import nu.marginalia.process.control.FakeProcessHeartbeat;
-import nu.marginalia.ranking.DomainRankings;
 import nu.marginalia.test.TestUtil;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;

code/index/query/build.gradle

@@ -13,10 +13,13 @@ java {
 
 dependencies {
     implementation project(':code:libraries:array')
+    implementation project(':code:common:model')
 
     implementation libs.bundles.slf4j
 
     implementation libs.prometheus
+    implementation libs.fastutil
+    implementation libs.bundles.mariadb
 
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit

code/index/query/src/main/java/nu/marginalia/index/domainrankings/DomainRankings.java

@@ -1,4 +1,4 @@
-package nu.marginalia.ranking;
+package nu.marginalia.index.domainrankings;
 
 import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
 import it.unimi.dsi.fastutil.ints.Int2ShortOpenHashMap;

code/index/readme.md

@@ -0,0 +1,70 @@
+# Index
+
+These are components that offer functionality for the [index-service](../../services-core/index-service).
+
+## Indexes
+
+There are two indexes with accompanying tools for constructing them.
+
+* [index-reverse](index-reverse/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. 
+
+* [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. 
+
+These indices rely heavily on the [libraries/btree](../../libraries/btree) and [libraries/array](../../libraries/array) components.
+
+## Algorithms
+
+* [domain-ranking](domain-ranking/) contains domain ranking algorithms.
+* [result-ranking](result-ranking/) contains logic for ranking search results by relevance.
+
+# Libraries
+
+* [index-query](index-query/) contains structures for evaluating search queries.
+* [index-journal](index-journal/) contains tools for writing and reading index data.
+
+# Domain Ranking
+
+Contains domain ranking algorithms.  The domain ranking algorithms are based on
+the JGraphT library.
+
+Two principal algorithms are available, the standard PageRank algorithm,
+and personalized pagerank; each are available for two graphs, the link graph
+and a similarity graph where each edge corresponds to the similarity between
+the sets of incident links to two domains, their cosine similarity acting as
+the weight of the links.
+
+With the standard PageRank algorithm, the similarity graph does not produce
+anything useful, but something magical happens when you apply Personalized PageRank
+to this graph.  It turns into a very good "vibe"-sensitive ranking algorithm.
+
+It's unclear if this is a well known result, but it's a very interesting one
+for creating a ranking algorithm that is focused on a particular segment of the web.
+
+## Central Classes
+
+* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java) - Ranks domains using the
+  PageRank or Personalized PageRank algorithm depending on whether a list of influence domains is provided.
+
+### Data sources
+
+* [LinkGraphSource](src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java) - fetches the link graph
+* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java) - fetches the inverted link graph
+* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java) - fetches the similarity graph from the database
+
+Note that the similarity graph needs to be precomputed and stored in the database for
+the similarity graph source to be available.
+
+## Useful Resources
+
+* [The PageRank Citation Ranking: Bringing Order to the Web](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf)
+
+# Result Ranking
+
+Contains various heuristics for deciding which search results are important
+with regard to a query. In broad strokes [BM-25](https://nlp.stanford.edu/IR-book/html/htmledition/okapi-bm25-a-non-binary-model-1.html)
+is used, with a number of additional bonuses and penalties to rank the appropriate search
+results higher.
+
+## Central Classes
+
+* [ResultValuator](src/main/java/nu/marginalia/ranking/ResultValuator.java)

code/index/reverse-index/build.gradle

@@ -15,9 +15,8 @@ dependencies {
     implementation project(':code:libraries:array')
     implementation project(':code:libraries:btree')
     implementation project(':code:libraries:random-write-funnel')
-    implementation project(':code:functions:index:domain-ranking')
-    implementation project(':code:functions:index:query')
-    implementation project(':code:functions:index:index-journal')
+    implementation project(':code:index:query')
+    implementation project(':code:index:index-journal')
     implementation project(':code:common:model')
     implementation project(':code:common:process')
 

code/index/src/main/java/nu/marginalia/index/IndexQueryGrpcService.java

@@ -1,4 +1,4 @@
-package nu.marginalia.functions.index;
+package nu.marginalia.index;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
@@ -13,13 +13,13 @@ import nu.marginalia.api.searchquery.model.query.SearchSubquery;
 import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
 import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
 import nu.marginalia.api.searchquery.model.results.SearchResultSet;
-import nu.marginalia.functions.index.index.IndexQueryService;
-import nu.marginalia.functions.index.index.StatefulIndex;
-import nu.marginalia.functions.index.model.IndexSearchParameters;
-import nu.marginalia.functions.index.results.IndexResultValuatorService;
-import nu.marginalia.functions.index.searchset.SearchSetsService;
-import nu.marginalia.functions.index.searchset.SmallSearchSet;
-import nu.marginalia.functions.index.searchset.SearchSet;
+import nu.marginalia.index.index.IndexQueryService;
+import nu.marginalia.index.index.StatefulIndex;
+import nu.marginalia.index.model.IndexSearchParameters;
+import nu.marginalia.index.results.IndexResultValuatorService;
+import nu.marginalia.index.searchset.SearchSetsService;
+import nu.marginalia.index.searchset.SmallSearchSet;
+import nu.marginalia.index.searchset.SearchSet;
 import nu.marginalia.service.module.ServiceConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

code/index/src/main/java/nu/marginalia/index/IndexServicesFactory.java

@@ -1,12 +1,9 @@
-package nu.marginalia.functions.index;
+package nu.marginalia.index;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import nu.marginalia.IndexLocations;
-import nu.marginalia.functions.index.index.CombinedIndexReader;
-import nu.marginalia.index.ReverseIndexFullFileNames;
-import nu.marginalia.index.ReverseIndexPrioFileNames;
-import nu.marginalia.index.ReverseIndexReader;
+import nu.marginalia.index.index.CombinedIndexReader;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.index.forward.ForwardIndexFileNames;
 import nu.marginalia.index.forward.ForwardIndexReader;

code/index/src/main/java/nu/marginalia/index/SearchTermsUtil.java

@@ -1,9 +1,9 @@
-package nu.marginalia.functions.index;
+package nu.marginalia.index;
 
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.longs.LongList;
 import nu.marginalia.api.searchquery.model.query.SearchSubquery;
-import nu.marginalia.functions.index.model.IndexSearchTerms;
+import nu.marginalia.index.model.IndexSearchTerms;
 import nu.marginalia.hash.MurmurHash3_128;
 
 import java.util.ArrayList;

code/index/src/main/java/nu/marginalia/index/index/CombinedIndexReader.java

@@ -1,6 +1,6 @@
-package nu.marginalia.functions.index.index;
+package nu.marginalia.index.index;
 
-import nu.marginalia.functions.index.model.IndexQueryParams;
+import nu.marginalia.index.model.IndexQueryParams;
 import nu.marginalia.index.ReverseIndexReader;
 import nu.marginalia.index.forward.ForwardIndexReader;
 import nu.marginalia.index.query.*;

code/index/src/main/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java

@@ -1,7 +1,7 @@
-package nu.marginalia.functions.index.index;
+package nu.marginalia.index.index;
 
 import gnu.trove.set.hash.TLongHashSet;
-import nu.marginalia.functions.index.model.IndexQueryParams;
+import nu.marginalia.index.model.IndexQueryParams;
 import nu.marginalia.index.ReverseIndexReader;
 import nu.marginalia.index.forward.ForwardIndexReader;
 import nu.marginalia.index.query.IndexQuery;

code/index/src/main/java/nu/marginalia/index/index/IndexQueryService.java

@@ -1,4 +1,4 @@
-package nu.marginalia.functions.index.index;
+package nu.marginalia.index.index;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
@@ -6,9 +6,9 @@ import gnu.trove.list.TLongList;
 import gnu.trove.list.array.TLongArrayList;
 import nu.marginalia.api.searchquery.model.query.SearchSubquery;
 import nu.marginalia.array.buffer.LongQueryBuffer;
-import nu.marginalia.functions.index.model.IndexSearchTerms;
-import nu.marginalia.functions.index.model.IndexSearchParameters;
-import nu.marginalia.functions.index.SearchTermsUtil;
+import nu.marginalia.index.model.IndexSearchTerms;
+import nu.marginalia.index.model.IndexSearchParameters;
+import nu.marginalia.index.SearchTermsUtil;
 import nu.marginalia.index.query.IndexQuery;
 import nu.marginalia.index.query.IndexQueryPriority;
 import org.slf4j.Logger;