From 0307c55f9f995cbabc63714dc6d6ee021a2f159f Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 11:41:14 +0100 Subject: [PATCH 01/56] (refac) Zookeeper for service-discovery, kill service-client lib (WIP) To avoid having to either hard-code or manually configure service addresses (possibly several dozen), and to reduce the project's dependency on docker to deal with routing and discovery, the option to use [Zookeeper](https://zookeeper.apache.org/) to manage services and discovery has been added. A service registry interface was added, with a Zookeeper implementation and a basic implementation that only works on docker and hard-codes everything. The last remaining REST service, the assistant-service, has been migrated to gRPC. This also proved a good time to clear out primordial technical debt from the root of the codebase. The 'service-client' library has been taken behind the barn and given a last farewell. It's replaced by a small library for managing gRPC channels. Since it's no longer used by anything, RxJava has been removed as a dependency from the project. Although the current state seems reasonably stable, this is a work-in-progress commit. --- code/api/assistant-api/build.gradle | 18 +- .../assistant/client/AssistantClient.java | 190 +++++--- .../client/AssistantProtobufCodec.java | 136 ++++++ .../src/main/protobuf/assistant-api.proto | 119 +++++ code/api/executor-api/build.gradle | 2 - .../executor/client/ExecutorClient.java | 70 ++- code/api/index-api/build.gradle | 3 +- .../marginalia/index/client/IndexClient.java | 50 +- .../index/client/IndexProtobufCodec.java | 2 +- .../results/ResultRankingParameters.java | 2 +- .../index/client/IndexProtobufCodecTest.java | 1 - .../mqapi/converting/ConvertRequest.java | 1 - code/api/query-api/build.gradle | 2 - .../marginalia/query/QueryProtobufCodec.java | 48 ++ .../marginalia/query/client/QueryClient.java | 126 ++--- code/common/config/build.gradle | 1 - .../src/main/java/nu/marginalia/WmsaHome.java | 2 - .../nodecfg/NodeConfigurationServiceTest.java | 6 - code/common/db/build.gradle | 1 - .../db/DomainRankingSetsService.java | 2 +- .../java/nu/marginalia/db/DomainTypes.java | 2 +- .../linkdb/dlinks/DomainLinkDb.java | 3 - code/common/model/build.gradle | 2 - .../java/nu/marginalia/model/EdgeUrl.java | 11 +- .../model/crawl/UrlIndexingState.java | 2 +- .../ProcessConfigurationModule.java | 4 - .../marginalia/renderer/MustacheRenderer.java | 2 - code/common/service-client/build.gradle | 37 -- code/common/service-client/readme.md | 10 - .../marginalia/client/AbortingScheduler.java | 49 -- .../nu/marginalia/client/AbstractClient.java | 438 ------------------ .../client/AbstractDynamicClient.java | 38 -- .../java/nu/marginalia/client/Context.java | 89 ---- .../marginalia/client/ContextScrambler.java | 85 ---- .../client/EndpointLivenessMonitor.java | 84 ---- .../nu/marginalia/client/HttpStatusCode.java | 19 - .../client/exception/LocalException.java | 15 - .../client/exception/MessagingException.java | 20 - .../client/exception/NetworkException.java | 15 - .../client/exception/RemoteException.java | 16 - .../RouteNotConfiguredException.java | 15 - .../client/exception/TimeoutException.java | 15 - .../client/grpc/GrpcChannelPool.java | 92 ---- .../client/route/RouteProvider.java | 25 - .../marginalia/client/route/ServiceRoute.java | 12 - .../client/route/ServiceRoutes.java | 22 - .../marginalia/client/AbstractClientTest.java | 189 -------- .../java/nu/marginalia/client/TestServer.java | 59 --- code/common/service-discovery/build.gradle | 10 + code/common/service-discovery/readme.md | 86 +++- .../service}/NodeConfigurationWatcher.java | 26 +- .../service/SearchServiceDescriptors.java | 21 - .../marginalia/service}/ServiceMonitors.java | 2 +- .../client/GrpcChannelPoolFactory.java | 55 +++ .../client/GrpcMultiNodeChannelPool.java | 93 ++++ .../client/GrpcSingleNodeChannelPool.java | 132 ++++++ .../service/client/NodeSelectionStrategy.java | 17 + .../client/ServiceNotAvailableException.java | 12 + .../service/descriptor/ServiceDescriptor.java | 33 -- .../descriptor/ServiceDescriptors.java | 33 -- .../discovery/FixedServiceRegistry.java | 122 +++++ .../service/discovery/ServiceRegistryIf.java | 70 +++ .../service/discovery/ZkServiceRegistry.java | 288 ++++++++++++ .../monitor/ServiceChangeMonitor.java | 23 + .../ServiceGrpcEndpointChangeMonitor.java | 25 + .../discovery/monitor/ServiceMonitorIf.java | 16 + .../monitor/ServiceNodeChangeMonitor.java | 25 + .../ServiceRestEndpointChangeMonitor.java | 25 + .../service/discovery/property/ApiSchema.java | 6 + .../discovery/property/ServiceEndpoint.java | 80 ++++ .../discovery/ZkServiceRegistryTest.java | 124 +++++ code/common/service/build.gradle | 4 +- code/common/service/readme.md | 3 +- .../nu/marginalia/service/ConfigLoader.java | 2 - .../java/nu/marginalia/service/MainClass.java | 27 -- .../service/module/ServiceConfiguration.java | 7 +- .../module/ServiceConfigurationModule.java | 43 +- .../service/server/BaseServiceParams.java | 5 + .../service/server/Initialization.java | 8 +- .../service/server/RateLimiter.java | 26 +- .../nu/marginalia/service/server/Service.java | 103 ++-- .../service/server/StaticResources.java | 4 - .../nu/marginalia/extractor/AtagExporter.java | 2 +- .../keyword/extractors/UrlKeywords.java | 1 - .../keyword/extractors/WordsTfIdfCounts.java | 2 - .../keyword/SentenceExtractorTest.java | 6 - .../reddit/RedditEntryReaderTest.java | 3 - .../integration/reddit/db/RedditDbTest.java | 2 - .../crawl-blocklist/build.gradle | 1 - .../domain-ranking/build.gradle | 1 - .../index/forward/ForwardIndexFileNames.java | 2 +- .../features-index/index-journal/build.gradle | 1 - .../index/query/filter/QueryFilterStepIf.java | 2 - .../index/ReverseIndexFullFileNames.java | 2 +- .../index/ReverseIndexPrioFileNames.java | 2 +- .../index/ReverseIndexSelfTest.java | 2 - .../ReversePreindexReference.java | 1 - .../ReversePreindexFinalizeTest.java | 1 - .../ReversePreindexWordSegmentsTest.java | 1 - .../nu/marginalia/ranking/ResultValuator.java | 2 - .../browse/DbBrowseDomainsRandom.java | 2 +- .../marginalia/array/page/FoldBenchmark.java | 1 - .../array/page/QuicksortBenchmark.java | 1 - .../java/nu/marginalia/array/IntArray.java | 1 - .../java/nu/marginalia/array/LongArray.java | 1 - .../marginalia/array/algo/IntArraySort.java | 1 - .../array/algo/TwoArrayOperations.java | 2 - .../array/algo/IntArraySearchTest.java | 1 - .../array/algo/IntArraySortTest.java | 1 - .../algo/IntArrayTransformations2Test.java | 1 - .../algo/IntArrayTransformationsTest.java | 2 - .../array/algo/LongArraySearchTest.java | 2 - .../array/algo/LongArraySortNTest.java | 1 - .../array/algo/LongArraySortTest.java | 1 - .../algo/LongArrayTransformations2Test.java | 2 - .../algo/LongArrayTransformationsTest.java | 2 - .../marginalia/bigstring/PlainBigString.java | 2 - .../java/nu/marginalia/btree/BTreeReader.java | 1 - .../java/nu/marginalia/btree/BTreeWriter.java | 1 - .../language/model/DocumentSentence.java | 2 - code/libraries/message-queue/build.gradle | 1 - .../marginalia/actor/ActorStateMachine.java | 1 - .../mq/persistence/MqPersistence.java | 2 +- .../actor/ActorRecordMachineTest.java | 2 - .../rwf/RandomFileAssemblerTest.java | 1 - .../crawling-model/build.gradle | 1 - .../crawling/body/HttpFetchResult.java | 10 +- ...ibleLegacySerializableCrawlDataStream.java | 1 - .../crawling/model/CrawledDocument.java | 1 - .../DomainLinkRecordParquetFileReader.java | 1 - .../processes/converting-process/build.gradle | 1 - .../marginalia/converting/ConverterMain.java | 1 - .../converting/ConverterModule.java | 1 - .../processor/logic/PlainTextLogic.java | 1 - .../MariadbKbSpecialization.java | 1 - .../specialization/PhpBBSpecialization.java | 3 - .../converting/sideload/SideloadSource.java | 1 - .../sideload/reddit/RedditSideloader.java | 2 +- .../converting/writer/ConverterWriter.java | 1 - .../ConvertingIntegrationTestModule.java | 2 +- .../WikiSpecializationTest.java | 1 - code/processes/crawling-process/build.gradle | 2 - .../java/nu/marginalia/crawl/CrawlerMain.java | 2 +- .../nu/marginalia/crawl/CrawlerModule.java | 4 - .../retreival/CrawlerWarcResynchronizer.java | 3 - .../crawl/retreival/DomainProber.java | 3 +- .../retreival/fetcher/HttpFetcherImpl.java | 2 - .../CrawlerWarcResynchronizerTest.java | 1 - code/processes/loading-process/build.gradle | 1 - .../nu/marginalia/loading/LoaderModule.java | 4 - .../domains/DomainLoaderServiceTest.java | 3 - .../build.gradle | 2 +- .../WebsiteAdjacenciesCalculator.java | 14 +- .../adjacencies/AdjacenciesLoaderTest.java | 2 +- .../api-service/build.gradle | 2 - .../main/java/nu/marginalia/api/ApiMain.java | 3 +- .../nu/marginalia/api/ApiSearchOperator.java | 6 +- .../java/nu/marginalia/api/ApiService.java | 4 +- .../dating-service/build.gradle | 2 - .../java/nu/marginalia/dating/DatingMain.java | 3 +- .../explorer-service/build.gradle | 2 - .../nu/marginalia/explorer/ExplorerMain.java | 3 +- .../marginalia/explorer/ExplorerService.java | 4 +- .../search-service/build.gradle | 2 - .../java/nu/marginalia/search/SearchMain.java | 3 +- .../nu/marginalia/search/SearchModule.java | 2 +- .../nu/marginalia/search/SearchOperator.java | 63 ++- .../nu/marginalia/search/SearchService.java | 4 +- .../search/command/CommandEvaluator.java | 7 +- .../command/SearchCommandInterface.java | 3 +- .../search/command/commands/BangCommand.java | 3 +- .../command/commands/BrowseCommand.java | 9 +- .../command/commands/ConvertCommand.java | 5 +- .../command/commands/DefinitionCommand.java | 25 +- .../command/commands/SearchCommand.java | 6 +- .../command/commands/SiteRedirectCommand.java | 3 +- .../search/results/BrowseResultCleaner.java | 1 - .../search/results/UrlDeduplicator.java | 1 - .../search/svc/SearchBrowseService.java | 15 +- .../search/svc/SearchCrosstalkService.java | 5 +- .../search/svc/SearchErrorPageService.java | 3 +- .../search/svc/SearchFlagSiteService.java | 2 +- .../search/svc/SearchQueryService.java | 8 +- .../search/svc/SearchSiteInfoService.java | 83 ++-- .../svc/SearchUnitConversionService.java | 43 +- .../command/commands/BangCommandTest.java | 4 +- .../assistant-service/build.gradle | 3 +- .../assistant/AssistantGrpcService.java | 131 ++++++ .../marginalia/assistant/AssistantMain.java | 3 +- .../assistant/AssistantService.java | 103 +--- .../domains/DomainInformationService.java | 40 +- .../domains/SimilarDomainsService.java | 89 ++-- .../control-service/build.gradle | 2 - .../nu/marginalia/control/ControlMain.java | 3 +- .../control/ControlProcessModule.java | 4 - .../nu/marginalia/control/ControlService.java | 2 +- .../actor/precession/RecrawlAllActor.java | 1 - .../control/app/svc/SearchToBanService.java | 14 +- .../node/svc/ControlFileStorageService.java | 5 +- .../control/node/svc/ControlNodeService.java | 6 +- .../control/sys/model/EventLogEntry.java | 1 - .../control/sys/model/MessageQueueEntry.java | 2 - .../control/sys/svc/DataSetsService.java | 1 - .../control/sys/svc/EventLogService.java | 1 - .../control/sys/svc/HeartbeatService.java | 1 - .../control/svc/HeartbeatServiceTest.java | 5 - .../executor-service/build.gradle | 2 - .../marginalia/actor/task/ConvertActor.java | 23 +- .../actor/task/ConvertAndLoadActor.java | 7 +- .../actor/task/ExportDataActor.java | 4 +- .../nu/marginalia/executor/ExecutorMain.java | 3 +- .../nu/marginalia/executor/ExecutorSvc.java | 41 +- .../executor/svc/TransferService.java | 161 ------- .../nu/marginalia/process/ProcessService.java | 2 +- .../sideload/RedditSideloadHelper.java | 1 - .../svc/DomainListRefreshServiceTest.java | 3 +- code/services-core/index-service/build.gradle | 1 - .../java/nu/marginalia/index/IndexMain.java | 3 +- .../nu/marginalia/index/IndexService.java | 30 +- .../index/index/SearchIndexReader.java | 1 - .../index/results/IndexMetadataService.java | 2 - .../index/svc/IndexQueryService.java | 52 --- .../index/svc/SearchParameters.java | 5 - .../index/svc/searchset/RankingSearchSet.java | 2 - .../service/util/DictionaryDataTest.java | 17 - .../service/util/DictionaryHashMapTest.java | 58 --- ...ndexQueryServiceIntegrationTestModule.java | 2 +- .../svc/searchset/RankingSearchSetTest.java | 1 - code/services-core/query-service/build.gradle | 2 - .../marginalia/query/QueryBasicInterface.java | 26 +- .../query/QueryGRPCDomainLinksService.java | 22 +- .../nu/marginalia/query/QueryGRPCService.java | 27 +- .../java/nu/marginalia/query/QueryMain.java | 3 +- .../nu/marginalia/query/QueryService.java | 69 +-- .../query/svc/QueryFactoryTest.java | 1 - .../java/nu/marginalia/tools/Experiment.java | 3 +- .../tools/experiments/AdblockExperiment.java | 1 - .../experiments/DebugConverterExperiment.java | 1 - .../SentenceStatisticsExperiment.java | 1 - .../tools/experiments/TestExperiment.java | 1 - .../docker-compose-barebones-1.yml.template | 6 + settings.gradle | 9 +- .../blue/strategic/parquet/ParquetReader.java | 1 - 243 files changed, 2345 insertions(+), 2844 deletions(-) create mode 100644 code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java create mode 100644 code/api/assistant-api/src/main/protobuf/assistant-api.proto delete mode 100644 code/common/service-client/build.gradle delete mode 100644 code/common/service-client/readme.md delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/AbortingScheduler.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/AbstractClient.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/AbstractDynamicClient.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/Context.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/ContextScrambler.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/EndpointLivenessMonitor.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/HttpStatusCode.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/exception/LocalException.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/exception/MessagingException.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/exception/NetworkException.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/exception/RemoteException.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/exception/RouteNotConfiguredException.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/exception/TimeoutException.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/grpc/GrpcChannelPool.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/route/RouteProvider.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoute.java delete mode 100644 code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoutes.java delete mode 100644 code/common/service-client/src/test/java/nu/marginalia/client/AbstractClientTest.java delete mode 100644 code/common/service-client/src/test/java/nu/marginalia/client/TestServer.java rename code/{services-core/query-service/src/main/java/nu/marginalia/query => common/service-discovery/src/main/java/nu/marginalia/service}/NodeConfigurationWatcher.java (55%) delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java rename code/common/{service-client/src/main/java/nu/marginalia/client => service-discovery/src/main/java/nu/marginalia/service}/ServiceMonitors.java (99%) create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptors.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java create mode 100644 code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java create mode 100644 code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java delete mode 100644 code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryDataTest.java delete mode 100644 code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryHashMapTest.java diff --git a/code/api/assistant-api/build.gradle b/code/api/assistant-api/build.gradle index 02bc8780..e87165d7 100644 --- a/code/api/assistant-api/build.gradle +++ b/code/api/assistant-api/build.gradle @@ -1,7 +1,7 @@ plugins { id 'java' - + id "com.google.protobuf" version "0.9.4" id 'jvm-test-suite' } @@ -10,19 +10,31 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } + +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + } +} + +apply from: "$rootProject.projectDir/protobuf.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:config') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation libs.bundles.slf4j implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.gson + implementation libs.protobuf + implementation libs.javax.annotation + implementation libs.bundles.grpc testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java b/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java index 2e6b209d..a311aab1 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java +++ b/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java @@ -1,95 +1,159 @@ package nu.marginalia.assistant.client; -import com.google.gson.reflect.TypeToken; import com.google.inject.Inject; import com.google.inject.Singleton; -import io.reactivex.rxjava3.core.Observable; +import nu.marginalia.assistant.api.AssistantApiGrpc; import nu.marginalia.assistant.client.model.DictionaryResponse; import nu.marginalia.assistant.client.model.DomainInformation; import nu.marginalia.assistant.client.model.SimilarDomain; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.exception.RouteNotConfiguredException; -import nu.marginalia.model.gson.GsonFactory; -import nu.marginalia.service.descriptor.ServiceDescriptors; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; import nu.marginalia.service.id.ServiceId; -import nu.marginalia.client.Context; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +import static nu.marginalia.assistant.client.AssistantProtobufCodec.*; @Singleton -public class AssistantClient extends AbstractDynamicClient { +public class AssistantClient { + private static final Logger logger = LoggerFactory.getLogger(AssistantClient.class); + private final GrpcSingleNodeChannelPool channelPool; + private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); @Inject - public AssistantClient(ServiceDescriptors descriptors) { - super(descriptors.forId(ServiceId.Assistant), GsonFactory::get); + public AssistantClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle(ServiceId.Assistant, AssistantApiGrpc::newBlockingStub); + } - public Observable dictionaryLookup(Context ctx, String word) { - try { - return super.get(ctx, 0, "/dictionary/" + URLEncoder.encode(word, StandardCharsets.UTF_8), DictionaryResponse.class); - } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } + public Future dictionaryLookup(String word) { + return virtualExecutorService.submit(() -> { + var rsp = channelPool.api().dictionaryLookup( + DictionaryLookup.createRequest(word) + ); + + return DictionaryLookup.convertResponse(rsp); + }); } @SuppressWarnings("unchecked") - public Observable> spellCheck(Context ctx, String word) { - try { - return (Observable>) (Object) super.get(ctx, 0, "/spell-check/" + URLEncoder.encode(word, StandardCharsets.UTF_8), List.class); - } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } - } - public Observable unitConversion(Context ctx, String value, String from, String to) { - try { - return super.get(ctx, 0, "/unit-conversion?value=" + value + "&from=" + from + "&to=" + to); - } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } + public Future> spellCheck(String word) { + return virtualExecutorService.submit(() -> { + var rsp = channelPool.api().spellCheck( + SpellCheck.createRequest(word) + ); + + return SpellCheck.convertResponse(rsp); + }); } - public Observable evalMath(Context ctx, String expression) { - try { - return super.get(ctx, 0, "/eval-expression?value=" + URLEncoder.encode(expression, StandardCharsets.UTF_8)); + public Map> spellCheck(List words, Duration timeout) throws InterruptedException { + List>>> tasks = new ArrayList<>(); + + for (String w : words) { + tasks.add(() -> { + var rsp = channelPool.api().spellCheck( + SpellCheck.createRequest(w) + ); + + return Map.entry(w, SpellCheck.convertResponse(rsp)); + }); } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); + + var futures = virtualExecutorService.invokeAll(tasks, timeout.toMillis(), TimeUnit.MILLISECONDS); + Map> results = new HashMap<>(); + + for (var f : futures) { + if (!f.isDone()) + continue; + + var entry = f.resultNow(); + + results.put(entry.getKey(), entry.getValue()); } + + return results; } - public Observable> similarDomains(Context ctx, int domainId, int count) { - try { - return super.get(ctx, 0, STR."/domain/\{domainId}/similar?count=\{count}", new TypeToken>() {}) - .onErrorResumeWith(Observable.just(new ArrayList<>())); - } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } + public Future unitConversion(String value, String from, String to) { + return virtualExecutorService.submit(() -> { + var rsp = channelPool.api().unitConversion( + UnitConversion.createRequest(from, to, value) + ); + + return UnitConversion.convertResponse(rsp); + }); } - public Observable> linkedDomains(Context ctx, int domainId, int count) { - try { - return super.get(ctx, 0, STR."/domain/\{domainId}/linking?count=\{count}", new TypeToken>() {}) - .onErrorResumeWith(Observable.just(new ArrayList<>())); - } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } + public Future evalMath(String expression) { + return virtualExecutorService.submit(() -> { + var rsp = channelPool.api().evalMath( + EvalMath.createRequest(expression) + ); + + return EvalMath.convertResponse(rsp); + }); } - public Observable domainInformation(Context ctx, int domainId) { - try { - return super.get(ctx, 0, STR."/domain/\{domainId}/info", DomainInformation.class) - .onErrorResumeWith(Observable.just(new DomainInformation())); - } - catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } + public Future> similarDomains(int domainId, int count) { + return virtualExecutorService.submit(() -> { + try { + var rsp = channelPool.api().getSimilarDomains( + DomainQueries.createRequest(domainId, count) + ); + + return DomainQueries.convertResponse(rsp); + } + catch (Exception e) { + logger.warn("Failed to get similar domains", e); + + throw e; + } + }); + } + + public Future> linkedDomains(int domainId, int count) { + return virtualExecutorService.submit(() -> { + try { + var rsp = channelPool.api().getLinkingDomains( + DomainQueries.createRequest(domainId, count) + ); + + return DomainQueries.convertResponse(rsp); + } + catch (Exception e) { + logger.warn("Failed to get linked domains", e); + throw e; + } + }); + + } + + public Future domainInformation(int domainId) { + return virtualExecutorService.submit(() -> { + try { + var rsp = channelPool.api().getDomainInfo( + DomainInfo.createRequest(domainId) + ); + + return DomainInfo.convertResponse(rsp); + } + catch (Exception e) { + logger.warn("Failed to get domain information", e); + + throw e; + } + }); + } + + public boolean isAccepting() { + return channelPool.hasChannel(); } } diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java b/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java new file mode 100644 index 00000000..cd2d61fa --- /dev/null +++ b/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java @@ -0,0 +1,136 @@ +package nu.marginalia.assistant.client; + +import lombok.SneakyThrows; +import nu.marginalia.assistant.api.*; +import nu.marginalia.assistant.client.model.DictionaryEntry; +import nu.marginalia.assistant.client.model.DictionaryResponse; +import nu.marginalia.assistant.client.model.DomainInformation; +import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.model.EdgeDomain; +import nu.marginalia.model.EdgeUrl; + +import java.util.ArrayList; +import java.util.List; + +public class AssistantProtobufCodec { + + public static class DictionaryLookup { + public static RpcDictionaryLookupRequest createRequest(String word) { + return RpcDictionaryLookupRequest.newBuilder() + .setWord(word) + .build(); + } + public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) { + return new DictionaryResponse( + rsp.getWord(), + rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList() + ); + } + + private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) { + return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition()); + } + } + + public static class SpellCheck { + public static RpcSpellCheckRequest createRequest(String text) { + return RpcSpellCheckRequest.newBuilder() + .setText(text) + .build(); + } + + public static List convertResponse(RpcSpellCheckResponse rsp) { + return rsp.getSuggestionsList(); + } + } + + public static class UnitConversion { + public static RpcUnitConversionRequest createRequest(String from, String to, String unit) { + return RpcUnitConversionRequest.newBuilder() + .setFrom(from) + .setTo(to) + .setUnit(unit) + .build(); + } + + public static String convertResponse(RpcUnitConversionResponse rsp) { + return rsp.getResult(); + } + } + + public static class EvalMath { + public static RpcEvalMathRequest createRequest(String expression) { + return RpcEvalMathRequest.newBuilder() + .setExpression(expression) + .build(); + } + + public static String convertResponse(RpcEvalMathResponse rsp) { + return rsp.getResult(); + } + } + + public static class DomainQueries { + public static RpcDomainLinksRequest createRequest(int domainId, int count) { + return RpcDomainLinksRequest.newBuilder() + .setDomainId(domainId) + .setCount(count) + .build(); + } + + public static List convertResponse(RpcSimilarDomains rsp) { + List ret = new ArrayList<>(rsp.getDomainsCount()); + + for (RpcSimilarDomain sd : rsp.getDomainsList()) { + ret.add(convertResponseEntry(sd)); + } + + return ret; + } + + @SneakyThrows + private static SimilarDomain convertResponseEntry(RpcSimilarDomain sd) { + return new SimilarDomain( + new EdgeUrl(sd.getUrl()), + sd.getDomainId(), + sd.getRelatedness(), + sd.getRank(), + sd.getIndexed(), + sd.getActive(), + sd.getScreenshot(), + SimilarDomain.LinkType.valueOf(sd.getLinkType().name()) + ); + } + } + + public static class DomainInfo { + public static RpcDomainId createRequest(int domainId) { + return RpcDomainId.newBuilder() + .setDomainId(domainId) + .build(); + } + + public static DomainInformation convertResponse(RpcDomainInfoResponse rsp) { + return new DomainInformation( + new EdgeDomain(rsp.getDomain()), + rsp.getBlacklisted(), + rsp.getPagesKnown(), + rsp.getPagesFetched(), + rsp.getPagesIndexed(), + rsp.getIncomingLinks(), + rsp.getOutboundLinks(), + rsp.getNodeAffinity(), + rsp.getRanking(), + rsp.getSuggestForCrawling(), + rsp.getInCrawlQueue(), + rsp.getUnknownDomain(), + rsp.getIp(), + rsp.getAsn(), + rsp.getAsnOrg(), + rsp.getAsnCountry(), + rsp.getIpCountry(), + rsp.getState() + ); + } + } +} diff --git a/code/api/assistant-api/src/main/protobuf/assistant-api.proto b/code/api/assistant-api/src/main/protobuf/assistant-api.proto new file mode 100644 index 00000000..12aa7ce1 --- /dev/null +++ b/code/api/assistant-api/src/main/protobuf/assistant-api.proto @@ -0,0 +1,119 @@ +syntax="proto3"; +package assistantapi; + +option java_package="nu.marginalia.assistant.api"; +option java_multiple_files=true; + +service AssistantApi { + /** Looks up a word in the dictionary. */ + rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {} + /** Checks the spelling of a text. */ + rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {} + /** Converts a unit from one to another. */ + rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {} + /** Evaluates a mathematical expression. */ + rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {} + + /** Fetches information about a domain. */ + rpc getDomainInfo(RpcDomainId) returns (RpcDomainInfoResponse) {} + + /** Fetches similar domains to the given domain. */ + rpc getSimilarDomains(RpcDomainLinksRequest) returns (RpcSimilarDomains) {} + + /** Fetches domains with links to or from the given domain. */ + rpc getLinkingDomains(RpcDomainLinksRequest) returns (RpcSimilarDomains) {} +} + +message RpcDomainLinksRequest { + int32 domainId = 1; + int32 count = 2; +} + +message RpcDomainId { + int32 domainId = 1; +} + +message RpcDictionaryLookupRequest { + string word = 1; +} + +message RpcDictionaryLookupResponse { + string word = 1; + repeated RpcDictionaryEntry entries = 2; +} + +message RpcDictionaryEntry { + string type = 1; + string word = 2; + string definition = 3; +} + +message RpcSpellCheckRequest { + string text = 1; +} + +message RpcSpellCheckResponse { + repeated string suggestions = 1; +} + +message RpcUnitConversionRequest { + string unit = 1; + string from = 2; + string to = 3; +} + +message RpcUnitConversionResponse { + string result = 1; +} + +message RpcEvalMathRequest { + string expression = 1; +} + +message RpcEvalMathResponse { + string result = 1; +} + +message RpcDomainInfoResponse { + int32 domainId = 1; + string domain = 2; + bool blacklisted = 3; + int32 pagesKnown = 4; + int32 pagesFetched = 5; + int32 pagesIndexed = 6; + int32 incomingLinks = 7; + int32 outboundLinks = 8; + int32 nodeAffinity = 9; + double ranking = 10; + bool suggestForCrawling = 11; + bool inCrawlQueue = 12; + bool unknownDomain = 13; + string ip = 14; + int32 asn = 15; + string asnOrg = 16; + string asnCountry = 17; + string ipCountry = 18; + string state = 19; +} + +message RpcSimilarDomains { + repeated RpcSimilarDomain domains = 1; +} + +message RpcSimilarDomain { + string url = 1; + int32 domainId = 2; + double relatedness = 3; + double rank = 4; + bool indexed = 5; + bool active = 6; + bool screenshot = 7; + LINK_TYPE linkType = 8; + + enum LINK_TYPE { + BACKWARD = 0; + FOWARD = 1; + BIDIRECTIONAL = 2; + NONE = 3; + }; +} \ No newline at end of file diff --git a/code/api/executor-api/build.gradle b/code/api/executor-api/build.gradle index 257c0285..aca172ac 100644 --- a/code/api/executor-api/build.gradle +++ b/code/api/executor-api/build.gradle @@ -29,14 +29,12 @@ dependencies { implementation project(':code:common:db') implementation project(':code:libraries:message-queue') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation libs.bundles.slf4j implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.protobuf implementation libs.bundles.grpc implementation libs.javax.annotation diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 658f6b37..3c585263 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -2,60 +2,44 @@ package nu.marginalia.executor.client; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.Context; -import nu.marginalia.client.grpc.GrpcChannelPool; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; import nu.marginalia.executor.api.*; import nu.marginalia.executor.api.ExecutorApiGrpc.ExecutorApiBlockingStub; import nu.marginalia.executor.model.ActorRunState; import nu.marginalia.executor.model.ActorRunStates; -import nu.marginalia.executor.model.transfer.TransferItem; -import nu.marginalia.executor.model.transfer.TransferSpec; import nu.marginalia.executor.storage.FileStorageContent; import nu.marginalia.executor.storage.FileStorageFile; import nu.marginalia.executor.upload.UploadDirContents; import nu.marginalia.executor.upload.UploadDirItem; -import nu.marginalia.model.gson.GsonFactory; -import nu.marginalia.nodecfg.NodeConfigurationService; -import nu.marginalia.nodecfg.model.NodeConfiguration; -import nu.marginalia.service.descriptor.ServiceDescriptors; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.ApiSchema; import nu.marginalia.service.id.ServiceId; import nu.marginalia.storage.model.FileStorageId; -import io.grpc.ManagedChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.OutputStream; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.List; -import java.util.concurrent.TimeUnit; @Singleton -public class ExecutorClient extends AbstractDynamicClient { - private final GrpcChannelPool channelPool; +public class ExecutorClient { + private final GrpcMultiNodeChannelPool channelPool; private static final Logger logger = LoggerFactory.getLogger(ExecutorClient.class); + private final ServiceRegistryIf registry; @Inject - public ExecutorClient(ServiceDescriptors descriptors, NodeConfigurationService nodeConfigurationService) { - super(descriptors.forId(ServiceId.Executor), GsonFactory::get); - - channelPool = new GrpcChannelPool<>(ServiceId.Executor) { - @Override - public ExecutorApiBlockingStub createStub(ManagedChannel channel) { - return ExecutorApiGrpc.newBlockingStub(channel); - } - - @Override - public List getEligibleNodes() { - return nodeConfigurationService.getAll() - .stream() - .map(NodeConfiguration::node) - .toList(); - } - }; + public ExecutorClient(ServiceRegistryIf registry, + GrpcChannelPoolFactory grpcChannelPoolFactory) + { + this.registry = registry; + this.channelPool = grpcChannelPoolFactory + .createMulti(ServiceId.Executor, ExecutorApiGrpc::newBlockingStub); } public void startFsm(int node, String actorName) { @@ -286,22 +270,20 @@ public class ExecutorClient extends AbstractDynamicClient { } } - public void transferFile(Context context, int node, FileStorageId fileId, String path, OutputStream destOutputStream) { - String endpoint = "/transfer/file/%d?path=%s".formatted(fileId.id(), URLEncoder.encode(path, StandardCharsets.UTF_8)); + public void transferFile(int node, FileStorageId fileId, String path, OutputStream destOutputStream) { + String uriPath = STR."/transfer/file/\{fileId.id()}"; + String uriQuery = STR."path=\{URLEncoder.encode(path, StandardCharsets.UTF_8)}"; - get(context, node, endpoint, - destOutputStream) - .blockingSubscribe(); + var service = registry.getEndpoints(ApiSchema.REST, ServiceId.Executor, node) + .stream().findFirst().orElseThrow(); + + try (var urlStream = service.endpoint().toURL(uriPath, uriQuery).openStream()) { + urlStream.transferTo(destOutputStream); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } } - public TransferSpec getTransferSpec(Context context, int node, int count) { - return get(context, node, "/transfer/spec?count="+count, TransferSpec.class) - .timeout(30, TimeUnit.MINUTES) - .blockingFirst(); - } - - public void yieldDomain(Context context, int node, TransferItem item) { - post(context, node, "/transfer/yield", item).blockingSubscribe(); - } } diff --git a/code/api/index-api/build.gradle b/code/api/index-api/build.gradle index 08e629a4..a38fb9f0 100644 --- a/code/api/index-api/build.gradle +++ b/code/api/index-api/build.gradle @@ -10,6 +10,7 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } + sourceSets { main { proto { @@ -24,7 +25,6 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:config') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:libraries:message-queue') implementation project(':code:features-index:index-query') @@ -33,7 +33,6 @@ dependencies { implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.protobuf implementation libs.fastutil implementation libs.javax.annotation diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java index 7c334b8e..20a8891a 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java +++ b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java @@ -3,80 +3,34 @@ package nu.marginalia.index.client; import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; -import io.prometheus.client.Summary; -import io.reactivex.rxjava3.core.Observable; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.Context; -import nu.marginalia.client.exception.RouteNotConfiguredException; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.results.SearchResultSet; -import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.mq.outbox.MqOutbox; -import nu.marginalia.service.descriptor.ServiceDescriptors; import nu.marginalia.service.id.ServiceId; -import java.util.List; - -import javax.annotation.CheckReturnValue; import java.util.UUID; @Singleton -public class IndexClient extends AbstractDynamicClient { +public class IndexClient { - private static final Summary wmsa_search_index_api_time = Summary.build().name("wmsa_search_index_api_time").help("-").register(); private final MessageQueueFactory messageQueueFactory; MqOutbox outbox; @Inject - public IndexClient(ServiceDescriptors descriptors, - MessageQueueFactory messageQueueFactory, + public IndexClient(MessageQueueFactory messageQueueFactory, @Named("wmsa-system-node") Integer nodeId) { - super(descriptors.forId(ServiceId.Index), GsonFactory::get); this.messageQueueFactory = messageQueueFactory; String inboxName = ServiceId.Index.serviceName; String outboxName = "pp:"+System.getProperty("service-name", UUID.randomUUID().toString()); outbox = messageQueueFactory.createOutbox(inboxName, nodeId, outboxName, nodeId, UUID.randomUUID()); - setTimeout(30); } public MqOutbox outbox() { return outbox; } - @CheckReturnValue - public SearchResultSet query(Context ctx, int node, SearchSpecification specs) { - return wmsa_search_index_api_time.time( - () -> this.postGet(ctx, node,"/search/", specs, SearchResultSet.class).blockingFirst() - ); - } - - @CheckReturnValue - public SearchResultSet query(Context ctx, List nodes, SearchSpecification specs) { - return Observable.fromIterable(nodes) - .flatMap(node -> { - try { - return this - .postGet(ctx, node, "/search/", specs, SearchResultSet.class).onErrorReturn(t -> new SearchResultSet()) - .observeOn(Schedulers.io()); - } catch (RouteNotConfiguredException ex) { - return Observable.empty(); - } - }) - .reduce(SearchResultSet::combine) - .blockingGet(); - } - - - @CheckReturnValue - public Observable isBlocked(Context ctx, int node) { - return super.get(ctx, node, "/is-blocked", Boolean.class); - } - public long triggerRepartition(int node) throws Exception { return messageQueueFactory.sendSingleShotRequest( ServiceId.Index.withNode(node), diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java index 633da26e..3b8650ad 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java +++ b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java @@ -91,7 +91,7 @@ public class IndexProtobufCodec { ResultRankingParameters.TemporalBias.valueOf(params.getTemporalBias().getBias().name()), params.getTemporalBiasWeight() ); - }; + } public static RpcResultRankingParameters convertRankingParameterss(ResultRankingParameters rankingParams, RpcTemporalBias temporalBias) diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java b/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java index b7bb9075..112db234 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java +++ b/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java @@ -58,5 +58,5 @@ public class ResultRankingParameters { public enum TemporalBias { RECENT, OLD, NONE - }; + } } diff --git a/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java b/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java index 2ea6d9ea..7178a4c9 100644 --- a/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java +++ b/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java @@ -1,6 +1,5 @@ package nu.marginalia.index.client; -import nu.marginalia.index.api.RpcTemporalBias; import nu.marginalia.index.client.model.query.SearchSubquery; import nu.marginalia.index.client.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java index 51d678e3..3064ceca 100644 --- a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java +++ b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java @@ -1,7 +1,6 @@ package nu.marginalia.mqapi.converting; import lombok.AllArgsConstructor; -import lombok.NonNull; import nu.marginalia.storage.model.FileStorageId; import java.nio.file.Path; diff --git a/code/api/query-api/build.gradle b/code/api/query-api/build.gradle index ed893ae1..81b8a55e 100644 --- a/code/api/query-api/build.gradle +++ b/code/api/query-api/build.gradle @@ -16,7 +16,6 @@ dependencies { implementation project(':code:libraries:message-queue') implementation project(':code:features-index:index-query') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation libs.bundles.slf4j @@ -25,7 +24,6 @@ dependencies { implementation libs.notnull implementation libs.trove implementation libs.guice - implementation libs.rxjava implementation libs.gson implementation libs.bundles.grpc implementation libs.protobuf diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java b/code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java index d24a727e..f1b9b6f4 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java @@ -53,6 +53,37 @@ public class QueryProtobufCodec { return builder.build(); } + public static RpcIndexQuery convertQuery(String humanQuery, ProcessedQuery query) { + var builder = RpcIndexQuery.newBuilder(); + + for (var subquery : query.specs.subqueries) { + builder.addSubqueries(IndexProtobufCodec.convertSearchSubquery(subquery)); + } + + builder.setSearchSetIdentifier(query.specs.searchSetIdentifier); + builder.setHumanQuery(humanQuery); + + builder.setQuality(convertSpecLimit(query.specs.quality)); + builder.setYear(convertSpecLimit(query.specs.year)); + builder.setSize(convertSpecLimit(query.specs.size)); + builder.setRank(convertSpecLimit(query.specs.rank)); + builder.setDomainCount(convertSpecLimit(query.specs.domainCount)); + + builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); + + // Query strategy may be overridden by the query, but if not, use the one from the request + builder.setQueryStrategy(query.specs.queryStrategy.name()); + + builder.setParameters(IndexProtobufCodec.convertRankingParameterss( + query.specs.rankingParams, + RpcTemporalBias.newBuilder().setBias( + RpcTemporalBias.Bias.NONE) + .build()) + ); + + return builder.build(); + } + public static QueryParams convertRequest(RpcQsQuery request) { return new QueryParams( request.getHumanQuery(), @@ -179,4 +210,21 @@ public class QueryProtobufCodec { return builder.build(); } + + @SneakyThrows + public static DecoratedSearchResultItem convertQueryResult(RpcDecoratedResultItem rpcDecoratedResultItem) { + return new DecoratedSearchResultItem( + convertRawResult(rpcDecoratedResultItem.getRawItem()), + new EdgeUrl(rpcDecoratedResultItem.getUrl()), + rpcDecoratedResultItem.getTitle(), + rpcDecoratedResultItem.getDescription(), + rpcDecoratedResultItem.getUrlQuality(), + rpcDecoratedResultItem.getFormat(), + rpcDecoratedResultItem.getFeatures(), + rpcDecoratedResultItem.getPubYear(), + rpcDecoratedResultItem.getDataHash(), + rpcDecoratedResultItem.getWordsTotal(), + rpcDecoratedResultItem.getRankingScore() + ); + } } diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java index 6c6e63a4..6667e5b0 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java @@ -2,26 +2,18 @@ package nu.marginalia.query.client; import com.google.inject.Inject; import com.google.inject.Singleton; -import gnu.trove.list.array.TIntArrayList; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; import io.prometheus.client.Summary; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.Context; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; import nu.marginalia.index.api.Empty; import nu.marginalia.index.api.IndexDomainLinksApiGrpc; import nu.marginalia.index.api.QueryApiGrpc; import nu.marginalia.index.api.RpcDomainId; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.results.SearchResultSet; -import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.query.QueryProtobufCodec; import nu.marginalia.query.model.QueryParams; import nu.marginalia.query.model.QueryResponse; -import nu.marginalia.service.descriptor.ServiceDescriptor; -import nu.marginalia.service.descriptor.ServiceDescriptors; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; import nu.marginalia.service.id.ServiceId; -import org.roaringbitmap.PeekableCharIterator; import org.roaringbitmap.longlong.PeekableLongIterator; import org.roaringbitmap.longlong.Roaring64Bitmap; import org.slf4j.Logger; @@ -29,78 +21,37 @@ import org.slf4j.LoggerFactory; import javax.annotation.CheckReturnValue; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; @Singleton -public class QueryClient extends AbstractDynamicClient { +public class QueryClient { - private static final Summary wmsa_qs_api_delegate_time = Summary.build() - .name("wmsa_qs_api_delegate_time") - .help("query service delegate time") - .register(); private static final Summary wmsa_qs_api_search_time = Summary.build() .name("wmsa_qs_api_search_time") .help("query service search time") .register(); - private final Map channels = new ConcurrentHashMap<>(); - private final Map queryIndexApis = new ConcurrentHashMap<>(); - private final Map domainLinkApis = new ConcurrentHashMap<>(); - - record ServiceAndNode(String service, int node) { - public String getHostName() { - return service; - } - } - - private ManagedChannel getChannel(ServiceAndNode serviceAndNode) { - return channels.computeIfAbsent(serviceAndNode, - san -> ManagedChannelBuilder - .forAddress(serviceAndNode.getHostName(), 81) - .usePlaintext() - .build()); - } - - public QueryApiGrpc.QueryApiBlockingStub queryApi(int node) { - return queryIndexApis.computeIfAbsent(new ServiceAndNode("query-service", node), n -> - QueryApiGrpc.newBlockingStub( - getChannel(n) - ) - ); - } + private final GrpcSingleNodeChannelPool queryApiPool; + private final GrpcMultiNodeChannelPool domainLinkApiPool; public IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub domainApi(int node) { - return domainLinkApis.computeIfAbsent(new ServiceAndNode("query-service", node), n -> - IndexDomainLinksApiGrpc.newBlockingStub( - getChannel(n) - ) - ); + return domainLinkApiPool.apiForNode(node); } private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject - public QueryClient(ServiceDescriptors descriptors) { - - super(descriptors.forId(ServiceId.Query), GsonFactory::get); - } - public QueryClient() { - super(new ServiceDescriptor(ServiceId.Query, "query-service"), GsonFactory::get); - } - - /** Delegate an Index API style query directly to the index service */ - @CheckReturnValue - public SearchResultSet delegate(Context ctx, SearchSpecification specs) { - return wmsa_qs_api_delegate_time.time( - () -> this.postGet(ctx, 0, "/delegate/", specs, SearchResultSet.class).blockingFirst() - ); + public QueryClient(GrpcChannelPoolFactory channelPoolFactory) { + this.queryApiPool = channelPoolFactory.createSingle(ServiceId.Query, QueryApiGrpc::newBlockingStub); + this.domainLinkApiPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub); } @CheckReturnValue - public QueryResponse search(Context ctx, QueryParams params) { + public QueryResponse search(QueryParams params) { return wmsa_qs_api_search_time.time( - () -> QueryProtobufCodec.convertQueryResponse(queryApi(0).query(QueryProtobufCodec.convertQueryParams(params))) + () -> QueryProtobufCodec.convertQueryResponse(queryApiPool + .api() + .query(QueryProtobufCodec.convertQueryParams(params)) + ) ); } @@ -118,11 +69,16 @@ public class QueryClient extends AbstractDynamicClient { public List getLinksToDomain(int domainId) { try { - return domainApi(0).getLinksToDomain(RpcDomainId + return domainLinkApiPool.callEachSequential( + api -> api.getLinksToDomain(RpcDomainId .newBuilder() .setDomainId(domainId) .build()) - .getDomainIdList(); + .getDomainIdList()) + .flatMap(List::stream) + .sorted() + .toList(); + } catch (Exception e) { logger.error("API Exception", e); @@ -132,11 +88,15 @@ public class QueryClient extends AbstractDynamicClient { public List getLinksFromDomain(int domainId) { try { - return domainApi(0).getLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getDomainIdList(); + return domainLinkApiPool.callEachSequential( + api -> api.getLinksFromDomain(RpcDomainId + .newBuilder() + .setDomainId(domainId) + .build()) + .getDomainIdList()) + .flatMap(List::stream) + .sorted() + .toList(); } catch (Exception e) { logger.error("API Exception", e); @@ -146,11 +106,13 @@ public class QueryClient extends AbstractDynamicClient { public int countLinksToDomain(int domainId) { try { - return domainApi(0).countLinksToDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getIdCount(); + return domainLinkApiPool.callEachSequential( + api -> api.countLinksToDomain(RpcDomainId + .newBuilder() + .setDomainId(domainId) + .build()).getIdCount()) + .mapToInt(Integer::valueOf) + .sum(); } catch (Exception e) { logger.error("API Exception", e); @@ -160,11 +122,13 @@ public class QueryClient extends AbstractDynamicClient { public int countLinksFromDomain(int domainId) { try { - return domainApi(0).countLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getIdCount(); + return domainLinkApiPool.callEachSequential( + api -> api.countLinksFromDomain(RpcDomainId + .newBuilder() + .setDomainId(domainId) + .build()).getIdCount()) + .mapToInt(Integer::valueOf) + .sum(); } catch (Exception e) { logger.error("API Exception", e); diff --git a/code/common/config/build.gradle b/code/common/config/build.gradle index d8eb085b..5d4b5ecc 100644 --- a/code/common/config/build.gradle +++ b/code/common/config/build.gradle @@ -13,7 +13,6 @@ java { dependencies { implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:common:db') implementation project(':code:common:model') diff --git a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java index 00270102..e65c5529 100644 --- a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java +++ b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java @@ -3,8 +3,6 @@ package nu.marginalia; import nu.marginalia.service.ServiceHomeNotConfiguredException; -import java.io.FileNotFoundException; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java b/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java index 13ea555d..63a18c93 100644 --- a/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java +++ b/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java @@ -2,7 +2,6 @@ package nu.marginalia.nodecfg; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.storage.FileStorageService; import nu.marginalia.test.TestMigrationLoader; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -13,12 +12,7 @@ import org.testcontainers.containers.MariaDBContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; -import java.nio.file.Path; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; import static org.junit.jupiter.api.Assertions.*; diff --git a/code/common/db/build.gradle b/code/common/db/build.gradle index 320a58af..a31e21fb 100644 --- a/code/common/db/build.gradle +++ b/code/common/db/build.gradle @@ -40,7 +40,6 @@ dependencies { implementation libs.trove - implementation libs.rxjava implementation libs.bundles.mariadb flywayMigration 'org.flywaydb:flyway-mysql:10.0.1' diff --git a/code/common/db/src/main/java/nu/marginalia/db/DomainRankingSetsService.java b/code/common/db/src/main/java/nu/marginalia/db/DomainRankingSetsService.java index a977e0de..9c4ef4c8 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/DomainRankingSetsService.java +++ b/code/common/db/src/main/java/nu/marginalia/db/DomainRankingSetsService.java @@ -129,7 +129,7 @@ public class DomainRankingSetsService { ADJACENCY_CHEIRANK, /** For reserved names. Use special algorithm, function of name */ SPECIAL - }; + } /** Defines a domain ranking set, parameters for the ranking algorithms. * diff --git a/code/common/db/src/main/java/nu/marginalia/db/DomainTypes.java b/code/common/db/src/main/java/nu/marginalia/db/DomainTypes.java index c4405aab..9a1fe265 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/DomainTypes.java +++ b/code/common/db/src/main/java/nu/marginalia/db/DomainTypes.java @@ -24,7 +24,7 @@ public class DomainTypes { BLOG, CRAWL, TEST - }; + } private final Logger logger = LoggerFactory.getLogger(DomainTypes.class); diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java b/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java index bb7c43c1..ed6f0a1d 100644 --- a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java +++ b/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java @@ -2,10 +2,7 @@ package nu.marginalia.linkdb.dlinks; import gnu.trove.list.array.TIntArrayList; -import java.io.IOException; import java.nio.file.Path; -import java.sql.SQLException; -import java.util.Arrays; /** A database of source-destination pairs of domain IDs. The database is loaded into memory from * a source. The database is then kept in memory, reloading it upon switchInput(). diff --git a/code/common/model/build.gradle b/code/common/model/build.gradle index b737f02d..9a3b81d8 100644 --- a/code/common/model/build.gradle +++ b/code/common/model/build.gradle @@ -12,7 +12,6 @@ java { dependencies { implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:libraries:big-string') implementation project(':code:libraries:braille-block-punch-cards') @@ -28,7 +27,6 @@ dependencies { implementation libs.trove implementation libs.fastutil - implementation libs.rxjava implementation libs.bundles.mariadb testImplementation libs.bundles.slf4j.test diff --git a/code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java b/code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java index c09ed550..c86611fb 100644 --- a/code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java +++ b/code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java @@ -32,7 +32,16 @@ public class EdgeUrl implements Serializable { } public EdgeUrl(String url) throws URISyntaxException { - this(new URI(urlencodeFixer(url))); + this(parseURI(url)); + } + + private static URI parseURI(String url) throws URISyntaxException { + try { + return new URI(urlencodeFixer(url)); + } + catch (URISyntaxException ex) { + throw new URISyntaxException(STR."Failed to parse URI '\{url}'", ex.getMessage()); + } } public static Optional parse(@Nullable String url) { diff --git a/code/common/model/src/main/java/nu/marginalia/model/crawl/UrlIndexingState.java b/code/common/model/src/main/java/nu/marginalia/model/crawl/UrlIndexingState.java index f4312480..25cc6265 100644 --- a/code/common/model/src/main/java/nu/marginalia/model/crawl/UrlIndexingState.java +++ b/code/common/model/src/main/java/nu/marginalia/model/crawl/UrlIndexingState.java @@ -5,6 +5,6 @@ public enum UrlIndexingState { OK, REDIRECT, DEAD, - DISQUALIFIED; + DISQUALIFIED } diff --git a/code/common/process/src/main/java/nu/marginalia/ProcessConfigurationModule.java b/code/common/process/src/main/java/nu/marginalia/ProcessConfigurationModule.java index 3d0023c0..cf87cdf8 100644 --- a/code/common/process/src/main/java/nu/marginalia/ProcessConfigurationModule.java +++ b/code/common/process/src/main/java/nu/marginalia/ProcessConfigurationModule.java @@ -1,12 +1,8 @@ package nu.marginalia; import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.name.Named; import com.google.inject.name.Names; -import java.util.Objects; import java.util.UUID; public class ProcessConfigurationModule extends AbstractModule { diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java b/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java index 0dae086c..f55cdecb 100644 --- a/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java +++ b/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java @@ -8,10 +8,8 @@ import lombok.SneakyThrows; import nu.marginalia.renderer.config.HandlebarsConfigurator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import spark.Response; import java.io.*; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; diff --git a/code/common/service-client/build.gradle b/code/common/service-client/build.gradle deleted file mode 100644 index da1b1a71..00000000 --- a/code/common/service-client/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -plugins { - id 'java' - id "de.undercouch.download" version "5.1.0" - id 'jvm-test-suite' -} - -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(21)) - } -} - -dependencies { - implementation project(':code:common:service-discovery') - - implementation libs.bundles.slf4j - - implementation libs.commons.lang3 - implementation libs.spark - implementation libs.guice - implementation libs.rxjava - - implementation libs.okhttp3 - implementation libs.bundles.httpcomponents - - implementation libs.bundles.gson - implementation libs.bundles.grpc - implementation libs.protobuf - - implementation libs.bundles.prometheus - - implementation libs.bundles.mariadb - - testImplementation libs.bundles.slf4j.test - testImplementation libs.bundles.junit - testImplementation libs.mockito -} diff --git a/code/common/service-client/readme.md b/code/common/service-client/readme.md deleted file mode 100644 index 09c535b2..00000000 --- a/code/common/service-client/readme.md +++ /dev/null @@ -1,10 +0,0 @@ -# Service Client - -These are base classes for all the [API](../../api) clients for talking to other [services](../service). - -## Central Classes - -* [AbstractDynamicClient](src/main/java/nu/marginalia/client/AbstractDynamicClient.java) base class for API clients -* [AbstractClient](src/main/java/nu/marginalia/client/AbstractClient.java) handles requests at a lower level -* [Context](src/main/java/nu/marginalia/client/Context.java) handles request tracking -* [ContextScrambler](src/main/java/nu/marginalia/client/ContextScrambler.java) handles anonymization of public IPs \ No newline at end of file diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/AbortingScheduler.java b/code/common/service-client/src/main/java/nu/marginalia/client/AbortingScheduler.java deleted file mode 100644 index d603a546..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/AbortingScheduler.java +++ /dev/null @@ -1,49 +0,0 @@ -package nu.marginalia.client; - -import io.reactivex.rxjava3.core.Scheduler; -import io.reactivex.rxjava3.schedulers.Schedulers; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class AbortingScheduler { - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Nullable - private ExecutorService executorService; - - public AbortingScheduler() { - } - - - public synchronized Scheduler get() { - return Schedulers.from(getExecutorService(), - true, - false); - } - - public synchronized void abort() { - if (null != executorService) { - executorService.shutdownNow(); - executorService = Executors.newVirtualThreadPerTaskExecutor(); - } - } - - @Nonnull - private synchronized ExecutorService getExecutorService() { - if (null == executorService) { - executorService = Executors.newVirtualThreadPerTaskExecutor(); - } - return executorService; - } - - public synchronized void close() { - if (null != executorService) { - executorService.shutdown(); - } - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/AbstractClient.java b/code/common/service-client/src/main/java/nu/marginalia/client/AbstractClient.java deleted file mode 100644 index 697671f0..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/AbstractClient.java +++ /dev/null @@ -1,438 +0,0 @@ -package nu.marginalia.client; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import com.google.protobuf.GeneratedMessageV3; -import io.reactivex.rxjava3.core.Observable; -import io.reactivex.rxjava3.core.ObservableSource; -import io.reactivex.rxjava3.plugins.RxJavaPlugins; -import lombok.SneakyThrows; -import nu.marginalia.client.exception.LocalException; -import nu.marginalia.client.exception.NetworkException; -import nu.marginalia.client.exception.RemoteException; -import nu.marginalia.client.exception.RouteNotConfiguredException; -import nu.marginalia.client.route.RouteProvider; -import nu.marginalia.client.route.ServiceRoutes; -import nu.marginalia.service.descriptor.ServiceDescriptor; -import okhttp3.*; -import org.apache.logging.log4j.ThreadContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.utils.IOUtils; - -import java.io.OutputStream; -import java.lang.reflect.Type; -import java.net.ConnectException; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; - -public abstract class AbstractClient implements AutoCloseable { - private final Logger logger = LoggerFactory.getLogger(getClass()); - - public static final String CONTEXT_OUTBOUND_REQUEST = "outbound-request"; - private final Gson gson; - private final OkHttpClient client; - - private boolean quiet; - final ServiceRoutes serviceRoutes; - private int timeout; - - private final EndpointLivenessMonitor livenessMonitor; - - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public AbstractClient(ServiceDescriptor service, int timeout, Supplier gsonProvider) { - this(new RouteProvider(service), timeout, gsonProvider); - } - - public AbstractClient(RouteProvider routeProvider, - int timeout, - Supplier gsonProvider) - { - this.gson = gsonProvider.get(); - - this.timeout = timeout; - client = new OkHttpClient.Builder() - .connectTimeout(100, TimeUnit.MILLISECONDS) - .readTimeout(6000, TimeUnit.SECONDS) - .retryOnConnectionFailure(true) - .followRedirects(true) - .build(); - - serviceRoutes = new ServiceRoutes(routeProvider); - - RxJavaPlugins.setErrorHandler(e -> { - if (e.getMessage() == null) { - logger.error("Error", e); - } - else { - logger.error("Error {}: {}", e.getClass().getSimpleName(), e.getMessage()); - } - }); - - logger.info("Finished creating client for {}", getClass().getSimpleName()); - livenessMonitor = new EndpointLivenessMonitor(this); - } - - @Override - public void close() { - livenessMonitor.close(); - scheduler().close(); - } - - public abstract AbortingScheduler scheduler(); - - public void setQuiet(boolean quiet) { - this.quiet = quiet; - } - - public abstract String name(); - - - public synchronized boolean isAccepting() { - Context ctx = Context.internal("ready"); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(0) + "/internal/ready").get().build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(Call::execute) - .map(this::getResponseStatus) - .flatMap(line -> validateStatus(line, req)) - .timeout(100, TimeUnit.MILLISECONDS) - .onErrorReturn(error -> 500) - .map(HttpStatusCode::new) - .map(HttpStatusCode::isGood) - .blockingFirst(); - } - - public synchronized boolean isResponsive(int node) { - Context ctx = Context.internal("ping"); - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + "/internal/ping").get().build(); - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(Call::execute) - .map(AbstractClient.this::getResponseStatus) - .flatMap(line -> validateStatus(line, req).timeout(5000, TimeUnit.SECONDS).onErrorReturn(e -> 500)) - .onErrorReturn(error -> 500) - .map(HttpStatusCode::new) - .map(HttpStatusCode::isGood) - .blockingFirst(); - } - - @SneakyThrows - protected synchronized Observable post(Context ctx, - int node, - String endpoint, - Object data) { - - ensureAlive(node); - - RequestBody body = RequestBody.create(json(data), MediaType.parse("application/json; charset=utf-8")); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).post(body).build(); - - return Observable - .just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(this::getResponseStatus) - .retryWhen(this::retryHandler) - .flatMap(line -> validateStatus(line, req)) - .map(HttpStatusCode::new) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - @SneakyThrows - protected synchronized Observable post(Context ctx, int node, String endpoint, GeneratedMessageV3 data) { - - ensureAlive(node); - - RequestBody body = RequestBody.create(data.toByteArray(), MediaType.parse("application/protobuf")); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).post(body).build(); - var call = client.newCall(req); - - logInbound(call); - ThreadContext.put("outbound-request", serviceRoutes.get(node) + endpoint); - try (var rsp = call.execute()) { - logOutbound(rsp); - int code = rsp.code(); - - return validateStatus(code, req).map(HttpStatusCode::new); - } - finally { - ThreadContext.remove("outbound-request"); - } - } - - - @SneakyThrows - protected synchronized Observable postGet(Context ctx, int node, String endpoint, Object data, Class returnType) { - - ensureAlive(node); - - RequestBody body = RequestBody.create(json(data), MediaType.parse("application/json")); - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).post(body).build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .retryWhen(this::retryHandler) - .map(rsp -> validateResponseStatus(rsp, req, 200)) - .map(rsp -> getEntity(rsp, returnType)) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - protected synchronized Observable post(Context ctx, int node, String endpoint, String data, MediaType mediaType) { - ensureAlive(node); - - var body = RequestBody.create(data, mediaType); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).post(body).build(); - var call = client.newCall(req); - - - return Observable.just(call) - .map((c) -> { - ThreadContext.put(CONTEXT_OUTBOUND_REQUEST, serviceRoutes.get(node) + endpoint); - return c; - }) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(this::getResponseStatus) - .retryWhen(this::retryHandler) - .flatMap(line -> validateStatus(line, req)) - .map(HttpStatusCode::new) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - protected synchronized Observable get(Context ctx, int node, String endpoint, Class type) { - ensureAlive(node); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).get().build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(rsp -> validateResponseStatus(rsp, req, 200)) - .map(rsp -> getEntity(rsp, type)) - .retryWhen(this::retryHandler) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - protected synchronized Observable get(Context ctx, int node, String endpoint, TypeToken type) { - ensureAlive(node); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).get().build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(rsp -> validateResponseStatus(rsp, req, 200)) - .map(rsp -> getEntity(rsp, type)) - .retryWhen(this::retryHandler) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - protected synchronized Observable get(Context ctx, int node, String endpoint, OutputStream outputStream) { - ensureAlive(node); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).get().build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(rsp -> validateResponseStatus(rsp, req, 200)) - .map(rsp -> copyToOutputStream(rsp, outputStream)) - .retryWhen(this::retryHandler) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - @SuppressWarnings("unchecked") - protected synchronized Observable get(Context ctx, int node, String endpoint) { - ensureAlive(node); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).get().build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(rsp -> validateResponseStatus(rsp, req,200)) - .map(this::getText) - .retryWhen(this::retryHandler) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - protected synchronized Observable delete(Context ctx, int node, String endpoint) { - ensureAlive(node); - - var req = ctx.paint(new Request.Builder()).url(serviceRoutes.get(node) + endpoint).delete().build(); - - return Observable.just(client.newCall(req)) - .subscribeOn(scheduler().get()) - .map(this::logInbound) - .map(Call::execute) - .map(this::logOutbound) - .map(this::getResponseStatus) - .flatMap(line -> validateStatus(line, req)) - .map(HttpStatusCode::new) - .retryWhen(this::retryHandler) - .timeout(timeout, TimeUnit.SECONDS) - .doFinally(() -> ThreadContext.remove("outbound-request")); - } - - - @SneakyThrows - private Call logInbound(Call outgoing) { - return outgoing; - } - - @SneakyThrows - private Response logOutbound(Response incoming) { - return incoming; - } - - @SneakyThrows - private void ensureAlive(int node) { - if (!isAlive(node)) { - var route = serviceRoutes.get(node); - - logger.error("Route not configured for {}:{}", name(), node); - - throw new RouteNotConfiguredException("Route not configured for " + name() + ":" + node + " -- tried " + route); - } - } - - - private ObservableSource retryHandler(Observable error) { - return error.flatMap(this::filterRetryableExceptions); - } - - private Observable filterRetryableExceptions(Throwable error) throws Throwable { - - synchronized (livenessMonitor) { - // Signal to the liveness monitor that we may have an outage - livenessMonitor.notifyAll(); - } - - if (error.getClass().equals(RouteNotConfiguredException.class)) { - logger.error("Network error {}", error.getMessage()); - return Observable.empty().delay(50, TimeUnit.MILLISECONDS); - } - else if (error.getClass().equals(NetworkException.class)) { - logger.error("Network error {}", error.getMessage()); - return Observable.empty().delay(1, TimeUnit.SECONDS); - } - else if (error.getClass().equals(ConnectException.class)) { - logger.error("Network error {}", error.getMessage()); - return Observable.empty().delay(1, TimeUnit.SECONDS); - } - - if (!quiet) { - if (error.getMessage() != null) { - logger.error("{} {}", error.getClass().getSimpleName(), error.getMessage()); - } - else { - logger.error("Error ", error); - } - } - throw error; - } - - private Observable validateStatus(int status, Request request) { - if (status == org.apache.http.HttpStatus.SC_OK) - return Observable.just(status); - if (status == org.apache.http.HttpStatus.SC_ACCEPTED) - return Observable.just(status); - if (status == org.apache.http.HttpStatus.SC_CREATED) - return Observable.just(status); - - return Observable.error(new RemoteException(name() + " responded status code " + status + " " + request.url())); - } - - private Response validateResponseStatus(Response response, Request req, int expected) { - if (expected != response.code()) { - response.close(); - - throw new RemoteException(name() + " responded status code " + response.code() + ", " + req.method() + " " + req.url().toString()); - } - return response; - } - - private int getResponseStatus(Response response) { - try (response) { - return response.code(); - } - } - - - @SneakyThrows - private Integer copyToOutputStream(Response response, OutputStream outputStream) { - try (response) { - return IOUtils.copy(response.body().byteStream(), outputStream); - } - } - - @SneakyThrows - private T getEntity(Response response, Class clazz) { - try (response) { - return gson.fromJson(response.body().charStream(), clazz); - } - catch (Exception ex) { - throw ex; - } - } - @SneakyThrows - private T getEntity(Response response, TypeToken clazz) { - try (response) { - return gson.fromJson(response.body().charStream(), clazz); - } - catch (Exception ex) { - throw ex; - } - } - @SneakyThrows - private String getText(Response response) { - try (response) { - return response.body().string(); - } - - } - - public boolean isAlive(int node) { - return livenessMonitor.isAlive(node); - } - - private String json(Object o) { - try { - return gson.toJson(o); - } - catch (Exception ex) { - throw new LocalException(ex); - } - } - -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/AbstractDynamicClient.java b/code/common/service-client/src/main/java/nu/marginalia/client/AbstractDynamicClient.java deleted file mode 100644 index 5d2e3ef6..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/AbstractDynamicClient.java +++ /dev/null @@ -1,38 +0,0 @@ -package nu.marginalia.client; - -import com.google.gson.Gson; -import nu.marginalia.service.descriptor.ServiceDescriptor; - -import javax.annotation.Nonnull; -import java.util.function.Supplier; - -public class AbstractDynamicClient extends AbstractClient { - private final ServiceDescriptor service; - private final AbortingScheduler scheduler; - - public AbstractDynamicClient(@Nonnull ServiceDescriptor service, Supplier gsonProvider) { - super( - service, - 10000, - gsonProvider - ); - - this.service = service; - this.scheduler = new AbortingScheduler(); - } - - @Override - public String name() { - return service.name; - } - - public ServiceDescriptor getService() { - return service; - } - - @Override - public AbortingScheduler scheduler() { - return scheduler; - } - -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/Context.java b/code/common/service-client/src/main/java/nu/marginalia/client/Context.java deleted file mode 100644 index 101f4c2e..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/Context.java +++ /dev/null @@ -1,89 +0,0 @@ -package nu.marginalia.client; - -import io.reactivex.rxjava3.schedulers.Schedulers; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; - -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class Context { - public static final String CONTEXT_HEADER = "X-Context"; - public static final String SESSION_HEADER = "Cookie"; - public static final String PUBLIC_HEADER = "X-Public"; - - private final String id; - private final String session; - private boolean treatAsPublic; - - private Context(String id, String session) { - this.id = Objects.requireNonNull(id, "Context missing"); - this.session = session; - } - - public Context treatAsPublic() { - this.treatAsPublic = true; - return this; - } - - public static Context internal() { - return new Context(UUID.randomUUID().toString(), null); - } - public static Context internal(String why) { - return new Context(why + ":" + System.nanoTime(), null); - } - - public static Context fromRequest(Request request) { - - if (Boolean.getBoolean("unit-test")) { - return Context.internal(); - } - - final var ctxHeader = anonymizeContext(request); - final var sessHeader = request.headers(SESSION_HEADER); - - return new Context(ctxHeader, sessHeader); - } - - private static String anonymizeContext(Request request) { - String header = request.headers(CONTEXT_HEADER); - if (header != null && header.contains("-") && !header.startsWith("#")) { - // The public X-Context header contains info that traces to the - // external user's IP. Anonymize this by running it through a - // hash code blender with rotating salt - - return ContextScrambler.anonymize(header, request); - } - else if (header != null) { - return header; - } - else { - // When no X-Context is provided, synthesize one from path - return request.pathInfo() + ":" + Thread.currentThread().getId(); - } - } - - public okhttp3.Request.Builder paint(okhttp3.Request.Builder requestBuilder) { - requestBuilder.addHeader(CONTEXT_HEADER, id); - - if (session != null) { - requestBuilder.addHeader(SESSION_HEADER, session); - } - - if (treatAsPublic) { - requestBuilder.header(PUBLIC_HEADER, "1"); - } - - return requestBuilder; - } - - public String getContextId() { - return id; - } - - public boolean isPublic() { - return id.startsWith("#"); - } - -} \ No newline at end of file diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/ContextScrambler.java b/code/common/service-client/src/main/java/nu/marginalia/client/ContextScrambler.java deleted file mode 100644 index c10b5389..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/ContextScrambler.java +++ /dev/null @@ -1,85 +0,0 @@ -package nu.marginalia.client; - -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; -import io.reactivex.rxjava3.schedulers.Schedulers; -import spark.Request; - -import java.util.Arrays; -import java.util.Objects; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -public class ContextScrambler { - private static final Random random; - private static final HashFunction hf = Hashing.sha512(); - private static volatile byte[] seed = new byte[12]; - - static { - random = new Random(); - int gr = random.nextInt(10000, 20000); - for (int i = 0; i < gr; i++) { - random.nextLong(); - } - random.nextBytes(seed); - - updateSalt(); - } - - /** Anonymize the string by running it through a hash function - * together with a salt that is rotated at random intervals. - *

- * This is probably not cryptographically secure, but should at least - * be fairly annoying to reverse-engineer. - */ - public static String anonymize(String connectionInfo, Request request) { - byte[] hashData = Arrays.copyOf(seed, seed.length+4); - int hashi = Objects.hash(connectionInfo.split("-", 2)[0]); - - for (int i = 0; i < 4; i++) { - hashData[seed.length] = (byte)(hashi & 0xFF); - hashData[seed.length+1] = (byte)(hashi>>>8 & 0xFF); - hashData[seed.length+2] = (byte)(hashi>>>16 & 0xFF); - hashData[seed.length+3] = (byte)(hashi>>>24 & 0xFF); - } - - final int connHash = hf.hashBytes(hashData).asInt(); - final int requestHash = Objects.hash(request.url(), request.queryString()); - - return String.format("#%08x:%08x", connHash, requestHash); - } - - /** Generate a humongous salt with as many moving parts as possible, - * as creating a rainbow table of all IP-addresses is fairly easy - */ - private static byte[] generateSalt() { - byte[] oldHash = seed; - - int hash1 = random.nextInt(); - int hash2 = hf.hashLong(System.nanoTime()).asInt(); - int hash3 = hf.hashBytes(oldHash).asInt(); - - return new byte[]{ - (byte) (hash1 & 0xFF), - (byte) (hash1 >>> 8 & 0xFF), - (byte) (hash1 >>> 16 & 0xFF), - (byte) (hash1 >>> 24 & 0xFF), - (byte) (hash2 & 0xFF), - (byte) (hash2 >>> 8 & 0xFF), - (byte) (hash2 >>> 16 & 0xFF), - (byte) (hash2 >>> 24 & 0xFF), - (byte) (hash3 & 0xFF), - (byte) (hash3 >>> 8 & 0xFF), - (byte) (hash3 >>> 16 & 0xFF), - (byte) (hash3 >>> 24 & 0xFF) - }; - } - - private static void updateSalt() { - seed = generateSalt(); - - int delay = (int) (1000 * (300 + 600*Math.random())); - Schedulers.computation().scheduleDirect(ContextScrambler::updateSalt, delay, TimeUnit.MILLISECONDS); - } - -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/EndpointLivenessMonitor.java b/code/common/service-client/src/main/java/nu/marginalia/client/EndpointLivenessMonitor.java deleted file mode 100644 index 4c98b28c..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/EndpointLivenessMonitor.java +++ /dev/null @@ -1,84 +0,0 @@ -package nu.marginalia.client; - -import lombok.SneakyThrows; -import nu.marginalia.client.route.ServiceRoutes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.ConcurrentHashMap; - -/** Keep tabs on which endpoints are accessible via polling. This permits us to reduce the chances of - * synchronous requests blocking on timeout. - */ -public class EndpointLivenessMonitor { - - private final ConcurrentHashMap alivenessMap = new ConcurrentHashMap<>(); - private final AbstractClient client; - private final ServiceRoutes serviceRoutes; - - private static final Logger logger = LoggerFactory.getLogger(EndpointLivenessMonitor.class); - private static Thread daemonThread; - - public EndpointLivenessMonitor(AbstractClient client) { - this.client = client; - this.serviceRoutes = client.serviceRoutes; - - daemonThread = new Thread(this::run, client.getClass().getSimpleName()+":Liveness"); - daemonThread.setDaemon(true); - daemonThread.start(); - } - - @SneakyThrows - public void run() { - Thread.sleep(100); // Wait for initialization - - try { - while (!Thread.interrupted()) { - if (updateLivenessMap()) { - synchronized (this) { - wait(1000); - } - } - else Thread.sleep(100); - } - } catch (InterruptedException ex) { - // nothing to see here - } - } - - private boolean updateLivenessMap() { - boolean allAlive = true; - - for (int node : serviceRoutes.getNodes()) { - allAlive &= alivenessMap.compute(node, this::isResponsive); - } - - return allAlive; - } - - private boolean isResponsive(int node, Boolean oldValue) { - try { - boolean wasAlive = Boolean.TRUE.equals(oldValue); - boolean isAlive = client.isResponsive(node); - if (wasAlive != isAlive) { - logger.info("Liveness change {}:{} -- {}", client.name(), node, isAlive ? "UP":"DOWN"); - } - return isAlive; - } - catch (Exception ex) { - logger.warn("Oops", ex); - return false; - } - } - - public boolean isAlive(int node) { - // compute-if-absence ensures we do a synchronous status check if this is a cold start, - // that way we don't have to wait for the polling loop to find out if the service is up - return alivenessMap.computeIfAbsent(node, client::isResponsive); - } - - - public void close() { - daemonThread.interrupt(); - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/HttpStatusCode.java b/code/common/service-client/src/main/java/nu/marginalia/client/HttpStatusCode.java deleted file mode 100644 index aa23e71d..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/HttpStatusCode.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.client; - -public final class HttpStatusCode { - public final int code; - - public HttpStatusCode(int code) { - this.code = code; - } - - public boolean isGood() { - if (code == org.apache.http.HttpStatus.SC_OK) - return true; - if (code == org.apache.http.HttpStatus.SC_ACCEPTED) - return true; - if (code == org.apache.http.HttpStatus.SC_CREATED) - return true; - return false; - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/exception/LocalException.java b/code/common/service-client/src/main/java/nu/marginalia/client/exception/LocalException.java deleted file mode 100644 index bcaa2982..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/exception/LocalException.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.client.exception; - -public class LocalException extends MessagingException { - public LocalException() { - } - public LocalException(String message) { - super(message); - } - public LocalException(Throwable cause) { - super(cause); - } - public LocalException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/exception/MessagingException.java b/code/common/service-client/src/main/java/nu/marginalia/client/exception/MessagingException.java deleted file mode 100644 index 6151381e..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/exception/MessagingException.java +++ /dev/null @@ -1,20 +0,0 @@ -package nu.marginalia.client.exception; - -public class MessagingException extends RuntimeException { - public MessagingException() { - } - public MessagingException(String message) { - super(message); - } - public MessagingException(Throwable cause) { - super(cause); - } - public MessagingException(String message, Throwable cause) { - super(message, cause); - } - - @Override - public Throwable fillInStackTrace() { - return this; - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/exception/NetworkException.java b/code/common/service-client/src/main/java/nu/marginalia/client/exception/NetworkException.java deleted file mode 100644 index 593333ad..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/exception/NetworkException.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.client.exception; - -public class NetworkException extends MessagingException { - public NetworkException() { - } - public NetworkException(String message) { - super(message); - } - public NetworkException(Throwable cause) { - super(cause); - } - public NetworkException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/exception/RemoteException.java b/code/common/service-client/src/main/java/nu/marginalia/client/exception/RemoteException.java deleted file mode 100644 index d26df9b3..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/exception/RemoteException.java +++ /dev/null @@ -1,16 +0,0 @@ -package nu.marginalia.client.exception; - -public class RemoteException extends MessagingException { - public RemoteException() { - } - public RemoteException(String message) { - super(message); - } - public RemoteException(Throwable cause) { - super(cause); - } - public RemoteException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/exception/RouteNotConfiguredException.java b/code/common/service-client/src/main/java/nu/marginalia/client/exception/RouteNotConfiguredException.java deleted file mode 100644 index c3155dcf..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/exception/RouteNotConfiguredException.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.client.exception; - -public class RouteNotConfiguredException extends MessagingException { - public RouteNotConfiguredException() { - } - public RouteNotConfiguredException(String message) { - super(message); - } - public RouteNotConfiguredException(Throwable cause) { - super(cause); - } - public RouteNotConfiguredException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/exception/TimeoutException.java b/code/common/service-client/src/main/java/nu/marginalia/client/exception/TimeoutException.java deleted file mode 100644 index 17632758..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/exception/TimeoutException.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.client.exception; - -public class TimeoutException extends MessagingException { - public TimeoutException() { - } - public TimeoutException(String message) { - super(message); - } - public TimeoutException(Throwable cause) { - super(cause); - } - public TimeoutException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/grpc/GrpcChannelPool.java b/code/common/service-client/src/main/java/nu/marginalia/client/grpc/GrpcChannelPool.java deleted file mode 100644 index 6c23a841..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/grpc/GrpcChannelPool.java +++ /dev/null @@ -1,92 +0,0 @@ -package nu.marginalia.client.grpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import nu.marginalia.service.id.ServiceId; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.*; -import java.util.function.Function; -import java.util.stream.Stream; - -import static io.grpc.ConnectivityState.SHUTDOWN; - -/** A pool of gRPC stubs for a service, with a separate stub for each node. - * Manages broadcast-style request. */ -public abstract class GrpcChannelPool { - public GrpcChannelPool(String serviceName) { - this.serviceName = serviceName; - } - - protected record ServiceAndNode(String service, int node) { - public String getHostName() { - return service+"-"+node; - } - } - - private final Map channels = new ConcurrentHashMap<>(); - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); - - private final String serviceName; - - public GrpcChannelPool(ServiceId serviceId) { - this.serviceName = serviceId.serviceName; - } - - /** Get an API stub for the given node */ - public STUB apiForNode(int node) { - return createStub( - channels.compute( - new ServiceAndNode(serviceName, node), - this::refreshChannel) - ); - } - - private ManagedChannel refreshChannel(ServiceAndNode serviceAndNode, ManagedChannel old) { - if (old == null) - return createChannel(serviceAndNode); - - // If the channel is in SHUTDOWN state, we need to create a new one - // (shouldn't really happen in practice, but it's a good idea to be safe) - if (old.getState(true) == SHUTDOWN) { - old.shutdown(); - return createChannel(serviceAndNode); - } - - return old; - } - - protected ManagedChannel createChannel(ServiceAndNode serviceAndNode) { - return ManagedChannelBuilder.forAddress(serviceAndNode.getHostName(), 81) - .usePlaintext() - .build(); - } - - /** Invoke a function on each node, returning a list of futures in a terminal state, as per - * ExecutorService$invokeAll */ - public List> invokeAll(Function> callF) throws InterruptedException { - List> calls = getEligibleNodes().stream() - .map(id -> callF.apply(apiForNode(id))) - .toList(); - - return virtualExecutorService.invokeAll(calls); - } - - /** Invoke a function on each node, returning a stream of results */ - public Stream callEachSequential(Function call) { - return getEligibleNodes().stream() - .map(id -> call.apply(apiForNode(id))); - } - - - /** Create a stub for the given channel, this is an operation - * that needs to be implemented for the particular API this - * pool is intended for - */ - public abstract STUB createStub(ManagedChannel channel); - - /** Get the list of nodes that are eligible for broadcast-style requests */ - public abstract List getEligibleNodes(); - -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/route/RouteProvider.java b/code/common/service-client/src/main/java/nu/marginalia/client/route/RouteProvider.java deleted file mode 100644 index 56e2a564..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/route/RouteProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.client.route; - -import nu.marginalia.service.descriptor.ServiceDescriptor; - -public class RouteProvider { - private static int defaultPort = 80; - - private final ServiceDescriptor descriptor; - - public RouteProvider(ServiceDescriptor descriptor) { - this.descriptor = descriptor; - } - - ServiceRoute findRoute(int node) { - return new ServiceRoute(descriptor.getHostName(node), defaultPort); - } - - // Access exists for testing - public static void setDefaultPort(int port) { - defaultPort = port; - } - public static void resetDefaultPort() { - defaultPort = 80; - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoute.java b/code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoute.java deleted file mode 100644 index c9c6ffab..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoute.java +++ /dev/null @@ -1,12 +0,0 @@ -package nu.marginalia.client.route; - -import org.apache.http.HttpHost; - -public record ServiceRoute(String hostname, int port) { - public String toString() { - if (port == 80) { - return "http://" + hostname; - } - return new HttpHost(hostname(), port()).toURI(); - } -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoutes.java b/code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoutes.java deleted file mode 100644 index 6026bc16..00000000 --- a/code/common/service-client/src/main/java/nu/marginalia/client/route/ServiceRoutes.java +++ /dev/null @@ -1,22 +0,0 @@ -package nu.marginalia.client.route; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -public class ServiceRoutes { - private final ConcurrentHashMap knownRoutes = new ConcurrentHashMap<>(); - private final RouteProvider provider; - - public ServiceRoutes(RouteProvider provider) { - this.provider = provider; - } - - public ServiceRoute get(int node) { - return knownRoutes.computeIfAbsent(node, provider::findRoute); - } - - public List getNodes() { - return new ArrayList<>(knownRoutes.keySet()); - } -} diff --git a/code/common/service-client/src/test/java/nu/marginalia/client/AbstractClientTest.java b/code/common/service-client/src/test/java/nu/marginalia/client/AbstractClientTest.java deleted file mode 100644 index 2f02a8a7..00000000 --- a/code/common/service-client/src/test/java/nu/marginalia/client/AbstractClientTest.java +++ /dev/null @@ -1,189 +0,0 @@ -package nu.marginalia.client; - -import com.google.gson.Gson; -import io.reactivex.rxjava3.core.Observable; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.SneakyThrows; -import nu.marginalia.client.route.RouteProvider; -import nu.marginalia.client.route.ServiceRoute; -import nu.marginalia.service.descriptor.ServiceDescriptor; -import nu.marginalia.service.id.ServiceId; -import org.junit.jupiter.api.*; -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.*; - -public class AbstractClientTest { - - static TestServer testServer; - static AbstractClient client; - Gson gson = new Gson(); - - @Data @AllArgsConstructor - private static class DummyObject { - public int num; - public String str; - } - - @BeforeAll - public static void setUp() { - int port = new Random().nextInt(6000, 10000); - RouteProvider.setDefaultPort(port); - testServer = new TestServer(port); - - - client = new AbstractClient(new RouteProvider(new ServiceDescriptor(ServiceId.Api, "localhost")), 1, Gson::new) { - @Override - public AbortingScheduler scheduler() { - return new AbortingScheduler(); - } - - @Override - public String name() { - return "test"; - } - }; - client.setTimeout(1); - } - - - @AfterAll - public static void tearDown() { - testServer.close(); - client.close(); - RouteProvider.resetDefaultPort(); - } - - private void assertError(Observable observable) { - try { - observable.blockingSubscribe(); - } - catch (RuntimeException ex) { - System.out.println("Got exception " + ex.getClass().getSimpleName() + " -- as expected!" ); - return; - } - Assertions.fail("Expected exception"); - } - @SneakyThrows - private Object timeout(Request request, Response response) { - Thread.sleep(5000); - return "yawn"; - } - @SneakyThrows - private Object error404(Request request, Response response) { - Spark.halt(404); - return ""; - } - - @Test - public void testGetTimeout() { - testServer.get(this::timeout); - - assertError(client.get(Context.internal(), 0, "/get")); - } - - @Test - public void testPostTimeout() { - testServer.post(this::timeout); - - assertError(client.post(Context.internal(), 0, "/post", "test")); - } - - @Test - public void testDeleteTimeout() { - testServer.delete(this::timeout); - - assertError(client.delete(Context.internal(), 0,"/post")); - } - - @Test - public void testPost404() { - testServer.post(this::error404); - - assertError(client.post(Context.internal(), 0,"/post", "test")); - } - - @Test - public void testGet404() { - testServer.get(this::error404); - - assertError(client.get(Context.internal(), 0,"/get")); - } - - @Test - public void testDelete404() { - testServer.delete(this::error404); - - assertError(client.delete(Context.internal(),0, "/delete")); - } - - @Test - public void testGet() { - testServer.get((req, rsp) -> "Hello World"); - - assertEquals("Hello World", client.get(Context.internal(), 0,"/get").blockingFirst()); - } - - @Test - public void testAcceptingUp() { - testServer.setReady(true); - assertTrue(client.isAccepting()); - } - - @Test - public void testAcceptingDown() { - testServer.setReady(false); - assertFalse(client.isAccepting()); - } - - @Test - public void testGetJson() { - testServer.get((req, rsp) -> new DummyObject(5, "23"), new Gson()::toJson); - - assertEquals(client.get(Context.internal(), 0,"/get", DummyObject.class).blockingFirst(), - new DummyObject(5, "23")); - } - - - @Test - public void testDelete() { - testServer.delete((req, rsp) -> "Hello World"); - - assertTrue(client.delete(Context.internal(), 0,"/delete").blockingFirst().isGood()); - } - - - @Test - public void testPost() { - List inbox = new ArrayList<>(); - testServer.post((req, rsp) -> { - inbox.add(gson.fromJson(req.body(), DummyObject.class)); - return "ok"; - }); - - client.post(Context.internal(),0, "/post", new DummyObject(5, "23")).blockingSubscribe(); - assertEquals(1, inbox.size()); - assertEquals(new DummyObject(5, "23"), inbox.get(0)); - } - - @Test - public void testPostGet() { - List inbox = new ArrayList<>(); - testServer.post((req, rsp) -> { - inbox.add(gson.fromJson(req.body(), DummyObject.class)); - return new DummyObject(1, "ret"); - }, gson::toJson); - - var ret = client.postGet(Context.internal(), 0,"/post", new DummyObject(5, "23"), DummyObject.class).blockingFirst(); - assertEquals(1, inbox.size()); - assertEquals(new DummyObject(5, "23"), inbox.get(0)); - assertEquals(new DummyObject(1, "ret"), ret); - } -} diff --git a/code/common/service-client/src/test/java/nu/marginalia/client/TestServer.java b/code/common/service-client/src/test/java/nu/marginalia/client/TestServer.java deleted file mode 100644 index cbfe1075..00000000 --- a/code/common/service-client/src/test/java/nu/marginalia/client/TestServer.java +++ /dev/null @@ -1,59 +0,0 @@ -package nu.marginalia.client; - -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.util.function.BiFunction; -import java.util.function.Function; - -public class TestServer { - BiFunction onGet; - BiFunction onPost; - BiFunction onDelete; - - - boolean isReady; - - public TestServer(int port) { - Spark.port(port); - Spark.get("/internal/ping", (r,q) -> "pong"); - Spark.get("/internal/ready", this::ready); - Spark.get("/get", (request, response) -> onGet.apply(request, response)); - Spark.post("/post", (request, response) -> onPost.apply(request, response)); - Spark.delete("/delete", (request, response) -> onDelete.apply(request, response)); - } - - private Object ready(Request request, Response response) { - if (isReady) { - return ""; - } - else { - response.status(401); - return "bad"; - } - } - - public void close() { - Spark.stop(); - } - - public boolean isReady() { - return isReady; - } - - public void setReady(boolean ready) { - isReady = ready; - } - - public TestServer get(BiFunction onGet) { this.onGet = onGet; return this; } - public TestServer get(BiFunction onGet, Function transform) { - this.onGet = onGet.andThen(transform); - return this; - } - public TestServer delete(BiFunction onDelete) { this.onDelete = onDelete; return this; } - public TestServer post(BiFunction onPost) { this.onPost = onPost; return this; } - public TestServer post(BiFunction onPost, Function transform) { - this.onPost = onPost.andThen(transform); return this; - } -} diff --git a/code/common/service-discovery/build.gradle b/code/common/service-discovery/build.gradle index d0214416..c274b66a 100644 --- a/code/common/service-discovery/build.gradle +++ b/code/common/service-discovery/build.gradle @@ -16,7 +16,17 @@ java { dependencies { implementation libs.bundles.slf4j + implementation libs.bundles.curator + implementation libs.guice + implementation libs.bundles.mariadb + implementation libs.bundles.grpc + implementation libs.notnull + testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit testImplementation libs.mockito + + testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') + testImplementation 'org.testcontainers:mariadb:1.17.4' + testImplementation 'org.testcontainers:junit-jupiter:1.17.4' } diff --git a/code/common/service-discovery/readme.md b/code/common/service-discovery/readme.md index c1480192..617bdf3f 100644 --- a/code/common/service-discovery/readme.md +++ b/code/common/service-discovery/readme.md @@ -1,3 +1,87 @@ # Service Discovery -Contains classes for helping services discover each other. \ No newline at end of file +Contains classes for helping services discover each other, +and managing connections between them. + +## Service Registry + +The service registry is a class that keeps track of the services +that are currently running, and their connection information. + +There are two implementations: + +* A simple implementation that effectively hard-codes the services. +This is sufficient when running in docker, although loses some +smart features. It is fundamentally incompatible with running +the system bare-metal, and does not permit multiple instances +of a service to run. + +* A more advanced implementation that is based on [Zookeeper](https://zookeeper.apache.org/), +which is a distributed coordination service. This implementation +lets services register themselves and announce their liveness, +and then discover each other. It supports multiple instances of +a service running, and supports running the system bare-metal, +where it will assign ports to the services from a range. + +To be discoverable, the caller must first register their +services: + +```java +// Register one or more services +registry.registerService(ApiSchema.GRPC, + ServiceId.Test, + nodeId, + instanceUUID, //must be unique to the runtime + "127.0.0.1"); // bind-address +// (+ any other services) +``` + +Then, the caller must announce their instance. Before this is done, +the service is not discoverable. + +```java +registry.announceInstance(ServiceId.Test, + nodeId, + instanceUUID); +``` + +All of this is done automatically by the `Service` base class +in the [service](../service/) module. + +To discover a service, the caller can query the registry: + +```java +Set> endpoints = registry.getEndpoints(ApiSchema.GRPC, ServiceId.Test, nodeId); + +for (var endpoint : endpoints) { + System.out.println(endpoint.getHost() + ":" + endpoint.getPort()); +} +``` + +It's also possible to subscribe to changes in the registry, so that +the caller can be notified when a service comes or goes, with `registry.registerMonitor()`. + +## gRPC Channel Pool + +From the GrpcChannelPoolFactory, two types of channel pools can be created +that are aware of the service registry: + +* GrpcMultiNodeChannelPool - This pool permits 1-n style communication with partitioned services +* GrpcSingleNodeChannelPool - This pool permits 1-1 style communication with non-partitioned services + +The pools manage the lifecycle of the gRPC channels, and will permit the caller +to access Stub interfaces for the services. + +# Central Classes + +## Service Registry + +* [ServiceRegistryIf](src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java) +* [ZkServiceRegistry](src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java) +* [FixedServiceRegistry](src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java) + +## gRPC Channel Pool + +* [GrpcChannelPoolFactory](src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java) +* [GrpcMultiNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java) +* [GrpcSingleNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java) \ No newline at end of file diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/NodeConfigurationWatcher.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/NodeConfigurationWatcher.java similarity index 55% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/NodeConfigurationWatcher.java rename to code/common/service-discovery/src/main/java/nu/marginalia/service/NodeConfigurationWatcher.java index a7117387..aae5080e 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/NodeConfigurationWatcher.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/NodeConfigurationWatcher.java @@ -1,24 +1,25 @@ -package nu.marginalia.query; +package nu.marginalia.service; import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.nodecfg.NodeConfigurationService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; public class NodeConfigurationWatcher { private static final Logger logger = LoggerFactory.getLogger(NodeConfigurationWatcher.class); + private final HikariDataSource dataSource; private volatile List queryNodes = new ArrayList<>(); - private final NodeConfigurationService configurationService; @Inject - public NodeConfigurationWatcher(NodeConfigurationService configurationService) { - this.configurationService = configurationService; + public NodeConfigurationWatcher(HikariDataSource dataSource) { + this.dataSource = dataSource; var watcherThread = new Thread(this::pollConfiguration, "Node Configuration Watcher"); watcherThread.setDaemon(true); @@ -30,11 +31,20 @@ public class NodeConfigurationWatcher { for (;;) { List goodNodes = new ArrayList<>(); - for (var cfg : configurationService.getAll()) { - if (!cfg.disabled() && cfg.acceptQueries()) { - goodNodes.add(cfg.node()); + try (var conn = dataSource.getConnection()) { + var stmt = conn.prepareStatement(""" + SELECT ID FROM NODE_CONFIGURATION + WHERE ACCEPT_QUERIES AND NOT DISABLED + """); + var rs = stmt.executeQuery(); + while (rs.next()) { + goodNodes.add(rs.getInt(1)); } } + catch (SQLException ex) { + logger.error("Error polling node configuration", ex); + } + queryNodes = goodNodes; TimeUnit.SECONDS.sleep(10); diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java deleted file mode 100644 index d5f7fb8d..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java +++ /dev/null @@ -1,21 +0,0 @@ -package nu.marginalia.service; - -import nu.marginalia.service.descriptor.ServiceDescriptor; -import nu.marginalia.service.descriptor.ServiceDescriptors; -import nu.marginalia.service.id.ServiceId; - -import java.util.List; - -public class SearchServiceDescriptors { - public static ServiceDescriptors descriptors = new ServiceDescriptors( - List.of(new ServiceDescriptor(ServiceId.Api), - new ServiceDescriptor(ServiceId.Index), - new ServiceDescriptor(ServiceId.Query), - new ServiceDescriptor(ServiceId.Search), - new ServiceDescriptor(ServiceId.Executor), - new ServiceDescriptor(ServiceId.Assistant), - new ServiceDescriptor(ServiceId.Dating), - new ServiceDescriptor(ServiceId.Explorer), - new ServiceDescriptor(ServiceId.Control) - )); -} diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceMonitors.java similarity index 99% rename from code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java rename to code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceMonitors.java index b09ffa0c..b3b497ef 100644 --- a/code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceMonitors.java @@ -1,4 +1,4 @@ -package nu.marginalia.client; +package nu.marginalia.service; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java new file mode 100644 index 00000000..087fc666 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java @@ -0,0 +1,55 @@ +package nu.marginalia.service.client; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import nu.marginalia.service.NodeConfigurationWatcher; +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; +import nu.marginalia.service.id.ServiceId; + +import java.util.function.Function; + +@Singleton +public class GrpcChannelPoolFactory { + + private final NodeConfigurationWatcher nodeConfigurationWatcher; + private final ServiceRegistryIf serviceRegistryIf; + + @Inject + public GrpcChannelPoolFactory(NodeConfigurationWatcher nodeConfigurationWatcher, + ServiceRegistryIf serviceRegistryIf) + { + this.nodeConfigurationWatcher = nodeConfigurationWatcher; + this.serviceRegistryIf = serviceRegistryIf; + } + + /** Create a new multi-node channel pool for the given service. */ + public GrpcMultiNodeChannelPool createMulti(ServiceId serviceId, + Function stubConstructor) + { + return new GrpcMultiNodeChannelPool<>(serviceRegistryIf, + serviceId, + this::createChannel, + stubConstructor, + nodeConfigurationWatcher); + } + + /** Create a new single-node channel pool for the given service. */ + public GrpcSingleNodeChannelPool createSingle(ServiceId serviceId, + Function stubConstructor) + { + return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, serviceId, + new NodeSelectionStrategy.Any(), + this::createChannel, + stubConstructor); + } + + private ManagedChannel createChannel(InstanceAddress route) { + return ManagedChannelBuilder + .forAddress(route.host(), route.port()) + .usePlaintext() + .build(); + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java new file mode 100644 index 00000000..e816bd1a --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -0,0 +1,93 @@ +package nu.marginalia.service.client; + +import io.grpc.ManagedChannel; +import lombok.SneakyThrows; +import nu.marginalia.service.NodeConfigurationWatcher; +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.ServiceEndpoint; +import nu.marginalia.service.id.ServiceId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.*; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Stream; + +/** A pool of gRPC channels for a service, with a separate channel for each node. + *

+ * Manages broadcast-style request. */ +public class GrpcMultiNodeChannelPool { + private final ConcurrentHashMap> pools = + new ConcurrentHashMap<>(); + private static final Logger logger = LoggerFactory.getLogger(GrpcMultiNodeChannelPool.class); + private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + private final ServiceRegistryIf serviceRegistryIf; + private final ServiceId serviceId; + private final Function, ManagedChannel> channelConstructor; + private final Function stubConstructor; + private final NodeConfigurationWatcher nodeConfigurationWatcher; + + @SneakyThrows + public GrpcMultiNodeChannelPool(ServiceRegistryIf serviceRegistryIf, + ServiceId serviceId, + Function, ManagedChannel> channelConstructor, + Function stubConstructor, + NodeConfigurationWatcher nodeConfigurationWatcher) { + this.serviceRegistryIf = serviceRegistryIf; + this.serviceId = serviceId; + this.channelConstructor = channelConstructor; + this.stubConstructor = stubConstructor; + this.nodeConfigurationWatcher = nodeConfigurationWatcher; + } + + + + /** Get an API stub for the given node */ + public STUB apiForNode(int node) { + return pools.computeIfAbsent(node, _ -> + new GrpcSingleNodeChannelPool<>( + serviceRegistryIf, + serviceId, + new NodeSelectionStrategy.Just(node), + channelConstructor, + stubConstructor) + ).api(); + } + + + /** Invoke a function on each node, returning a list of futures in a terminal state, as per + * ExecutorService$invokeAll */ + public List> invokeAll(Function> callF) throws InterruptedException { + List> calls = getEligibleNodes().stream() + .mapMulti(this::passNodeIfOk) + .map(callF) + .toList(); + + return virtualExecutorService.invokeAll(calls); + } + + /** Invoke a function on each node, returning a stream of results */ + public Stream callEachSequential(Function call) { + return getEligibleNodes().stream() + .mapMulti(this::passNodeIfOk) + .map(call); + } + + // Eat connectivity exceptions and log them when doing a broadcast-style calls + private void passNodeIfOk(Integer nodeId, Consumer consumer) { + try { + consumer.accept(apiForNode(nodeId)); + } + catch (Exception ex) { + logger.error("Error calling node {}", nodeId, ex); + } + } + + /** Get the list of nodes that are eligible for broadcast-style requests */ + public List getEligibleNodes() { + return nodeConfigurationWatcher.getQueryNodes(); + } + +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java new file mode 100644 index 00000000..69941e51 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -0,0 +1,132 @@ +package nu.marginalia.service.client; + +import com.google.common.collect.Sets; +import io.grpc.ConnectivityState; +import io.grpc.ManagedChannel; +import lombok.SneakyThrows; +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor; +import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; +import nu.marginalia.service.id.ServiceId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.*; +import java.util.function.Function; + +/** A pool of gRPC channels for a service, with a separate channel for each node. + *

+ * Manages unicast-style requests */ +public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { + private final Map, ManagedChannel> channels = new ConcurrentHashMap<>(); + private volatile Set> routes = Set.of(); + + private static final Logger logger = LoggerFactory.getLogger(GrpcSingleNodeChannelPool.class); + + private final ServiceRegistryIf serviceRegistryIf; + private final ServiceId serviceId; + private final NodeSelectionStrategy nodeSelectionStrategy; + private final Function, ManagedChannel> channelConstructor; + private final Function stubConstructor; + + @SneakyThrows + public GrpcSingleNodeChannelPool(ServiceRegistryIf serviceRegistryIf, + ServiceId serviceId, + NodeSelectionStrategy nodeSelectionStrategy, + Function, ManagedChannel> channelConstructor, + Function stubConstructor) { + super(serviceId); + + this.serviceRegistryIf = serviceRegistryIf; + this.serviceId = serviceId; + this.nodeSelectionStrategy = nodeSelectionStrategy; + this.channelConstructor = channelConstructor; + this.stubConstructor = stubConstructor; + + serviceRegistryIf.registerMonitor(this); + + onChange(); + } + + + @Override + public boolean onChange() { + switch (nodeSelectionStrategy) { + case NodeSelectionStrategy.Any() -> + serviceRegistryIf + .getServiceNodes(serviceId) + .forEach(this::refreshNode); + case NodeSelectionStrategy.Just(int node) -> + refreshNode(node); + } + + return true; + } + + private void refreshNode(int node) { + + Set> newRoutes = serviceRegistryIf.getEndpoints(ApiSchema.GRPC, serviceId, node); + Set> oldRoutes = routes; + + if (!oldRoutes.equals(newRoutes)) { + // Find the routes that have been added or removed + for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { + + ManagedChannel oldChannel; + + if (newRoutes.contains(route)) { + logger.info(STR."Adding channel for \{serviceId.serviceName}-\{node} \{route.host()}:\{route.port()}"); + + var newChannel = channelConstructor.apply(route); + oldChannel = channels.put(route, newChannel); + } else { + logger.info(STR."Removing channel for \{serviceId.serviceName}-\{node} \{route.host()}:\{route.port()}"); + + oldChannel = channels.remove(route); + } + + if (oldChannel != null) + oldChannel.shutdown(); + } + + routes = newRoutes; + } + } + + public boolean hasChannel() { + return !channels.isEmpty(); + } + + /** Get an API stub for the given node */ + public STUB api() { + return stubConstructor.apply(getChannel()); + } + + /** Get the channel that is most ready to use */ + public ManagedChannel getChannel() { + return channels + .values() + .stream() + .min(this::compareChannelsByState) + .orElseThrow(() -> new ServiceNotAvailableException(serviceId)); + } + + /** Sort the channels by how ready they are to use */ + private int compareChannelsByState(ManagedChannel a, ManagedChannel b) { + var aState = a.getState(true); + var bState = b.getState(true); + + if (aState == ConnectivityState.READY) return -1; + if (bState == ConnectivityState.READY) return 1; + if (aState == ConnectivityState.CONNECTING) return -1; + if (bState == ConnectivityState.CONNECTING) return 1; + if (aState == ConnectivityState.IDLE) return -1; + if (bState == ConnectivityState.IDLE) return 1; + + return 0; + } + + +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java new file mode 100644 index 00000000..7e040597 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java @@ -0,0 +1,17 @@ +package nu.marginalia.service.client; + +public sealed interface NodeSelectionStrategy { + boolean test(int node); + record Any() implements NodeSelectionStrategy { + @Override + public boolean test(int node) { + return true; + } + } + record Just(int node) implements NodeSelectionStrategy { + @Override + public boolean test(int node) { + return this.node == node; + } + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java new file mode 100644 index 00000000..e11287ab --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java @@ -0,0 +1,12 @@ +package nu.marginalia.service.client; + +import nu.marginalia.service.id.ServiceId; + +public class ServiceNotAvailableException extends RuntimeException { + public ServiceNotAvailableException(ServiceId id, int node) { + super(STR."Service \{id} not available on node \{node}"); + } + public ServiceNotAvailableException(ServiceId id) { + super(STR."Service \{id} not available"); + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java deleted file mode 100644 index 68008bb3..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java +++ /dev/null @@ -1,33 +0,0 @@ -package nu.marginalia.service.descriptor; - -import nu.marginalia.service.id.ServiceId; - -public class ServiceDescriptor { - public final ServiceId id; - public final String name; - - public ServiceDescriptor(ServiceId id) { - this.id = id; - this.name = id.serviceName; - } - - public ServiceDescriptor(ServiceId id, String host) { - this.id = id; - this.name = host; - } - - public String getHostName(int node) { - if (node > 0) - return name + "-" + node; - - return name; - } - - public String toString() { - return name; - } - - public String describeService() { - return String.format("%s", name); - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptors.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptors.java deleted file mode 100644 index 88f2a693..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptors.java +++ /dev/null @@ -1,33 +0,0 @@ -package nu.marginalia.service.descriptor; - -import nu.marginalia.service.SearchServiceDescriptors; -import nu.marginalia.service.id.ServiceId; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** @see SearchServiceDescriptors */ -public class ServiceDescriptors { - private final Map descriptorsAll = new LinkedHashMap<>(); - - public ServiceDescriptors() { - - } - - public ServiceDescriptors(List descriptors) { - descriptors.forEach(d -> descriptorsAll.put(d.id, d)); - } - - public ServiceDescriptor[] values() { - return descriptorsAll.values().toArray(ServiceDescriptor[]::new); - } - - public ServiceDescriptor forId(ServiceId id) { - return Objects.requireNonNull(descriptorsAll.get(id), - "No service descriptor defined for " + id + " -- did you forget to " - + "bind(ServiceDescriptors.class).toInstance(SearchServiceDescriptors.descriptors); ?"); - } - -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java new file mode 100644 index 00000000..e8254cfd --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java @@ -0,0 +1,122 @@ +package nu.marginalia.service.discovery; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.service.discovery.monitor.*; +import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceEndpoint; +import nu.marginalia.service.discovery.property.ServiceEndpoint.GrpcEndpoint; +import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; +import nu.marginalia.service.discovery.property.ServiceEndpoint.RestEndpoint; +import nu.marginalia.service.id.ServiceId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.time.Duration; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +/** A service registry that returns fixed endpoints for all services. + *

+ * This is for backwards-compatibility with old docker-compose files with no + * ZooKeeper configured. + * */ +public class FixedServiceRegistry implements ServiceRegistryIf { + private static final Logger logger = LoggerFactory.getLogger(FixedServiceRegistry.class); + + private final HikariDataSource dataSource; + + @Inject + public FixedServiceRegistry(HikariDataSource dataSource) { + this.dataSource = dataSource; + } + + @Override + public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, int node, UUID instanceUUID, String externalAddress) throws Exception { + return switch (schema) { + case REST -> new ServiceEndpoint.RestEndpoint(externalAddress, 80); + case GRPC -> new ServiceEndpoint.GrpcEndpoint(externalAddress, 81); + }; + } + + @Override + public void announceInstance(ServiceId id, int node, UUID instanceUUID) { + // No-op + } + + @Override + public Set getServiceNodes(ServiceId id) { + + if (id == ServiceId.Executor || id == ServiceId.Index) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement("SELECT ID FROM NODE_CONFIGURATION")) { + Set ret = new HashSet<>(); + var rs = stmt.executeQuery(); + while (rs.next()) { + ret.add(rs.getInt(1)); + } + return ret; + } + catch (SQLException ex) { + return Set.of(); + } + } + + else return Set.of(0); + } + + @Override + public int requestPort(String externalHost, ApiSchema schema, ServiceId id, int node) { + return switch(schema) { + case REST -> 80; + case GRPC -> 81; + }; + } + + @Override + public Set> getEndpoints(ApiSchema schema, ServiceId id, int node) { + return switch (schema) { + case REST -> Set.of(new InstanceAddress<>( + new RestEndpoint(id.serviceName + "-" + node, 80), + UUID.randomUUID())); + case GRPC -> Set.of(new InstanceAddress<>( + new GrpcEndpoint(id.serviceName + "-" + node, 81), + UUID.randomUUID())); + }; + } + + public void registerMonitor(ServiceMonitorIf monitor) throws Exception { + // We don't have any notification mechanism, so we just periodically + // invoke the monitor's onChange method to simulate it. + + periodicallyInvoke(monitor, Duration.ofSeconds(15)); + } + + + void periodicallyInvoke(ServiceMonitorIf monitor, Duration d) { + Thread.ofPlatform().name("PeriodicInvoker").start(() -> { + for (;;) { + try { + Thread.sleep(d); + } catch (InterruptedException e) { + break; + } + + boolean reRegister; + try { + reRegister = monitor.onChange(); + } + catch (Exception ex) { + logger.error("Monitor failed", ex); + reRegister = true; + } + + if (!reRegister) { + break; + } + } + }); + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java new file mode 100644 index 00000000..b3460418 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java @@ -0,0 +1,70 @@ +package nu.marginalia.service.discovery; + +import nu.marginalia.service.discovery.monitor.*; +import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceEndpoint; +import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; +import nu.marginalia.service.id.ServiceId; + +import java.util.Set; +import java.util.UUID; + +/** A service registry that allows services to register themselves and + * be discovered by other services on the network. + */ +public interface ServiceRegistryIf { + /** + * Register a service with the registry. + *

+ * Once the instance has announced itself with {@link #announceInstance(ServiceId id, int node, UUID instanceUUID) announceInstance(...)}, + * the service will be available for discovery with {@link #getEndpoints(ApiSchema schema, ServiceId id, int node) getEndpoints(...)}. + * + * @param schema the API schema + * @param id the service identifier + * @param node the node number + * @param instanceUUID the unique UUID of the instance + * @param externalAddress the public address of the service + */ + ServiceEndpoint registerService(ApiSchema schema, + ServiceId id, + int node, + UUID instanceUUID, + String externalAddress) throws Exception; + + /** Let the world know that the service is running + * and ready to accept requests. */ + void announceInstance(ServiceId id, int node, UUID instanceUUID); + + /** Return all nodes that are running for the specified service. */ + Set getServiceNodes(ServiceId id); + + /** At the discretion of the implementation, provide a port that is unique + * across (externalHost, serviceId, schema, node). It may be randomly selected + * or hard-coded or some combination of behaviors. + */ + int requestPort(String externalHost, + ApiSchema schema, + ServiceId id, + int node); + + /** Get all endpoints for the service on the specified node and schema. */ + Set> + getEndpoints(ApiSchema schema, ServiceId id, int node); + + /** Register a monitor to be notified when the service registry changes. + *

+ * {@link ServiceMonitorIf#onChange()} will be called when the registry changes. + * Spurious calls to {@link ServiceMonitorIf#onChange()} are allowed depending + * on the implementation. + *

+ * Behavior of the monitor depends on the implementation of the registry, and the + * monitor type. + *
    + *
  • {@link ServiceChangeMonitor} is notified when any node for the service changes.
    • + *
  • {@link ServiceNodeChangeMonitor} is notified when a specific node for the service changes.
    • + *
  • {@link ServiceRestEndpointChangeMonitor} is notified when the REST endpoints for the specified node service changes.
    • + *
  • {@link ServiceGrpcEndpointChangeMonitor} is notified when the gRPC endpoints for the specified node service changes.
    • + *
+ * */ + void registerMonitor(ServiceMonitorIf monitor) throws Exception; +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java new file mode 100644 index 00000000..b91e691f --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -0,0 +1,288 @@ +package nu.marginalia.service.discovery; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import lombok.SneakyThrows; +import nu.marginalia.service.discovery.monitor.*; +import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceEndpoint; +import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; +import nu.marginalia.service.id.ServiceId; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.CuratorWatcher; +import org.apache.curator.utils.ZKPaths; +import org.apache.zookeeper.CreateMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** A versatile service registry that uses ZooKeeper to store service endpoints. + * It is used to register services and to look up the endpoints of other services. + *

+ * It may also be used to assign ports to services, if the system property + * service.random-port is set to true. This is useful + * for running the system in a bare-metal environment, where the ports are not + * managed by Docker, and there are enough services that managing them manually + * will be a serious headache. + * */ +@Singleton +public class ZkServiceRegistry implements ServiceRegistryIf { + private final CuratorFramework curatorFramework; + private static final Logger logger = LoggerFactory.getLogger(ZkServiceRegistry.class); + private volatile boolean stopped = false; + + @Inject + @SneakyThrows + public ZkServiceRegistry(CuratorFramework curatorFramework) { + this.curatorFramework = curatorFramework; + + curatorFramework.start(); + if (!curatorFramework.blockUntilConnected(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("Failed to connect to zookeeper after 30s"); + } + + Runtime.getRuntime().addShutdownHook( + new Thread(this::shutDown, "ZkServiceRegistry shutdown hook") + ); + } + + @Override + public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, + int node, + UUID instanceUUID, + String externalAddress) + throws Exception + { + var ephemeralProperty = curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL); + + var endpoint = ServiceEndpoint.forSchema(schema, externalAddress, + requestPort(externalAddress, schema, id, node) + ); + + String path; + byte[] payload; + + switch (endpoint) { + case ServiceEndpoint.GrpcEndpoint(String host, int port) -> { + path = STR."/services/\{id.serviceName}/\{node}/grpc/\{instanceUUID.toString()}"; + payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8); + } + case ServiceEndpoint.RestEndpoint(String host, int port) -> { + path = STR."/services/\{id.serviceName}/\{node}/rest/\{instanceUUID.toString()}"; + payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8); + } + } + + logger.info("Registering {} -> {}", path, instanceUUID); + + ephemeralProperty.forPath(path, payload); + + return endpoint; + } + + @Override + public void announceInstance(ServiceId id, int node, UUID instanceUUID) { + try { + String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}"; + curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL) + .forPath(serviceRoot); + } + catch (Exception ex) { + logger.error("Failed to create service root for {}", id.serviceName); + } + } + + /** + * Returns true if the service has announced itself as up and running. + */ + public boolean isInstanceRunning(ServiceId id, int node, UUID instanceUUID) { + try { + String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}"; + return null != curatorFramework.checkExists().forPath(serviceRoot); + } + catch (Exception ex) { + logger.error("Failed to check if service is running {}", id.serviceName); + return false; + } + } + + @Override + public Set getServiceNodes(ServiceId id) { + try { + String serviceRoot = STR."/services/\{id.serviceName}"; + return curatorFramework.getChildren().forPath(serviceRoot) + .stream().map(Integer::parseInt) + .collect(Collectors.toSet()); + } + catch (Exception ex) { + logger.error("Failed to get nodes for service {}", id.serviceName); + return Set.of(); + } + } + + @Override + public int requestPort(String externalHost, + ApiSchema schema, + ServiceId id, + int node) + { + if (!Boolean.getBoolean("service.random-port")) { + return switch(schema) { + case REST -> 80; + case GRPC -> 81; + }; + } + + int portRangeLow = 12_000; + int portRangeHigh = 12_999; + + var random = new Random(); + + String host = STR."\{id.serviceName}-\{node}"; + + byte[] payload = STR."\{schema}://\{host}".getBytes(StandardCharsets.UTF_8); + + for (int iter = 0; iter < 1000; iter++) { + try { + int port = random.nextInt(portRangeLow, portRangeHigh); + + curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL) + .forPath(STR."/port-registry/\{externalHost}/\{port}", payload); + return port; + } + catch (Exception ex) { + logger.error(STR."Still negotiating port for \{schema}://\{id.serviceName}:\{node}"); + } + } + + throw new IllegalStateException("Failed to negotiate a port for host " + externalHost); + } + + @Override + public Set> getEndpoints(ApiSchema schema, ServiceId id, int node) { + return switch (schema) { + case REST -> getRestEndpoints(id, node); + case GRPC -> getGrpcEndpoints(id, node); + }; + } + + public Set> getRestEndpoints(ServiceId id, int node) { + try { + Set> ret = new HashSet<>(); + String restRoot = STR."/services/\{id.serviceName}/\{node}/rest"; + for (var uuid : curatorFramework + .getChildren() + .forPath(restRoot)) { + + if (!isInstanceRunning(id, node, UUID.fromString(uuid))) { + continue; + } + + byte[] data = curatorFramework + .getData() + .forPath(ZKPaths.makePath(restRoot, uuid)); + String hostAndPort = new String(data); + ret.add(RestEndpoint + .parse(hostAndPort) + .asInstance(UUID.fromString(uuid)) + ); + + } + + return ret; + } + catch (Exception ex) { + return Set.of(); + } + } + + public Set> getGrpcEndpoints(ServiceId id, int node) { + try { + Set> ret = new HashSet<>(); + String restRoot = STR."/services/\{id.serviceName}/\{node}/grpc"; + for (var uuid : curatorFramework + .getChildren() + .forPath(restRoot)) { + + if (!isInstanceRunning(id, node, UUID.fromString(uuid))) { + continue; + } + + byte[] data = curatorFramework + .getData() + .forPath(ZKPaths.makePath(restRoot, uuid)); + String hostAndPort = new String(data); + ret.add(GrpcEndpoint + .parse(hostAndPort) + .asInstance(UUID.fromString(uuid)) + ); + + } + + return ret; + } + catch (Exception ex) { + return Set.of(); + } + } + + public void registerMonitor(ServiceMonitorIf monitor) throws Exception { + monitor.register(this); + } + + public void registerMonitor(ServiceChangeMonitor monitor) throws Exception { + installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}"); + } + + public void registerMonitor(ServiceNodeChangeMonitor monitor) throws Exception { + installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}"); + } + + public void registerMonitor(ServiceRestEndpointChangeMonitor monitor) throws Exception { + installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/rest"); + } + + public void registerMonitor(ServiceGrpcEndpointChangeMonitor monitor) throws Exception { + installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/grpc"); + } + + private void installMonitor(ServiceMonitorIf monitor, String path) throws Exception { + CuratorWatcher watcher = _ -> { + boolean reRegister; + + try { + reRegister = monitor.onChange(); + } + catch (Exception ex) { + logger.error("Monitor for path {} failed", path, ex); + reRegister = true; + } + + if (reRegister) { + registerMonitor(monitor); + } + }; + + curatorFramework.watchers().add() + .usingWatcher(watcher) + .forPath(path); + } + + /* Exposed for tests */ + public synchronized void shutDown() { + if (!stopped) { + curatorFramework.close(); + stopped = true; + } + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java new file mode 100644 index 00000000..63e35891 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java @@ -0,0 +1,23 @@ +package nu.marginalia.service.discovery.monitor; + +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.ZkServiceRegistry; +import nu.marginalia.service.id.ServiceId; + +public abstract class ServiceChangeMonitor implements ServiceMonitorIf { + public final ServiceId serviceId; + + public ServiceChangeMonitor(ServiceId serviceId) { + this.serviceId = serviceId; + } + + public abstract boolean onChange(); + public void register(ServiceRegistryIf registry) throws Exception { + if (registry instanceof ZkServiceRegistry zkServiceRegistry) { + zkServiceRegistry.registerMonitor(this); + } + else { + registry.registerMonitor(this); + } + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java new file mode 100644 index 00000000..2b33a40e --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java @@ -0,0 +1,25 @@ +package nu.marginalia.service.discovery.monitor; + +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.ZkServiceRegistry; +import nu.marginalia.service.id.ServiceId; + +public abstract class ServiceGrpcEndpointChangeMonitor implements ServiceMonitorIf { + public final ServiceId serviceId; + public final int node; + public ServiceGrpcEndpointChangeMonitor(ServiceId serviceId, int node) { + this.serviceId = serviceId; + this.node = node; + } + + public abstract boolean onChange(); + + public void register(ServiceRegistryIf registry) throws Exception { + if (registry instanceof ZkServiceRegistry zkServiceRegistry) { + zkServiceRegistry.registerMonitor(this); + } + else { + registry.registerMonitor(this); + } + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java new file mode 100644 index 00000000..af4cabb8 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java @@ -0,0 +1,16 @@ +package nu.marginalia.service.discovery.monitor; + +import nu.marginalia.service.discovery.ServiceRegistryIf; + +public interface ServiceMonitorIf { + /** Called when the monitored service has changed. + * @return true if the monitor is to be refreshed + */ + boolean onChange(); + + /** Register this monitor with the given registry. + * It is preferred to use {@link ServiceRegistryIf}'s + * registerMonitor function. + * */ + void register(ServiceRegistryIf registry) throws Exception; +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java new file mode 100644 index 00000000..42e04e89 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java @@ -0,0 +1,25 @@ +package nu.marginalia.service.discovery.monitor; + +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.ZkServiceRegistry; +import nu.marginalia.service.id.ServiceId; + +public abstract class ServiceNodeChangeMonitor implements ServiceMonitorIf { + public final ServiceId serviceId; + public final int node; + public ServiceNodeChangeMonitor(ServiceId serviceId, int node) { + this.serviceId = serviceId; + this.node = node; + } + + public abstract boolean onChange(); + + public void register(ServiceRegistryIf registry) throws Exception { + if (registry instanceof ZkServiceRegistry zkServiceRegistry) { + zkServiceRegistry.registerMonitor(this); + } + else { + registry.registerMonitor(this); + } + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java new file mode 100644 index 00000000..1e8c8564 --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java @@ -0,0 +1,25 @@ +package nu.marginalia.service.discovery.monitor; + +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.ZkServiceRegistry; +import nu.marginalia.service.id.ServiceId; + +public abstract class ServiceRestEndpointChangeMonitor implements ServiceMonitorIf { + public final ServiceId serviceId; + public final int node; + public ServiceRestEndpointChangeMonitor(ServiceId serviceId, int node) { + this.serviceId = serviceId; + this.node = node; + } + + public abstract boolean onChange(); + + public void register(ServiceRegistryIf registry) throws Exception { + if (registry instanceof ZkServiceRegistry zkServiceRegistry) { + zkServiceRegistry.registerMonitor(this); + } + else { + registry.registerMonitor(this); + } + } +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java new file mode 100644 index 00000000..dc8300af --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java @@ -0,0 +1,6 @@ +package nu.marginalia.service.discovery.property; + +public enum ApiSchema { + REST, + GRPC +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java new file mode 100644 index 00000000..8102f52f --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -0,0 +1,80 @@ +package nu.marginalia.service.discovery.property; + + +import lombok.SneakyThrows; + +import java.net.URI; +import java.net.URL; +import java.util.UUID; + +public sealed interface ServiceEndpoint { + String host(); + int port(); + + URL toURL(String endpoint, String query); + + static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) { + return switch (schema) { + case REST -> new RestEndpoint(host, port); + case GRPC -> new GrpcEndpoint(host, port); + }; + } + + record RestEndpoint(String host, int port) implements ServiceEndpoint { + public static RestEndpoint parse(String hostColonPort) { + String[] parts = hostColonPort.split(":"); + + if (parts.length != 2) { + throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'"); + } + + return new RestEndpoint( + parts[0], + Integer.parseInt(parts[1]) + ); + } + + @SneakyThrows + public URL toURL(String endpoint, String query) { + return new URI("http", null, host, port, endpoint, null, null) + .toURL(); + } + + public InstanceAddress asInstance(UUID uuid) { + return new InstanceAddress<>(this, uuid); + } + } + + record GrpcEndpoint(String host, int port) implements ServiceEndpoint { + public static GrpcEndpoint parse(String hostColonPort) { + String[] parts = hostColonPort.split(":"); + + if (parts.length != 2) { + throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'"); + } + + return new GrpcEndpoint( + parts[0], + Integer.parseInt(parts[1]) + ); + } + + public InstanceAddress asInstance(UUID uuid) { + return new InstanceAddress<>(this, uuid); + } + + @Override + public URL toURL(String endpoint, String query) { + throw new UnsupportedOperationException(); + } + } + + record InstanceAddress(T endpoint, UUID instance) { + public String host() { + return endpoint.host(); + } + public int port() { + return endpoint.port(); + } + } +} diff --git a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java new file mode 100644 index 00000000..3684a75a --- /dev/null +++ b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java @@ -0,0 +1,124 @@ +package nu.marginalia.service.discovery; + +import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.id.ServiceId; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.util.*; + +import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +@Tag("slow") +class ZkServiceRegistryTest { + private static final int ZOOKEEPER_PORT = 2181; + private static final GenericContainer zookeeper = + new GenericContainer<>("zookeeper:3.8.0") + .withExposedPorts(ZOOKEEPER_PORT); + + List registries = new ArrayList<>(); + String connectString; + @BeforeEach + public void setUp() { + zookeeper.start(); + connectString = STR."\{zookeeper.getHost()}:\{zookeeper.getMappedPort(ZOOKEEPER_PORT)}"; + } + + @AfterEach + public void tearDown() { + for (var registry : registries) { + registry.shutDown(); + } + zookeeper.stop(); + + System.clearProperty("service.random-port"); + + } + + ZkServiceRegistry createRegistry() { + return new ZkServiceRegistry(CuratorFrameworkFactory.newClient( + connectString, + new ExponentialBackoffRetry(10, 10, 1000))); + } + + @Test + void getPort() { + System.setProperty("service.random-port", "true"); + + var registry1 = createRegistry(); + var registry2 = createRegistry(); + + List ports = new ArrayList<>(); + Set portsSet = new HashSet<>(); + for (int i = 0; i < 500; i++) { + int port = registry1.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0); + ports.add(port); + + // Ensure we get unique ports + assertTrue(portsSet.add(port)); + } + for (int i = 0; i < 50; i++) { + int port = registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0); + ports.add(port); + + // Ensure we get unique ports + assertTrue(portsSet.add(port)); + } + registry1.shutDown(); + for (int i = 0; i < 500; i++) { + // Verify we can reclaim ports + ports.add(registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0)); + } + assertEquals(1050, ports.size()); + } + + @Test + void getInstances() throws Exception { + var uuid1 = UUID.randomUUID(); + var uuid2 = UUID.randomUUID(); + + var registry1 = createRegistry(); + var registry2 = createRegistry(); + + var endpoint1 = (RestEndpoint) registry1.registerService(ApiSchema.REST, ServiceId.Search, 0, uuid1, "127.0.0.1"); + var endpoint2 = (GrpcEndpoint) registry2.registerService(ApiSchema.GRPC, ServiceId.Search, 0, uuid2, "127.0.0.2"); + + registry1.announceInstance(ServiceId.Search, 0, uuid1); + registry2.announceInstance(ServiceId.Search, 0, uuid2); + + assertEquals(Set.of(endpoint1.asInstance(uuid1)), + registry1.getRestEndpoints(ServiceId.Search, 0)); + + assertEquals(Set.of(endpoint2.asInstance(uuid2)), + registry1.getGrpcEndpoints(ServiceId.Search, 0)); + + + registry1.shutDown(); + Thread.sleep(100); + + assertEquals(Set.of(), + registry2.getRestEndpoints(ServiceId.Search, 0)); + assertEquals(Set.of(endpoint2.asInstance(uuid2)), + registry2.getGrpcEndpoints(ServiceId.Search, 0)); + } + + @Test + public void announceLiveness() throws Exception { + var registry1 = createRegistry(); + var uuid1 = UUID.randomUUID(); + + assertFalse(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1)); + registry1.announceInstance(ServiceId.Search, 0, uuid1); + assertTrue(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1)); + + registry1.shutDown(); + } +} \ No newline at end of file diff --git a/code/common/service/build.gradle b/code/common/service/build.gradle index 64bb7cd5..4e4e8ed1 100644 --- a/code/common/service/build.gradle +++ b/code/common/service/build.gradle @@ -10,7 +10,6 @@ java { } dependencies { - implementation project(':code:common:service-client') implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') implementation project(':code:common:db') @@ -18,8 +17,9 @@ dependencies { implementation libs.spark implementation libs.guice - implementation libs.rxjava + implementation libs.bundles.curator + implementation libs.bundles.grpc implementation libs.bundles.prometheus implementation libs.bundles.slf4j implementation libs.bucket4j diff --git a/code/common/service/readme.md b/code/common/service/readme.md index 91741dc0..17f07d15 100644 --- a/code/common/service/readme.md +++ b/code/common/service/readme.md @@ -21,8 +21,7 @@ public class FoobarMain extends MainClass { Injector injector = Guice.createInjector( new FoobarModule(), /* optional custom bindings go here */ new DatabaseModule(), - new ConfigurationModule(SearchServiceDescriptors.descriptors, - ServiceId.Foobar)); + new ConfigurationModule(ServiceId.Foobar)); injector.getInstance(FoobarMain.class); diff --git a/code/common/service/src/main/java/nu/marginalia/service/ConfigLoader.java b/code/common/service/src/main/java/nu/marginalia/service/ConfigLoader.java index 44512264..a418655f 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/ConfigLoader.java +++ b/code/common/service/src/main/java/nu/marginalia/service/ConfigLoader.java @@ -1,8 +1,6 @@ package nu.marginalia.service; import nu.marginalia.WmsaHome; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Files; diff --git a/code/common/service/src/main/java/nu/marginalia/service/MainClass.java b/code/common/service/src/main/java/nu/marginalia/service/MainClass.java index be3080fb..18bedc51 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/MainClass.java +++ b/code/common/service/src/main/java/nu/marginalia/service/MainClass.java @@ -1,16 +1,10 @@ package nu.marginalia.service; import io.prometheus.client.hotspot.DefaultExports; -import io.reactivex.rxjava3.exceptions.UndeliverableException; -import io.reactivex.rxjava3.plugins.RxJavaPlugins; import nu.marginalia.service.id.ServiceId; -import nu.marginalia.client.exception.NetworkException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; - /** Each main class of a service should extend this class. * They must also invoke init() in their main method. */ @@ -25,29 +19,8 @@ public abstract class MainClass { } public MainClass() { - RxJavaPlugins.setErrorHandler(this::handleError); } - protected void handleError(Throwable ex) { - if (ex instanceof UndeliverableException) { - ex = ex.getCause(); - } - - if (ex instanceof SocketTimeoutException) { - logger.warn("SocketTimeoutException"); - } - else if (ex instanceof UnknownHostException) { - logger.warn("UnknownHostException"); - } - else if (ex instanceof NetworkException) { - logger.warn("NetworkException", ex); - } - else { - logger.error("Uncaught exception", ex); - } - } - - protected static void init(ServiceId id, String... args) { System.setProperty("log4j2.isThreadContextMapInheritable", "true"); System.setProperty("isThreadContextMapInheritable", "true"); diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java index 6e2b3399..e64c88a8 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java @@ -10,15 +10,14 @@ import java.util.UUID; * * @param serviceId - service descriptor * @param node - always 0 for now, for future service partitioning - * @param host - the bind address of the service - * @param port - main port of the service + * @param bindAddress - the bind address of the service * @param metricsPort - prometheus metrics server port * @param instanceUuid - unique identifier for this instance of the service */ public record ServiceConfiguration(ServiceId serviceId, int node, - String host, - int port, + String bindAddress, + String externalAddress, int metricsPort, UUID instanceUuid) { public String serviceName() { diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java index 5ea41d47..c5484066 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java @@ -2,46 +2,49 @@ package nu.marginalia.service.module; import com.google.inject.AbstractModule; import com.google.inject.name.Names; -import nu.marginalia.service.descriptor.ServiceDescriptors; +import nu.marginalia.service.discovery.FixedServiceRegistry; +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.ZkServiceRegistry; import nu.marginalia.service.id.ServiceId; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; import java.util.Objects; import java.util.UUID; public class ServiceConfigurationModule extends AbstractModule { - private final ServiceDescriptors descriptors; private final ServiceId id; - public ServiceConfigurationModule(ServiceDescriptors descriptors, ServiceId id) { - this.descriptors = descriptors; + public ServiceConfigurationModule(ServiceId id) { this.id = id; } public void configure() { - bind(ServiceDescriptors.class).toInstance(descriptors); - int node = getNode(); var configObject = new ServiceConfiguration(id, node, + getBindAddress(), getHost(), - getBasePort(), getPrometheusPort(), UUID.randomUUID() ); bind(Integer.class).annotatedWith(Names.named("wmsa-system-node")).toInstance(node); bind(ServiceConfiguration.class).toInstance(configObject); - } - private int getBasePort() { - String port = System.getenv("WMSA_SERVICE_PORT"); + if (Boolean.getBoolean("system.useZookeeper")) { + CuratorFramework client = CuratorFrameworkFactory + .newClient(System.getProperty("zookeeper-hosts", "zookeeper:2181"), + new ExponentialBackoffRetry(100, 10, 1000)); - if (port != null) { - return Integer.parseInt(port); + bind(CuratorFramework.class).toInstance(client); + bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); + } + else { + bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class); } - - return 80; } private int getPrometheusPort() { @@ -61,7 +64,17 @@ public class ServiceConfigurationModule extends AbstractModule { } private String getHost() { - return System.getProperty("service-host", "127.0.0.1"); + int node = getNode(); + final String defaultValue; + + if (node > 0) defaultValue = STR."\{id.serviceName}-\{node}"; + else defaultValue = id.serviceName; + + return System.getProperty("service.host", defaultValue); + } + + private String getBindAddress() { + return System.getProperty("service.bind-address", "0.0.0.0"); } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/BaseServiceParams.java b/code/common/service/src/main/java/nu/marginalia/service/server/BaseServiceParams.java index 9db8d82e..b1661143 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/BaseServiceParams.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/BaseServiceParams.java @@ -5,6 +5,7 @@ import com.google.inject.Singleton; import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.control.ServiceHeartbeatImpl; +import nu.marginalia.service.discovery.ServiceRegistryIf; import nu.marginalia.service.module.ServiceConfiguration; /** This class exists to reduce Service boilerplate */ @@ -15,19 +16,23 @@ public class BaseServiceParams { public final MetricsServer metricsServer; public final ServiceHeartbeatImpl heartbeat; public final ServiceEventLog eventLog; + public final ServiceRegistryIf serviceRegistry; public final MessageQueueFactory messageQueueInboxFactory; + @Inject public BaseServiceParams(ServiceConfiguration configuration, Initialization initialization, MetricsServer metricsServer, ServiceHeartbeatImpl heartbeat, ServiceEventLog eventLog, + ServiceRegistryIf serviceRegistry, MessageQueueFactory messageQueueInboxFactory) { this.configuration = configuration; this.initialization = initialization; this.metricsServer = metricsServer; this.heartbeat = heartbeat; this.eventLog = eventLog; + this.serviceRegistry = serviceRegistry; this.messageQueueInboxFactory = messageQueueInboxFactory; } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Initialization.java b/code/common/service/src/main/java/nu/marginalia/service/server/Initialization.java index e75db6fe..717b49cf 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/Initialization.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/Initialization.java @@ -8,9 +8,14 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; +/** A class for delayed initialization of services. + *

+ * This is useful for tasks that need to be performed after the service has been + * fully initialized, such as registering with a service registry. + */ @Singleton public class Initialization { - boolean initialized; + private boolean initialized; private final Logger logger = LoggerFactory.getLogger(getClass()); private final List callbacks = new ArrayList<>(); @@ -25,7 +30,6 @@ public class Initialization { logger.info("Initialized"); initialized = true; notifyAll(); - } callbacks.forEach(Runnable::run); diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/RateLimiter.java b/code/common/service/src/main/java/nu/marginalia/service/server/RateLimiter.java index f9de1cb5..3e6b9c21 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/RateLimiter.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/RateLimiter.java @@ -3,12 +3,9 @@ package nu.marginalia.service.server; import io.github.bucket4j.Bandwidth; import io.github.bucket4j.Bucket; import io.github.bucket4j.Refill; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.client.Context; import java.time.Duration; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -23,7 +20,18 @@ public class RateLimiter { this.capacity = capacity; this.refillRate = refillRate; - Schedulers.io().schedulePeriodicallyDirect(this::cleanIdleBuckets, 30, 30, TimeUnit.MINUTES); + Thread.ofPlatform() + .name("rate-limiter-cleaner") + .start(() -> { + while (true) { + cleanIdleBuckets(); + try { + TimeUnit.MINUTES.sleep(30); + } catch (InterruptedException e) { + break; + } + } + }); } @@ -48,19 +56,11 @@ public class RateLimiter { bucketMap.clear(); } - public boolean isAllowed(Context ctx) { - if (!ctx.isPublic()) { // Internal server->server request - return true; - } - - return bucketMap.computeIfAbsent(ctx.getContextId(), - (ip) -> createBucket()).tryConsume(1); - } - public boolean isAllowed() { return bucketMap.computeIfAbsent("any", (ip) -> createBucket()).tryConsume(1); } + private Bucket createBucket() { var refill = Refill.greedy(1, Duration.ofSeconds(refillRate)); var bw = Bandwidth.classic(capacity, refill); diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java index 63552cf7..73cd5977 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java @@ -1,10 +1,12 @@ package nu.marginalia.service.server; +import io.grpc.BindableService; +import io.grpc.ServerBuilder; import io.prometheus.client.Counter; -import nu.marginalia.client.Context; -import nu.marginalia.client.exception.MessagingException; +import lombok.SneakyThrows; import nu.marginalia.mq.inbox.*; -import nu.marginalia.service.server.mq.MqRequest; +import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceEndpoint; import nu.marginalia.service.server.mq.ServiceMqSubscription; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +16,7 @@ import spark.Request; import spark.Response; import spark.Spark; +import java.util.List; import java.util.Optional; public class Service { @@ -42,9 +45,10 @@ public class Service { protected final MqInboxIf messageQueueInbox; private final int node; + @SneakyThrows public Service(BaseServiceParams params, - Runnable configureStaticFiles - ) { + Runnable configureStaticFiles, + List grpcServices) { this.initialization = params.initialization; var config = params.configuration; node = config.node(); @@ -52,6 +56,16 @@ public class Service { String inboxName = config.serviceName(); logger.info("Inbox name: {}", inboxName); + var serviceRegistry = params.serviceRegistry; + + var restEndpoint = + serviceRegistry.registerService( + ApiSchema.REST, config.serviceId(), + config.node(), + config.instanceUuid(), + config.externalAddress() + ); + var mqInboxFactory = params.messageQueueInboxFactory; messageQueueInbox = mqInboxFactory.createSynchronousInbox(inboxName, config.node(), config.instanceUuid()); messageQueueInbox.subscribe(new ServiceMqSubscription(this)); @@ -61,50 +75,62 @@ public class Service { initialization.addCallback(params.heartbeat::start); initialization.addCallback(messageQueueInbox::start); initialization.addCallback(() -> params.eventLog.logEvent("SVC-INIT", serviceName + ":" + config.node())); + initialization.addCallback(() -> serviceRegistry.announceInstance(config.serviceId(), config.node(), config.instanceUuid())); if (!initialization.isReady() && ! initialized ) { initialized = true; Spark.threadPool(32, 4, 60_000); - Spark.ipAddress(params.configuration.host()); - Spark.port(params.configuration.port()); - logger.info("{} Listening to {}:{}", getClass().getSimpleName(), - params.configuration.host(), - params.configuration.port()); + Spark.ipAddress(config.bindAddress()); + Spark.port(restEndpoint.port()); + + logger.info("{} Listening to {}:{} ({})", getClass().getSimpleName(), + params.configuration.bindAddress(), + restEndpoint.port(), + params.configuration.externalAddress()); configureStaticFiles.run(); Spark.before(this::auditRequestIn); Spark.before(this::filterPublicRequests); Spark.after(this::auditRequestOut); - Spark.exception(MessagingException.class, this::handleException); + // Live and ready endpoints Spark.get("/internal/ping", (rq,rp) -> "pong"); Spark.get("/internal/started", this::isInitialized); Spark.get("/internal/ready", this::isReady); - Spark.get("/public/who", (rq,rp) -> getClass().getSimpleName()); + + ServiceEndpoint.GrpcEndpoint grpcEndpoint = (ServiceEndpoint.GrpcEndpoint) params.serviceRegistry.registerService( + ApiSchema.GRPC, config.serviceId(), + config.node(), + config.instanceUuid(), + config.externalAddress() + ); + + var grpcServerBuilder = ServerBuilder.forPort(grpcEndpoint.port()); + for (var grpcService : grpcServices) { + grpcServerBuilder.addService(grpcService); + } + grpcServerBuilder.build().start(); } } + public Service(BaseServiceParams params, + List grpcServices) { + this(params, Service::defaultSparkConfig, grpcServices); + } + public Service(BaseServiceParams params) { - this(params, () -> { - // configureStaticFiles can't be an overridable method in Service because it may - // need to depend on parameters to the constructor, and super-constructors - // must run first - Spark.staticFiles.expireTime(3600); - Spark.staticFiles.header("Cache-control", "public"); - }); + this(params, Service::defaultSparkConfig, List.of()); } - @MqRequest(endpoint = "SVC-READY") - public boolean mqIsReady() { - return initialization.isReady(); - } - - @MqRequest(endpoint = "SVC-PING") - public String mqPing() { - return "pong"; + private static void defaultSparkConfig() { + // configureStaticFiles can't be an overridable method in Service because it may + // need to depend on parameters to the constructor, and super-constructors + // must run first + Spark.staticFiles.expireTime(3600); + Spark.staticFiles.header("Cache-control", "public"); } private void filterPublicRequests(Request request, Response response) { @@ -149,16 +175,10 @@ public class Service { } private void auditRequestIn(Request request, Response response) { - // Paint context - paintThreadName(request, "req:"); - request_counter.labels(serviceName, Integer.toString(node)).inc(); } private void auditRequestOut(Request request, Response response) { - - paintThreadName(request, "rsp:"); - if (response.status() < 400) { request_counter_good.labels(serviceName, Integer.toString(node)).inc(); } @@ -170,21 +190,6 @@ public class Service { } - private void paintThreadName(Request request, String prefix) { - var ctx = Context.fromRequest(request); - Thread.currentThread().setName(prefix + ctx.getContextId()); - } - - protected void handleException(Exception ex, Request request, Response response) { - request_counter_err.labels(serviceName, Integer.toString(node)).inc(); - if (ex instanceof MessagingException) { - logger.error("{} {}", ex.getClass().getSimpleName(), ex.getMessage()); - } - else { - logger.error("Uncaught exception", ex); - } - } - /** Log the request on the HTTP log */ protected void logRequest(Request request) { String url = request.pathInfo(); @@ -192,7 +197,7 @@ public class Service { url = url + "?" + request.queryString(); } - logger.info(httpMarker, "PUBLIC {}: {} {}", Context.fromRequest(request).getContextId(), request.requestMethod(), url); + logger.info(httpMarker, "PUBLIC: {} {}", request.requestMethod(), url); } /** Log the response on the HTTP log */ diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/StaticResources.java b/code/common/service/src/main/java/nu/marginalia/service/server/StaticResources.java index ff684525..71460ecc 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/StaticResources.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/StaticResources.java @@ -10,10 +10,6 @@ import spark.staticfiles.MimeType; import java.io.FileNotFoundException; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.Map; -import java.util.TreeMap; - -import static java.util.Map.entry; public class StaticResources { private final long startTime = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); diff --git a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/AtagExporter.java b/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/AtagExporter.java index dc286323..3db0a284 100644 --- a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/AtagExporter.java +++ b/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/AtagExporter.java @@ -48,7 +48,7 @@ public class AtagExporter implements ExporterIf { Path inputDir = storageService.getStorage(crawlId).asPath(); - try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))); + try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))) ) { Path crawlerLogFile = inputDir.resolve("crawler.log"); diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/UrlKeywords.java b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/UrlKeywords.java index 0b91a050..83c17078 100644 --- a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/UrlKeywords.java +++ b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/UrlKeywords.java @@ -1,7 +1,6 @@ package nu.marginalia.keyword.extractors; import ca.rmen.porterstemmer.PorterStemmer; -import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.EdgeUrl; import java.util.Arrays; diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java index 559f69a6..8904e16e 100644 --- a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java +++ b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java @@ -3,10 +3,8 @@ package nu.marginalia.keyword.extractors; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; import nu.marginalia.keyword.WordReps; import nu.marginalia.language.model.DocumentLanguageData; -import nu.marginalia.language.model.DocumentSentence; import nu.marginalia.language.model.WordRep; import nu.marginalia.keyword.KeywordExtractor; -import nu.marginalia.language.model.WordSpan; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import org.apache.commons.lang3.StringUtils; diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/SentenceExtractorTest.java b/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/SentenceExtractorTest.java index c561eae7..dabad6d1 100644 --- a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/SentenceExtractorTest.java +++ b/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/SentenceExtractorTest.java @@ -2,23 +2,17 @@ package nu.marginalia.keyword; import lombok.SneakyThrows; import nu.marginalia.LanguageModels; -import nu.marginalia.language.model.WordRep; -import nu.marginalia.language.model.WordSpan; import nu.marginalia.language.sentence.SentenceExtractor; import nu.marginalia.term_frequency_dict.TermFrequencyDict; -import nu.marginalia.language.model.WordSeparator; import nu.marginalia.WmsaHome; import nu.marginalia.model.EdgeUrl; import nu.marginalia.test.util.TestLanguageModels; import org.jsoup.Jsoup; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.regex.Pattern; diff --git a/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/RedditEntryReaderTest.java b/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/RedditEntryReaderTest.java index 7201eb33..f8f2cff7 100644 --- a/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/RedditEntryReaderTest.java +++ b/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/RedditEntryReaderTest.java @@ -5,9 +5,6 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; class RedditEntryReaderTest { /** This test case exists for debugging, reddit sideloading. It requires local reddit data, diff --git a/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/db/RedditDbTest.java b/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/db/RedditDbTest.java index bff2aab6..3670e7cf 100644 --- a/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/db/RedditDbTest.java +++ b/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/db/RedditDbTest.java @@ -10,8 +10,6 @@ import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; -import static org.junit.jupiter.api.Assertions.*; - class RedditDbTest { @Disabled diff --git a/code/features-crawl/crawl-blocklist/build.gradle b/code/features-crawl/crawl-blocklist/build.gradle index 8288aa0c..fa849eef 100644 --- a/code/features-crawl/crawl-blocklist/build.gradle +++ b/code/features-crawl/crawl-blocklist/build.gradle @@ -20,7 +20,6 @@ dependencies { implementation libs.notnull implementation libs.bundles.gson - implementation libs.rxjava implementation libs.bundles.slf4j testImplementation libs.bundles.slf4j.test diff --git a/code/features-index/domain-ranking/build.gradle b/code/features-index/domain-ranking/build.gradle index 885787eb..ba0c614b 100644 --- a/code/features-index/domain-ranking/build.gradle +++ b/code/features-index/domain-ranking/build.gradle @@ -17,7 +17,6 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:model') implementation project(':code:common:service') - implementation project(':code:common:service-client') implementation project(':code:api:query-api') implementation libs.bundles.slf4j diff --git a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java index 89cd0d6d..e16e8618 100644 --- a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java +++ b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java @@ -19,7 +19,7 @@ public class ForwardIndexFileNames { public enum FileVersion { CURRENT, NEXT - }; + } public enum FileIdentifier { DOC_DATA, diff --git a/code/features-index/index-journal/build.gradle b/code/features-index/index-journal/build.gradle index f9197954..26e7855a 100644 --- a/code/features-index/index-journal/build.gradle +++ b/code/features-index/index-journal/build.gradle @@ -18,7 +18,6 @@ dependencies { implementation libs.prometheus implementation libs.notnull - implementation libs.rxjava implementation libs.guava implementation libs.trove implementation libs.zstd diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java index a13615ce..748fdde6 100644 --- a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java +++ b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java @@ -2,8 +2,6 @@ package nu.marginalia.index.query.filter; import nu.marginalia.array.buffer.LongQueryBuffer; -import java.util.List; - public interface QueryFilterStepIf extends Comparable { boolean test(long value); diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java index c8005fdb..3d0f2499 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java @@ -19,7 +19,7 @@ public class ReverseIndexFullFileNames { public enum FileVersion { CURRENT, NEXT - }; + } public enum FileIdentifier { WORDS, diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java index 6e73df43..e99841d4 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java @@ -19,7 +19,7 @@ public class ReverseIndexPrioFileNames { public enum FileVersion { CURRENT, NEXT - }; + } public enum FileIdentifier { WORDS, diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java index ffb25782..61dee824 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java @@ -6,8 +6,6 @@ import nu.marginalia.btree.BTreeReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.HashSet; import java.util.Random; public class ReverseIndexSelfTest { diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java index 1acaa514..16c542d5 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java @@ -1,6 +1,5 @@ package nu.marginalia.index.construction; -import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; import java.io.IOException; diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java index 882bc34a..049c9bad 100644 --- a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java +++ b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java @@ -1,7 +1,6 @@ package nu.marginalia.index.construction; -import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; import nu.marginalia.array.algo.SortingContext; import nu.marginalia.btree.BTreeReader; diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java index 1f5556ac..0ad3205a 100644 --- a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java +++ b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java @@ -1,7 +1,6 @@ package nu.marginalia.index.construction; import nu.marginalia.array.LongArray; -import nu.marginalia.array.algo.SortingContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java index 961a9e81..5698e64c 100644 --- a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java +++ b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java @@ -17,8 +17,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; -import static java.lang.Math.min; - @Singleton public class ResultValuator { final static double scalingFactor = 500.; diff --git a/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java index 385d8eb5..138230f9 100644 --- a/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java +++ b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java @@ -38,7 +38,7 @@ public class DbBrowseDomainsRandom { List domains = new ArrayList<>(count); try (var conn = dataSource.getConnection()) { try (var stmt = conn.prepareStatement(q)) { - stmt.setInt(1, set);; + stmt.setInt(1, set); stmt.setInt(2, count); var rsp = stmt.executeQuery(); while (rsp.next()) { diff --git a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/FoldBenchmark.java b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/FoldBenchmark.java index 31eb1359..50f6e925 100644 --- a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/FoldBenchmark.java +++ b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/FoldBenchmark.java @@ -2,7 +2,6 @@ package nu.marginalia.array.page; import nu.marginalia.array.LongArray; import nu.marginalia.array.SimulatedNaiveArray; -import nu.marginalia.array.scheme.ArrayPartitioningScheme; import org.openjdk.jmh.annotations.*; import java.lang.foreign.Arena; diff --git a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/QuicksortBenchmark.java b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/QuicksortBenchmark.java index 84905577..e90777d8 100644 --- a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/QuicksortBenchmark.java +++ b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/QuicksortBenchmark.java @@ -2,7 +2,6 @@ package nu.marginalia.array.page; import nu.marginalia.array.LongArray; import nu.marginalia.array.SimulatedNaiveArray; -import nu.marginalia.array.scheme.ArrayPartitioningScheme; import org.openjdk.jmh.annotations.*; import java.lang.foreign.Arena; diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java b/code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java index 7f16bb63..355dc2dc 100644 --- a/code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java +++ b/code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java @@ -7,7 +7,6 @@ import nu.marginalia.array.algo.IntArrayTransformations; import nu.marginalia.array.delegate.ShiftedIntArray; import nu.marginalia.array.page.SegmentIntArray; -import java.io.IOException; import java.lang.foreign.Arena; public interface IntArray extends IntArrayBase, IntArrayTransformations, IntArraySearch, IntArraySort { diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java b/code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java index c58014d0..98059f9f 100644 --- a/code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java +++ b/code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java @@ -5,7 +5,6 @@ import nu.marginalia.array.algo.LongArraySearch; import nu.marginalia.array.algo.LongArraySort; import nu.marginalia.array.algo.LongArrayTransformations; import nu.marginalia.array.delegate.ShiftedLongArray; -import nu.marginalia.array.page.SegmentLongArray; import nu.marginalia.array.page.UnsafeLongArray; import java.lang.foreign.Arena; diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySort.java b/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySort.java index 89905ea6..5eec50ef 100644 --- a/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySort.java +++ b/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySort.java @@ -1,7 +1,6 @@ package nu.marginalia.array.algo; import java.io.IOException; -import java.nio.IntBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/TwoArrayOperations.java b/code/libraries/array/src/main/java/nu/marginalia/array/algo/TwoArrayOperations.java index 5eafa361..c4af9090 100644 --- a/code/libraries/array/src/main/java/nu/marginalia/array/algo/TwoArrayOperations.java +++ b/code/libraries/array/src/main/java/nu/marginalia/array/algo/TwoArrayOperations.java @@ -2,8 +2,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.LongArray; -import java.util.function.LongBinaryOperator; - /** Functions for operating on pairs of arrays. */ diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySearchTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySearchTest.java index 85ec4410..9c1680a9 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySearchTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySearchTest.java @@ -3,7 +3,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.IntArray; import nu.marginalia.array.buffer.IntQueryBuffer; import nu.marginalia.array.page.SegmentIntArray; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySortTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySortTest.java index dc3985c6..8dc16dab 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySortTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySortTest.java @@ -1,7 +1,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.IntArray; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import nu.marginalia.util.test.TestUtil; import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.BeforeEach; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java index b9b374ac..b44ae07e 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java @@ -1,7 +1,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.IntArray; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformationsTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformationsTest.java index 7c2dc120..37954f27 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformationsTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformationsTest.java @@ -1,8 +1,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.IntArray; -import nu.marginalia.array.LongArray; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySearchTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySearchTest.java index c09a0a01..a515917b 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySearchTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySearchTest.java @@ -3,8 +3,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; import nu.marginalia.array.buffer.LongQueryBuffer; -import nu.marginalia.array.page.SegmentLongArray; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java index aa963070..8094d956 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java @@ -4,7 +4,6 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import nu.marginalia.util.test.TestUtil; import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.BeforeEach; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java index 22f7528c..81916b61 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java @@ -3,7 +3,6 @@ package nu.marginalia.array.algo; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import nu.marginalia.util.test.TestUtil; import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.BeforeEach; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java index eb9aafe2..01af58cc 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java @@ -1,8 +1,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.LongArray; -import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformationsTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformationsTest.java index 489ae54b..92fb3842 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformationsTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformationsTest.java @@ -1,8 +1,6 @@ package nu.marginalia.array.algo; import nu.marginalia.array.LongArray; -import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/PlainBigString.java b/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/PlainBigString.java index 5ff08c04..f5fef370 100644 --- a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/PlainBigString.java +++ b/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/PlainBigString.java @@ -1,7 +1,5 @@ package nu.marginalia.bigstring; -import java.nio.charset.StandardCharsets; - public class PlainBigString implements BigString { private final String value; diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java b/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java index d12b21be..048e0301 100644 --- a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java +++ b/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java @@ -3,7 +3,6 @@ package nu.marginalia.btree; import nu.marginalia.array.LongArray; import nu.marginalia.array.algo.LongArraySearch; import nu.marginalia.array.buffer.LongQueryBuffer; -import nu.marginalia.array.delegate.ShiftedLongArray; import nu.marginalia.btree.model.BTreeContext; import nu.marginalia.btree.model.BTreeHeader; diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java b/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java index c8d36c7b..1804ff40 100644 --- a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java +++ b/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java @@ -1,7 +1,6 @@ package nu.marginalia.btree; import nu.marginalia.array.LongArray; -import nu.marginalia.array.delegate.ShiftedLongArray; import nu.marginalia.btree.model.BTreeContext; import nu.marginalia.btree.model.BTreeHeader; diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentSentence.java b/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentSentence.java index 71b9fab4..ef5bc0a9 100644 --- a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentSentence.java +++ b/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentSentence.java @@ -8,8 +8,6 @@ import java.lang.ref.SoftReference; import java.util.BitSet; import java.util.Iterator; import java.util.StringJoiner; -import java.util.stream.Collectors; -import java.util.stream.IntStream; public class DocumentSentence implements Iterable{ public final String originalSentence; diff --git a/code/libraries/message-queue/build.gradle b/code/libraries/message-queue/build.gradle index 61f0292f..e3572cf6 100644 --- a/code/libraries/message-queue/build.gradle +++ b/code/libraries/message-queue/build.gradle @@ -13,7 +13,6 @@ dependencies { implementation libs.spark implementation libs.guice implementation libs.gson - implementation libs.rxjava implementation libs.bundles.slf4j implementation libs.bucket4j diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java b/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java index 27215f50..c97f0641 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java @@ -8,7 +8,6 @@ import nu.marginalia.mq.inbox.MqInboxResponse; import nu.marginalia.mq.inbox.MqSubscription; import nu.marginalia.mq.inbox.MqSynchronousInbox; import nu.marginalia.mq.outbox.MqOutbox; -import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.actor.state.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java index 02740232..6e3dd46c 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java @@ -211,7 +211,7 @@ public class MqPersistence { AND OWNER_INSTANCE IS NULL AND STATE='NEW' ORDER BY ID ASC LIMIT ? - """); + """) ) { updateStmt.setString(1, instanceUUID); updateStmt.setLong(2, tick); diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java index cbd1d17c..7c9f7d61 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java @@ -5,9 +5,7 @@ import com.google.gson.GsonBuilder; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.prototype.RecordActorPrototype; -import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.actor.state.ActorStep; -import nu.marginalia.actor.state.Resume; import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.mq.MqTestUtil; import nu.marginalia.mq.persistence.MqPersistence; diff --git a/code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomFileAssemblerTest.java b/code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomFileAssemblerTest.java index 76f5e822..bcbc7383 100644 --- a/code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomFileAssemblerTest.java +++ b/code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomFileAssemblerTest.java @@ -7,7 +7,6 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; import static org.junit.jupiter.api.Assertions.*; diff --git a/code/process-models/crawling-model/build.gradle b/code/process-models/crawling-model/build.gradle index ab4e8a8a..62ac447f 100644 --- a/code/process-models/crawling-model/build.gradle +++ b/code/process-models/crawling-model/build.gradle @@ -20,7 +20,6 @@ dependencies { implementation project(':code:libraries:big-string') implementation project(':code:api:index-api') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:features-crawl:content-type') implementation project(':code:libraries:language-processing') implementation project(':third-party:parquet-floor') diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/HttpFetchResult.java b/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/HttpFetchResult.java index f0db28e8..6bafaf5c 100644 --- a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/HttpFetchResult.java +++ b/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/HttpFetchResult.java @@ -106,7 +106,7 @@ public sealed interface HttpFetchResult { return headers.get(name); } - }; + } /** This is a special case where the document was not fetched * because it was already in the database. In this case, we @@ -128,14 +128,14 @@ public sealed interface HttpFetchResult { return Optional.empty(); } } - }; + } /** Fetching resulted in an exception */ record ResultException(Exception ex) implements HttpFetchResult { public boolean isOk() { return false; } - }; + } /** Fetching resulted in a HTTP 304, the remote content is identical to * our reference copy. This will be replaced with a Result304ReplacedWithReference @@ -147,7 +147,7 @@ public sealed interface HttpFetchResult { public boolean isOk() { return false; } - }; + } /** No result. This is typically injected at a later stage * of processing, e.g. after filtering out irrelevant responses. @@ -156,5 +156,5 @@ public sealed interface HttpFetchResult { public boolean isOk() { return false; } - }; + } } diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java b/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java index 0b5b7aa8..76ecf7e7 100644 --- a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java +++ b/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java @@ -10,7 +10,6 @@ import nu.marginalia.crawling.model.SerializableCrawlData; import java.io.*; import java.nio.file.Path; -import java.util.Objects; import static java.util.Objects.*; diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java b/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java index 497f7a00..bb344dfb 100644 --- a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java +++ b/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java @@ -3,7 +3,6 @@ package nu.marginalia.crawling.model; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.ToString; -import nu.marginalia.bigstring.BigString; import nu.marginalia.model.EdgeUrl; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.Nullable; diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java b/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java index e778169e..efa109cc 100644 --- a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java +++ b/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java @@ -3,7 +3,6 @@ package nu.marginalia.io.processed; import blue.strategic.parquet.HydratorSupplier; import blue.strategic.parquet.ParquetReader; import nu.marginalia.model.processed.DomainLinkRecord; -import nu.marginalia.model.processed.DomainRecord; import org.jetbrains.annotations.NotNull; import java.io.IOException; diff --git a/code/processes/converting-process/build.gradle b/code/processes/converting-process/build.gradle index 5b1dbde5..4def16db 100644 --- a/code/processes/converting-process/build.gradle +++ b/code/processes/converting-process/build.gradle @@ -34,7 +34,6 @@ dependencies { implementation project(':code:libraries:message-queue') implementation project(':code:libraries:blocking-thread-pool') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:libraries:guarded-regex') implementation project(':code:libraries:easy-lsh') diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java index c72e284a..a570e72d 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java @@ -28,7 +28,6 @@ import nu.marginalia.converting.processor.DomainProcessor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.nio.file.Path; import java.sql.SQLException; import java.util.Collection; diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterModule.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterModule.java index 903834f4..5f04c86f 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterModule.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterModule.java @@ -4,7 +4,6 @@ import com.google.gson.Gson; import com.google.inject.AbstractModule; import com.google.inject.name.Names; import nu.marginalia.LanguageModels; -import nu.marginalia.ProcessConfiguration; import nu.marginalia.WmsaHome; import nu.marginalia.model.gson.GsonFactory; diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java index 27c493db..2fc9068f 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java @@ -5,7 +5,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; import java.util.stream.Collectors; public class PlainTextLogic { diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java index b26288e9..3aa35973 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java @@ -5,7 +5,6 @@ import com.google.inject.Singleton; import nu.marginalia.keyword.model.DocumentKeywordsBuilder; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.summary.SummaryExtractor; -import org.apache.commons.lang3.StringUtils; import org.jsoup.nodes.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java index b92ebcbc..947cc4c0 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java @@ -4,12 +4,9 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.model.EdgeUrl; import nu.marginalia.summary.SummaryExtractor; -import org.jsoup.nodes.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Set; - @Singleton public class PhpBBSpecialization extends DefaultSpecialization { private static final Logger logger = LoggerFactory.getLogger(PhpBBSpecialization.class); diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSource.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSource.java index a74cf456..5b64b798 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSource.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSource.java @@ -2,7 +2,6 @@ package nu.marginalia.converting.sideload; import nu.marginalia.converting.model.ProcessedDocument; import nu.marginalia.converting.model.ProcessedDomain; -import nu.marginalia.model.EdgeUrl; import java.util.Iterator; diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java index 24706317..d564b308 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java @@ -175,5 +175,5 @@ public class RedditSideloader implements SideloadSource { return doc; - }; + } } diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterWriter.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterWriter.java index 6bac2804..bc86e369 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterWriter.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterWriter.java @@ -1,7 +1,6 @@ package nu.marginalia.converting.writer; import lombok.SneakyThrows; -import nu.marginalia.converting.model.ProcessedDomain; import nu.marginalia.worklog.BatchingWorkLog; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTestModule.java b/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTestModule.java index 3d171a53..83f28882 100644 --- a/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTestModule.java +++ b/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTestModule.java @@ -16,7 +16,7 @@ public class ConvertingIntegrationTestModule extends AbstractModule { bind(Integer.class).annotatedWith(Names.named("max-title-length")).toInstance(128); bind(Integer.class).annotatedWith(Names.named("max-summary-length")).toInstance(255); bind(ServiceConfiguration.class).toInstance(new ServiceConfiguration( - null, 1, "localhost", 0, 0, null + null, 1, "localhost", "localhost", 0, null )); bind(ProcessConfiguration.class).toInstance(new ProcessConfiguration( "converting-process", 1, null diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java b/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java index d48ceeb2..1fc23148 100644 --- a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java +++ b/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java @@ -1,6 +1,5 @@ package nu.marginalia.converting.processor.plugin.specialization; -import nu.marginalia.converting.processor.logic.DocumentGeneratorExtractor; import nu.marginalia.summary.SummaryExtractor; import nu.marginalia.test.CommonTestData; import org.jsoup.Jsoup; diff --git a/code/processes/crawling-process/build.gradle b/code/processes/crawling-process/build.gradle index 2e99b8a7..f9cf5029 100644 --- a/code/processes/crawling-process/build.gradle +++ b/code/processes/crawling-process/build.gradle @@ -30,7 +30,6 @@ dependencies { implementation project(':code:api:index-api') implementation project(':code:api:process-mqapi') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:libraries:message-queue') implementation project(':code:libraries:language-processing') implementation project(':code:libraries:easy-lsh') @@ -55,7 +54,6 @@ dependencies { implementation libs.okhttp3 implementation libs.jsoup implementation libs.opencsv - implementation libs.rxjava implementation libs.fastutil implementation libs.bundles.mariadb diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java index 580ac3c7..4461a85a 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java @@ -332,7 +332,7 @@ public class CrawlerMain extends ProcessMainClass { if (request.specStorage != null) { var specData = fileStorageService.getStorage(request.specStorage); - var parquetProvider = new ParquetCrawlSpecProvider(CrawlSpecFileNames.resolve(specData));; + var parquetProvider = new ParquetCrawlSpecProvider(CrawlSpecFileNames.resolve(specData)); // Ensure the parquet domains are loaded into the database to avoid // rare data-loss scenarios diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerModule.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerModule.java index 91251694..1d27f08f 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerModule.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerModule.java @@ -2,15 +2,11 @@ package nu.marginalia.crawl; import com.google.gson.Gson; import com.google.inject.AbstractModule; -import com.google.inject.name.Names; import lombok.SneakyThrows; -import nu.marginalia.ProcessConfiguration; import nu.marginalia.UserAgent; import nu.marginalia.WmsaHome; import nu.marginalia.model.gson.GsonFactory; -import java.util.UUID; - public class CrawlerModule extends AbstractModule { @SneakyThrows public void configure() { diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java index bfbcab14..ab1ce5ef 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java @@ -1,7 +1,6 @@ package nu.marginalia.crawl.retreival; import nu.marginalia.crawling.body.DocumentBodyExtractor; -import nu.marginalia.crawling.body.DocumentBodyResult; import nu.marginalia.crawling.body.HttpFetchResult; import nu.marginalia.crawl.retreival.fetcher.warc.WarcRecorder; import nu.marginalia.model.EdgeUrl; @@ -10,8 +9,6 @@ import org.netpreserve.jwarc.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Path; /** diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainProber.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainProber.java index fcc005a8..57147aec 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainProber.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainProber.java @@ -5,7 +5,6 @@ import com.google.inject.Singleton; import nu.marginalia.crawl.retreival.fetcher.FetchResultState; import nu.marginalia.crawl.retreival.fetcher.HttpFetcher; import nu.marginalia.crawling.model.CrawlerDomainStatus; -import nu.marginalia.ip_blocklist.GeoIpBlocklist; import nu.marginalia.ip_blocklist.IpBlockList; import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.EdgeUrl; @@ -57,7 +56,7 @@ public class DomainProber { return new ProbeResultError(CrawlerDomainStatus.ERROR, "Bad status"); } - public sealed interface ProbeResult permits ProbeResultError, ProbeResultRedirect, ProbeResultOk {}; + public sealed interface ProbeResult permits ProbeResultError, ProbeResultRedirect, ProbeResultOk {} /** The probing failed for one reason or another * @param status Machine readable status diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java index 2bc8482b..94494402 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java @@ -1,7 +1,6 @@ package nu.marginalia.crawl.retreival.fetcher; import com.google.inject.Inject; -import com.google.inject.name.Named; import crawlercommons.robots.SimpleRobotRules; import crawlercommons.robots.SimpleRobotRulesParser; import lombok.SneakyThrows; @@ -22,7 +21,6 @@ import okhttp3.ConnectionPool; import okhttp3.Dispatcher; import okhttp3.OkHttpClient; import okhttp3.Request; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java b/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java index ae3d9be4..e3720941 100644 --- a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java +++ b/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java @@ -15,7 +15,6 @@ import org.netpreserve.jwarc.WarcResponse; import java.io.IOException; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; diff --git a/code/processes/loading-process/build.gradle b/code/processes/loading-process/build.gradle index 116b1d59..030e6e28 100644 --- a/code/processes/loading-process/build.gradle +++ b/code/processes/loading-process/build.gradle @@ -26,7 +26,6 @@ dependencies { implementation project(':code:common:config') implementation project(':code:common:service') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:common:linkdb') implementation project(':code:features-index:index-journal') implementation project(':code:libraries:message-queue') diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java index 35c98fc6..4f9765a4 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java @@ -13,8 +13,6 @@ import nu.marginalia.linkdb.dlinks.DomainLinkDbWriter; import nu.marginalia.storage.FileStorageService; import nu.marginalia.linkdb.docs.DocumentDbWriter; import nu.marginalia.model.gson.GsonFactory; -import nu.marginalia.service.SearchServiceDescriptors; -import nu.marginalia.service.descriptor.ServiceDescriptors; import java.io.IOException; import java.nio.file.Files; @@ -30,8 +28,6 @@ public class LoaderModule extends AbstractModule { } public void configure() { - bind(ServiceDescriptors.class).toInstance(SearchServiceDescriptors.descriptors); - bind(Gson.class).toProvider(this::createGson); bind(Path.class).annotatedWith(Names.named("local-index-path")).toInstance(Path.of(System.getProperty("local-index-path", "/vol"))); bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels()); diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java b/code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java index a751ecdf..fda0e9b6 100644 --- a/code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java +++ b/code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java @@ -11,11 +11,8 @@ import nu.marginalia.model.processed.DomainLinkRecord; import nu.marginalia.model.processed.DomainRecord; import nu.marginalia.process.control.ProcessAdHocTaskHeartbeat; import nu.marginalia.process.control.ProcessHeartbeat; -import nu.marginalia.test.TestMigrationLoader; import org.junit.jupiter.api.*; import org.mockito.Mockito; -import org.testcontainers.containers.MariaDBContainer; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.io.IOException; diff --git a/code/processes/website-adjacencies-calculator/build.gradle b/code/processes/website-adjacencies-calculator/build.gradle index 113e784a..a007c105 100644 --- a/code/processes/website-adjacencies-calculator/build.gradle +++ b/code/processes/website-adjacencies-calculator/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:process') - implementation project(':code:common:service-client') + implementation project(':code:common:service-discovery') implementation project(':code:common:service') implementation project(':code:api:query-api') diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index df87d03b..1e680321 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -9,6 +9,9 @@ import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeatImpl; import nu.marginalia.query.client.QueryClient; import nu.marginalia.service.MainClass; +import nu.marginalia.service.NodeConfigurationWatcher; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.discovery.FixedServiceRegistry; import nu.marginalia.service.module.DatabaseModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,8 +141,9 @@ public class WebsiteAdjacenciesCalculator extends MainClass { return weightedProduct(weights, a, b) / Math.sqrt(a.mulAndSum(weights) * b.mulAndSum(weights)); } - public record DomainSimilarities(int domainId, List similarities) {}; - public record DomainSimilarity(int domainId, double value) {}; + public record DomainSimilarities(int domainId, List similarities) {} + + public record DomainSimilarity(int domainId, double value) {} @SneakyThrows private void findAdjacentDtoS(int domainId, Consumer andThen) { @@ -190,7 +194,11 @@ public class WebsiteAdjacenciesCalculator extends MainClass { DatabaseModule dm = new DatabaseModule(false); var dataSource = dm.provideConnection(); - var qc = new QueryClient(); + + // FIXME: we should use zookeeper when configured here: + var qc = new QueryClient(new GrpcChannelPoolFactory( + new NodeConfigurationWatcher(dataSource), + new FixedServiceRegistry(dataSource))); var main = new WebsiteAdjacenciesCalculator(qc, dataSource); diff --git a/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java b/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java index 34fa618e..4ce29080 100644 --- a/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java +++ b/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java @@ -70,7 +70,7 @@ public class AdjacenciesLoaderTest { } try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement("SELECT * FROM EC_DOMAIN_NEIGHBORS_2 WHERE DOMAIN_ID=1"); + var stmt = conn.prepareStatement("SELECT * FROM EC_DOMAIN_NEIGHBORS_2 WHERE DOMAIN_ID=1") ) { var rs = stmt.executeQuery(); Assertions.assertTrue(rs.next()); diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index 53fb91a3..a3e402f7 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -27,7 +27,6 @@ dependencies { implementation project(':code:common:service') implementation project(':code:common:config') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:api:query-api') implementation project(':code:api:index-api') implementation project(':code:features-index:index-query') @@ -37,7 +36,6 @@ dependencies { implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.spark implementation libs.opencsv implementation libs.trove diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java index aea51959..f9c4347f 100644 --- a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java +++ b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -21,7 +20,7 @@ public class ApiMain extends MainClass { Injector injector = Guice.createInjector( new DatabaseModule(false), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Api)); + new ServiceConfigurationModule(ServiceId.Api)); injector.getInstance(ApiMain.class); injector.getInstance(Initialization.class).setReady(); } diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java index a1180f4c..0938dc34 100644 --- a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java +++ b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java @@ -5,7 +5,6 @@ import com.google.inject.Singleton; import nu.marginalia.api.model.ApiSearchResult; import nu.marginalia.api.model.ApiSearchResultQueryDetails; import nu.marginalia.api.model.ApiSearchResults; -import nu.marginalia.client.Context; import nu.marginalia.index.client.model.query.SearchSetIdentifier; import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; import nu.marginalia.index.client.model.results.SearchResultKeywordScore; @@ -29,12 +28,11 @@ public class ApiSearchOperator { this.queryClient = queryClient; } - public ApiSearchResults query(Context context, - String query, + public ApiSearchResults query(String query, int count, int index) { - var rsp = queryClient.search(context, createParams(query, count, index)); + var rsp = queryClient.search(createParams(query, count, index)); return new ApiSearchResults("RESTRICTED", query, rsp.results() diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java index 35d9a7a7..85a19473 100644 --- a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java +++ b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java @@ -9,9 +9,7 @@ import nu.marginalia.api.model.ApiSearchResults; import nu.marginalia.api.svc.LicenseService; import nu.marginalia.api.svc.RateLimiterService; import nu.marginalia.api.svc.ResponseCache; -import nu.marginalia.client.Context; import nu.marginalia.model.gson.GsonFactory; -import nu.marginalia.query.client.QueryClient; import nu.marginalia.service.server.*; import nu.marginalia.service.server.mq.MqRequest; import org.slf4j.Logger; @@ -126,7 +124,7 @@ public class ApiService extends Service { .labels(license.key) .time(() -> searchOperator - .query(Context.fromRequest(request), query, count, index) + .query(query, count, index) .withLicense(license.getLicense()) ); } diff --git a/code/services-application/dating-service/build.gradle b/code/services-application/dating-service/build.gradle index 6ad45833..a59476c8 100644 --- a/code/services-application/dating-service/build.gradle +++ b/code/services-application/dating-service/build.gradle @@ -25,7 +25,6 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:service') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:common:renderer') implementation project(':code:features-search:screenshots') implementation project(':code:features-search:random-websites') @@ -36,7 +35,6 @@ dependencies { implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.spark implementation libs.opencsv implementation libs.trove diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java b/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java index 00f31cb5..92065da7 100644 --- a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java +++ b/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -26,7 +25,7 @@ public class DatingMain extends MainClass { Injector injector = Guice.createInjector( new DatingModule(), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Dating), + new ServiceConfigurationModule(ServiceId.Dating), new DatabaseModule(false) ); diff --git a/code/services-application/explorer-service/build.gradle b/code/services-application/explorer-service/build.gradle index 807ca8a1..596afdf0 100644 --- a/code/services-application/explorer-service/build.gradle +++ b/code/services-application/explorer-service/build.gradle @@ -24,7 +24,6 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:service') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:common:renderer') implementation project(':code:features-search:random-websites') @@ -34,7 +33,6 @@ dependencies { implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.spark implementation libs.opencsv implementation libs.trove diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java index 2b47657b..4ce949f6 100644 --- a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java +++ b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -25,7 +24,7 @@ public class ExplorerMain extends MainClass { Spark.staticFileLocation("/static/explore/"); Injector injector = Guice.createInjector( - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Explorer), + new ServiceConfigurationModule(ServiceId.Explorer), new ExplorerModule(), new DatabaseModule(false) ); diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerService.java b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerService.java index a753eed9..0b6ae0de 100644 --- a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerService.java +++ b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerService.java @@ -136,9 +136,9 @@ public class ExplorerService extends Service { GROUP BY NV.DOMAIN_ID ORDER BY NV.RELATEDNESS DESC """ - ); + ) - ) { + ) { stmt.setInt(1, domainIdInformation.domainId); stmt.setInt(2, domainIdInformation.aliasId); diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index ee504bcb..80936e48 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -42,7 +42,6 @@ dependencies { implementation project(':code:api:query-api') implementation project(':code:api:index-api') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:common:renderer') implementation project(':code:features-search:screenshots') @@ -55,7 +54,6 @@ dependencies { implementation libs.prometheus implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.handlebars implementation libs.spark implementation libs.opencsv diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java index 30dc1723..9d25d278 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -27,7 +26,7 @@ public class SearchMain extends MainClass { Injector injector = Guice.createInjector( new SearchModule(), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Search), + new ServiceConfigurationModule(ServiceId.Search), new DatabaseModule(false) ); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchModule.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchModule.java index 4a32f9e1..52d1cbea 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchModule.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchModule.java @@ -25,7 +25,7 @@ public class SearchModule extends AbstractModule { @Provides public FeedlotClient provideFeedlotClient() { return new FeedlotClient( - System.getProperty("ext-svc-feedlot-host", "feedlot"), + System.getProperty("ext-svc-feedlot-bindAddress", "feedlot"), Integer.getInteger("ext-svc-feedlot-port", 80), GsonFactory.get(), Duration.ofMillis(250), diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java index b3597950..9c38ffef 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java @@ -2,8 +2,7 @@ package nu.marginalia.search; import com.google.inject.Inject; import com.google.inject.Singleton; -import io.reactivex.rxjava3.core.Observable; -import io.reactivex.rxjava3.schedulers.Schedulers; +import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; import nu.marginalia.assistant.client.AssistantClient; import nu.marginalia.model.EdgeDomain; @@ -12,7 +11,6 @@ import nu.marginalia.query.client.QueryClient; import nu.marginalia.query.model.QueryResponse; import nu.marginalia.search.command.SearchParameters; import nu.marginalia.search.model.*; -import nu.marginalia.client.Context; import nu.marginalia.search.svc.SearchQueryIndexService; import nu.marginalia.search.svc.SearchUnitConversionService; import org.apache.logging.log4j.util.Strings; @@ -22,6 +20,7 @@ import org.slf4j.Marker; import org.slf4j.MarkerFactory; import javax.annotation.Nullable; +import java.time.Duration; import java.util.*; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -64,37 +63,35 @@ public class SearchOperator { this.searchUnitConversionService = searchUnitConversionService; } - public List doSiteSearch(Context ctx, - String domain, + public List doSiteSearch(String domain, int count) { var queryParams = paramFactory.forSiteSearch(domain, count); - var queryResponse = queryClient.search(ctx, queryParams); + var queryResponse = queryClient.search(queryParams); return searchQueryService.getResultsFromQuery(queryResponse); } - public List doBacklinkSearch(Context ctx, - String domain) { + public List doBacklinkSearch(String domain) { var queryParams = paramFactory.forBacklinkSearch(domain); - var queryResponse = queryClient.search(ctx, queryParams); + var queryResponse = queryClient.search(queryParams); return searchQueryService.getResultsFromQuery(queryResponse); } - public List doLinkSearch(Context context, String source, String dest) { + public List doLinkSearch(String source, String dest) { var queryParams = paramFactory.forLinkSearch(source, dest); - var queryResponse = queryClient.search(context, queryParams); + var queryResponse = queryClient.search(queryParams); return searchQueryService.getResultsFromQuery(queryResponse); } - public DecoratedSearchResults doSearch(Context ctx, SearchParameters userParams) { + public DecoratedSearchResults doSearch(SearchParameters userParams) { - Future eval = searchUnitConversionService.tryEval(ctx, userParams.query()); + Future eval = searchUnitConversionService.tryEval(userParams.query()); var queryParams = paramFactory.forRegularSearch(userParams); - var queryResponse = queryClient.search(ctx, queryParams); + var queryResponse = queryClient.search(queryParams); List queryResults = searchQueryService.getResultsFromQuery(queryResponse); @@ -109,7 +106,7 @@ public class SearchOperator { return DecoratedSearchResults.builder() .params(userParams) - .problems(getProblems(ctx, evalResult, queryResults, queryResponse)) + .problems(getProblems(evalResult, queryResults, queryResponse)) .evalResult(evalResult) .results(clusteredResults) .filters(new SearchFilters(websiteUrl, userParams)) @@ -139,13 +136,13 @@ public class SearchOperator { return domainQueries.tryGetDomainId(new EdgeDomain(domain)).orElse(-1); } - private List getProblems(Context ctx, String evalResult, List queryResults, QueryResponse response) { + private List getProblems(String evalResult, List queryResults, QueryResponse response) { final List problems = new ArrayList<>(response.problems()); boolean siteSearch = response.domain() != null; if (!siteSearch) { if (queryResults.size() <= 5 && null == evalResult) { - spellCheckTerms(ctx, response).forEach(problems::add); + spellCheckTerms(response); } if (queryResults.size() <= 5) { @@ -163,30 +160,22 @@ public class SearchOperator { } - private Iterable spellCheckTerms(Context ctx, QueryResponse response) { - return Observable.fromIterable(response.searchTermsHuman()) - .subscribeOn(Schedulers.io()) - .flatMap(term -> assistantClient.spellCheck(ctx, term) - .onErrorReturn(e -> Collections.emptyList()) - .filter(results -> hasSpellSuggestions(term, results)) - .map(suggestions -> searchTermToProblemDescription(term, suggestions)) - ) - .blockingIterable(); - } + @SneakyThrows + private void spellCheckTerms(QueryResponse response) { + var suggestions = assistantClient + .spellCheck(response.searchTermsHuman(), Duration.ofMillis(20)); - private boolean hasSpellSuggestions(String term, List results) { - if (results.size() > 1) { - return true; - } - else if (results.size() == 1) { - return !term.equalsIgnoreCase(results.get(0)); - } - return false; + suggestions.entrySet() + .stream() + .filter(e -> e.getValue().size() > 1) + .map(e -> searchTermToProblemDescription(e.getKey(), e.getValue())) + .forEach(response.problems()::add); } private String searchTermToProblemDescription(String term, List suggestions) { - return "\"" + term + "\" could be spelled " + - suggestions.stream().map(s -> "\""+s+"\"").collect(Collectors.joining(", ")); + String suggestionsStr = suggestions.stream().map(s -> STR."\"\{s}\"").collect(Collectors.joining(", ")); + + return STR."\"\{term}\" could be spelled \{suggestionsStr}"; } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java index 509f015d..8e8594be 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java @@ -5,8 +5,6 @@ import io.prometheus.client.Counter; import io.prometheus.client.Histogram; import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; -import nu.marginalia.client.Context; -import nu.marginalia.search.svc.SearchFrontPageService; import nu.marginalia.search.svc.*; import nu.marginalia.service.server.*; import org.slf4j.Logger; @@ -75,7 +73,7 @@ public class SearchService extends Service { Spark.exception(Exception.class, (e,p,q) -> { logger.error("Error during processing", e); wmsa_search_service_error_count.labels(p.pathInfo(), p.requestMethod()).inc(); - errorPageService.serveError(Context.fromRequest(p), p, q); + errorPageService.serveError(p, q); }); Spark.awaitInitialization(); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/CommandEvaluator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/CommandEvaluator.java index 53f8a56e..eb352a93 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/CommandEvaluator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/CommandEvaluator.java @@ -2,7 +2,6 @@ package nu.marginalia.search.command; import com.google.inject.Inject; import nu.marginalia.search.command.commands.*; -import nu.marginalia.client.Context; import spark.Response; import java.util.ArrayList; @@ -31,14 +30,14 @@ public class CommandEvaluator { defaultCommand = search; } - public Object eval(Context ctx, Response response, SearchParameters parameters) { + public Object eval(Response response, SearchParameters parameters) { for (var cmd : specialCommands) { - var maybe = cmd.process(ctx, response, parameters); + var maybe = cmd.process(response, parameters); if (maybe.isPresent()) return maybe.get(); } - return defaultCommand.process(ctx, response, parameters).orElse(""); + return defaultCommand.process(response, parameters).orElse(""); } } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchCommandInterface.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchCommandInterface.java index c2693dd3..d69bacbd 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchCommandInterface.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchCommandInterface.java @@ -1,11 +1,10 @@ package nu.marginalia.search.command; -import nu.marginalia.client.Context; import spark.Response; import java.util.Optional; public interface SearchCommandInterface { - Optional process(Context ctx, Response response, SearchParameters parameters); + Optional process(Response response, SearchParameters parameters); } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BangCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BangCommand.java index 5dbbec1c..92f493c5 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BangCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BangCommand.java @@ -3,7 +3,6 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; -import nu.marginalia.client.Context; import nu.marginalia.search.exceptions.RedirectException; import spark.Response; @@ -25,7 +24,7 @@ public class BangCommand implements SearchCommandInterface { } @Override - public Optional process(Context ctx, Response response, SearchParameters parameters) { + public Optional process(Response response, SearchParameters parameters) { for (var entry : bangsToPattern.entrySet()) { String bangPattern = entry.getKey(); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BrowseCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BrowseCommand.java index aa6b19ea..a889ec3d 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BrowseCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BrowseCommand.java @@ -2,7 +2,6 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; import nu.marginalia.browse.model.BrowseResultSet; -import nu.marginalia.client.Context; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import nu.marginalia.search.command.SearchCommandInterface; @@ -35,12 +34,12 @@ public class BrowseCommand implements SearchCommandInterface { } @Override - public Optional process(Context ctx, Response response, SearchParameters parameters) { + public Optional process(Response response, SearchParameters parameters) { if (!queryPatternPredicate.test(parameters.query())) { return Optional.empty(); } - var model = browseSite(ctx, parameters.query()); + var model = browseSite(parameters.query()); if (null == model) return Optional.empty(); @@ -53,7 +52,7 @@ public class BrowseCommand implements SearchCommandInterface { } - private BrowseResultSet browseSite(Context ctx, String humanQuery) { + private BrowseResultSet browseSite(String humanQuery) { String definePrefix = "browse:"; String word = humanQuery.substring(definePrefix.length()).toLowerCase(); @@ -66,7 +65,7 @@ public class BrowseCommand implements SearchCommandInterface { return browseService.getRandomEntries(set); } else { - return browseService.getRelatedEntries(ctx, word); + return browseService.getRelatedEntries(word); } } catch (Exception ex) { diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/ConvertCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/ConvertCommand.java index 14119ed7..38f98af2 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/ConvertCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/ConvertCommand.java @@ -5,7 +5,6 @@ import lombok.SneakyThrows; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; import nu.marginalia.search.svc.SearchUnitConversionService; -import nu.marginalia.client.Context; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import spark.Response; @@ -27,8 +26,8 @@ public class ConvertCommand implements SearchCommandInterface { @Override @SneakyThrows - public Optional process(Context ctx, Response response, SearchParameters parameters) { - var conversion = searchUnitConversionService.tryConversion(ctx, parameters.query()); + public Optional process(Response response, SearchParameters parameters) { + var conversion = searchUnitConversionService.tryConversion(parameters.query()); return conversion.map(s -> conversionRenderer.render(Map.of( "query", parameters.query(), "result", s, diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java index d0c65ed0..52b0dfdb 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java @@ -2,10 +2,8 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; -import lombok.SneakyThrows; import nu.marginalia.assistant.client.AssistantClient; import nu.marginalia.assistant.client.model.DictionaryResponse; -import nu.marginalia.client.Context; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; import nu.marginalia.renderer.MustacheRenderer; @@ -17,6 +15,7 @@ import spark.Response; import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -39,12 +38,12 @@ public class DefinitionCommand implements SearchCommandInterface { } @Override - public Optional process(Context ctx, Response response, SearchParameters parameters) { + public Optional process(Response response, SearchParameters parameters) { if (!queryPatternPredicate.test(parameters.query())) { return Optional.empty(); } - var results = lookupDefinition(ctx, parameters.query()); + var results = lookupDefinition(parameters.query()); return Optional.of(dictionaryRenderer.render(results, Map.of("query", parameters.query(), @@ -53,17 +52,19 @@ public class DefinitionCommand implements SearchCommandInterface { } - @SneakyThrows - private DictionaryResponse lookupDefinition(Context ctx, String humanQuery) { + private DictionaryResponse lookupDefinition(String humanQuery) { String definePrefix = "define:"; String word = humanQuery.substring(definePrefix.length()).toLowerCase(); - logger.info("Defining: {}", word); - var results = assistantClient - .dictionaryLookup(ctx, word) - .blockingFirst(); - logger.debug("Results = {}", results); + try { + return assistantClient + .dictionaryLookup(word) + .get(100, TimeUnit.MILLISECONDS); + } + catch (Exception e) { + logger.error("Failed to lookup definition for word: " + word, e); - return results; + throw new RuntimeException(e); + } } } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java index aeede238..76454538 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java @@ -1,13 +1,11 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; -import nu.marginalia.client.Context; import nu.marginalia.db.DomainBlacklist; import nu.marginalia.search.SearchOperator; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; import nu.marginalia.search.model.DecoratedSearchResults; -import nu.marginalia.search.model.UrlDetails; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import spark.Response; @@ -33,8 +31,8 @@ public class SearchCommand implements SearchCommandInterface { } @Override - public Optional process(Context ctx, Response response, SearchParameters parameters) { - DecoratedSearchResults results = searchOperator.doSearch(ctx, parameters); + public Optional process(Response response, SearchParameters parameters) { + DecoratedSearchResults results = searchOperator.doSearch(parameters); return Optional.of(searchResultsRenderer.render(results)); } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java index cea76c36..dc6855ee 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java @@ -2,7 +2,6 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; import lombok.SneakyThrows; -import nu.marginalia.client.Context; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; import org.slf4j.Logger; @@ -25,7 +24,7 @@ public class SiteRedirectCommand implements SearchCommandInterface { @SneakyThrows @Override - public Optional process(Context ctx, Response response, SearchParameters parameters) { + public Optional process(Response response, SearchParameters parameters) { if (!queryPatternPredicate.test(parameters.query())) { return Optional.empty(); } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/BrowseResultCleaner.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/results/BrowseResultCleaner.java index 7ebefb55..8f4e5644 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/BrowseResultCleaner.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/results/BrowseResultCleaner.java @@ -2,7 +2,6 @@ package nu.marginalia.search.results; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.assistant.client.model.SimilarDomain; import nu.marginalia.browse.model.BrowseResult; import nu.marginalia.screenshot.ScreenshotService; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java index e64f8517..16fe8aa3 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java @@ -5,7 +5,6 @@ import gnu.trove.list.array.TLongArrayList; import gnu.trove.map.hash.TObjectIntHashMap; import gnu.trove.set.hash.TIntHashSet; import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; -import nu.marginalia.search.model.UrlDetails; import nu.marginalia.lsh.EasyLSH; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java index 187a9081..2dcbbe96 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java @@ -6,7 +6,6 @@ import nu.marginalia.assistant.client.model.SimilarDomain; import nu.marginalia.browse.DbBrowseDomainsRandom; import nu.marginalia.browse.model.BrowseResult; import nu.marginalia.browse.model.BrowseResultSet; -import nu.marginalia.client.Context; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.db.DomainBlacklist; import nu.marginalia.model.EdgeDomain; @@ -16,6 +15,9 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static java.util.Collections.shuffle; @@ -48,16 +50,21 @@ public class SearchBrowseService { return new BrowseResultSet(results); } - public BrowseResultSet getRelatedEntries(Context ctx, String domainName) { + public BrowseResultSet getRelatedEntries(String domainName) throws ExecutionException, InterruptedException, TimeoutException { var domain = domainQueries.getDomainId(new EdgeDomain(domainName)); - var neighbors = assistantClient.similarDomains(ctx, domain, 50).blockingFirst(); + var neighbors = assistantClient.similarDomains(domain, 50) + .get(100, TimeUnit.MILLISECONDS); + neighbors.removeIf(sd -> !sd.screenshot()); // If the results are very few, supplement with the alternative shitty algorithm if (neighbors.size() < 25) { Set allNeighbors = new HashSet<>(neighbors); - allNeighbors.addAll(assistantClient.linkedDomains(ctx, domain, 50).blockingFirst()); + allNeighbors.addAll(assistantClient + .linkedDomains(domain, 50) + .get(100, TimeUnit.MILLISECONDS) + ); neighbors.clear(); neighbors.addAll(allNeighbors); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchCrosstalkService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchCrosstalkService.java index 910348e0..ddce56ac 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchCrosstalkService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchCrosstalkService.java @@ -1,7 +1,6 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.client.Context; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import nu.marginalia.search.SearchOperator; @@ -43,8 +42,8 @@ public class SearchCrosstalkService { parts[i] = parts[i].trim(); } - var resAtoB = searchOperator.doLinkSearch(Context.fromRequest(request), parts[0], parts[1]); - var resBtoA = searchOperator.doLinkSearch(Context.fromRequest(request), parts[1], parts[0]); + var resAtoB = searchOperator.doLinkSearch(parts[0], parts[1]); + var resBtoA = searchOperator.doLinkSearch(parts[1], parts[0]); var model = new CrosstalkResult(parts[0], parts[1], resAtoB, resBtoA); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java index 68f55b4f..7af3cabe 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java @@ -1,7 +1,6 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.client.Context; import nu.marginalia.index.client.IndexClient; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; @@ -27,7 +26,7 @@ public class SearchErrorPageService { this.indexClient = indexClient; } - public void serveError(Context ctx, Request request, Response rsp) { + public void serveError(Request request, Response rsp) { rsp.body(renderError(request, "Internal error", """ An error occurred when communicating with the search engine index. diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java index e4ffce78..c7ccfa34 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java @@ -81,5 +81,5 @@ public class SearchFlagSiteService { public record CategoryItem(String categoryName, String categoryDesc) {} public record FlagSiteComplaintModel(String category, String submitTime, boolean isReviewed, String decision) {} - public record FlagSiteFormData(int domainId, String category, String description, String sampleQuery) {}; + public record FlagSiteFormData(int domainId, String category, String description, String sampleQuery) {} } diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryService.java index 906c4caf..948b0af7 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryService.java @@ -5,7 +5,6 @@ import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; import nu.marginalia.search.command.*; import nu.marginalia.search.model.SearchProfile; -import nu.marginalia.client.Context; import nu.marginalia.search.exceptions.RedirectException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,18 +30,15 @@ public class SearchQueryService { @SneakyThrows public Object pathSearch(Request request, Response response) { - - final var ctx = Context.fromRequest(request); - try { - return searchCommandEvaulator.eval(ctx, response, parseParameters(request)); + return searchCommandEvaulator.eval(response, parseParameters(request)); } catch (RedirectException ex) { response.redirect(ex.newUrl); } catch (Exception ex) { logger.error("Error", ex); - errorPageService.serveError(ctx, request, response); + errorPageService.serveError(request, response); } return ""; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java index 23ac2843..7f228d7f 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java @@ -3,7 +3,6 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; import nu.marginalia.assistant.client.AssistantClient; import nu.marginalia.assistant.client.model.SimilarDomain; -import nu.marginalia.client.Context; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.feedlot.model.FeedItems; import nu.marginalia.model.EdgeDomain; @@ -24,6 +23,10 @@ import java.io.IOException; import java.sql.SQLException; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; public class SearchSiteInfoService { private static final Logger logger = LoggerFactory.getLogger(SearchSiteInfoService.class); @@ -64,14 +67,12 @@ public class SearchSiteInfoService { return null; } - var ctx = Context.fromRequest(request); - var model = switch (view) { - case "links" -> listLinks(ctx, domainName); - case "docs" -> listDocs(ctx, domainName); - case "info" -> listInfo(ctx, domainName); - case "report" -> reportSite(ctx, domainName); - default -> listInfo(ctx, domainName); + case "links" -> listLinks(domainName); + case "docs" -> listDocs(domainName); + case "info" -> listInfo(domainName); + case "report" -> reportSite(domainName); + default -> listInfo(domainName); }; return renderer.render(model); @@ -105,7 +106,7 @@ public class SearchSiteInfoService { return renderer.render(model); } - private Object reportSite(Context ctx, String domainName) throws SQLException { + private Object reportSite(String domainName) throws SQLException { int domainId = domainQueries.getDomainId(new EdgeDomain(domainName)); var existingComplaints = flagSiteService.getExistingComplaints(domainId); @@ -117,40 +118,37 @@ public class SearchSiteInfoService { } - private Backlinks listLinks(Context ctx, String domainName) { + private Backlinks listLinks(String domainName) { return new Backlinks(domainName, domainQueries.tryGetDomainId(new EdgeDomain(domainName)).orElse(-1), - searchOperator.doBacklinkSearch(ctx, domainName)); + searchOperator.doBacklinkSearch(domainName)); } - private SiteInfoWithContext listInfo(Context ctx, String domainName) { + private SiteInfoWithContext listInfo(String domainName) { final int domainId = domainQueries.tryGetDomainId(new EdgeDomain(domainName)).orElse(-1); - final DomainInformation domainInfo; - final List similarSet; - final List linkingDomains; - String url = "https://" + domainName + "/";; + final Future domainInfoFuture; + final Future> similarSetFuture; + final Future> linkingDomainsFuture; + + String url = "https://" + domainName + "/"; boolean hasScreenshot = screenshotService.hasScreenshot(domainId); var feedItemsFuture = feedlotClient.getFeedItems(domainName); if (domainId < 0 || !assistantClient.isAccepting()) { - domainInfo = createDummySiteInfo(domainName); - similarSet = List.of(); - linkingDomains = List.of(); + domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); + similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); + linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); } else { - domainInfo = assistantClient.domainInformation(ctx, domainId).blockingFirst(); - similarSet = assistantClient - .similarDomains(ctx, domainId, 25) - .blockingFirst(); - linkingDomains = assistantClient - .linkedDomains(ctx, domainId, 25) - .blockingFirst(); + domainInfoFuture = assistantClient.domainInformation(domainId); + similarSetFuture = assistantClient.similarDomains(domainId, 25); + linkingDomainsFuture = assistantClient.linkedDomains(domainId, 25); } - List sampleResults = searchOperator.doSiteSearch(ctx, domainName, 5); + List sampleResults = searchOperator.doSiteSearch(domainName, 5); if (!sampleResults.isEmpty()) { url = sampleResults.getFirst().url.withPathAndParam("/", null).toString(); } @@ -166,26 +164,35 @@ public class SearchSiteInfoService { domainId, url, hasScreenshot, - domainInfo, - similarSet, - linkingDomains, + waitForFuture(domainInfoFuture, () -> createDummySiteInfo(domainName)), + waitForFuture(similarSetFuture, List::of), + waitForFuture(linkingDomainsFuture, List::of), feedItems, sampleResults ); } - private DomainInformation createDummySiteInfo(String domainName) { - return DomainInformation.builder() - .domain(new EdgeDomain(domainName)) - .suggestForCrawling(true) - .unknownDomain(true) - .build(); + private T waitForFuture(Future future, Supplier fallback) { + try { + return future.get(50, TimeUnit.MILLISECONDS); + } catch (Exception e) { + logger.info("Failed to get domain data: {}", e.getMessage()); + return fallback.get(); + } } - private Docs listDocs(Context ctx, String domainName) { + private DomainInformation createDummySiteInfo(String domainName) { + return DomainInformation.builder() + .domain(new EdgeDomain(domainName)) + .suggestForCrawling(true) + .unknownDomain(true) + .build(); + } + + private Docs listDocs(String domainName) { return new Docs(domainName, domainQueries.tryGetDomainId(new EdgeDomain(domainName)).orElse(-1), - searchOperator.doSiteSearch(ctx, domainName, 100)); + searchOperator.doSiteSearch(domainName, 100)); } public record Docs(Map view, diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java index 110ece21..d581c674 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java @@ -1,9 +1,6 @@ package nu.marginalia.search.svc; -import io.reactivex.rxjava3.schedulers.Schedulers; import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.client.exception.RemoteException; -import nu.marginalia.client.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,7 +8,10 @@ import javax.annotation.CheckForNull; import com.google.inject.Inject; import com.google.inject.Singleton; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -28,10 +28,10 @@ public class SearchUnitConversionService { this.assistantClient = assistantClient; } - public Optional tryConversion(Context context, String query) { + public Optional tryConversion(String query) { var matcher = conversionPattern.matcher(query); if (!matcher.matches()) - return Optional.empty(); + return Optional.empty(); String value = matcher.group(1); String from = matcher.group(3); @@ -40,24 +40,22 @@ public class SearchUnitConversionService { logger.info("{} -> '{}' '{}' '{}'", query, value, from, to); try { - return Optional.of(assistantClient.unitConversion(context, value, from, to).blockingFirst()); - } - catch (RemoteException ex) { - return Optional.empty(); + var resultFuture = assistantClient.unitConversion(value, from, to); + return Optional.of( + resultFuture.get(100, TimeUnit.MILLISECONDS) + ); + } catch (ExecutionException e) { + logger.error("Error in unit conversion", e); + } catch (InterruptedException e) { + logger.error("Interrupted while waiting for unit conversion", e); + } catch (TimeoutException e) { + // Ignore } + return Optional.empty(); } - public boolean isNumeric(String str) { - try { - Double.parseDouble(str); - return true; - } - catch (NumberFormatException ex) { - return false; - } - } - public @CheckForNull Future tryEval(Context context, String query) { + public @CheckForNull Future tryEval(String query) { if (!evalPredicate.test(query)) { return null; } @@ -70,11 +68,6 @@ public class SearchUnitConversionService { logger.info("eval({})", expr); - try { - return assistantClient.evalMath(context, expr).subscribeOn(Schedulers.io()).toFuture(); - } - catch (RemoteException ex) { - return null; - } + return assistantClient.evalMath(expr); } } diff --git a/code/services-application/search-service/src/test/java/nu/marginalia/search/command/commands/BangCommandTest.java b/code/services-application/search-service/src/test/java/nu/marginalia/search/command/commands/BangCommandTest.java index 3394eb8e..3912a51c 100644 --- a/code/services-application/search-service/src/test/java/nu/marginalia/search/command/commands/BangCommandTest.java +++ b/code/services-application/search-service/src/test/java/nu/marginalia/search/command/commands/BangCommandTest.java @@ -1,6 +1,5 @@ package nu.marginalia.search.command.commands; -import nu.marginalia.client.Context; import nu.marginalia.search.command.SearchParameters; import nu.marginalia.search.exceptions.RedirectException; import org.junit.jupiter.api.Assertions; @@ -14,8 +13,7 @@ class BangCommandTest { @Test public void testG() { try { - bangCommand.process(Context.internal(), - null, + bangCommand.process(null, new SearchParameters(" !g test", null, null, null, null, null) ); diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index 950dc359..b9f7625a 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -30,7 +30,6 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:features-search:screenshots') @@ -41,9 +40,9 @@ dependencies { implementation libs.bundles.slf4j implementation libs.prometheus + implementation libs.bundles.grpc implementation libs.notnull implementation libs.guice - implementation libs.rxjava implementation libs.spark implementation libs.opencsv implementation libs.trove diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java new file mode 100644 index 00000000..7cf82cd1 --- /dev/null +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java @@ -0,0 +1,131 @@ +package nu.marginalia.assistant; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.assistant.api.*; +import nu.marginalia.assistant.dict.DictionaryService; +import nu.marginalia.assistant.dict.SpellChecker; +import nu.marginalia.assistant.domains.DomainInformationService; +import nu.marginalia.assistant.domains.SimilarDomainsService; +import nu.marginalia.assistant.eval.MathParser; +import nu.marginalia.assistant.eval.Units; + +public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase { + + private final DictionaryService dictionaryService; + private final SpellChecker spellChecker; + private final Units units; + private final MathParser mathParser; + private final DomainInformationService domainInformationService; + private final SimilarDomainsService similarDomainsService; + @Inject + public AssistantGrpcService(DictionaryService dictionaryService, + SpellChecker spellChecker, Units units, MathParser mathParser, DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService) + { + + this.dictionaryService = dictionaryService; + this.spellChecker = spellChecker; + this.units = units; + this.mathParser = mathParser; + this.domainInformationService = domainInformationService; + this.similarDomainsService = similarDomainsService; + } + + @Override + public void dictionaryLookup(RpcDictionaryLookupRequest request, + StreamObserver responseObserver) + { + var definition = dictionaryService.define(request.getWord()); + + var responseBuilder = RpcDictionaryLookupResponse + .newBuilder() + .setWord(request.getWord()); + + for (var def : definition.entries) { + responseBuilder.addEntries( + RpcDictionaryEntry.newBuilder() + .setWord(def.word) + .setDefinition(def.definition) + .setType(def.type) + ); + } + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void spellCheck(RpcSpellCheckRequest request, + StreamObserver responseObserver) + { + var result = spellChecker.correct(request.getText()); + var response = RpcSpellCheckResponse.newBuilder() + .addAllSuggestions(result) + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + + @Override + public void unitConversion(RpcUnitConversionRequest request, StreamObserver responseObserver) { + var res = units.convert(request.getUnit(), + request.getFrom(), + request.getTo()); + + res.ifPresent(s -> { + var response = RpcUnitConversionResponse.newBuilder() + .setResult(s) + .build(); + responseObserver.onNext(response); + + }); + + responseObserver.onCompleted(); + } + + @Override + public void evalMath(RpcEvalMathRequest request, StreamObserver responseObserver) { + var ret = mathParser.eval(request.getExpression()); + + responseObserver.onNext(RpcEvalMathResponse.newBuilder() + .setResult(Double.toString(ret)) + .build()); + + responseObserver.onCompleted(); + } + + @Override + public void getDomainInfo(RpcDomainId request, StreamObserver responseObserver) { + var ret = domainInformationService.domainInfo(request.getDomainId()); + + ret.ifPresent(responseObserver::onNext); + + responseObserver.onCompleted(); + } + + @Override + public void getSimilarDomains(RpcDomainLinksRequest request, + StreamObserver responseObserver) { + var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount()); + + var responseBuilder = RpcSimilarDomains + .newBuilder() + .addAllDomains(ret); + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver responseObserver) { + var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount()); + + var responseBuilder = RpcSimilarDomains + .newBuilder() + .addAllDomains(ret); + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } +} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java index 24f94fa8..db4b7ff1 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -23,7 +22,7 @@ public class AssistantMain extends MainClass { Injector injector = Guice.createInjector( new AssistantModule(), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Assistant), + new ServiceConfigurationModule(ServiceId.Assistant), new DatabaseModule(false) ); diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java index 592e6308..51240c7a 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java @@ -3,14 +3,9 @@ package nu.marginalia.assistant; import com.google.gson.Gson; import com.google.inject.Inject; import lombok.SneakyThrows; -import nu.marginalia.assistant.domains.DomainInformationService; -import nu.marginalia.assistant.domains.SimilarDomainsService; -import nu.marginalia.assistant.eval.Units; import nu.marginalia.assistant.suggest.Suggestions; -import nu.marginalia.assistant.eval.MathParser; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.screenshot.ScreenshotService; -import nu.marginalia.assistant.dict.DictionaryService; import nu.marginalia.service.server.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,99 +13,33 @@ import spark.Request; import spark.Response; import spark.Spark; -import java.util.ArrayList; -import java.util.Objects; +import java.util.List; public class AssistantService extends Service { private final Logger logger = LoggerFactory.getLogger(getClass()); private final Gson gson = GsonFactory.get(); - private final Units units; - private final MathParser mathParser; - private final SimilarDomainsService similarDomainsService; - private final DomainInformationService domainInformationService; private final Suggestions suggestions; @SneakyThrows @Inject public AssistantService(BaseServiceParams params, - DictionaryService dictionaryService, - MathParser mathParser, - Units units, ScreenshotService screenshotService, - SimilarDomainsService similarDomainsService, - DomainInformationService domainInformationService, + AssistantGrpcService assistantGrpcService, Suggestions suggestions) { - super(params); + super(params, List.of(assistantGrpcService)); - this.mathParser = mathParser; - this.units = units; - this.similarDomainsService = similarDomainsService; - this.domainInformationService = domainInformationService; this.suggestions = suggestions; Spark.staticFiles.expireTime(600); Spark.get("/public/screenshot/:id", screenshotService::serveScreenshotRequest); Spark.get("/screenshot/:id", screenshotService::serveScreenshotRequest); - - Spark.get("/dictionary/:word", (req, rsp) -> dictionaryService.define(req.params("word")), this::convertToJson); - Spark.get("/spell-check/:term", (req, rsp) -> dictionaryService.spellCheck(req.params("term").toLowerCase()), this::convertToJson); - Spark.get("/unit-conversion", (req, rsp) -> unitConversion( - rsp, - req.queryParams("value"), - req.queryParams("from"), - req.queryParams("to") - - )); - Spark.get("/eval-expression", (req, rsp) -> evalExpression( - rsp, - req.queryParams("value") - )); - Spark.get("/domain/:id/similar", this::getSimilarDomains, this::convertToJson); - Spark.get("/domain/:id/linking", this::getLinkingDomains, this::convertToJson); - Spark.get("/domain/:id/info", this::getDomainInformation, this::convertToJson); Spark.get("/public/suggest/", this::getSuggestions, this::convertToJson); Spark.awaitInitialization(); } - private Object getSimilarDomains(Request request, Response response) { - int domainId = Integer.parseInt(request.params("id")); - int count = Integer.parseInt(Objects.requireNonNullElse(request.queryParams("count"), "25")); - - response.type("application/json"); - - if (!similarDomainsService.isReady()) { - return new ArrayList<>(); - } - - return similarDomainsService.getSimilarDomains(domainId, count); - } - - private Object getLinkingDomains(Request request, Response response) { - int domainId = Integer.parseInt(request.params("id")); - int count = Integer.parseInt(Objects.requireNonNullElse(request.queryParams("count"), "25")); - - response.type("application/json"); - if (!similarDomainsService.isReady()) { - return new ArrayList<>(); - } - return similarDomainsService.getLinkingDomains(domainId, count); - } - - private Object getDomainInformation(Request request, Response response) { - int domainId = Integer.parseInt(request.params("id")); - - response.type("application/json"); - - var maybeDomainInfo = domainInformationService.domainInfo(domainId); - if (maybeDomainInfo.isEmpty()) { - Spark.halt(404); - } - return maybeDomainInfo.get(); - } - private Object getSuggestions(Request request, Response response) { response.type("application/json"); var param = request.queryParams("partial"); @@ -121,32 +50,6 @@ public class AssistantService extends Service { return suggestions.getSuggestions(10, param); } - private Object evalExpression(Response rsp, String value) { - try { - var val = mathParser.evalFormatted(value); - if (val.isBlank()) { - Spark.halt(400); - return null; - } - return val; - } - catch (Exception ex) { - Spark.halt(400); - return null; - } - } - - private Object unitConversion(Response rsp, String value, String fromUnit, String toUnit) { - var result = units.convert(value, fromUnit, toUnit); - if (result.isPresent()) { - return result.get(); - } - { - Spark.halt(400); - return null; - } - } - private String convertToJson(Object o) { return gson.toJson(o); } diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java index b99c3abf..f2590e7e 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java @@ -1,10 +1,10 @@ package nu.marginalia.assistant.domains; import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.assistant.api.RpcDomainInfoResponse; import nu.marginalia.geoip.GeoIpDictionary; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; -import nu.marginalia.assistant.client.model.DomainInformation; import nu.marginalia.query.client.QueryClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ public class DomainInformationService { } - public Optional domainInfo(int domainId) { + public Optional domainInfo(int domainId) { Optional domain = dbDomainQueries.getDomain(domainId); if (domain.isEmpty()) { @@ -46,9 +46,9 @@ public class DomainInformationService { } - var builder = DomainInformation.builder(); + var builder = RpcDomainInfoResponse.newBuilder(); try (var connection = dataSource.getConnection(); - var stmt = connection.createStatement(); + var stmt = connection.createStatement() ) { boolean inCrawlQueue; int outboundLinks = 0; @@ -63,18 +63,18 @@ public class DomainInformationService { if (rs.next()) { String ip = rs.getString("IP"); - builder.ip(ip); + builder.setIp(ip); geoIpDictionary.getAsnInfo(ip).ifPresent(asnInfo -> { - builder.asn(asnInfo.asn()); - builder.asnOrg(asnInfo.org()); - builder.asnCountry(asnInfo.country()); + builder.setAsn(asnInfo.asn()); + builder.setAsnOrg(asnInfo.org()); + builder.setAsnCountry(asnInfo.country()); }); - builder.ipCountry(geoIpDictionary.getCountry(ip)); + builder.setIpCountry(geoIpDictionary.getCountry(ip)); - builder.nodeAffinity(rs.getInt("NODE_AFFINITY")); - builder.domain(new EdgeDomain(rs.getString("DOMAIN_NAME"))); - builder.state(rs.getString("STATE")); - builder.ranking(Math.round(100.0*(1.0-rs.getDouble("RANK")))); + builder.setNodeAffinity(rs.getInt("NODE_AFFINITY")); + builder.setDomain(rs.getString("DOMAIN_NAME")); + builder.setState(rs.getString("STATE")); + builder.setRanking(Math.round(100.0*(1.0-rs.getDouble("RANK")))); } rs = stmt.executeQuery(STR.""" SELECT 1 FROM CRAWL_QUEUE @@ -82,10 +82,10 @@ public class DomainInformationService { WHERE EC_DOMAIN.ID=\{domainId} """); inCrawlQueue = rs.next(); - builder.inCrawlQueue(inCrawlQueue); + builder.setInCrawlQueue(inCrawlQueue); - builder.incomingLinks(queryClient.countLinksToDomain(domainId)); - builder.outboundLinks(queryClient.countLinksFromDomain(domainId)); + builder.setIncomingLinks(queryClient.countLinksToDomain(domainId)); + builder.setOutboundLinks(queryClient.countLinksFromDomain(domainId)); rs = stmt.executeQuery(STR.""" SELECT KNOWN_URLS, GOOD_URLS, VISITED_URLS FROM DOMAIN_METADATA WHERE ID=\{domainId} @@ -93,12 +93,12 @@ public class DomainInformationService { if (rs.next()) { pagesVisited = rs.getInt("VISITED_URLS"); - builder.pagesKnown(rs.getInt("KNOWN_URLS")); - builder.pagesIndexed(rs.getInt("GOOD_URLS")); - builder.pagesFetched(rs.getInt("VISITED_URLS")); + builder.setPagesKnown(rs.getInt("KNOWN_URLS")); + builder.setPagesIndexed(rs.getInt("GOOD_URLS")); + builder.setPagesFetched(rs.getInt("VISITED_URLS")); } - builder.suggestForCrawling((pagesVisited == 0 && outboundLinks == 0 && !inCrawlQueue)); + builder.setSuggestForCrawling((pagesVisited == 0 && outboundLinks == 0 && !inCrawlQueue)); return Optional.of(builder.build()); } diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java index ddcc2e98..4d00864f 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java @@ -8,6 +8,7 @@ import gnu.trove.map.hash.TIntDoubleHashMap; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; +import nu.marginalia.assistant.api.RpcSimilarDomain; import nu.marginalia.assistant.client.model.SimilarDomain; import nu.marginalia.model.EdgeDomain; import nu.marginalia.query.client.QueryClient; @@ -168,7 +169,7 @@ public class SimilarDomainsService { } - public List getSimilarDomains(int domainId, int count) { + public List getSimilarDomains(int domainId, int count) { int domainIdx = domainIdToIdx.get(domainId); TIntList allIdsList = domainNeighbors[domainIdx]; @@ -204,23 +205,30 @@ public class SimilarDomainsService { .limit(count) .toArray(); - List domains = new ArrayList<>(); + List domains = new ArrayList<>(); + for (int idx : resultIds) { int id = domainIdxToId[idx]; - domains.add(new SimilarDomain( - new EdgeDomain(domainNames[idx]).toRootUrl(), - id, - getRelatedness(domainId, id), - domainRanks[idx], - indexedDomains.get(idx), - activeDomains.get(idx), - screenshotDomains.get(idx), - SimilarDomain.LinkType.find( - linkingIdsStoD.contains(idx), - linkingIdsDtoS.contains(idx) - ) - )); + if (domainNames[idx].length() > 32) + continue; + + var linkType = SimilarDomain.LinkType.find( + linkingIdsStoD.contains(idx), + linkingIdsDtoS.contains(idx) + ); + + domains.add(RpcSimilarDomain.newBuilder() + .setDomainId(id) + .setUrl(new EdgeDomain(domainNames[idx]).toRootUrl().toString()) + .setRelatedness(getRelatedness(domainId, id)) + .setRank(domainRanks[idx]) + .setIndexed(indexedDomains.get(idx)) + .setActive(activeDomains.get(idx)) + .setScreenshot(screenshotDomains.get(idx)) + .setLinkType(RpcSimilarDomain.LINK_TYPE.valueOf(linkType.name())) + .build()); + } domains.removeIf(this::shouldRemove); @@ -228,21 +236,18 @@ public class SimilarDomainsService { return domains; } - private boolean shouldRemove(SimilarDomain domainResult) { - if (domainResult.url().domain.toString().length() > 32) - return true; - + private boolean shouldRemove(RpcSimilarDomain domainResult) { // Remove domains that have a relatively high likelihood of being dead links // or not very interesting - if (!(domainResult.indexed() && domainResult.active()) - && domainResult.relatedness() <= 50) + if (!(domainResult.getIndexed() && domainResult.getActive()) + && domainResult.getRelatedness() <= 50) { return true; } // Remove domains that are not very similar if there is no mutual link - if (domainResult.linkType() == SimilarDomain.LinkType.NONE - && domainResult.relatedness() <= 25) + if (domainResult.getLinkType() == RpcSimilarDomain.LINK_TYPE.NONE + && domainResult.getRelatedness() <= 25) return true; return false; @@ -268,7 +273,7 @@ public class SimilarDomainsService { return items; } - public List getLinkingDomains(int domainId, int count) { + public List getLinkingDomains(int domainId, int count) { int domainIdx = domainIdToIdx.get(domainId); TIntSet linkingIdsDtoS = getLinkingIdsDToS(domainIdx); @@ -286,7 +291,7 @@ public class SimilarDomainsService { double[] ranksArray = new double[idsArray.length]; for (int i = 0; i < idxArray.length; i++) { - ranksArray[i] = domainRanks[idxArray[i]]; + ranksArray[i] = this.domainRanks[idxArray[i]]; } double[] relatednessArray = new double[idsArray.length]; for (int i = 0; i < idsArray.length; i++) { @@ -316,23 +321,29 @@ public class SimilarDomainsService { .limit(count) .toArray(); - List domains = new ArrayList<>(); + List domains = new ArrayList<>(); for (int id : resultIds) { int idx = domainIdToIdx.get(id); - domains.add(new SimilarDomain( - new EdgeDomain(domainNames[idx]).toRootUrl(), - id, - getRelatedness(domainId, id), - domainRanks[idx], - indexedDomains.get(idx), - activeDomains.get(idx), - screenshotDomains.get(idx), - SimilarDomain.LinkType.find( - linkingIdsStoD.contains(idx), - linkingIdsDtoS.contains(idx) - ) - )); + if (domainNames[idx].length() > 32) + continue; + + var linkType = SimilarDomain.LinkType.find( + linkingIdsStoD.contains(idx), + linkingIdsDtoS.contains(idx) + ); + + domains.add(RpcSimilarDomain.newBuilder() + .setDomainId(id) + .setUrl(new EdgeDomain(domainNames[idx]).toRootUrl().toString()) + .setRelatedness(getRelatedness(domainId, id)) + .setRank(domainRanks[idx]) + .setIndexed(indexedDomains.get(idx)) + .setActive(activeDomains.get(idx)) + .setScreenshot(screenshotDomains.get(idx)) + .setLinkType(RpcSimilarDomain.LINK_TYPE.valueOf(linkType.name())) + .build()); + } domains.removeIf(this::shouldRemove); diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 4b98d378..f55610b7 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -31,7 +31,6 @@ dependencies { implementation project(':code:common:renderer') implementation project(':code:libraries:message-queue') implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') implementation project(':code:api:index-api') implementation project(':code:api:query-api') implementation project(':code:api:executor-api') @@ -48,7 +47,6 @@ dependencies { implementation libs.notnull implementation libs.guice implementation libs.zstd - implementation libs.rxjava implementation libs.handlebars implementation libs.trove diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java index a37cb9c4..48531a91 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -22,7 +21,7 @@ public class ControlMain extends MainClass { Injector injector = Guice.createInjector( new DatabaseModule(true), new ControlProcessModule(), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Control)); + new ServiceConfigurationModule(ServiceId.Control)); injector.getInstance(ControlMain.class); injector.getInstance(Initialization.class).setReady(); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java index 76c773b8..8c7d6c40 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java @@ -1,12 +1,8 @@ package nu.marginalia.control; import com.google.inject.AbstractModule; -import com.google.inject.Module; -import com.google.inject.name.Names; import nu.marginalia.renderer.config.HandlebarsConfigurator; -import java.nio.file.Path; - public class ControlProcessModule extends AbstractModule { @Override protected void configure() { diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index 91e5d3d5..264174a5 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -2,7 +2,7 @@ package nu.marginalia.control; import com.google.gson.Gson; import com.google.inject.Inject; -import nu.marginalia.client.ServiceMonitors; +import nu.marginalia.service.ServiceMonitors; import nu.marginalia.control.actor.ControlActorService; import nu.marginalia.control.app.svc.*; import nu.marginalia.control.node.svc.ControlNodeActionsService; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java index e0268901..b6e857dc 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java @@ -9,7 +9,6 @@ import nu.marginalia.actor.state.ActorStep; import nu.marginalia.actor.state.Resume; import nu.marginalia.control.actor.PrecessionNodes; import nu.marginalia.executor.client.ExecutorRemoteActorFactory; -import nu.marginalia.nodecfg.NodeConfigurationService; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageType; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java index 6ff198da..69537a37 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java @@ -1,9 +1,7 @@ package nu.marginalia.control.app.svc; import com.google.inject.Inject; -import nu.marginalia.client.Context; import nu.marginalia.control.ControlRendererFactory; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.EdgeUrl; import nu.marginalia.nodecfg.NodeConfigurationService; @@ -49,17 +47,17 @@ public class SearchToBanService { if (Objects.equals(request.requestMethod(), "POST")) { executeBlacklisting(request); - return findResults(Context.fromRequest(request), request.queryParams("query")); + return findResults(request.queryParams("query")); } - return findResults(Context.fromRequest(request), request.queryParams("q")); + return findResults(request.queryParams("q")); } - private Object findResults(Context ctx, String q) { + private Object findResults(String q) { if (q == null || q.isBlank()) { return Map.of(); } else { - return executeQuery(ctx, q); + return executeQuery(q); } } @@ -76,8 +74,8 @@ public class SearchToBanService { } } - private Object executeQuery(Context ctx, String query) { - return queryClient.search(ctx, new QueryParams( + private Object executeQuery(String query) { + return queryClient.search(new QueryParams( query, new QueryLimits(2, 200, 250, 8192), "NONE" )); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java index 7de00d0a..03a7baab 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java @@ -3,7 +3,6 @@ package nu.marginalia.control.node.svc; import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; -import nu.marginalia.client.Context; import nu.marginalia.control.Redirects; import nu.marginalia.executor.client.ExecutorClient; import nu.marginalia.storage.FileStorageService; @@ -16,8 +15,6 @@ import spark.Response; import spark.Spark; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.sql.SQLException; @Singleton @@ -75,7 +72,7 @@ public class ControlFileStorageService { else response.type("application/octet-stream"); - executorClient.transferFile(Context.fromRequest(request), nodeId, fileStorageId, path, response.raw().getOutputStream()); + executorClient.transferFile(nodeId, fileStorageId, path, response.raw().getOutputStream()); return ""; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java index e4859eb3..95a0ec40 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java @@ -3,8 +3,7 @@ package nu.marginalia.control.node.svc; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.client.Context; -import nu.marginalia.client.ServiceMonitors; +import nu.marginalia.service.ServiceMonitors; import nu.marginalia.control.ControlRendererFactory; import nu.marginalia.control.RedirectControl; import nu.marginalia.control.Redirects; @@ -211,7 +210,7 @@ public class ControlNodeService { private Object nodeStorageDetailsModel(Request request, Response response) throws SQLException { int nodeId = Integer.parseInt(request.params("id")); - var storage = getFileStorageWithRelatedEntries(Context.fromRequest(request), nodeId, FileStorageId.parse(request.queryParams("fid"))); + var storage = getFileStorageWithRelatedEntries(nodeId, FileStorageId.parse(request.queryParams("fid"))); String view = switch(storage.type()) { case BACKUP -> "backup"; @@ -390,7 +389,6 @@ public class ControlNodeService { public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries( - Context context, int node, FileStorageId fileId ) throws SQLException { diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java index 48bf2a07..1de52852 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java @@ -1,6 +1,5 @@ package nu.marginalia.control.sys.model; -import java.sql.Timestamp; import java.time.LocalDate; public record EventLogEntry( diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java index bea285a1..1705a994 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java @@ -1,7 +1,5 @@ package nu.marginalia.control.sys.model; -import java.time.LocalDate; - public record MessageQueueEntry ( long id, long relatedId, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java index 491f1b62..d1636623 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java @@ -2,7 +2,6 @@ package nu.marginalia.control.sys.svc; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.control.ControlRendererFactory; import nu.marginalia.db.DomainTypes; import spark.Request; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java index 446b1c32..7f6a58ea 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java @@ -11,7 +11,6 @@ import spark.Request; import spark.Response; import java.sql.SQLException; -import java.sql.Timestamp; import java.util.*; @Singleton diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java index 6fb9931e..875c4bd1 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java @@ -6,7 +6,6 @@ import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.control.sys.model.ProcessHeartbeat; import nu.marginalia.control.sys.model.ServiceHeartbeat; import nu.marginalia.control.sys.model.TaskHeartbeat; -import nu.marginalia.service.control.ServiceEventLog; import java.sql.SQLException; import java.util.ArrayList; diff --git a/code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java b/code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java index 0d6c52ab..8593d20f 100644 --- a/code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java +++ b/code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java @@ -4,22 +4,17 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.control.sys.model.TaskHeartbeat; import nu.marginalia.control.sys.svc.HeartbeatService; -import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.test.TestMigrationLoader; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; -import org.mockito.Mockito; import org.testcontainers.containers.MariaDBContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; import java.sql.SQLException; -import java.util.List; -import java.util.Objects; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index f2d9678d..5ee7ef74 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -29,7 +29,6 @@ dependencies { implementation project(':code:common:linkdb') implementation project(':code:common:service') - implementation project(':code:common:service-client') implementation project(':code:common:service-discovery') implementation project(':third-party:commons-codec') @@ -59,7 +58,6 @@ dependencies { implementation libs.guice implementation libs.trove implementation libs.protobuf - implementation libs.rxjava implementation libs.zstd implementation libs.jsoup implementation libs.commons.io diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java index 7a187f5f..f7d5cf1f 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java @@ -34,16 +34,23 @@ public class ConvertActor extends RecordActorPrototype { private final MqOutbox mqConverterOutbox; private final FileStorageService storageService; - public record Convert(FileStorageId fid) implements ActorStep {}; - public record ConvertEncyclopedia(String source, String baseUrl) implements ActorStep {}; - public record PredigestEncyclopedia(String source, String dest, String baseUrl) implements ActorStep {}; - public record ConvertDirtree(String source) implements ActorStep {}; - public record ConvertWarc(String source) implements ActorStep {}; - public record ConvertReddit(String source) implements ActorStep {}; - public record ConvertStackexchange(String source) implements ActorStep {}; + public record Convert(FileStorageId fid) implements ActorStep {} + + public record ConvertEncyclopedia(String source, String baseUrl) implements ActorStep {} + + public record PredigestEncyclopedia(String source, String dest, String baseUrl) implements ActorStep {} + + public record ConvertDirtree(String source) implements ActorStep {} + + public record ConvertWarc(String source) implements ActorStep {} + + public record ConvertReddit(String source) implements ActorStep {} + + public record ConvertStackexchange(String source) implements ActorStep {} + @Resume(behavior = ActorResumeBehavior.RETRY) public record ConvertWait(FileStorageId destFid, - long msgId) implements ActorStep {}; + long msgId) implements ActorStep {} @Override public ActorStep transition(ActorStep self) throws Exception { diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java index 62aa8929..698c8c27 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java @@ -58,9 +58,9 @@ public class ConvertAndLoadActor extends RecordActorPrototype { public List processedStorageId = null; public long converterMsgId = 0L; public long loaderMsgId = 0L; - }; + } - public record Initial(FileStorageId fid) implements ActorStep {}; + public record Initial(FileStorageId fid) implements ActorStep {} @Resume(behavior = ActorResumeBehavior.RETRY) public record Convert(FileStorageId crawlId, FileStorageId processedId, long msgId) implements ActorStep { @@ -69,7 +69,8 @@ public class ConvertAndLoadActor extends RecordActorPrototype { @Resume(behavior = ActorResumeBehavior.RETRY) public record Load(List processedId, long msgId) implements ActorStep { public Load(List processedId) { this(processedId, -1); } - }; + } + @Resume(behavior = ActorResumeBehavior.RETRY) public record Backup(List processedIds) implements ActorStep { } @Resume(behavior = ActorResumeBehavior.RETRY) diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java index 042a3ec7..537a0969 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java @@ -54,7 +54,7 @@ public class ExportDataActor extends RecordActorPrototype { try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))); var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement("SELECT URL_DOMAIN FROM EC_DOMAIN_BLACKLIST"); + var stmt = conn.prepareStatement("SELECT URL_DOMAIN FROM EC_DOMAIN_BLACKLIST") ) { stmt.setFetchSize(1000); @@ -82,7 +82,7 @@ public class ExportDataActor extends RecordActorPrototype { try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))); var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement("SELECT DOMAIN_NAME, ID, INDEXED, STATE FROM EC_DOMAIN"); + var stmt = conn.prepareStatement("SELECT DOMAIN_NAME, ID, INDEXED, STATE FROM EC_DOMAIN") ) { stmt.setFetchSize(1000); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java index 9d9d7d2b..e9c8478a 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.DatabaseModule; import nu.marginalia.service.module.ServiceConfigurationModule; @@ -25,7 +24,7 @@ public class ExecutorMain extends MainClass { Injector injector = Guice.createInjector( new ExecutorModule(), new DatabaseModule(false), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Executor) + new ServiceConfigurationModule(ServiceId.Executor) ); injector.getInstance(NodeStatusWatcher.class); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java index ac567467..0091ac9f 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java @@ -2,7 +2,6 @@ package nu.marginalia.executor; import com.google.gson.Gson; import com.google.inject.Inject; -import io.grpc.ServerBuilder; import nu.marginalia.actor.ExecutorActor; import nu.marginalia.actor.ExecutorActorControlService; import nu.marginalia.executor.svc.TransferService; @@ -13,8 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Spark; -import java.io.IOException; -import java.sql.SQLException; +import java.util.List; // Weird name for this one to not have clashes with java.util.concurrent.ExecutorService public class ExecutorSvc extends Service { @@ -26,26 +24,20 @@ public class ExecutorSvc extends Service { private static final Logger logger = LoggerFactory.getLogger(ExecutorSvc.class); @Inject + public ExecutorSvc(BaseServiceParams params, ExecutorActorControlService actorControlService, ExecutorGrpcService executorGrpcService, Gson gson, - TransferService transferService) throws IOException { - super(params); + TransferService transferService) + { + super(params, List.of(executorGrpcService)); this.params = params; this.gson = gson; this.actorControlService = actorControlService; this.transferService = transferService; - var grpcServer = ServerBuilder.forPort(params.configuration.port() + 1) - .addService(executorGrpcService) - .build(); - grpcServer.start(); - Spark.get("/transfer/file/:fid", transferService::transferFile); - - Spark.get("/transfer/spec", transferService::getTransferSpec, gson::toJson); - Spark.post("/transfer/yield", transferService::yieldDomain); } @MqRequest(endpoint="FIRST-BOOT") @@ -60,27 +52,4 @@ public class ExecutorSvc extends Service { actorControlService.start(ExecutorActor.PROC_LOADER_SPAWNER); } - @MqRequest(endpoint="TRANSFER-DOMAINS") - public String transferDomains(String message) throws Exception { - - var spec = gson.fromJson(message, TransferService.TransferReq.class); - - synchronized (this) { - transferService.transferMqEndpoint(spec.sourceNode(), spec.count()); - } - - return "OK"; - } - - - @MqRequest(endpoint="PRUNE-CRAWL-DATA") - public String pruneCrawlData(String message) throws SQLException, IOException { - - synchronized (this) { // would not be great if this ran in parallel with itself - transferService.pruneCrawlDataMqEndpoint(); - } - - return "OK"; - } - } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java index 377bd354..0fdd6c65 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java @@ -4,20 +4,14 @@ import com.google.gson.Gson; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.client.Context; import nu.marginalia.executor.api.RpcFileStorageContent; import nu.marginalia.executor.api.RpcFileStorageEntry; import nu.marginalia.executor.api.RpcFileStorageId; import nu.marginalia.executor.client.ExecutorClient; -import nu.marginalia.executor.model.transfer.TransferItem; -import nu.marginalia.executor.model.transfer.TransferSpec; -import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mq.persistence.MqPersistence; -import nu.marginalia.process.log.WorkLog; import nu.marginalia.service.module.ServiceConfiguration; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; -import nu.marginalia.storage.model.FileStorageType; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,12 +21,8 @@ import spark.Response; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Comparator; -import java.util.List; -import java.util.UUID; public class TransferService { private final Gson gson; @@ -102,157 +92,6 @@ public class TransferService { .build(); } - public TransferSpec getTransferSpec(Request request, Response response) throws SQLException { - List fileStorageIds = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.CRAWL_DATA); - if (fileStorageIds.isEmpty()) { - logger.warn("No ACTIVE crawl data"); - return new TransferSpec(); - } - int count = Integer.parseInt(request.queryParams("count")); - - logger.info("Preparing a transfer of {} domains", count); - - List items = new ArrayList<>(); - var storage = fileStorageService.getStorage(fileStorageIds.get(0)); - - try (var conn = dataSource.getConnection(); - var query = conn.prepareStatement("SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=? AND NODE_AFFINITY=" + nodeId) - ) { - for (var item : WorkLog.iterable(storage.asPath().resolve("crawler.log"))) { - if (items.size() >= count) - break; - - if (!Files.isRegularFile(storage.asPath().resolve(item.relPath()))) { - logger.info("Ignoring absent item {}", item); - continue; - } - - query.setString(1, item.id()); - var rs = query.executeQuery(); - if (rs.next()) { - items.add(new TransferItem( - item.id(), - rs.getInt(1), - fileStorageIds.get(0), - item.relPath() - )); - } - else { - logger.info("Rejected {}", item); - } - } - } - - logger.info("Found {} eligible domains", items.size()); - - return new TransferSpec(items); - } - - public Object yieldDomain(Request request, Response response) throws SQLException, IOException { - var item = gson.fromJson(request.body(), TransferItem.class); - var storage = fileStorageService.getStorage(item.fileStorageId()); - Files.delete(storage.asPath().resolve(storage.asPath().resolve(item.path()))); - return ""; - } - - public void pruneCrawlDataMqEndpoint() throws SQLException, IOException { - List fileStorageIds = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.CRAWL_DATA); - if (fileStorageIds.isEmpty()) { - return; - } - var storage = fileStorageService.getStorage(fileStorageIds.get(0)); - - Path newCrawlLogPath = storage.asPath().resolve("crawler.log-new"); - Path oldCrawlLogPath = storage.asPath().resolve("crawler.log"); - - int pruned = 0; - try (var newWorkLog = new WorkLog(newCrawlLogPath)) { - for (var item : WorkLog.iterable(oldCrawlLogPath)) { - if (Files.exists(storage.asPath().resolve(item.relPath()))) { - newWorkLog.setJobToFinished(item.id(), item.path(), item.cnt()); - } - else { - pruned++; - } - } - } - if (pruned > 0) { - logger.info("Pruned {} items from the crawl log!", pruned); - } - - Files.move(newCrawlLogPath, oldCrawlLogPath, StandardCopyOption.REPLACE_EXISTING); - } - - public void transferMqEndpoint(int sourceNode, int count) throws Exception { - var storages = fileStorageService.getOnlyActiveFileStorage(FileStorageType.CRAWL_DATA); - - // Ensure crawl data exists to receive into - if (storages.isEmpty()) { - var storage = fileStorageService.allocateStorage( - FileStorageType.CRAWL_DATA, - "crawl-data", - "Crawl Data" - ); - fileStorageService.enableFileStorage(storage.id()); - } - - var storageId = fileStorageService - .getOnlyActiveFileStorage(FileStorageType.CRAWL_DATA) - .orElseThrow(AssertionError::new); // This Shouldn't Happen (tm) - - var storage = fileStorageService.getStorage(storageId); - - var spec = executorClient.getTransferSpec(Context.internal(), sourceNode, count); - if (spec.size() == 0) { - return; - } - - Path basePath = storage.asPath(); - try (var workLog = new WorkLog(basePath.resolve("crawler.log")); - var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement("UPDATE EC_DOMAIN SET NODE_AFFINITY=? WHERE ID=?"); - ) { - for (var item : spec.items()) { - logger.info("{}", item); - logger.info("Transferring {}", item.domainName()); - - Path dest = basePath.resolve(item.path()); - Files.createDirectories(dest.getParent()); - try (var fileStream = Files.newOutputStream(dest)) { - executorClient.transferFile(Context.internal(), - sourceNode, - item.fileStorageId(), - item.path(), - fileStream); - - stmt.setInt(1, nodeId); - stmt.setInt(2, item.domainId()); - stmt.executeUpdate(); - - executorClient.yieldDomain(Context.internal(), sourceNode, item); - workLog.setJobToFinished(item.domainName(), item.path(), 1); - } - catch (IOException ex) { - Files.deleteIfExists(dest); - throw new RuntimeException(ex); - } - catch (Exception ex) { - throw new RuntimeException(ex); - } - } - } - - var outbox = new MqOutbox(persistence, executorServiceName, sourceNode, - getClass().getSimpleName(), nodeId, UUID.randomUUID()); - - try { - outbox.send("PRUNE-CRAWL-DATA", ":-)"); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - outbox.stop(); - } - } public record TransferReq(int sourceNode, int count) { } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java index aadfa625..d53cd041 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java @@ -54,7 +54,7 @@ public class ProcessService { ProcessId(String path) { this.path = path; } - }; + } @Inject public ProcessService(BaseServiceParams params, diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java b/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java index 23d7e544..e5c68120 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java @@ -10,7 +10,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.*; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; diff --git a/code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java b/code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java index 95f4801b..e9c085aa 100644 --- a/code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java +++ b/code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java @@ -15,7 +15,6 @@ import org.testcontainers.containers.MariaDBContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; import java.sql.SQLException; import java.util.*; @@ -55,7 +54,7 @@ class DomainListRefreshServiceTest { void downloadDomainsList() throws SQLException { DomainTypes domainTypes = new DomainTypes(dataSource); DomainListRefreshService service = new DomainListRefreshService(dataSource, - domainTypes, new ServiceConfiguration(null, 1, null, -1, -1, null)); + domainTypes, new ServiceConfiguration(null, 1, null, null, -1, null)); domainTypes.updateUrlForSelection(DomainTypes.Type.CRAWL, "https://downloads.marginalia.nu/domain-list-test.txt"); service.synchronizeDomainList(); diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 4523dc27..160773ea 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -51,7 +51,6 @@ dependencies { implementation libs.bundles.httpcomponents implementation libs.roaringbitmap implementation libs.snakeyaml - implementation libs.rxjava implementation libs.spark implementation libs.opencsv implementation libs.trove diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java index 9e4042ea..3dc13fca 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -25,7 +24,7 @@ public class IndexMain extends MainClass { Injector injector = Guice.createInjector( new IndexModule(), new DatabaseModule(false), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Index) + new ServiceConfigurationModule(ServiceId.Index) ); injector.getInstance(NodeStatusWatcher.class); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index b5f5a7f8..242c6cef 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -2,8 +2,6 @@ package nu.marginalia.index; import com.google.gson.Gson; import com.google.inject.Inject; -import io.grpc.ServerBuilder; -import io.reactivex.rxjava3.schedulers.Schedulers; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; import nu.marginalia.index.svc.IndexDomainLinksService; @@ -21,18 +19,14 @@ import nu.marginalia.service.server.mq.MqRequest; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; import spark.Spark; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.concurrent.TimeUnit; +import java.util.List; import static nu.marginalia.linkdb.LinkdbFileNames.DOCDB_FILE_NAME; import static nu.marginalia.linkdb.LinkdbFileNames.DOMAIN_LINKS_FILE_NAME; -import static spark.Spark.get; public class IndexService extends Service { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -48,6 +42,7 @@ public class IndexService extends Service { private final ServiceEventLog eventLog; + @SneakyThrows @Inject public IndexService(BaseServiceParams params, IndexOpsService opsService, @@ -57,8 +52,9 @@ public class IndexService extends Service { DocumentDbReader documentDbReader, DomainLinkDb domainLinkDb, IndexDomainLinksService indexDomainLinksService, - ServiceEventLog eventLog) throws IOException { - super(params); + ServiceEventLog eventLog) + { + super(params, List.of(indexQueryService, indexDomainLinksService)); this.opsService = opsService; this.searchIndex = searchIndex; @@ -71,14 +67,6 @@ public class IndexService extends Service { this.init = params.initialization; - var grpcServer = ServerBuilder.forPort(params.configuration.port() + 1) - .addService(indexQueryService) - .addService(indexDomainLinksService) - .build(); - grpcServer.start(); - - Spark.post("/search/", indexQueryService::search, gson::toJson); - Spark.get("/public/debug/docmeta", indexQueryService::debugEndpointDocMetadata, gson::toJson); Spark.get("/public/debug/wordmeta", indexQueryService::debugEndpointWordMetadata, gson::toJson); Spark.get("/public/debug/word", indexQueryService::debugEndpointWordEncoding, gson::toJson); @@ -86,13 +74,7 @@ public class IndexService extends Service { Spark.post("/ops/repartition", opsService::repartitionEndpoint); Spark.post("/ops/reindex", opsService::reindexEndpoint); - get("/is-blocked", this::isBlocked, gson::toJson); - - Schedulers.newThread().scheduleDirect(this::initialize, 1, TimeUnit.MICROSECONDS); - } - - private Object isBlocked(Request request, Response response) { - return !initialized || opsService.isBusy(); + Thread.ofPlatform().name("initialize-index").start(this::initialize); } volatile boolean initialized = false; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java index bef389b0..23b9d954 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java @@ -5,7 +5,6 @@ import nu.marginalia.index.forward.ForwardIndexReader; import nu.marginalia.index.forward.ParamMatchingQueryFilter; import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepIf; -import nu.marginalia.model.EdgeUrl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java index b9380883..6be5f1df 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java @@ -1,10 +1,8 @@ package nu.marginalia.index.results; import com.google.inject.Inject; -import gnu.trove.map.hash.TObjectIntHashMap; import gnu.trove.map.hash.TObjectLongHashMap; import gnu.trove.set.hash.TLongHashSet; -import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap; import it.unimi.dsi.fastutil.longs.LongArrayList; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java index 08c7c1c3..c3f088f6 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java @@ -1,6 +1,5 @@ package nu.marginalia.index.svc; -import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Singleton; import gnu.trove.list.TLongList; @@ -11,7 +10,6 @@ import io.prometheus.client.Histogram; import lombok.SneakyThrows; import nu.marginalia.index.api.*; import nu.marginalia.index.api.IndexApiGrpc.IndexApiImplBase; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; import nu.marginalia.index.client.model.query.SearchSubquery; import nu.marginalia.index.client.model.results.ResultRankingParameters; import nu.marginalia.index.client.model.results.SearchResultItem; @@ -28,7 +26,6 @@ import nu.marginalia.index.results.IndexResultValuator; import nu.marginalia.index.query.IndexQuery; import nu.marginalia.index.results.IndexResultDomainDeduplicator; import nu.marginalia.index.svc.searchset.SmallSearchSet; -import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.model.idx.DocumentMetadata; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.service.module.ServiceConfiguration; @@ -36,10 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; import org.slf4j.MarkerFactory; -import spark.HaltException; import spark.Request; import spark.Response; -import spark.Spark; import java.sql.SQLException; import java.util.*; @@ -72,7 +67,6 @@ public class IndexQueryService extends IndexApiImplBase { .register(); private final IndexQueryExecutor queryExecutor; - private final Gson gson = GsonFactory.get(); private final SearchIndex index; private final IndexResultDecorator resultDecorator; @@ -126,52 +120,6 @@ public class IndexQueryService extends IndexApiImplBase { return Long.toHexString(searchTermsSvc.getWordId(word)); } - public Object search(Request request, Response response) { - final String json = request.body(); - final SearchSpecification specsSet = gson.fromJson(json, SearchSpecification.class); - - if (!index.isAvailable()) { - Spark.halt(503, "Index is not loaded"); - } - - final String nodeName = Integer.toString(nodeId); - - try { - return wmsa_query_time - .labels(nodeName, "REST") - .time(() -> { - var params = new SearchParameters(specsSet, getSearchSet(specsSet)); - - SearchResultSet results = executeSearch(params); - - logger.info(queryMarker, "Index Result Count: {}", results.size()); - - wmsa_query_cost - .labels(nodeName, "REST") - .set(params.getDataCost()); - - if (!params.hasTimeLeft()) { - wmsa_query_timeouts - .labels(nodeName, "REST") - .inc(); - } - - return results; - }); - } - catch (HaltException ex) { - logger.warn("Halt", ex); - throw ex; - } - catch (Exception ex) { - logger.info("Error during search {}({}) (query: {})", ex.getClass().getSimpleName(), ex.getMessage(), json); - logger.info("Error", ex); - Spark.halt(500, "Error"); - return null; - } - } - - // GRPC endpoint @SneakyThrows public void query(nu.marginalia.index.api.RpcIndexQuery request, diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java index 52195a6b..7403979c 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java @@ -2,21 +2,16 @@ package nu.marginalia.index.svc; import gnu.trove.set.hash.TLongHashSet; import nu.marginalia.index.api.RpcIndexQuery; -import nu.marginalia.index.api.RpcSpecLimit; import nu.marginalia.index.client.IndexProtobufCodec; import nu.marginalia.index.client.model.query.SearchSpecification; import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.Bm25Parameters; import nu.marginalia.index.client.model.results.ResultRankingParameters; import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.index.SearchIndexSearchTerms; import nu.marginalia.index.query.IndexQuery; import nu.marginalia.index.query.IndexQueryParams; import nu.marginalia.index.query.IndexSearchBudget; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.index.query.limit.SpecificationLimit; -import nu.marginalia.index.query.limit.SpecificationLimitType; import nu.marginalia.index.searchset.SearchSet; import java.util.ArrayList; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java index 89018493..3437bf57 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java @@ -3,8 +3,6 @@ package nu.marginalia.index.svc.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import nu.marginalia.index.client.model.query.SearchSetIdentifier; import nu.marginalia.index.searchset.SearchSet; -import nu.marginalia.model.idx.DocumentFlags; -import nu.marginalia.model.idx.DocumentMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryDataTest.java b/code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryDataTest.java deleted file mode 100644 index 92f134cf..00000000 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryDataTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.index.service.util; - -class DictionaryDataTest { - -// @Test -// public void testDataBankGrow2() { -// var dataBank = new DictionaryData(65535); -// for (int i = 0; i < 64; i++) { -// String s = "" + i; -// int offset = dataBank.add(s.getBytes(), i); -// System.out.println(s + " " + offset + " " + new String(dataBank.getKey(i)) + " " + dataBank.getValue(i)); -// -// Assertions.assertEquals(s, new String(dataBank.getKey(i))); -// Assertions.assertEquals(i, dataBank.getValue(i)); -// } -// } -} \ No newline at end of file diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryHashMapTest.java b/code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryHashMapTest.java deleted file mode 100644 index a290f33f..00000000 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/service/util/DictionaryHashMapTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package nu.marginalia.index.service.util; - -class DictionaryHashMapTest { -// -// @Test -// public void testDictionaryHashMap() { -// var dhm = new DictionaryHashMap(1<<6); -// System.out.println(dhm.put("hello".getBytes(), 23)); -// System.out.println(dhm.put("hello".getBytes(), 23)); -// System.out.println(dhm.put("world".getBytes(), 54)); -// assertEquals(23, dhm.get("hello".getBytes())); -// assertEquals(54, dhm.get("world".getBytes())); -// -// } -// -// @Test -// public void testDictionaryHashMapMissing() { -// var dhm = new DictionaryHashMap(1<<8); -// assertEquals(DictionaryHashMap.NO_VALUE, dhm.get(new byte[] { 1,2,3})); -// -// } -// -// @Test -// public void randomTest() { -// Set strings = new HashSet<>(); -// var dhm = new DictionaryHashMap(1<<14); -// -// for (int i = 0; i < 10000; i++) { -// strings.add(Double.toString(Math.random())); -// } -// -// for (String s : strings) { -// dhm.put(s.getBytes(), s.hashCode()); -// } -// -// for (String s : strings) { -// assertEquals(s.hashCode(), dhm.get(s.getBytes())); -// } -// -// assertEquals(strings.size(), dhm.size()); -// } -// -// @Test -// public void fillHerUp2() { -// var dhm = new DictionaryHashMap(1<<13); -// -// try { -// for (int i = 0; i < 10000; i++) { -// dhm.put(Double.toString(Math.random()).getBytes(), i); -// } -// Assertions.fail("Expected exception"); -// } -// catch (IllegalStateException ex) { -// ex.printStackTrace(); -// } -// } - -} \ No newline at end of file diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java b/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java index a7f67bbe..438b3f0f 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java +++ b/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java @@ -82,7 +82,7 @@ public class IndexQueryServiceIntegrationTestModule extends AbstractModule { ServiceId.Index, 0, "127.0.0.1", - randomPort(), + "127.0.0.1", randomPort(), UUID.randomUUID() )); diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java b/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java index 577c1fb3..01837bd2 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java +++ b/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java @@ -1,7 +1,6 @@ package nu.marginalia.index.svc.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; import org.junit.jupiter.api.Test; import java.io.IOException; diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 8f7cea9a..4f87c11a 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -27,7 +27,6 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:service') implementation project(':code:common:renderer') - implementation project(':code:common:service-client') implementation project(':code:api:index-api') implementation project(':code:api:query-api') implementation project(':code:common:service-discovery') @@ -44,7 +43,6 @@ dependencies { implementation libs.notnull implementation libs.guice implementation libs.protobuf - implementation libs.rxjava implementation libs.bundles.mariadb implementation libs.bundles.grpc diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java index 6558f1b8..2db3dcab 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java @@ -2,8 +2,6 @@ package nu.marginalia.query; import com.google.gson.Gson; import com.google.inject.Inject; -import nu.marginalia.client.Context; -import nu.marginalia.index.client.IndexClient; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.query.model.QueryParams; @@ -18,23 +16,21 @@ import java.util.Map; public class QueryBasicInterface { private final MustacheRenderer renderer; - private final NodeConfigurationWatcher nodeConfigurationWatcher; - private final IndexClient indexClient; private final QueryFactory queryFactory; private final Gson gson = GsonFactory.get(); + private final QueryGRPCService queryGRPCService; + @Inject public QueryBasicInterface(RendererFactory rendererFactory, - NodeConfigurationWatcher nodeConfigurationWatcher, - IndexClient indexClient, - QueryFactory queryFactory - ) throws IOException + QueryFactory queryFactory, + QueryGRPCService queryGRPCService + ) throws IOException { this.renderer = rendererFactory.renderer("search"); - this.nodeConfigurationWatcher = nodeConfigurationWatcher; - this.indexClient = indexClient; this.queryFactory = queryFactory; + this.queryGRPCService = queryGRPCService; } public Object handle(Request request, Response response) { @@ -51,11 +47,9 @@ public class QueryBasicInterface { domainCount, count, 250, 8192 ), set)); - var rsp = indexClient.query( - Context.fromRequest(request), - nodeConfigurationWatcher.getQueryNodes(), - query.specs - ); + var rsp = queryGRPCService.executeQueries(QueryProtobufCodec.convertQuery(queryParam, query), count); + + var results = rsp.stream().map(QueryProtobufCodec::convertQueryResult).toList(); if (request.headers("Accept").contains("application/json")) { response.type("application/json"); @@ -64,7 +58,7 @@ public class QueryBasicInterface { else { return renderer.render( Map.of("query", queryParam, - "results", rsp.results) + "results", results) ); } } diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java index a2b6b780..89de784c 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java @@ -1,37 +1,25 @@ package nu.marginalia.query; import com.google.inject.Inject; -import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; -import nu.marginalia.client.grpc.GrpcChannelPool; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; import nu.marginalia.index.api.IndexDomainLinksApiGrpc; import nu.marginalia.index.api.RpcDomainIdCount; import nu.marginalia.index.api.RpcDomainIdList; import nu.marginalia.index.api.RpcDomainIdPairs; +import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.id.ServiceId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - public class QueryGRPCDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase { private static final Logger logger = LoggerFactory.getLogger(QueryGRPCDomainLinksService.class); - private final GrpcChannelPool channelPool; + private final GrpcMultiNodeChannelPool channelPool; @Inject - public QueryGRPCDomainLinksService(NodeConfigurationWatcher nodeConfigurationWatcher) { - channelPool = new GrpcChannelPool<>(ServiceId.Index) { - @Override - public IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub createStub(ManagedChannel channel) { - return IndexDomainLinksApiGrpc.newBlockingStub(channel); - } - - @Override - public List getEligibleNodes() { - return nodeConfigurationWatcher.getQueryNodes(); - } - }; + public QueryGRPCDomainLinksService(GrpcChannelPoolFactory channelPoolFactory) { + channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub); } @Override diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java index bd2c0452..b2bb9391 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java @@ -1,14 +1,15 @@ package nu.marginalia.query; import com.google.inject.Inject; -import io.grpc.ManagedChannel; +import com.google.inject.Singleton; import io.prometheus.client.Histogram; import lombok.SneakyThrows; -import nu.marginalia.client.grpc.GrpcChannelPool; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; import nu.marginalia.db.DomainBlacklist; import nu.marginalia.index.api.*; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.query.svc.QueryFactory; +import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.id.ServiceId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.util.*; import java.util.concurrent.*; +@Singleton public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { private final Logger logger = LoggerFactory.getLogger(QueryGRPCService.class); @@ -27,30 +29,19 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { .help("QS-side query time (GRPC endpoint)") .register(); - private final GrpcChannelPool channelPool; + private final GrpcMultiNodeChannelPool channelPool; private final QueryFactory queryFactory; private final DomainBlacklist blacklist; @Inject public QueryGRPCService(QueryFactory queryFactory, - DomainBlacklist blacklist, - NodeConfigurationWatcher nodeConfigurationWatcher) + GrpcChannelPoolFactory channelPoolFactory, + DomainBlacklist blacklist) { this.queryFactory = queryFactory; this.blacklist = blacklist; - - channelPool = new GrpcChannelPool<>(ServiceId.Index) { - @Override - public IndexApiGrpc.IndexApiBlockingStub createStub(ManagedChannel channel) { - return IndexApiGrpc.newBlockingStub(channel); - } - - @Override - public List getEligibleNodes() { - return nodeConfigurationWatcher.getQueryNodes(); - } - }; + this.channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexApiGrpc::newBlockingStub); } public void query(nu.marginalia.index.api.RpcQsQuery request, @@ -88,7 +79,7 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { Comparator.comparing(RpcDecoratedResultItem::getRankingScore); @SneakyThrows - private List executeQueries(RpcIndexQuery indexRequest, int totalSize) { + List executeQueries(RpcIndexQuery indexRequest, int totalSize) { return channelPool.invokeAll(stub -> new QueryTask(stub, indexRequest)) .stream() .filter(f -> f.state() == Future.State.SUCCESS) diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java index f3f3d561..f3849515 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java @@ -4,7 +4,6 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; -import nu.marginalia.service.SearchServiceDescriptors; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -24,7 +23,7 @@ public class QueryMain extends MainClass { Injector injector = Guice.createInjector( new QueryModule(), new DatabaseModule(false), - new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Query) + new ServiceConfigurationModule(ServiceId.Query) ); injector.getInstance(QueryMain.class); diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index bd5eaebd..0cf44d2b 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -2,21 +2,14 @@ package nu.marginalia.query; import com.google.gson.Gson; import com.google.inject.Inject; -import io.grpc.ServerBuilder; import io.prometheus.client.Histogram; -import nu.marginalia.client.Context; +import lombok.SneakyThrows; import nu.marginalia.db.DomainBlacklist; import nu.marginalia.index.client.IndexClient; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; -import nu.marginalia.index.client.model.results.SearchResultSet; -import nu.marginalia.query.model.QueryParams; -import nu.marginalia.query.model.QueryResponse; import nu.marginalia.query.svc.QueryFactory; +import nu.marginalia.service.NodeConfigurationWatcher; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; -import spark.Request; -import spark.Response; import spark.Spark; import java.io.IOException; @@ -37,6 +30,7 @@ public class QueryService extends Service { .register(); + @SneakyThrows @Inject public QueryService(BaseServiceParams params, IndexClient indexClient, @@ -46,11 +40,11 @@ public class QueryService extends Service { Gson gson, DomainBlacklist blacklist, QueryBasicInterface queryBasicInterface, - QueryFactory queryFactory) throws IOException + QueryFactory queryFactory) { - super(params, () -> { - Spark.staticFileLocation("/static/"); - }); + super(params, + () -> Spark.staticFileLocation("/static/"), + List.of(queryGRPCService, domainLinksService)); this.indexClient = indexClient; this.nodeWatcher = nodeWatcher; @@ -58,15 +52,6 @@ public class QueryService extends Service { this.blacklist = blacklist; this.queryFactory = queryFactory; - var grpcServer = ServerBuilder.forPort(params.configuration.port() + 1) - .addService(queryGRPCService) - .addService(domainLinksService) - .build(); - grpcServer.start(); - - Spark.post("/delegate/", this::delegateToIndex, gson::toJson); - Spark.post("/search/", this::search, gson::toJson); - Spark.get("/public/search", queryBasicInterface::handle); Spark.exception(Exception.class, (e, request, response) -> { @@ -79,44 +64,4 @@ public class QueryService extends Service { }); } - private Object search(Request request, Response response) { - return wmsa_qs_query_time_rest.time(() -> { - String json = request.body(); - QueryParams params = gson.fromJson(json, QueryParams.class); - - var query = queryFactory.createQuery(params); - var rsp = executeQuery(Context.fromRequest(request), query.specs); - - rsp.results.removeIf(this::isBlacklisted); - - response.type("application/json"); - - return new QueryResponse( - query.specs, - rsp.results, - query.searchTermsHuman, - List.of(), // no problems - query.domain - ); - }); - } - - private SearchResultSet delegateToIndex(Request request, Response response) { - String json = request.body(); - SearchSpecification specsSet = gson.fromJson(json, SearchSpecification.class); - - response.type("application/json"); - - return executeQuery(Context.fromRequest(request), specsSet); - } - - private SearchResultSet executeQuery(Context ctx, SearchSpecification query) { - var nodes = nodeWatcher.getQueryNodes(); - - return indexClient.query(ctx, nodes, query); - } - - private boolean isBlacklisted(DecoratedSearchResultItem item) { - return blacklist.isBlacklisted(item.domainId()); - } } diff --git a/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java b/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java index 44cddced..7ce9635f 100644 --- a/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java +++ b/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java @@ -1,7 +1,6 @@ package nu.marginalia.query.svc; import nu.marginalia.WmsaHome; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; import nu.marginalia.index.client.model.query.SearchSpecification; import nu.marginalia.index.client.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/Experiment.java b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/Experiment.java index f08ccf53..b5f9ff40 100644 --- a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/Experiment.java +++ b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/Experiment.java @@ -1,7 +1,6 @@ package nu.marginalia.tools; import nu.marginalia.crawling.io.SerializableCrawlDataStream; -import nu.marginalia.crawling.model.CrawledDomain; import java.io.IOException; import java.util.HashSet; @@ -14,7 +13,7 @@ public abstract class Experiment { for (String domain : args) { domains.add(domain.toLowerCase()); } - }; + } public abstract boolean process(SerializableCrawlDataStream dataStream) throws IOException; diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AdblockExperiment.java b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AdblockExperiment.java index c5cb8dda..70856439 100644 --- a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AdblockExperiment.java +++ b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AdblockExperiment.java @@ -5,7 +5,6 @@ import nu.marginalia.adblock.AdblockSimulator; import nu.marginalia.converting.processor.DocumentProcessor; import nu.marginalia.crawling.model.CrawledDocument; import nu.marginalia.crawling.model.CrawledDomain; -import nu.marginalia.tools.Experiment; import nu.marginalia.tools.LegacyExperiment; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java index a7866193..8290a658 100644 --- a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java +++ b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java @@ -4,7 +4,6 @@ import com.google.inject.Inject; import nu.marginalia.converting.processor.DomainProcessor; import nu.marginalia.converting.processor.plugin.specialization.BlogSpecialization; import nu.marginalia.crawling.model.CrawledDomain; -import nu.marginalia.tools.Experiment; import nu.marginalia.tools.LegacyExperiment; import org.jsoup.Jsoup; diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java index f804ac2d..8614d1e6 100644 --- a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java +++ b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java @@ -9,7 +9,6 @@ import nu.marginalia.keyword.DocumentKeywordExtractor; import nu.marginalia.language.sentence.SentenceExtractor; import nu.marginalia.model.EdgeUrl; import nu.marginalia.term_frequency_dict.TermFrequencyDict; -import nu.marginalia.tools.Experiment; import nu.marginalia.tools.LegacyExperiment; import org.jsoup.Jsoup; diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TestExperiment.java b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TestExperiment.java index 65991ea4..521b36e8 100644 --- a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TestExperiment.java +++ b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TestExperiment.java @@ -1,7 +1,6 @@ package nu.marginalia.tools.experiments; import nu.marginalia.crawling.model.CrawledDomain; -import nu.marginalia.tools.Experiment; import nu.marginalia.tools.LegacyExperiment; public class TestExperiment extends LegacyExperiment { diff --git a/run/install/docker-compose-barebones-1.yml.template b/run/install/docker-compose-barebones-1.yml.template index 1a08cc31..11dc8dcf 100644 --- a/run/install/docker-compose-barebones-1.yml.template +++ b/run/install/docker-compose-barebones-1.yml.template @@ -103,6 +103,12 @@ services: - db:/var/lib/mysql networks: - wmsa + zookeeper: + image: zookeeper + container_name: "zookeeper" + restart: always + ports: + - "127.0.0.1:2181:2181" traefik: image: "traefik:v2.10" container_name: "traefik" diff --git a/settings.gradle b/settings.gradle index af0cad4f..79aeace2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -53,7 +53,6 @@ include 'code:features-index:index-forward' include 'code:features-index:index-reverse' include 'code:features-index:domain-ranking' -include 'code:api:actor-api' include 'code:api:query-api' include 'code:api:index-api' include 'code:api:assistant-api' @@ -61,7 +60,6 @@ include 'code:api:process-mqapi' include 'code:api:executor-api' include 'code:common:service-discovery' -include 'code:common:service-client' include 'code:common:db' include 'code:common:linkdb' include 'code:common:service' @@ -131,7 +129,6 @@ dependencyResolutionManagement { library('grpc-protobuf', 'io.grpc', 'grpc-protobuf').version('1.49.2') library('grpc-stub', 'io.grpc', 'grpc-stub').version('1.49.2') library('grpc-netty', 'io.grpc', 'grpc-netty-shaded').version('1.49.2') - library('rxjava', 'io.reactivex.rxjava3', 'rxjava').version('3.1.6') library('prometheus', 'io.prometheus', 'simpleclient').version('0.16.0') library('prometheus-servlet', 'io.prometheus', 'simpleclient_servlet').version('0.16.0') @@ -201,9 +198,13 @@ dependencyResolutionManagement { library('sqlite','org.xerial','sqlite-jdbc').version('3.41.2.2') library('javax.annotation','javax.annotation','javax.annotation-api').version('1.3.2') + library('parquet-column', 'org.apache.parquet','parquet-column').version('1.13.1') library('parquet-hadoop', 'org.apache.parquet','parquet-hadoop').version('1.13.1') + library('curator-framework', 'org.apache.curator','curator-framework').version('5.6.0') + library('curator-x-discovery', 'org.apache.curator','curator-x-discovery').version('5.6.0') + bundle('slf4j', ['slf4j.api', 'log4j.api', 'log4j.core', 'log4j.slf4j']) bundle('slf4j.test', ['slf4j.jdk14']) bundle('prometheus', ['prometheus', 'prometheus-servlet', 'prometheus-server', 'prometheus-hotspot']) @@ -217,6 +218,8 @@ dependencyResolutionManagement { bundle('parquet', ['parquet-column', 'parquet-hadoop']) bundle('junit', ['junit.jupiter', 'junit.jupiter.engine']) bundle('flyway', ['flyway.core', 'flyway.mysql']) + + bundle('curator', ['curator-framework', 'curator-x-discovery']) } diff --git a/third-party/parquet-floor/src/main/java/blue/strategic/parquet/ParquetReader.java b/third-party/parquet-floor/src/main/java/blue/strategic/parquet/ParquetReader.java index 45718fe8..0a5e8375 100644 --- a/third-party/parquet-floor/src/main/java/blue/strategic/parquet/ParquetReader.java +++ b/third-party/parquet-floor/src/main/java/blue/strategic/parquet/ParquetReader.java @@ -13,7 +13,6 @@ import org.apache.parquet.io.DelegatingSeekableInputStream; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.SeekableInputStream; import org.apache.parquet.io.api.GroupConverter; -import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; From 746a86510666beb4809ee3f1b6feeaaf23c12a3d Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 14:14:09 +0100 Subject: [PATCH 02/56] (client) Fix handling of channel refreshes The previous code made an incorrect assumption that all routes refer to the same node, and would overwrite the route list on each update. This lead to storms of closing and opening channels whenever an update was received. The new code is correctly aware that we may talk to multiple nodes. --- .../client/GrpcSingleNodeChannelPool.java | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index 69941e51..c59e2498 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -21,7 +21,7 @@ import java.util.function.Function; * Manages unicast-style requests */ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { private final Map, ManagedChannel> channels = new ConcurrentHashMap<>(); - private volatile Set> routes = Set.of(); + private final Map>> routes = new ConcurrentHashMap<>(); private static final Logger logger = LoggerFactory.getLogger(GrpcSingleNodeChannelPool.class); @@ -68,31 +68,25 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { private void refreshNode(int node) { Set> newRoutes = serviceRegistryIf.getEndpoints(ApiSchema.GRPC, serviceId, node); - Set> oldRoutes = routes; + Set> oldRoutes = routes.getOrDefault(node, Set.of()); - if (!oldRoutes.equals(newRoutes)) { - // Find the routes that have been added or removed - for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { + // Find the routes that have been added or removed + for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { - ManagedChannel oldChannel; + ManagedChannel oldChannel; - if (newRoutes.contains(route)) { - logger.info(STR."Adding channel for \{serviceId.serviceName}-\{node} \{route.host()}:\{route.port()}"); - - var newChannel = channelConstructor.apply(route); - oldChannel = channels.put(route, newChannel); - } else { - logger.info(STR."Removing channel for \{serviceId.serviceName}-\{node} \{route.host()}:\{route.port()}"); - - oldChannel = channels.remove(route); - } - - if (oldChannel != null) - oldChannel.shutdown(); + if (newRoutes.contains(route)) { + var newChannel = channelConstructor.apply(route); + oldChannel = channels.put(route, newChannel); + } else { + oldChannel = channels.remove(route); } - routes = newRoutes; + if (oldChannel != null) + oldChannel.shutdown(); } + + routes.put(node, newRoutes); } public boolean hasChannel() { From 07b625c58d52045e36d37640b91906058e96c765 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 14:16:05 +0100 Subject: [PATCH 03/56] (query-client) Add support for fault-tolerant requests to single node services Adding a method importantCall that will retry a failing request on each route until it succeeds or the routes run out. --- .../nu/marginalia/query/client/QueryClient.java | 5 +++-- .../client/GrpcSingleNodeChannelPool.java | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java index 6667e5b0..af32900c 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java @@ -47,10 +47,11 @@ public class QueryClient { @CheckReturnValue public QueryResponse search(QueryParams params) { + var query = QueryProtobufCodec.convertQueryParams(params); + return wmsa_qs_api_search_time.time( () -> QueryProtobufCodec.convertQueryResponse(queryApiPool - .api() - .query(QueryProtobufCodec.convertQueryParams(params)) + .importantCall((api) -> api.query(query)) ) ); } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index c59e2498..137e2933 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -98,6 +98,22 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { return stubConstructor.apply(getChannel()); } + /** Try to make the call go through. The function will cycle through + * available routes until exhaustion, and only then will it give up + */ + public T importantCall(Function function) { + for (int i = 0; i < channels.size(); i++) { + try { + return function.apply(api()); + } + catch (Exception e) { + logger.error("API Exception", e); + } + } + + throw new ServiceNotAvailableException(serviceId); + } + /** Get the channel that is most ready to use */ public ManagedChannel getChannel() { return channels From 36a5c8b44cd9b8498f9904dee9b1a5d9dc2549b8 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 14:17:18 +0100 Subject: [PATCH 04/56] (cleanup) Clean up code --- .../service/discovery/ZkServiceRegistry.java | 2 +- .../nu/marginalia/query/QueryService.java | 23 +------------------ run/env/service.env | 8 +++---- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java index b91e691f..a75d1f8d 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -79,7 +79,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } } - logger.info("Registering {} -> {}", path, instanceUUID); + logger.info("Registering {} -> {}", path, endpoint); ephemeralProperty.forPath(path, payload); diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index 0cf44d2b..0c21558f 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -1,13 +1,8 @@ package nu.marginalia.query; -import com.google.gson.Gson; import com.google.inject.Inject; import io.prometheus.client.Histogram; import lombok.SneakyThrows; -import nu.marginalia.db.DomainBlacklist; -import nu.marginalia.index.client.IndexClient; -import nu.marginalia.query.svc.QueryFactory; -import nu.marginalia.service.NodeConfigurationWatcher; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import spark.Spark; @@ -17,12 +12,6 @@ import java.util.List; public class QueryService extends Service { - private final IndexClient indexClient; - private final NodeConfigurationWatcher nodeWatcher; - private final Gson gson; - private final DomainBlacklist blacklist; - private final QueryFactory queryFactory; - private static final Histogram wmsa_qs_query_time_rest = Histogram.build() .name("wmsa_qs_query_time_rest") .linearBuckets(0.05, 0.05, 15) @@ -33,24 +22,14 @@ public class QueryService extends Service { @SneakyThrows @Inject public QueryService(BaseServiceParams params, - IndexClient indexClient, - NodeConfigurationWatcher nodeWatcher, QueryGRPCDomainLinksService domainLinksService, QueryGRPCService queryGRPCService, - Gson gson, - DomainBlacklist blacklist, - QueryBasicInterface queryBasicInterface, - QueryFactory queryFactory) + QueryBasicInterface queryBasicInterface) { super(params, () -> Spark.staticFileLocation("/static/"), List.of(queryGRPCService, domainLinksService)); - this.indexClient = indexClient; - this.nodeWatcher = nodeWatcher; - this.gson = gson; - this.blacklist = blacklist; - this.queryFactory = queryFactory; Spark.get("/public/search", queryBasicInterface::handle); diff --git a/run/env/service.env b/run/env/service.env index 8a1e311f..13151094 100644 --- a/run/env/service.env +++ b/run/env/service.env @@ -1,8 +1,8 @@ WMSA_HOME=run/ -JAVA_OPTS="--enable-preview -da -Dservice-host=0.0.0.0 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" +JAVA_OPTS="--enable-preview -da -Dservice.bind-address=0.0.0.0 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" JAVA_TOOL_OPTIONS="--enable-preview -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5000" EXECUTOR_SERVICE_OPTS="-DdistPath=/dist" -CONVERTER_PROCESS_OPTS="-Dservice-name=converter -Dservice-host=0.0.0.0" -CRAWLER_PROCESS_OPTS="-Dservice-name=crawler -Dservice-host=0.0.0.0" -LOADER_PROCESS_OPTS="-Dservice-name=loader -Dservice-host=0.0.0.0" +CONVERTER_PROCESS_OPTS="-Dservice-name=converter" +CRAWLER_PROCESS_OPTS="-Dservice-name=crawler" +LOADER_PROCESS_OPTS="-Dservice-name=loader" INDEX_CONSTRUCTION_PROCESS_OPTS="-Dservice-name=index-constructor -Djava.util.concurrent.ForkJoinPool.common.parallelism=4" \ No newline at end of file From 2ee492fb7443aa37d0ee0e964d2259d796c5bd6b Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 14:18:22 +0100 Subject: [PATCH 05/56] (gRPC) Bind gRPC services to an interface By default gRPC it magically decides on an interface. The change will explicitly tell it what to use. --- .../service/discovery/property/ServiceEndpoint.java | 4 ++++ .../main/java/nu/marginalia/service/server/Service.java | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java index 8102f52f..01caba90 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -3,6 +3,7 @@ package nu.marginalia.service.discovery.property; import lombok.SneakyThrows; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; import java.util.UUID; @@ -12,6 +13,9 @@ public sealed interface ServiceEndpoint { int port(); URL toURL(String endpoint, String query); + default InetSocketAddress toInetSocketAddress() { + return new InetSocketAddress(host(), port()); + } static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) { return switch (schema) { diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java index 73cd5977..b9bb082b 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java @@ -1,7 +1,7 @@ package nu.marginalia.service.server; -import io.grpc.BindableService; -import io.grpc.ServerBuilder; +import io.grpc.*; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.prometheus.client.Counter; import lombok.SneakyThrows; import nu.marginalia.mq.inbox.*; @@ -108,7 +108,8 @@ public class Service { config.externalAddress() ); - var grpcServerBuilder = ServerBuilder.forPort(grpcEndpoint.port()); + // Start the gRPC server + var grpcServerBuilder = NettyServerBuilder.forAddress(grpcEndpoint.toInetSocketAddress()); for (var grpcService : grpcServices) { grpcServerBuilder.addService(grpcService); } From 30bdb4b4e96cbaea0c759555c60743a7e04fde8c Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 14:21:09 +0100 Subject: [PATCH 06/56] (config) Clean up service configuration for IP addresses Adds new ways to configure the bind and external IP addresses for a service. Notably, if the environment variable WMSA_IN_DOCKER is present, the system will grab the HOSTNAME variable and announce that as the external address in the service registry. The default bind address is also changed to be 0.0.0.0 only if WMSA_IN_DOCKER is present, otherwise 127.0.0.1; as this is a more secure default. --- .../module/ServiceConfigurationModule.java | 49 ++++++++++++++++--- docker-service.gradle | 3 ++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java index c5484066..1502ec44 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java @@ -9,12 +9,15 @@ import nu.marginalia.service.id.ServiceId; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Objects; import java.util.UUID; public class ServiceConfigurationModule extends AbstractModule { private final ServiceId id; + private static final Logger logger = LoggerFactory.getLogger(ServiceConfigurationModule.class); public ServiceConfigurationModule(ServiceId id) { this.id = id; @@ -26,11 +29,13 @@ public class ServiceConfigurationModule extends AbstractModule { var configObject = new ServiceConfiguration(id, node, getBindAddress(), - getHost(), + getExternalHost(), getPrometheusPort(), UUID.randomUUID() ); + logger.info("Service configuration: {}", configObject); + bind(Integer.class).annotatedWith(Names.named("wmsa-system-node")).toInstance(node); bind(ServiceConfiguration.class).toInstance(configObject); @@ -63,18 +68,46 @@ public class ServiceConfigurationModule extends AbstractModule { return Integer.parseInt(nodeEnv); } - private String getHost() { - int node = getNode(); - final String defaultValue; + /** Get the external host for the service. This is announced via the service registry, + * and should be an IP address or hostname that resolves to this machine */ + private String getExternalHost() { + // Check for an environment variable override + String configuredValue; + if (null != (configuredValue = System.getenv("SERVICE_HOST"))) { + return configuredValue; + } - if (node > 0) defaultValue = STR."\{id.serviceName}-\{node}"; - else defaultValue = id.serviceName; + // Check for a system property override + if (null != (configuredValue = System.getProperty("service.host"))) { + return configuredValue; + } - return System.getProperty("service.host", defaultValue); + // If we're in docker, we'll use the hostname + if (isDocker()) { + return System.getenv("HOSTNAME"); + } + + // If we've not been told about a host, and we're not in docker, we'll fall back to localhost + // and hope the operator's remembered to enable random port assignment via zookeeper + return "127.0.0.1"; } + /** Get the bind address for the service. This is the address that the service will listen on. + */ private String getBindAddress() { - return System.getProperty("service.bind-address", "0.0.0.0"); + String configuredValue = System.getProperty("service.bind-address"); + if (configuredValue != null) { + return configuredValue; + } + + // If we're in docker, we'll bind to all interfaces + if (isDocker()) + return "0.0.0.0"; + else // If we're not in docker, we'll default to binding to localhost to avoid exposing services + return "127.0.0.1"; } + boolean isDocker() { + return System.getenv("WMSA_IN_DOCKER") != null; + } } diff --git a/docker-service.gradle b/docker-service.gradle index 9ac559c6..25dd0e17 100644 --- a/docker-service.gradle +++ b/docker-service.gradle @@ -18,6 +18,9 @@ RUN apt-get update && apt-get install -y curl ADD ${application.applicationName}.tar / RUN mkdir /wmsa +# This will make the service grab the hostname from the HOSTNAME variable +ENV WMSA_IN_DOCKER true + ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} """ } From fd5d121648f33d6cd8314324abd4b8a69f6ceee1 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:39:46 +0100 Subject: [PATCH 07/56] (minor) Add WMSA_IN_DOCKER to all docker files --- docker-service-with-dist.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-service-with-dist.gradle b/docker-service-with-dist.gradle index 4beb9ac2..c06bd4bd 100644 --- a/docker-service-with-dist.gradle +++ b/docker-service-with-dist.gradle @@ -25,6 +25,9 @@ ADD index-construction-process.tar /dist RUN mkdir /wmsa +# This will make the service grab the hostname from the HOSTNAME variable +ENV WMSA_IN_DOCKER true + ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} """ } From ee8e0497ae5f45d9dc73e005f1a328d0f24fc282 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:41:04 +0100 Subject: [PATCH 08/56] (refac) Move service discovery injection to a separate guice module --- .../service/ServiceDiscoveryModule.java | 42 +++++++++++++++++++ .../module/ServiceConfigurationModule.java | 18 -------- .../main/java/nu/marginalia/api/ApiMain.java | 2 + .../java/nu/marginalia/dating/DatingMain.java | 2 + .../nu/marginalia/explorer/ExplorerMain.java | 2 + .../java/nu/marginalia/search/SearchMain.java | 2 + .../marginalia/assistant/AssistantMain.java | 2 + .../nu/marginalia/control/ControlMain.java | 2 + .../nu/marginalia/executor/ExecutorMain.java | 2 + .../java/nu/marginalia/index/IndexMain.java | 2 + .../java/nu/marginalia/query/QueryMain.java | 2 + 11 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java new file mode 100644 index 00000000..161c2a9e --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java @@ -0,0 +1,42 @@ +package nu.marginalia.service; + +import com.google.inject.AbstractModule; +import nu.marginalia.service.discovery.FixedServiceRegistry; +import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.ZkServiceRegistry; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; + +/** Provides a Guice module for service discovery. */ +public class ServiceDiscoveryModule extends AbstractModule { + + private static final Logger logger = LoggerFactory.getLogger(ServiceDiscoveryModule.class); + + public void configure() { + getZookeeperHosts().ifPresentOrElse((hosts) -> { + logger.info("Using Zookeeper service registry at {}", hosts); + CuratorFramework client = CuratorFrameworkFactory + .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000)); + + bind(CuratorFramework.class).toInstance(client); + bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); + }, + () -> { + logger.info("Using fixed service registry"); + bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class); + }); + } + + private Optional getZookeeperHosts() { + if (System.getProperty("zookeeper-hosts") != null) { + return Optional.of(System.getProperty("zookeeper-hosts")); + } + return Optional.ofNullable(System.getenv("ZOOKEEPER_HOSTS")); + } + +} diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java index 1502ec44..37ce99d5 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java @@ -2,13 +2,7 @@ package nu.marginalia.service.module; import com.google.inject.AbstractModule; import com.google.inject.name.Names; -import nu.marginalia.service.discovery.FixedServiceRegistry; -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; import nu.marginalia.service.id.ServiceId; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,18 +32,6 @@ public class ServiceConfigurationModule extends AbstractModule { bind(Integer.class).annotatedWith(Names.named("wmsa-system-node")).toInstance(node); bind(ServiceConfiguration.class).toInstance(configObject); - - if (Boolean.getBoolean("system.useZookeeper")) { - CuratorFramework client = CuratorFrameworkFactory - .newClient(System.getProperty("zookeeper-hosts", "zookeeper:2181"), - new ExponentialBackoffRetry(100, 10, 1000)); - - bind(CuratorFramework.class).toInstance(client); - bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); - } - else { - bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class); - } } private int getPrometheusPort() { diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java index f9c4347f..cca846d4 100644 --- a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java +++ b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -20,6 +21,7 @@ public class ApiMain extends MainClass { Injector injector = Guice.createInjector( new DatabaseModule(false), + new ServiceDiscoveryModule(), new ServiceConfigurationModule(ServiceId.Api)); injector.getInstance(ApiMain.class); injector.getInstance(Initialization.class).setReady(); diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java b/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java index 92065da7..d5cf6f4b 100644 --- a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java +++ b/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -25,6 +26,7 @@ public class DatingMain extends MainClass { Injector injector = Guice.createInjector( new DatingModule(), + new ServiceDiscoveryModule(), new ServiceConfigurationModule(ServiceId.Dating), new DatabaseModule(false) ); diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java index 4ce949f6..e48320f7 100644 --- a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java +++ b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -25,6 +26,7 @@ public class ExplorerMain extends MainClass { Injector injector = Guice.createInjector( new ServiceConfigurationModule(ServiceId.Explorer), + new ServiceDiscoveryModule(), new ExplorerModule(), new DatabaseModule(false) ); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java index 9d25d278..f799759c 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -27,6 +28,7 @@ public class SearchMain extends MainClass { Injector injector = Guice.createInjector( new SearchModule(), new ServiceConfigurationModule(ServiceId.Search), + new ServiceDiscoveryModule(), new DatabaseModule(false) ); diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java index db4b7ff1..f8c2cc3b 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -23,6 +24,7 @@ public class AssistantMain extends MainClass { Injector injector = Guice.createInjector( new AssistantModule(), new ServiceConfigurationModule(ServiceId.Assistant), + new ServiceDiscoveryModule(), new DatabaseModule(false) ); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java index 48531a91..a8bbb4f4 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -21,6 +22,7 @@ public class ControlMain extends MainClass { Injector injector = Guice.createInjector( new DatabaseModule(true), new ControlProcessModule(), + new ServiceDiscoveryModule(), new ServiceConfigurationModule(ServiceId.Control)); injector.getInstance(ControlMain.class); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java index e9c8478a..86798d86 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.DatabaseModule; import nu.marginalia.service.module.ServiceConfigurationModule; @@ -24,6 +25,7 @@ public class ExecutorMain extends MainClass { Injector injector = Guice.createInjector( new ExecutorModule(), new DatabaseModule(false), + new ServiceDiscoveryModule(), new ServiceConfigurationModule(ServiceId.Executor) ); injector.getInstance(NodeStatusWatcher.class); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java index 3dc13fca..006f313e 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -24,6 +25,7 @@ public class IndexMain extends MainClass { Injector injector = Guice.createInjector( new IndexModule(), new DatabaseModule(false), + new ServiceDiscoveryModule(), new ServiceConfigurationModule(ServiceId.Index) ); diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java index f3849515..4034efde 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java @@ -4,6 +4,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; @@ -23,6 +24,7 @@ public class QueryMain extends MainClass { Injector injector = Guice.createInjector( new QueryModule(), new DatabaseModule(false), + new ServiceDiscoveryModule(), new ServiceConfigurationModule(ServiceId.Query) ); From 3c9234078a65bf267e83c5479eabade77d5eb084 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:42:16 +0100 Subject: [PATCH 09/56] (refac) Propagate ZOOKEEPER_HOSTS to spawned processes --- .../src/main/java/nu/marginalia/process/ProcessService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java index d53cd041..971b2ab9 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java @@ -132,6 +132,7 @@ public class ProcessService { * along with WMSA_HOME, but it has special logic */ private final List propagatedEnvironmentVariables = List.of( "JAVA_HOME", + "ZOOKEEPER_HOSTS", "WMSA_SERVICE_NODE", "CONVERTER_PROCESS_OPTS", "LOADER_PROCESS_OPTS", From c600d7aa47dfb8a524efc33f189f12c0cce44389 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:42:32 +0100 Subject: [PATCH 10/56] (refac) Inject ServiceRegistry into WebsiteAdjacenciesCalculator --- .../WebsiteAdjacenciesCalculator.java | 66 ++----------------- 1 file changed, 7 insertions(+), 59 deletions(-) diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index 1e680321..eef58571 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -1,5 +1,6 @@ package nu.marginalia.adjacencies; +import com.google.inject.Guice; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.ProcessConfiguration; @@ -9,9 +10,7 @@ import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeatImpl; import nu.marginalia.query.client.QueryClient; import nu.marginalia.service.MainClass; -import nu.marginalia.service.NodeConfigurationWatcher; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.discovery.FixedServiceRegistry; +import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.module.DatabaseModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,7 +18,6 @@ import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.util.*; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.IntStream; @@ -84,48 +82,6 @@ public class WebsiteAdjacenciesCalculator extends MainClass { loader.stop(); } - private static class ProgressPrinter { - - private final AtomicInteger progress; - private final int total; - volatile boolean running = false; - private Thread printerThread; - - private ProgressPrinter(int total) { - this.total = total; - this.progress = new AtomicInteger(0); - } - - public void advance() { - progress.incrementAndGet(); - } - - private void run() { - while (running) { - double value = 100 * progress.get() / (double) total; - System.out.printf("\u001b[2K\r%3.2f%%", value); - try { - TimeUnit.MILLISECONDS.sleep(100); - } catch (InterruptedException e) { - return; - } - } - } - public void start() { - running = true; - printerThread = new Thread(this::run); - printerThread.setDaemon(true); - printerThread.start(); - } - - public void stop() throws InterruptedException { - running = false; - printerThread.join(); - System.out.println(); - } - } - - public void findAdjacent(int domainId, Consumer andThen) { findAdjacentDtoS(domainId, andThen); } @@ -191,14 +147,13 @@ public class WebsiteAdjacenciesCalculator extends MainClass { public static void main(String[] args) throws SQLException { - DatabaseModule dm = new DatabaseModule(false); + var injector = Guice.createInjector( + new DatabaseModule(false), + new ServiceDiscoveryModule()); - var dataSource = dm.provideConnection(); - // FIXME: we should use zookeeper when configured here: - var qc = new QueryClient(new GrpcChannelPoolFactory( - new NodeConfigurationWatcher(dataSource), - new FixedServiceRegistry(dataSource))); + var dataSource = injector.getInstance(HikariDataSource.class); + var qc = injector.getInstance(QueryClient.class); var main = new WebsiteAdjacenciesCalculator(qc, dataSource); @@ -236,13 +191,6 @@ public class WebsiteAdjacenciesCalculator extends MainClass { } } -// -// if (args.length == 0) { -// main.loadAll(); -// } -// else { -// main.tryDomains(args); -// } } } From 14172312dc428cedf550665b1d1990e80b9983d5 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:44:07 +0100 Subject: [PATCH 11/56] (query-client) Fix query client The query service delegates and aggregates IndexDomainLinksApiGrpc messages to the index services. The query client was accidentally also doing this, instead of talking to the query client. Fixed so it correctly talks to the query client and nothing else. --- .../marginalia/query/client/QueryClient.java | 75 ++++++++----------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java index af32900c..74f78aa7 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java @@ -3,11 +3,7 @@ package nu.marginalia.query.client; import com.google.inject.Inject; import com.google.inject.Singleton; import io.prometheus.client.Summary; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; -import nu.marginalia.index.api.Empty; -import nu.marginalia.index.api.IndexDomainLinksApiGrpc; -import nu.marginalia.index.api.QueryApiGrpc; -import nu.marginalia.index.api.RpcDomainId; +import nu.marginalia.index.api.*; import nu.marginalia.query.QueryProtobufCodec; import nu.marginalia.query.model.QueryParams; import nu.marginalia.query.model.QueryResponse; @@ -31,18 +27,14 @@ public class QueryClient { .register(); private final GrpcSingleNodeChannelPool queryApiPool; - private final GrpcMultiNodeChannelPool domainLinkApiPool; - - public IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub domainApi(int node) { - return domainLinkApiPool.apiForNode(node); - } + private final GrpcSingleNodeChannelPool domainLinkApiPool; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject public QueryClient(GrpcChannelPoolFactory channelPoolFactory) { this.queryApiPool = channelPoolFactory.createSingle(ServiceId.Query, QueryApiGrpc::newBlockingStub); - this.domainLinkApiPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub); + this.domainLinkApiPool = channelPoolFactory.createSingle(ServiceId.Query, IndexDomainLinksApiGrpc::newBlockingStub); } @CheckReturnValue @@ -59,27 +51,28 @@ public class QueryClient { public AllLinks getAllDomainLinks() { AllLinks links = new AllLinks(); - domainApi(0).getAllLinks(Empty.newBuilder().build()).forEachRemaining(pairs -> { - for (int i = 0; i < pairs.getDestIdsCount(); i++) { - links.add(pairs.getSourceIds(i), pairs.getDestIds(i)); - } - }); + domainLinkApiPool.api() + .getAllLinks(Empty.getDefaultInstance()) + .forEachRemaining(pairs -> { + for (int i = 0; i < pairs.getDestIdsCount(); i++) { + links.add(pairs.getSourceIds(i), pairs.getDestIds(i)); + } + }); return links; } public List getLinksToDomain(int domainId) { try { - return domainLinkApiPool.callEachSequential( - api -> api.getLinksToDomain(RpcDomainId + return domainLinkApiPool.api() + .getLinksToDomain(RpcDomainId .newBuilder() .setDomainId(domainId) .build()) - .getDomainIdList()) - .flatMap(List::stream) + .getDomainIdList() + .stream() .sorted() .toList(); - } catch (Exception e) { logger.error("API Exception", e); @@ -89,13 +82,13 @@ public class QueryClient { public List getLinksFromDomain(int domainId) { try { - return domainLinkApiPool.callEachSequential( - api -> api.getLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getDomainIdList()) - .flatMap(List::stream) + return domainLinkApiPool.api() + .getLinksFromDomain(RpcDomainId + .newBuilder() + .setDomainId(domainId) + .build()) + .getDomainIdList() + .stream() .sorted() .toList(); } @@ -107,13 +100,12 @@ public class QueryClient { public int countLinksToDomain(int domainId) { try { - return domainLinkApiPool.callEachSequential( - api -> api.countLinksToDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()).getIdCount()) - .mapToInt(Integer::valueOf) - .sum(); + return domainLinkApiPool.api() + .countLinksToDomain(RpcDomainId + .newBuilder() + .setDomainId(domainId) + .build()) + .getIdCount(); } catch (Exception e) { logger.error("API Exception", e); @@ -123,13 +115,12 @@ public class QueryClient { public int countLinksFromDomain(int domainId) { try { - return domainLinkApiPool.callEachSequential( - api -> api.countLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()).getIdCount()) - .mapToInt(Integer::valueOf) - .sum(); + return domainLinkApiPool.api() + .countLinksFromDomain(RpcDomainId + .newBuilder() + .setDomainId(domainId) + .build()) + .getIdCount(); } catch (Exception e) { logger.error("API Exception", e); From 904f2587cdeba939032756fc869d3708e0cd2092 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:44:26 +0100 Subject: [PATCH 12/56] (minor) Add default ZOOKEEPER_HOSTS to service.env --- run/env/service.env | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run/env/service.env b/run/env/service.env index 13151094..d94f62a9 100644 --- a/run/env/service.env +++ b/run/env/service.env @@ -1,8 +1,9 @@ WMSA_HOME=run/ -JAVA_OPTS="--enable-preview -da -Dservice.bind-address=0.0.0.0 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" +JAVA_OPTS="--enable-preview -da -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" JAVA_TOOL_OPTIONS="--enable-preview -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5000" EXECUTOR_SERVICE_OPTS="-DdistPath=/dist" CONVERTER_PROCESS_OPTS="-Dservice-name=converter" CRAWLER_PROCESS_OPTS="-Dservice-name=crawler" LOADER_PROCESS_OPTS="-Dservice-name=loader" -INDEX_CONSTRUCTION_PROCESS_OPTS="-Dservice-name=index-constructor -Djava.util.concurrent.ForkJoinPool.common.parallelism=4" \ No newline at end of file +INDEX_CONSTRUCTION_PROCESS_OPTS="-Dservice-name=index-constructor -Djava.util.concurrent.ForkJoinPool.common.parallelism=4" +ZOOKEEPER_HOSTS="zookeeper:2181" From 453bd6064b6ae04d59c96792273cbbb52728ac28 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:45:16 +0100 Subject: [PATCH 13/56] (minor) Add warm-up to GrpcMultiNodeChannelPool to speed up the initial messages Without doing this, connections would be created lazily, which is probably never desirable. --- .../marginalia/service/client/GrpcMultiNodeChannelPool.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index e816bd1a..2552dcf0 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -40,6 +40,11 @@ public class GrpcMultiNodeChannelPool { this.channelConstructor = channelConstructor; this.stubConstructor = stubConstructor; this.nodeConfigurationWatcher = nodeConfigurationWatcher; + + // Warm up the pool to reduce latency for the initial request + for (var node : nodeConfigurationWatcher.getQueryNodes()) { + apiForNode(node); + } } From d1858582663ab77f33f9122013427d5e075dc7e0 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 15:49:43 +0100 Subject: [PATCH 14/56] (minor) Add missing query parameter to ServiceEndpoint.toURL --- .../marginalia/service/discovery/property/ServiceEndpoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java index 01caba90..e8ce621e 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -40,7 +40,7 @@ public sealed interface ServiceEndpoint { @SneakyThrows public URL toURL(String endpoint, String query) { - return new URI("http", null, host, port, endpoint, null, null) + return new URI("http", null, host, port, endpoint, query, null) .toURL(); } From 273aeb7bae55505a13531a20aba88ba7ed7aa44c Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 16:06:05 +0100 Subject: [PATCH 15/56] (doc) Update documentation with new gRPC service setup --- code/common/service/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/common/service/readme.md b/code/common/service/readme.md index 17f07d15..04d216e2 100644 --- a/code/common/service/readme.md +++ b/code/common/service/readme.md @@ -39,7 +39,7 @@ public class FoobarService extends Service { @Inject public FoobarService(BaseServiceParams params) { - super(params); + super(params, List.of(/* grpc services */)); // set up Spark endpoints here } From 6c764bceebb995d34c5503195fbccb61249bd330 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 16:09:49 +0100 Subject: [PATCH 16/56] (doc) Update documentation for `service-discovery` --- code/common/service-discovery/readme.md | 28 ++++++++++--------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/code/common/service-discovery/readme.md b/code/common/service-discovery/readme.md index 617bdf3f..3436db6a 100644 --- a/code/common/service-discovery/readme.md +++ b/code/common/service-discovery/readme.md @@ -61,20 +61,7 @@ for (var endpoint : endpoints) { It's also possible to subscribe to changes in the registry, so that the caller can be notified when a service comes or goes, with `registry.registerMonitor()`. -## gRPC Channel Pool - -From the GrpcChannelPoolFactory, two types of channel pools can be created -that are aware of the service registry: - -* GrpcMultiNodeChannelPool - This pool permits 1-n style communication with partitioned services -* GrpcSingleNodeChannelPool - This pool permits 1-1 style communication with non-partitioned services - -The pools manage the lifecycle of the gRPC channels, and will permit the caller -to access Stub interfaces for the services. - -# Central Classes - -## Service Registry +### Central Classes * [ServiceRegistryIf](src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java) * [ZkServiceRegistry](src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java) @@ -82,6 +69,13 @@ to access Stub interfaces for the services. ## gRPC Channel Pool -* [GrpcChannelPoolFactory](src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java) -* [GrpcMultiNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java) -* [GrpcSingleNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java) \ No newline at end of file +From the [GrpcChannelPoolFactory](src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java), two types of channel pools can be created +that are aware of the service registry: + +* [GrpcMultiNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java) - This pool permits 1-n style communication with partitioned services +* [GrpcSingleNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java) - This pool permits 1-1 style communication with non-partitioned services. + if multiple instances are running, it will use one of them and fall back + to another if the first is not available. + +The pools manage the lifecycle of the gRPC channels, and will permit the caller +to access Stub interfaces for the services. From a69c0b2718dd17fb275cd6e3d6961dbf6a5f5241 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 18:03:57 +0100 Subject: [PATCH 17/56] (grpc-client) Fix warmup crash The warmup would sometimes crash during a cold start-up, because it could not get an API. Changed the warmup to just create a GrpcSingleNodeChannelPool for the node. --- .../client/GrpcMultiNodeChannelPool.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index 2552dcf0..f3916338 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -43,22 +43,25 @@ public class GrpcMultiNodeChannelPool { // Warm up the pool to reduce latency for the initial request for (var node : nodeConfigurationWatcher.getQueryNodes()) { - apiForNode(node); + getPoolForNode(node); } } + private GrpcSingleNodeChannelPool getPoolForNode(int node) { + return pools.computeIfAbsent(node, _ -> + new GrpcSingleNodeChannelPool<>( + serviceRegistryIf, + serviceId, + new NodeSelectionStrategy.Just(node), + channelConstructor, + stubConstructor)); + } + /** Get an API stub for the given node */ public STUB apiForNode(int node) { - return pools.computeIfAbsent(node, _ -> - new GrpcSingleNodeChannelPool<>( - serviceRegistryIf, - serviceId, - new NodeSelectionStrategy.Just(node), - channelConstructor, - stubConstructor) - ).api(); + return pools.computeIfAbsent(node, this::getPoolForNode).api(); } From 73947d9ecae0287f0e0aed17d8944abfbef8ff82 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 20 Feb 2024 18:09:11 +0100 Subject: [PATCH 18/56] (zk-registry) Filter out phantom addresses in the registry The change adds a hostname validation step to remove endpoints from the ZkServiceRegistry when they do not resolve. This is a scenario that primarily happens when running in docker, and the entire system is started and stopped. --- .../service/discovery/ZkServiceRegistry.java | 29 +++++++++++++++---- .../discovery/property/ServiceEndpoint.java | 20 +++++++++++-- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java index a75d1f8d..48efc64f 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -192,10 +192,18 @@ public class ZkServiceRegistry implements ServiceRegistryIf { .getData() .forPath(ZKPaths.makePath(restRoot, uuid)); String hostAndPort = new String(data); - ret.add(RestEndpoint + var address = RestEndpoint .parse(hostAndPort) - .asInstance(UUID.fromString(uuid)) - ); + .asInstance(UUID.fromString(uuid)); + + // Ensure that the address is resolvable + // (this reduces the risk of exceptions when trying to connect to the service) + if (!address.endpoint().validateHost()) { + logger.warn("Omitting stale address {}, address does not resolve", address); + continue; + } + + ret.add(address); } @@ -221,11 +229,20 @@ public class ZkServiceRegistry implements ServiceRegistryIf { byte[] data = curatorFramework .getData() .forPath(ZKPaths.makePath(restRoot, uuid)); + String hostAndPort = new String(data); - ret.add(GrpcEndpoint + var address = GrpcEndpoint .parse(hostAndPort) - .asInstance(UUID.fromString(uuid)) - ); + .asInstance(UUID.fromString(uuid)); + + // Ensure that the address is resolvable + // (this reduces the risk of exceptions when trying to connect to the service) + if (!address.endpoint().validateHost()) { + logger.warn("Omitting stale address {}, address does not resolve", address); + continue; + } + + ret.add(address); } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java index e8ce621e..7763eca0 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -3,9 +3,7 @@ package nu.marginalia.service.discovery.property; import lombok.SneakyThrows; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URL; +import java.net.*; import java.util.UUID; public sealed interface ServiceEndpoint { @@ -17,6 +15,22 @@ public sealed interface ServiceEndpoint { return new InetSocketAddress(host(), port()); } + /** Validate the host by checking if it is a valid IP address or a hostname that can be resolved. + * + * @return true if the host is a valid + */ + default boolean validateHost() { + try { + // Throws UnknownHostException if the host is not a valid IP address or hostname + // (this should not be slow since the DNS lookup should be local, and if it isn't; + // should be cached by the OS or the JVM) + InetAddress.getByName(host()); + return true; + } catch (UnknownHostException e) { + return false; + } + } + static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) { return switch (schema) { case REST -> new RestEndpoint(host, port); From 66c1281301502c142942af4cb8c85ea52003e123 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 14:01:23 +0100 Subject: [PATCH 19/56] (zk-registry) epic jak shaving WIP Cleaning out a lot of old junk from the code, and one thing lead to another... * Build is improved, now constructing docker images with 'jib'. Clean build went from 3 minutes to 50 seconds. * The ProcessService's spawning is smarter. Will now just spawn a java process instead of relying on the application plugin's generated outputs. * Project is migrated to GraalVM * gRPC clients are re-written with a neat fluent/functional style. e.g. ```channelPool.call(grpcStub::method) .async(executor) // <-- optional .run(argument); ``` This change is primarily to allow handling ManagedChannel errors, but it turned out to be a pretty clean API overall. * For now the project is all in on zookeeper * Service discovery is now based on APIs and not services. Theoretically means we could ship the same code either a monolith or a service mesh. * To this end, began modularizing a few of the APIs so that they aren't strongly "living" in a service. WIP! Missing is documentation and testing, and some more breaking apart of code. --- build.gradle | 10 +- code/api/assistant-api/readme.md | 8 - .../assistant/client/AssistantClient.java | 159 ------------ .../executor/client/ExecutorClient.java | 197 ++++++++------- .../src/main/protobuf/index-api.proto | 22 -- .../marginalia/query/client/QueryClient.java | 127 +--------- .../src/main/java/nu/marginalia/WmsaHome.java | 44 ++-- code/common/service-discovery/build.gradle | 7 + .../service/ServiceDiscoveryModule.java | 18 +- .../ServiceHomeNotConfiguredException.java | 4 - .../client/GrpcChannelPoolFactory.java | 24 +- .../client/GrpcMultiNodeChannelPool.java | 114 +++++---- .../client/GrpcSingleNodeChannelPool.java | 230 +++++++++++------ .../service/client/NodeSelectionStrategy.java | 17 -- .../client/ServiceNotAvailableException.java | 9 +- .../discovery/FixedServiceRegistry.java | 122 --------- .../service/discovery/ServiceRegistryIf.java | 38 ++- .../service/discovery/ZkServiceRegistry.java | 235 +++++++----------- .../monitor/ServiceChangeMonitor.java | 20 +- .../ServiceGrpcEndpointChangeMonitor.java | 25 -- .../discovery/monitor/ServiceMonitorIf.java | 9 +- .../monitor/ServiceNodeChangeMonitor.java | 25 -- .../ServiceRestEndpointChangeMonitor.java | 25 -- .../service/discovery/property/ApiSchema.java | 6 - .../discovery/property/PartitionTraits.java | 8 + .../discovery/property/ServiceEndpoint.java | 82 ++---- .../discovery/property/ServiceKey.java | 69 +++++ .../discovery/property/ServicePartition.java | 29 +++ .../discovery/ZkServiceRegistryTest.java | 93 +++++-- .../service/control/ServiceHeartbeatImpl.java | 2 + .../service/module/DatabaseModule.java | 15 +- .../service/module/ServiceConfiguration.java | 1 + .../module/ServiceConfigurationModule.java | 11 +- .../nu/marginalia/service/server/Service.java | 53 ++-- .../src/main/resources/log4j2-prod.xml | 2 + .../src/main/resources/log4j2-test.xml | 2 + .../domain-ranking/build.gradle | 2 +- .../ranking/data/InvertedLinkGraphSource.java | 9 +- .../ranking/data/LinkGraphSource.java | 10 +- .../RankingAlgorithmsContainerTest.java | 19 +- code/functions/domain-info/api/build.gradle | 45 ++++ .../api/domains/DomainInfoClient.java | 56 +++++ .../api/domains/DomainsProtobufCodec.java} | 66 +---- .../api/domains}/model/DomainInformation.java | 2 +- .../api/domains}/model/SimilarDomain.java | 2 +- .../api/src/main/protobuf/domain-info.proto} | 14 +- code/functions/domain-info/build.gradle | 44 ++++ .../domains/DomainInfoGrpcService.java | 53 ++++ .../domains/DomainInformationService.java | 16 +- .../domains/SimilarDomainsService.java | 18 +- .../domain-links/aggregate/build.gradle | 36 +++ .../AggregateDomainLinksService.java | 96 +++++++ code/functions/domain-links/api/build.gradle | 46 ++++ .../AggregateDomainLinksClient.java | 138 ++++++++++ .../PartitionDomainLinksClient.java | 30 +++ .../api/src/main/protobuf/domain-links.proto | 29 +++ .../domain-links/partition/build.gradle | 42 ++++ .../PartitionDomainLinksService.java} | 13 +- .../math/api}/build.gradle | 2 + .../nu/marginalia/api/math/MathClient.java | 90 +++++++ .../api/math/MathProtobufCodec.java | 66 +++++ .../api/math}/model/DictionaryEntry.java | 2 +- .../api/math}/model/DictionaryResponse.java | 2 +- .../math/api/src/main/protobuf/math-api.proto | 57 +++++ code/functions/math/build.gradle | 32 +++ .../functions/math/MathGrpcService.java} | 57 +---- .../math}/dict/DictionaryService.java | 6 +- .../functions/math}/dict/SpellChecker.java | 2 +- .../functions/math}/eval/MathParser.java | 2 +- .../marginalia/functions/math}/eval/Unit.java | 2 +- .../functions/math}/eval/Units.java | 2 +- .../mq/persistence/MqPersistence.java | 8 + .../index/IndexConstructorMain.java | 1 + .../build.gradle | 2 +- .../adjacencies/AdjacenciesData.java | 7 +- .../WebsiteAdjacenciesCalculator.java | 20 +- .../api-service/build.gradle | 18 +- .../dating-service/build.gradle | 19 +- .../explorer-service/build.gradle | 19 +- .../search-service/build.gradle | 23 +- .../nu/marginalia/search/SearchOperator.java | 10 +- .../command/commands/DefinitionCommand.java | 16 +- .../command/commands/SearchCommand.java | 8 +- .../search/svc/SearchBrowseService.java | 14 +- .../search/svc/SearchSiteInfoService.java | 27 +- .../svc/SearchUnitConversionService.java | 14 +- .../assistant-service/build.gradle | 26 +- .../assistant/AssistantService.java | 10 +- .../assistant/suggest/Suggestions.java | 2 +- .../control-service/build.gradle | 20 +- .../nu/marginalia/control/ControlService.java | 1 + .../executor-service/build.gradle | 32 ++- .../actor/task/ExportDataActor.java | 10 +- .../marginalia/executor/ExecutorModule.java | 4 +- .../nu/marginalia/executor/ExecutorSvc.java | 13 +- .../nu/marginalia/process/ProcessService.java | 90 +++---- code/services-core/index-service/build.gradle | 24 +- .../nu/marginalia/index/IndexService.java | 11 +- code/services-core/query-service/build.gradle | 22 +- .../query/QueryGRPCDomainLinksService.java | 90 ------- .../nu/marginalia/query/QueryGRPCService.java | 53 ++-- .../nu/marginalia/query/QueryService.java | 5 +- .../screenshot-capture-tool/build.gradle | 18 +- docker-service-with-dist.gradle | 72 ------ docker-service.gradle | 50 ---- settings.gradle | 16 +- 106 files changed, 2082 insertions(+), 1663 deletions(-) delete mode 100644 code/api/assistant-api/readme.md delete mode 100644 code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java delete mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java create mode 100644 code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java create mode 100644 code/functions/domain-info/api/build.gradle create mode 100644 code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java rename code/{api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java => functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java} (51%) rename code/{api/assistant-api/src/main/java/nu/marginalia/assistant/client => functions/domain-info/api/src/main/java/nu/marginalia/api/domains}/model/DomainInformation.java (96%) rename code/{api/assistant-api/src/main/java/nu/marginalia/assistant/client => functions/domain-info/api/src/main/java/nu/marginalia/api/domains}/model/SimilarDomain.java (97%) rename code/{api/assistant-api/src/main/protobuf/assistant-api.proto => functions/domain-info/api/src/main/protobuf/domain-info.proto} (77%) create mode 100644 code/functions/domain-info/build.gradle create mode 100644 code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/domain-info/src/main/java/nu/marginalia/functions}/domains/DomainInformationService.java (87%) rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/domain-info/src/main/java/nu/marginalia/functions}/domains/SimilarDomainsService.java (95%) create mode 100644 code/functions/domain-links/aggregate/build.gradle create mode 100644 code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java create mode 100644 code/functions/domain-links/api/build.gradle create mode 100644 code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java create mode 100644 code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java create mode 100644 code/functions/domain-links/api/src/main/protobuf/domain-links.proto create mode 100644 code/functions/domain-links/partition/build.gradle rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java => functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java} (88%) rename code/{api/assistant-api => functions/math/api}/build.gradle (96%) create mode 100644 code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java create mode 100644 code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java rename code/{api/assistant-api/src/main/java/nu/marginalia/assistant/client => functions/math/api/src/main/java/nu/marginalia/api/math}/model/DictionaryEntry.java (84%) rename code/{api/assistant-api/src/main/java/nu/marginalia/assistant/client => functions/math/api/src/main/java/nu/marginalia/api/math}/model/DictionaryResponse.java (86%) create mode 100644 code/functions/math/api/src/main/protobuf/math-api.proto create mode 100644 code/functions/math/build.gradle rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java => functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java} (53%) rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/math/src/main/java/nu/marginalia/functions/math}/dict/DictionaryService.java (89%) rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/math/src/main/java/nu/marginalia/functions/math}/dict/SpellChecker.java (91%) rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/math/src/main/java/nu/marginalia/functions/math}/eval/MathParser.java (99%) rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/math/src/main/java/nu/marginalia/functions/math}/eval/Unit.java (86%) rename code/{services-core/assistant-service/src/main/java/nu/marginalia/assistant => functions/math/src/main/java/nu/marginalia/functions/math}/eval/Units.java (98%) delete mode 100644 code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java delete mode 100644 docker-service-with-dist.gradle delete mode 100644 docker-service.gradle diff --git a/build.gradle b/build.gradle index 9eb4ef2f..12ab5b31 100644 --- a/build.gradle +++ b/build.gradle @@ -3,6 +3,10 @@ plugins { id("org.jetbrains.gradle.plugin.idea-ext") version "1.0" id "io.freefair.lombok" version "8.3" id "me.champeau.jmh" version "0.6.6" + + // This is a workaround for a bug in the Jib plugin that causes it to stall randomly + // https://github.com/GoogleContainerTools/jib/issues/3347 + id 'com.google.cloud.tools.jib' version '3.4.0' apply(false) } group 'marginalia' @@ -29,11 +33,12 @@ subprojects.forEach {it -> reproducibleFileOrder = true } } - +ext { + dockerImageBase='container-registry.oracle.com/graalvm/jdk:21@sha256:1fd33d4d4eba3a9e1a41a728e39ea217178d257694eea1214fec68d2ed4d3d9b' +} allprojects { apply plugin: 'java' apply plugin: 'io.freefair.lombok' - dependencies { implementation libs.lombok testImplementation libs.lombok @@ -77,3 +82,4 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } + diff --git a/code/api/assistant-api/readme.md b/code/api/assistant-api/readme.md deleted file mode 100644 index 7a35a592..00000000 --- a/code/api/assistant-api/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# Assistant API - -Client and models for talking to the [assistant-service](../../services-core/assistant-service), -implemented with the base client from [service-client](../../common/service-client). - -## Central Classes - -* [AssistantClient](src/main/java/nu/marginalia/assistant/client/AssistantClient.java) \ No newline at end of file diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java b/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java deleted file mode 100644 index a311aab1..00000000 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java +++ /dev/null @@ -1,159 +0,0 @@ -package nu.marginalia.assistant.client; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.assistant.api.AssistantApiGrpc; -import nu.marginalia.assistant.client.model.DictionaryResponse; -import nu.marginalia.assistant.client.model.DomainInformation; -import nu.marginalia.assistant.client.model.SimilarDomain; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.client.GrpcSingleNodeChannelPool; -import nu.marginalia.service.id.ServiceId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.*; - -import static nu.marginalia.assistant.client.AssistantProtobufCodec.*; - -@Singleton -public class AssistantClient { - private static final Logger logger = LoggerFactory.getLogger(AssistantClient.class); - - private final GrpcSingleNodeChannelPool channelPool; - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); - @Inject - public AssistantClient(GrpcChannelPoolFactory factory) { - this.channelPool = factory.createSingle(ServiceId.Assistant, AssistantApiGrpc::newBlockingStub); - - } - - public Future dictionaryLookup(String word) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().dictionaryLookup( - DictionaryLookup.createRequest(word) - ); - - return DictionaryLookup.convertResponse(rsp); - }); - } - - @SuppressWarnings("unchecked") - public Future> spellCheck(String word) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().spellCheck( - SpellCheck.createRequest(word) - ); - - return SpellCheck.convertResponse(rsp); - }); - } - - public Map> spellCheck(List words, Duration timeout) throws InterruptedException { - List>>> tasks = new ArrayList<>(); - - for (String w : words) { - tasks.add(() -> { - var rsp = channelPool.api().spellCheck( - SpellCheck.createRequest(w) - ); - - return Map.entry(w, SpellCheck.convertResponse(rsp)); - }); - } - - var futures = virtualExecutorService.invokeAll(tasks, timeout.toMillis(), TimeUnit.MILLISECONDS); - Map> results = new HashMap<>(); - - for (var f : futures) { - if (!f.isDone()) - continue; - - var entry = f.resultNow(); - - results.put(entry.getKey(), entry.getValue()); - } - - return results; - } - - public Future unitConversion(String value, String from, String to) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().unitConversion( - UnitConversion.createRequest(from, to, value) - ); - - return UnitConversion.convertResponse(rsp); - }); - } - - public Future evalMath(String expression) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().evalMath( - EvalMath.createRequest(expression) - ); - - return EvalMath.convertResponse(rsp); - }); - } - - public Future> similarDomains(int domainId, int count) { - return virtualExecutorService.submit(() -> { - try { - var rsp = channelPool.api().getSimilarDomains( - DomainQueries.createRequest(domainId, count) - ); - - return DomainQueries.convertResponse(rsp); - } - catch (Exception e) { - logger.warn("Failed to get similar domains", e); - - throw e; - } - }); - } - - public Future> linkedDomains(int domainId, int count) { - return virtualExecutorService.submit(() -> { - try { - var rsp = channelPool.api().getLinkingDomains( - DomainQueries.createRequest(domainId, count) - ); - - return DomainQueries.convertResponse(rsp); - } - catch (Exception e) { - logger.warn("Failed to get linked domains", e); - throw e; - } - }); - - } - - public Future domainInformation(int domainId) { - return virtualExecutorService.submit(() -> { - try { - var rsp = channelPool.api().getDomainInfo( - DomainInfo.createRequest(domainId) - ); - - return DomainInfo.convertResponse(rsp); - } - catch (Exception e) { - logger.warn("Failed to get domain information", e); - - throw e; - } - }); - } - - public boolean isAccepting() { - return channelPool.hasChannel(); - } -} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 3c585263..225e0518 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -13,7 +13,8 @@ import nu.marginalia.executor.upload.UploadDirContents; import nu.marginalia.executor.upload.UploadDirItem; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import nu.marginalia.storage.model.FileStorageId; @@ -22,6 +23,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; +import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -39,180 +41,190 @@ public class ExecutorClient { { this.registry = registry; this.channelPool = grpcChannelPoolFactory - .createMulti(ServiceId.Executor, ExecutorApiGrpc::newBlockingStub); + .createMulti( + ServiceKey.forGrpcApi(ExecutorApiGrpc.class, ServicePartition.multi()), + ExecutorApiGrpc::newBlockingStub); } public void startFsm(int node, String actorName) { - channelPool.apiForNode(node).startFsm( - RpcFsmName.newBuilder() + channelPool.call(ExecutorApiBlockingStub::startFsm) + .forNode(node) + .run(RpcFsmName.newBuilder() .setActorName(actorName) - .build() - ); + .build()); + } public void stopFsm(int node, String actorName) { - channelPool.apiForNode(node).stopFsm( - RpcFsmName.newBuilder() + channelPool.call(ExecutorApiBlockingStub::stopFsm) + .forNode(node) + .run(RpcFsmName.newBuilder() .setActorName(actorName) - .build() - ); + .build()); } public void stopProcess(int node, String id) { - channelPool.apiForNode(node).stopProcess( - RpcProcessId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::stopProcess) + .forNode(node) + .run(RpcProcessId.newBuilder() .setProcessId(id) - .build() - ); + .build()); + } public void triggerCrawl(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerCrawl( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerCrawl) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void triggerRecrawl(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerRecrawl( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerRecrawl) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void triggerConvert(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerConvert( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerConvert) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void triggerConvertAndLoad(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerConvertAndLoad( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerConvertAndLoad) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void loadProcessedData(int node, List ids) { - channelPool.apiForNode(node).loadProcessedData( - RpcFileStorageIds.newBuilder() + channelPool.call(ExecutorApiBlockingStub::loadProcessedData) + .forNode(node) + .run(RpcFileStorageIds.newBuilder() .addAllFileStorageIds(ids.stream().map(FileStorageId::id).toList()) - .build() - ); + .build()); } public void calculateAdjacencies(int node) { - channelPool.apiForNode(node).calculateAdjacencies(Empty.getDefaultInstance()); + channelPool.call(ExecutorApiBlockingStub::calculateAdjacencies) + .forNode(node) + .run(Empty.getDefaultInstance()); } public void sideloadEncyclopedia(int node, Path sourcePath, String baseUrl) { - channelPool.apiForNode(node).sideloadEncyclopedia( - RpcSideloadEncyclopedia.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadEncyclopedia) + .forNode(node) + .run(RpcSideloadEncyclopedia.newBuilder() .setBaseUrl(baseUrl) .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadDirtree(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadDirtree( - RpcSideloadDirtree.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadDirtree) + .forNode(node) + .run(RpcSideloadDirtree.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadReddit(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadReddit( - RpcSideloadReddit.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadReddit) + .forNode(node) + .run(RpcSideloadReddit.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadWarc(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadWarc( - RpcSideloadWarc.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadWarc) + .forNode(node) + .run(RpcSideloadWarc.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadStackexchange(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadStackexchange( - RpcSideloadStackexchange.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadStackexchange) + .forNode(node) + .run(RpcSideloadStackexchange.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void createCrawlSpecFromDownload(int node, String description, String url) { - channelPool.apiForNode(node).createCrawlSpecFromDownload( - RpcCrawlSpecFromDownload.newBuilder() + channelPool.call(ExecutorApiBlockingStub::createCrawlSpecFromDownload) + .forNode(node) + .run(RpcCrawlSpecFromDownload.newBuilder() .setDescription(description) .setUrl(url) - .build() - ); + .build()); } public void exportAtags(int node, FileStorageId fid) { - channelPool.apiForNode(node).exportAtags( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportAtags) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void exportSampleData(int node, FileStorageId fid, int size, String name) { - channelPool.apiForNode(node).exportSampleData( - RpcExportSampleData.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportSampleData) + .forNode(node) + .run(RpcExportSampleData.newBuilder() .setFileStorageId(fid.id()) .setSize(size) .setName(name) - .build() - ); + .build()); } public void exportRssFeeds(int node, FileStorageId fid) { - channelPool.apiForNode(node).exportRssFeeds( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportRssFeeds) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void exportTermFrequencies(int node, FileStorageId fid) { - channelPool.apiForNode(node).exportTermFrequencies( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportTermFrequencies) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void downloadSampleData(int node, String sampleSet) { - channelPool.apiForNode(node).downloadSampleData( - RpcDownloadSampleData.newBuilder() + channelPool.call(ExecutorApiBlockingStub::downloadSampleData) + .forNode(node) + .run(RpcDownloadSampleData.newBuilder() .setSampleSet(sampleSet) - .build() - ); + .build()); } public void exportData(int node) { - channelPool.apiForNode(node).exportData(Empty.getDefaultInstance()); + channelPool.call(ExecutorApiBlockingStub::exportData) + .forNode(node) + .run(Empty.getDefaultInstance()); } public void restoreBackup(int node, FileStorageId fid) { - channelPool.apiForNode(node).restoreBackup( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::restoreBackup) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public ActorRunStates getActorStates(int node) { try { - var rs = channelPool.apiForNode(node).getActorStates(Empty.getDefaultInstance()); + var rs = channelPool.call(ExecutorApiBlockingStub::getActorStates) + .forNode(node) + .run(Empty.getDefaultInstance()); var states = rs.getActorRunStatesList().stream() .map(r -> new ActorRunState( r.getActorName(), @@ -236,7 +248,9 @@ public class ExecutorClient { public UploadDirContents listSideloadDir(int node) { try { - var rs = channelPool.apiForNode(node).listSideloadDir(Empty.getDefaultInstance()); + var rs = channelPool.call(ExecutorApiBlockingStub::listSideloadDir) + .forNode(node) + .run(Empty.getDefaultInstance()); var items = rs.getEntriesList().stream() .map(i -> new UploadDirItem(i.getName(), i.getLastModifiedTime(), i.getIsDirectory(), i.getSize())) .toList(); @@ -252,11 +266,12 @@ public class ExecutorClient { public FileStorageContent listFileStorage(int node, FileStorageId fileId) { try { - var rs = channelPool.apiForNode(node).listFileStorage( - RpcFileStorageId.newBuilder() + var rs = channelPool.call(ExecutorApiBlockingStub::listFileStorage) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fileId.id()) .build() - ); + ); return new FileStorageContent(rs.getEntriesList().stream() .map(e -> new FileStorageFile(e.getName(), e.getSize(), e.getLastModifiedTime())) @@ -274,13 +289,13 @@ public class ExecutorClient { String uriPath = STR."/transfer/file/\{fileId.id()}"; String uriQuery = STR."path=\{URLEncoder.encode(path, StandardCharsets.UTF_8)}"; - var service = registry.getEndpoints(ApiSchema.REST, ServiceId.Executor, node) + var service = registry.getEndpoints(ServiceKey.forRest(ServiceId.Executor, node)) .stream().findFirst().orElseThrow(); try (var urlStream = service.endpoint().toURL(uriPath, uriQuery).openStream()) { urlStream.transferTo(destOutputStream); } - catch (IOException ex) { + catch (IOException | URISyntaxException ex) { throw new RuntimeException(ex); } } diff --git a/code/api/index-api/src/main/protobuf/index-api.proto b/code/api/index-api/src/main/protobuf/index-api.proto index ad05152e..521362f5 100644 --- a/code/api/index-api/src/main/protobuf/index-api.proto +++ b/code/api/index-api/src/main/protobuf/index-api.proto @@ -4,28 +4,6 @@ package actorapi; option java_package="nu.marginalia.index.api"; option java_multiple_files=true; -service IndexDomainLinksApi { - rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {} - rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {} - rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {} - rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {} - rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {} -} - -message RpcDomainId { - int32 domainId = 1; -} -message RpcDomainIdList { - repeated int32 domainId = 1 [packed=true]; -} -message RpcDomainIdCount { - int32 idCount = 1; -} -message RpcDomainIdPairs { - repeated int32 sourceIds = 1 [packed=true]; - repeated int32 destIds = 2 [packed=true]; -} - service QueryApi { rpc query(RpcQsQuery) returns (RpcQsResponse) {} } diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java index 74f78aa7..616b84c2 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java @@ -9,14 +9,12 @@ import nu.marginalia.query.model.QueryParams; import nu.marginalia.query.model.QueryResponse; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.client.GrpcSingleNodeChannelPool; -import nu.marginalia.service.id.ServiceId; -import org.roaringbitmap.longlong.PeekableLongIterator; -import org.roaringbitmap.longlong.Roaring64Bitmap; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.CheckReturnValue; -import java.util.List; @Singleton public class QueryClient { @@ -27,134 +25,25 @@ public class QueryClient { .register(); private final GrpcSingleNodeChannelPool queryApiPool; - private final GrpcSingleNodeChannelPool domainLinkApiPool; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject public QueryClient(GrpcChannelPoolFactory channelPoolFactory) { - this.queryApiPool = channelPoolFactory.createSingle(ServiceId.Query, QueryApiGrpc::newBlockingStub); - this.domainLinkApiPool = channelPoolFactory.createSingle(ServiceId.Query, IndexDomainLinksApiGrpc::newBlockingStub); + this.queryApiPool = channelPoolFactory.createSingle( + ServiceKey.forGrpcApi(QueryApiGrpc.class, ServicePartition.any()), + QueryApiGrpc::newBlockingStub); } @CheckReturnValue public QueryResponse search(QueryParams params) { var query = QueryProtobufCodec.convertQueryParams(params); - return wmsa_qs_api_search_time.time( - () -> QueryProtobufCodec.convertQueryResponse(queryApiPool - .importantCall((api) -> api.query(query)) + return wmsa_qs_api_search_time.time(() -> + QueryProtobufCodec.convertQueryResponse( + queryApiPool.call(QueryApiGrpc.QueryApiBlockingStub::query).run(query) ) ); } - public AllLinks getAllDomainLinks() { - AllLinks links = new AllLinks(); - - domainLinkApiPool.api() - .getAllLinks(Empty.getDefaultInstance()) - .forEachRemaining(pairs -> { - for (int i = 0; i < pairs.getDestIdsCount(); i++) { - links.add(pairs.getSourceIds(i), pairs.getDestIds(i)); - } - }); - - return links; - } - - public List getLinksToDomain(int domainId) { - try { - return domainLinkApiPool.api() - .getLinksToDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getDomainIdList() - .stream() - .sorted() - .toList(); - } - catch (Exception e) { - logger.error("API Exception", e); - return List.of(); - } - } - - public List getLinksFromDomain(int domainId) { - try { - return domainLinkApiPool.api() - .getLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getDomainIdList() - .stream() - .sorted() - .toList(); - } - catch (Exception e) { - logger.error("API Exception", e); - return List.of(); - } - } - - public int countLinksToDomain(int domainId) { - try { - return domainLinkApiPool.api() - .countLinksToDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getIdCount(); - } - catch (Exception e) { - logger.error("API Exception", e); - return 0; - } - } - - public int countLinksFromDomain(int domainId) { - try { - return domainLinkApiPool.api() - .countLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getIdCount(); - } - catch (Exception e) { - logger.error("API Exception", e); - return 0; - } - } - public static class AllLinks { - private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap(); - - public void add(int source, int dest) { - sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest)); - } - - public Iterator iterator() { - return new Iterator(); - } - - public class Iterator { - private final PeekableLongIterator base = sourceToDest.getLongIterator(); - long val = Long.MIN_VALUE; - - public boolean advance() { - if (base.hasNext()) { - val = base.next(); - return true; - } - return false; - } - public int source() { - return (int) (val >>> 32); - } - public int dest() { - return (int) (val & 0xFFFF_FFFFL); - } - } - } } diff --git a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java index e65c5529..b520f6e2 100644 --- a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java +++ b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java @@ -5,8 +5,7 @@ import nu.marginalia.service.ServiceHomeNotConfiguredException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Optional; +import java.util.Objects; import java.util.stream.Stream; public class WmsaHome { @@ -26,15 +25,28 @@ public class WmsaHome { } public static Path getHomePath() { - var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElseGet(WmsaHome::findDefaultHomePath); + String[] possibleLocations = new String[] { + System.getenv("WMSA_HOME"), + System.getProperty("system.homePath"), + "/var/lib/wmsa", + "/wmsa" + }; + + String retStr = Stream.of(possibleLocations) + .filter(Objects::nonNull) + .map(Path::of) + .filter(Files::isDirectory) + .map(Path::toString) + .findFirst() + .orElseThrow(() -> + new ServiceHomeNotConfiguredException(""" + Could not find $WMSA_HOME, either set environment + variable, the 'system.homePath' property, + or ensure either /wmssa or /var/lib/wmsa exists + """)); var ret = Path.of(retStr); - if (!Files.isDirectory(ret)) { - throw new ServiceHomeNotConfiguredException("Could not find $WMSA_HOME, either set environment variable or ensure " + retStr + " exists"); - } - - if (!Files.isDirectory(ret.resolve("model"))) { throw new ServiceHomeNotConfiguredException("You need to run 'run/setup.sh' to download models to run/ before this will work!"); } @@ -42,22 +54,6 @@ public class WmsaHome { return ret; } - private static String findDefaultHomePath() { - - // Assume this is a local developer and not a production system, since it would have WMSA_HOME set. - // Developers probably have a "run/" somewhere upstream from cwd. - // - - return Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent) - .filter(p -> Files.exists(p.resolve("run/env"))) - .filter(p -> Files.exists(p.resolve("run/setup.sh"))) - .map(p -> p.resolve("run")) - .findAny() - .orElse(Path.of("/var/lib/wmsa")) - .toString(); - } - - public static Path getAdsDefinition() { return getHomePath().resolve("data").resolve("adblock.txt"); } diff --git a/code/common/service-discovery/build.gradle b/code/common/service-discovery/build.gradle index c274b66a..d86ef9f4 100644 --- a/code/common/service-discovery/build.gradle +++ b/code/common/service-discovery/build.gradle @@ -6,6 +6,11 @@ plugins { repositories { mavenLocal() mavenCentral() + + repositories { + mavenCentral() + maven { url 'https://jitpack.io' } + } } java { @@ -18,6 +23,7 @@ dependencies { implementation libs.bundles.curator implementation libs.guice + implementation libs.bundles.gson implementation libs.bundles.mariadb implementation libs.bundles.grpc implementation libs.notnull @@ -29,4 +35,5 @@ dependencies { testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') testImplementation 'org.testcontainers:mariadb:1.17.4' testImplementation 'org.testcontainers:junit-jupiter:1.17.4' + testImplementation project(':code:functions:math:api') } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java index 161c2a9e..cc783c72 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java @@ -1,7 +1,6 @@ package nu.marginalia.service; import com.google.inject.AbstractModule; -import nu.marginalia.service.discovery.FixedServiceRegistry; import nu.marginalia.service.discovery.ServiceRegistryIf; import nu.marginalia.service.discovery.ZkServiceRegistry; import org.apache.curator.framework.CuratorFramework; @@ -18,18 +17,13 @@ public class ServiceDiscoveryModule extends AbstractModule { private static final Logger logger = LoggerFactory.getLogger(ServiceDiscoveryModule.class); public void configure() { - getZookeeperHosts().ifPresentOrElse((hosts) -> { - logger.info("Using Zookeeper service registry at {}", hosts); - CuratorFramework client = CuratorFrameworkFactory - .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000)); + var hosts = getZookeeperHosts().orElseThrow(() -> new IllegalStateException("Zookeeper hosts not set")); + logger.info("Using Zookeeper service registry at {}", hosts); + CuratorFramework client = CuratorFrameworkFactory + .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000)); - bind(CuratorFramework.class).toInstance(client); - bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); - }, - () -> { - logger.info("Using fixed service registry"); - bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class); - }); + bind(CuratorFramework.class).toInstance(client); + bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); } private Optional getZookeeperHosts() { diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java index 181428e1..504abdc0 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java @@ -1,10 +1,6 @@ package nu.marginalia.service; public class ServiceHomeNotConfiguredException extends RuntimeException { - - public ServiceHomeNotConfiguredException() { - super("WMSA_HOME environment variable not set"); - } public ServiceHomeNotConfiguredException(String message) { super(message); } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java index 087fc666..a4d0035a 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java @@ -6,8 +6,10 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import nu.marginalia.service.NodeConfigurationWatcher; import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import java.util.function.Function; @@ -26,30 +28,32 @@ public class GrpcChannelPoolFactory { } /** Create a new multi-node channel pool for the given service. */ - public GrpcMultiNodeChannelPool createMulti(ServiceId serviceId, + public GrpcMultiNodeChannelPool createMulti(ServiceKey key, Function stubConstructor) { return new GrpcMultiNodeChannelPool<>(serviceRegistryIf, - serviceId, + key, this::createChannel, stubConstructor, nodeConfigurationWatcher); } /** Create a new single-node channel pool for the given service. */ - public GrpcSingleNodeChannelPool createSingle(ServiceId serviceId, + public GrpcSingleNodeChannelPool createSingle(ServiceKey key, Function stubConstructor) { - return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, serviceId, - new NodeSelectionStrategy.Any(), - this::createChannel, - stubConstructor); + return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, key, this::createChannel, stubConstructor); } - private ManagedChannel createChannel(InstanceAddress route) { - return ManagedChannelBuilder + private ManagedChannel createChannel(InstanceAddress route) { + + var mc = ManagedChannelBuilder .forAddress(route.host(), route.port()) .usePlaintext() .build(); + + mc.getState(true); + + return mc; } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index f3916338..de76ecc9 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -4,13 +4,17 @@ import io.grpc.ManagedChannel; import lombok.SneakyThrows; import nu.marginalia.service.NodeConfigurationWatcher; import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; @@ -22,21 +26,20 @@ public class GrpcMultiNodeChannelPool { private final ConcurrentHashMap> pools = new ConcurrentHashMap<>(); private static final Logger logger = LoggerFactory.getLogger(GrpcMultiNodeChannelPool.class); - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); private final ServiceRegistryIf serviceRegistryIf; - private final ServiceId serviceId; - private final Function, ManagedChannel> channelConstructor; + private final ServiceKey serviceKey; + private final Function channelConstructor; private final Function stubConstructor; private final NodeConfigurationWatcher nodeConfigurationWatcher; @SneakyThrows public GrpcMultiNodeChannelPool(ServiceRegistryIf serviceRegistryIf, - ServiceId serviceId, - Function, ManagedChannel> channelConstructor, + ServiceKey serviceKey, + Function channelConstructor, Function stubConstructor, NodeConfigurationWatcher nodeConfigurationWatcher) { this.serviceRegistryIf = serviceRegistryIf; - this.serviceId = serviceId; + this.serviceKey = serviceKey; this.channelConstructor = channelConstructor; this.stubConstructor = stubConstructor; this.nodeConfigurationWatcher = nodeConfigurationWatcher; @@ -51,51 +54,74 @@ public class GrpcMultiNodeChannelPool { return pools.computeIfAbsent(node, _ -> new GrpcSingleNodeChannelPool<>( serviceRegistryIf, - serviceId, - new NodeSelectionStrategy.Just(node), + serviceKey.forPartition(ServicePartition.partition(node)), channelConstructor, stubConstructor)); } - - /** Get an API stub for the given node */ - public STUB apiForNode(int node) { - return pools.computeIfAbsent(node, this::getPoolForNode).api(); - } - - - /** Invoke a function on each node, returning a list of futures in a terminal state, as per - * ExecutorService$invokeAll */ - public List> invokeAll(Function> callF) throws InterruptedException { - List> calls = getEligibleNodes().stream() - .mapMulti(this::passNodeIfOk) - .map(callF) - .toList(); - - return virtualExecutorService.invokeAll(calls); - } - - /** Invoke a function on each node, returning a stream of results */ - public Stream callEachSequential(Function call) { - return getEligibleNodes().stream() - .mapMulti(this::passNodeIfOk) - .map(call); - } - - // Eat connectivity exceptions and log them when doing a broadcast-style calls - private void passNodeIfOk(Integer nodeId, Consumer consumer) { - try { - consumer.accept(apiForNode(nodeId)); - } - catch (Exception ex) { - logger.error("Error calling node {}", nodeId, ex); - } - } - /** Get the list of nodes that are eligible for broadcast-style requests */ public List getEligibleNodes() { return nodeConfigurationWatcher.getQueryNodes(); } + public CallBuilderBase call(BiFunction method) { + return new CallBuilderBase<>(method); + } + + public class CallBuilderBase { + private final BiFunction method; + + private CallBuilderBase(BiFunction method) { + this.method = method; + } + + public GrpcSingleNodeChannelPool.CallBuilderBase forNode(int node) { + return getPoolForNode(node).call(method); + } + + public List run(I arg) { + return getEligibleNodes().stream() + .map(node -> getPoolForNode(node).call(method).run(arg)) + .toList(); + } + + public CallBuilderAsync async(ExecutorService service) { + return new CallBuilderAsync<>(service, method); + } + } + + public class CallBuilderAsync { + private final Executor executor; + private final BiFunction method; + + public CallBuilderAsync(Executor executor, BiFunction method) { + this.executor = executor; + this.method = method; + } + + public CompletableFuture> runAll(I arg) { + var futures = getEligibleNodes().stream() + .map(GrpcMultiNodeChannelPool.this::getPoolForNode) + .map(pool -> + pool.call(method) + .async(executor) + .run(arg) + ).toList(); + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> futures.stream().map(CompletableFuture::join).toList()); + } + + public List> runEach(I arg) { + return getEligibleNodes().stream() + .map(GrpcMultiNodeChannelPool.this::getPoolForNode) + .map(pool -> + pool.call(method) + .async(executor) + .run(arg) + ).toList(); + + } + } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index 137e2933..68c33277 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -1,142 +1,232 @@ package nu.marginalia.service.client; import com.google.common.collect.Sets; -import io.grpc.ConnectivityState; import io.grpc.ManagedChannel; import lombok.SneakyThrows; import nu.marginalia.service.discovery.ServiceRegistryIf; import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor; -import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Function; /** A pool of gRPC channels for a service, with a separate channel for each node. *

* Manages unicast-style requests */ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { - private final Map, ManagedChannel> channels = new ConcurrentHashMap<>(); - private final Map>> routes = new ConcurrentHashMap<>(); + private final Map channels = new ConcurrentHashMap<>(); private static final Logger logger = LoggerFactory.getLogger(GrpcSingleNodeChannelPool.class); private final ServiceRegistryIf serviceRegistryIf; - private final ServiceId serviceId; - private final NodeSelectionStrategy nodeSelectionStrategy; - private final Function, ManagedChannel> channelConstructor; + private final Function channelConstructor; private final Function stubConstructor; + @SneakyThrows public GrpcSingleNodeChannelPool(ServiceRegistryIf serviceRegistryIf, - ServiceId serviceId, - NodeSelectionStrategy nodeSelectionStrategy, - Function, ManagedChannel> channelConstructor, + ServiceKey serviceKey, + Function channelConstructor, Function stubConstructor) { - super(serviceId); + super(serviceKey); this.serviceRegistryIf = serviceRegistryIf; - this.serviceId = serviceId; - this.nodeSelectionStrategy = nodeSelectionStrategy; this.channelConstructor = channelConstructor; this.stubConstructor = stubConstructor; serviceRegistryIf.registerMonitor(this); onChange(); + + awaitChannel(Duration.ofSeconds(5)); } @Override - public boolean onChange() { - switch (nodeSelectionStrategy) { - case NodeSelectionStrategy.Any() -> - serviceRegistryIf - .getServiceNodes(serviceId) - .forEach(this::refreshNode); - case NodeSelectionStrategy.Just(int node) -> - refreshNode(node); + public synchronized boolean onChange() { + Set newRoutes = serviceRegistryIf.getEndpoints(serviceKey); + Set oldRoutes = new HashSet<>(channels.keySet()); + + // Find the routes that have been added or removed + for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { + ConnectionHolder oldChannel; + if (newRoutes.contains(route)) { + logger.info("Adding route {}", route); + oldChannel = channels.put(route, new ConnectionHolder(route)); + } else { + logger.info("Expelling route {}", route); + oldChannel = channels.remove(route); + } + if (oldChannel != null) { + oldChannel.close(); + } } return true; } - private void refreshNode(int node) { + private class ConnectionHolder implements Comparable { + private final AtomicReference channel = new AtomicReference<>(); + private final InstanceAddress address; - Set> newRoutes = serviceRegistryIf.getEndpoints(ApiSchema.GRPC, serviceId, node); - Set> oldRoutes = routes.getOrDefault(node, Set.of()); - - // Find the routes that have been added or removed - for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { - - ManagedChannel oldChannel; - - if (newRoutes.contains(route)) { - var newChannel = channelConstructor.apply(route); - oldChannel = channels.put(route, newChannel); - } else { - oldChannel = channels.remove(route); - } - - if (oldChannel != null) - oldChannel.shutdown(); + ConnectionHolder(InstanceAddress address) { + this.address = address; } - routes.put(node, newRoutes); + public ManagedChannel get() { + var value = channel.get(); + if (value != null) { + return value; + } + + try { + logger.info("Creating channel for {}:{}", serviceKey, address); + value = channelConstructor.apply(address); + if (channel.compareAndSet(null, value)) { + return value; + } + else { + value.shutdown(); + return channel.get(); + } + } + catch (Exception e) { + logger.error(STR."Failed to get channel for \{address}", e); + return null; + } + } + + public void close() { + ManagedChannel mc = channel.getAndSet(null); + if (mc != null) { + mc.shutdown(); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConnectionHolder that = (ConnectionHolder) o; + return Objects.equals(address, that.address); + } + + + @Override + public int hashCode() { + return Objects.hash(address); + } + + @Override + public int compareTo(@NotNull GrpcSingleNodeChannelPool.ConnectionHolder o) { + + return -Long.compare(address.cxTime(), o.address.cxTime()); // Reverse order + } } + + public boolean hasChannel() { return !channels.isEmpty(); } - /** Get an API stub for the given node */ - public STUB api() { - return stubConstructor.apply(getChannel()); + public synchronized boolean awaitChannel(Duration timeout) throws InterruptedException { + if (hasChannel()) return true; + + final long endTime = System.currentTimeMillis() + timeout.toMillis(); + + while (!hasChannel()) { + long timeLeft = endTime - System.currentTimeMillis(); + if (timeLeft <= 0) return false; + this.wait(timeLeft); + } + return hasChannel(); } - /** Try to make the call go through. The function will cycle through - * available routes until exhaustion, and only then will it give up - */ - public T importantCall(Function function) { - for (int i = 0; i < channels.size(); i++) { + private T call(BiFunction call, I arg) throws RuntimeException { + final List exceptions = new ArrayList<>(); + final List connectionHolders = new ArrayList<>(channels.values()); + + // Randomize the order of the connection holders to spread out the load + Collections.shuffle(connectionHolders); + + for (var channel : connectionHolders) { try { - return function.apply(api()); + return call.apply(stubConstructor.apply(channel.get()), arg); } catch (Exception e) { - logger.error("API Exception", e); + exceptions.add(e); } } - throw new ServiceNotAvailableException(serviceId); + for (var e : exceptions) { + logger.error("Failed to call service {}", serviceKey, e); + } + + throw new ServiceNotAvailableException(serviceKey); } - /** Get the channel that is most ready to use */ - public ManagedChannel getChannel() { - return channels - .values() - .stream() - .min(this::compareChannelsByState) - .orElseThrow(() -> new ServiceNotAvailableException(serviceId)); + public CallBuilderBase call(BiFunction method) { + return new CallBuilderBase<>(method); } - /** Sort the channels by how ready they are to use */ - private int compareChannelsByState(ManagedChannel a, ManagedChannel b) { - var aState = a.getState(true); - var bState = b.getState(true); + public class CallBuilderBase { + private final BiFunction method; + private CallBuilderBase(BiFunction method) { + this.method = method; + } - if (aState == ConnectivityState.READY) return -1; - if (bState == ConnectivityState.READY) return 1; - if (aState == ConnectivityState.CONNECTING) return -1; - if (bState == ConnectivityState.CONNECTING) return 1; - if (aState == ConnectivityState.IDLE) return -1; - if (bState == ConnectivityState.IDLE) return 1; + public T run(I arg) { + return call(method, arg); + } - return 0; + public List runFor(I... args) { + return runFor(List.of(args)); + } + + public List runFor(List args) { + List results = new ArrayList<>(); + for (var arg : args) { + results.add(call(method, arg)); + } + return results; + } + public CallBuilderAsync async(Executor executor) { + return new CallBuilderAsync<>(executor, method); + } } + public class CallBuilderAsync { + private final Executor executor; + private final BiFunction method; + public CallBuilderAsync(Executor executor, BiFunction method) { + this.executor = executor; + this.method = method; + } + public CompletableFuture run(I arg) { + return CompletableFuture.supplyAsync(() -> call(method, arg), executor); + } + public CompletableFuture> runFor(List args) { + List> results = new ArrayList<>(); + for (var arg : args) { + results.add(CompletableFuture.supplyAsync(() -> call(method, arg), executor)); + } + return CompletableFuture.allOf(results.toArray(new CompletableFuture[0])) + .thenApply(v -> results.stream().map(CompletableFuture::join).toList()); + } + public CompletableFuture> runFor(I... args) { + return runFor(List.of(args)); + } + } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java deleted file mode 100644 index 7e040597..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.service.client; - -public sealed interface NodeSelectionStrategy { - boolean test(int node); - record Any() implements NodeSelectionStrategy { - @Override - public boolean test(int node) { - return true; - } - } - record Just(int node) implements NodeSelectionStrategy { - @Override - public boolean test(int node) { - return this.node == node; - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java index e11287ab..b905beab 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java @@ -1,12 +1,9 @@ package nu.marginalia.service.client; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; public class ServiceNotAvailableException extends RuntimeException { - public ServiceNotAvailableException(ServiceId id, int node) { - super(STR."Service \{id} not available on node \{node}"); - } - public ServiceNotAvailableException(ServiceId id) { - super(STR."Service \{id} not available"); + public ServiceNotAvailableException(ServiceKey key) { + super(STR."Service \{key} not available"); } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java deleted file mode 100644 index e8254cfd..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java +++ /dev/null @@ -1,122 +0,0 @@ -package nu.marginalia.service.discovery; - -import com.google.inject.Inject; -import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.service.discovery.monitor.*; -import nu.marginalia.service.discovery.property.ApiSchema; -import nu.marginalia.service.discovery.property.ServiceEndpoint; -import nu.marginalia.service.discovery.property.ServiceEndpoint.GrpcEndpoint; -import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; -import nu.marginalia.service.discovery.property.ServiceEndpoint.RestEndpoint; -import nu.marginalia.service.id.ServiceId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.SQLException; -import java.time.Duration; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; - -/** A service registry that returns fixed endpoints for all services. - *

- * This is for backwards-compatibility with old docker-compose files with no - * ZooKeeper configured. - * */ -public class FixedServiceRegistry implements ServiceRegistryIf { - private static final Logger logger = LoggerFactory.getLogger(FixedServiceRegistry.class); - - private final HikariDataSource dataSource; - - @Inject - public FixedServiceRegistry(HikariDataSource dataSource) { - this.dataSource = dataSource; - } - - @Override - public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, int node, UUID instanceUUID, String externalAddress) throws Exception { - return switch (schema) { - case REST -> new ServiceEndpoint.RestEndpoint(externalAddress, 80); - case GRPC -> new ServiceEndpoint.GrpcEndpoint(externalAddress, 81); - }; - } - - @Override - public void announceInstance(ServiceId id, int node, UUID instanceUUID) { - // No-op - } - - @Override - public Set getServiceNodes(ServiceId id) { - - if (id == ServiceId.Executor || id == ServiceId.Index) { - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement("SELECT ID FROM NODE_CONFIGURATION")) { - Set ret = new HashSet<>(); - var rs = stmt.executeQuery(); - while (rs.next()) { - ret.add(rs.getInt(1)); - } - return ret; - } - catch (SQLException ex) { - return Set.of(); - } - } - - else return Set.of(0); - } - - @Override - public int requestPort(String externalHost, ApiSchema schema, ServiceId id, int node) { - return switch(schema) { - case REST -> 80; - case GRPC -> 81; - }; - } - - @Override - public Set> getEndpoints(ApiSchema schema, ServiceId id, int node) { - return switch (schema) { - case REST -> Set.of(new InstanceAddress<>( - new RestEndpoint(id.serviceName + "-" + node, 80), - UUID.randomUUID())); - case GRPC -> Set.of(new InstanceAddress<>( - new GrpcEndpoint(id.serviceName + "-" + node, 81), - UUID.randomUUID())); - }; - } - - public void registerMonitor(ServiceMonitorIf monitor) throws Exception { - // We don't have any notification mechanism, so we just periodically - // invoke the monitor's onChange method to simulate it. - - periodicallyInvoke(monitor, Duration.ofSeconds(15)); - } - - - void periodicallyInvoke(ServiceMonitorIf monitor, Duration d) { - Thread.ofPlatform().name("PeriodicInvoker").start(() -> { - for (;;) { - try { - Thread.sleep(d); - } catch (InterruptedException e) { - break; - } - - boolean reRegister; - try { - reRegister = monitor.onChange(); - } - catch (Exception ex) { - logger.error("Monitor failed", ex); - reRegister = true; - } - - if (!reRegister) { - break; - } - } - }); - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java index b3460418..04877bd3 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java @@ -1,10 +1,10 @@ package nu.marginalia.service.discovery; import nu.marginalia.service.discovery.monitor.*; -import nu.marginalia.service.discovery.property.ApiSchema; import nu.marginalia.service.discovery.property.ServiceEndpoint; import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; -import nu.marginalia.service.id.ServiceId; + +import nu.marginalia.service.discovery.property.ServiceKey; import java.util.Set; import java.util.UUID; @@ -16,40 +16,33 @@ public interface ServiceRegistryIf { /** * Register a service with the registry. *

- * Once the instance has announced itself with {@link #announceInstance(ServiceId id, int node, UUID instanceUUID) announceInstance(...)}, - * the service will be available for discovery with {@link #getEndpoints(ApiSchema schema, ServiceId id, int node) getEndpoints(...)}. + * Once the instance has announced itself with {@link #announceInstance(UUID instanceUUID) announceInstance(...)}, + * the service will be available for discovery with {@link #getEndpoints(ServiceKey key) getEndpoints(...)}. * - * @param schema the API schema - * @param id the service identifier - * @param node the node number + * @param key the key identifying the service * @param instanceUUID the unique UUID of the instance * @param externalAddress the public address of the service */ - ServiceEndpoint registerService(ApiSchema schema, - ServiceId id, - int node, + ServiceEndpoint registerService(ServiceKey key, UUID instanceUUID, String externalAddress) throws Exception; + + void declareFirstBoot(); + void waitForFirstBoot() throws InterruptedException; + /** Let the world know that the service is running * and ready to accept requests. */ - void announceInstance(ServiceId id, int node, UUID instanceUUID); - - /** Return all nodes that are running for the specified service. */ - Set getServiceNodes(ServiceId id); + void announceInstance(UUID instanceUUID); /** At the discretion of the implementation, provide a port that is unique - * across (externalHost, serviceId, schema, node). It may be randomly selected + * across (host, api-schema). It may be randomly selected * or hard-coded or some combination of behaviors. */ - int requestPort(String externalHost, - ApiSchema schema, - ServiceId id, - int node); + int requestPort(String externalHost, ServiceKey key); /** Get all endpoints for the service on the specified node and schema. */ - Set> - getEndpoints(ApiSchema schema, ServiceId id, int node); + Set getEndpoints(ServiceKey schema); /** Register a monitor to be notified when the service registry changes. *

@@ -61,9 +54,6 @@ public interface ServiceRegistryIf { * monitor type. *
    *
  • {@link ServiceChangeMonitor} is notified when any node for the service changes.
    • - *
  • {@link ServiceNodeChangeMonitor} is notified when a specific node for the service changes.
    • - *
  • {@link ServiceRestEndpointChangeMonitor} is notified when the REST endpoints for the specified node service changes.
    • - *
  • {@link ServiceGrpcEndpointChangeMonitor} is notified when the gRPC endpoints for the specified node service changes.
    • *
* */ void registerMonitor(ServiceMonitorIf monitor) throws Exception; diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java index 48efc64f..1af51d75 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -4,10 +4,10 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; import nu.marginalia.service.discovery.monitor.*; -import nu.marginalia.service.discovery.property.ApiSchema; import nu.marginalia.service.discovery.property.ServiceEndpoint; import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; -import nu.marginalia.service.id.ServiceId; + +import nu.marginalia.service.discovery.property.ServiceKey; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorWatcher; import org.apache.curator.utils.ZKPaths; @@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; /** A versatile service registry that uses ZooKeeper to store service endpoints. * It is used to register services and to look up the endpoints of other services. @@ -35,6 +34,8 @@ public class ZkServiceRegistry implements ServiceRegistryIf { private static final Logger logger = LoggerFactory.getLogger(ZkServiceRegistry.class); private volatile boolean stopped = false; + private final List livenessPaths = new ArrayList<>(); + @Inject @SneakyThrows public ZkServiceRegistry(CuratorFramework curatorFramework) { @@ -51,93 +52,98 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } @Override - public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, - int node, + public ServiceEndpoint registerService(ServiceKey key, UUID instanceUUID, String externalAddress) throws Exception { - var ephemeralProperty = curatorFramework.create() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL); + var endpoint = new ServiceEndpoint(externalAddress, requestPort(externalAddress, key)); - var endpoint = ServiceEndpoint.forSchema(schema, externalAddress, - requestPort(externalAddress, schema, id, node) - ); - - String path; - byte[] payload; - - switch (endpoint) { - case ServiceEndpoint.GrpcEndpoint(String host, int port) -> { - path = STR."/services/\{id.serviceName}/\{node}/grpc/\{instanceUUID.toString()}"; - payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8); - } - case ServiceEndpoint.RestEndpoint(String host, int port) -> { - path = STR."/services/\{id.serviceName}/\{node}/rest/\{instanceUUID.toString()}"; - payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8); - } - } + String path = STR."\{key.toPath()}/\{instanceUUID.toString()}"; + byte[] payload = STR."\{endpoint.host()}:\{endpoint.port()}".getBytes(StandardCharsets.UTF_8); logger.info("Registering {} -> {}", path, endpoint); - ephemeralProperty.forPath(path, payload); + curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL) + .forPath(path, payload); return endpoint; } + @SneakyThrows @Override - public void announceInstance(ServiceId id, int node, UUID instanceUUID) { + public void declareFirstBoot() { + if (!isFirstBoot()) { + curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(STR."/first-boot"); + } + } + + @Override + public void waitForFirstBoot() throws InterruptedException { + if (!isFirstBoot()) + logger.info("Waiting for first-boot flag"); + + while (true) { + if (isFirstBoot()) + return; + + Thread.sleep(1000); + } + } + + private boolean isFirstBoot() { try { - String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}"; + return curatorFramework.checkExists().forPath("/first-boot") != null; + } + catch (Exception ex) { + logger.error("Failed to check first-boot", ex); + return false; + } + } + + @Override + public void announceInstance(UUID instanceUUID) { + try { + String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}"; + + livenessPaths.add(serviceRoot); + curatorFramework.create() .creatingParentsIfNeeded() .withMode(CreateMode.EPHEMERAL) .forPath(serviceRoot); } catch (Exception ex) { - logger.error("Failed to create service root for {}", id.serviceName); + logger.error("Failed to create service root for {}", instanceUUID); } } /** * Returns true if the service has announced itself as up and running. */ - public boolean isInstanceRunning(ServiceId id, int node, UUID instanceUUID) { + public boolean isInstanceRunning(UUID instanceUUID) { try { - String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}"; + String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}"; return null != curatorFramework.checkExists().forPath(serviceRoot); } catch (Exception ex) { - logger.error("Failed to check if service is running {}", id.serviceName); + logger.error("Failed to check if instance is running {}", instanceUUID); return false; } } - @Override - public Set getServiceNodes(ServiceId id) { - try { - String serviceRoot = STR."/services/\{id.serviceName}"; - return curatorFramework.getChildren().forPath(serviceRoot) - .stream().map(Integer::parseInt) - .collect(Collectors.toSet()); - } - catch (Exception ex) { - logger.error("Failed to get nodes for service {}", id.serviceName); - return Set.of(); - } - } - @Override public int requestPort(String externalHost, - ApiSchema schema, - ServiceId id, - int node) - { + ServiceKey key) { if (!Boolean.getBoolean("service.random-port")) { - return switch(schema) { - case REST -> 80; - case GRPC -> 81; + return switch (key) { + case ServiceKey.Rest rest -> 80; + case ServiceKey.Grpc grpc -> 81; }; } @@ -146,9 +152,9 @@ public class ZkServiceRegistry implements ServiceRegistryIf { var random = new Random(); - String host = STR."\{id.serviceName}-\{node}"; + String identifier = key.toPath(); - byte[] payload = STR."\{schema}://\{host}".getBytes(StandardCharsets.UTF_8); + byte[] payload = identifier.getBytes(); for (int iter = 0; iter < 1000; iter++) { try { @@ -161,7 +167,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf { return port; } catch (Exception ex) { - logger.error(STR."Still negotiating port for \{schema}://\{id.serviceName}:\{node}"); + logger.error(STR."Still negotiating port for \{identifier}"); } } @@ -169,81 +175,30 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } @Override - public Set> getEndpoints(ApiSchema schema, ServiceId id, int node) { - return switch (schema) { - case REST -> getRestEndpoints(id, node); - case GRPC -> getGrpcEndpoints(id, node); - }; - } - - public Set> getRestEndpoints(ServiceId id, int node) { + public Set getEndpoints(ServiceKey key) { try { - Set> ret = new HashSet<>(); - String restRoot = STR."/services/\{id.serviceName}/\{node}/rest"; + Set ret = new HashSet<>(); for (var uuid : curatorFramework .getChildren() - .forPath(restRoot)) { + .forPath(key.toPath())) { - if (!isInstanceRunning(id, node, UUID.fromString(uuid))) { + if (!isInstanceRunning(UUID.fromString(uuid))) { continue; } + var path = ZKPaths.makePath(key.toPath(), uuid); byte[] data = curatorFramework .getData() - .forPath(ZKPaths.makePath(restRoot, uuid)); - String hostAndPort = new String(data); - var address = RestEndpoint - .parse(hostAndPort) - .asInstance(UUID.fromString(uuid)); + .forPath(path); - // Ensure that the address is resolvable - // (this reduces the risk of exceptions when trying to connect to the service) - if (!address.endpoint().validateHost()) { - logger.warn("Omitting stale address {}, address does not resolve", address); - continue; - } - - ret.add(address); - - } - - return ret; - } - catch (Exception ex) { - return Set.of(); - } - } - - public Set> getGrpcEndpoints(ServiceId id, int node) { - try { - Set> ret = new HashSet<>(); - String restRoot = STR."/services/\{id.serviceName}/\{node}/grpc"; - for (var uuid : curatorFramework - .getChildren() - .forPath(restRoot)) { - - if (!isInstanceRunning(id, node, UUID.fromString(uuid))) { - continue; - } - - byte[] data = curatorFramework - .getData() - .forPath(ZKPaths.makePath(restRoot, uuid)); + long cxTime = curatorFramework.checkExists().forPath(path).getMzxid(); String hostAndPort = new String(data); - var address = GrpcEndpoint + var address = ServiceEndpoint .parse(hostAndPort) - .asInstance(UUID.fromString(uuid)); - - // Ensure that the address is resolvable - // (this reduces the risk of exceptions when trying to connect to the service) - if (!address.endpoint().validateHost()) { - logger.warn("Omitting stale address {}, address does not resolve", address); - continue; - } + .asInstance(UUID.fromString(uuid), cxTime); ret.add(address); - } return ret; @@ -254,29 +209,13 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } public void registerMonitor(ServiceMonitorIf monitor) throws Exception { - monitor.register(this); - } + if (stopped) + logger.info("Not registering monitor for {} because the registry is stopped", monitor.getKey()); - public void registerMonitor(ServiceChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}"); - } + String path = monitor.getKey().toPath(); - public void registerMonitor(ServiceNodeChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}"); - } - - public void registerMonitor(ServiceRestEndpointChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/rest"); - } - - public void registerMonitor(ServiceGrpcEndpointChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/grpc"); - } - - private void installMonitor(ServiceMonitorIf monitor, String path) throws Exception { - CuratorWatcher watcher = _ -> { + CuratorWatcher watcher = change -> { boolean reRegister; - try { reRegister = monitor.onChange(); } @@ -293,13 +232,31 @@ public class ZkServiceRegistry implements ServiceRegistryIf { curatorFramework.watchers().add() .usingWatcher(watcher) .forPath(path); + + // Also register for updates to the running-instances list, + // as this will have an effect on the result of getEndpoints() + curatorFramework.watchers().add() + .usingWatcher(watcher) + .forPath("/running-instances"); } /* Exposed for tests */ public synchronized void shutDown() { - if (!stopped) { - curatorFramework.close(); - stopped = true; + if (!stopped) + return; + + stopped = true; + + // Delete all liveness paths + for (var path : livenessPaths) { + logger.info("Cleaning up {}", path); + + try { + curatorFramework.delete().forPath(path); + } + catch (Exception ex) { + logger.error("Failed to delete path {}", path, ex); + } } } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java index 63e35891..23faba58 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java @@ -1,23 +1,17 @@ package nu.marginalia.service.discovery.monitor; -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; public abstract class ServiceChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; + public final ServiceKey serviceKey; - public ServiceChangeMonitor(ServiceId serviceId) { - this.serviceId = serviceId; + public ServiceChangeMonitor(ServiceKey key) { + this.serviceKey = key; } public abstract boolean onChange(); - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } + public ServiceKey getKey() { + return serviceKey; } + } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java deleted file mode 100644 index 2b33a40e..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.service.discovery.monitor; - -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; - -public abstract class ServiceGrpcEndpointChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; - public final int node; - public ServiceGrpcEndpointChangeMonitor(ServiceId serviceId, int node) { - this.serviceId = serviceId; - this.node = node; - } - - public abstract boolean onChange(); - - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java index af4cabb8..8dca094a 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java @@ -1,16 +1,13 @@ package nu.marginalia.service.discovery.monitor; -import nu.marginalia.service.discovery.ServiceRegistryIf; + +import nu.marginalia.service.discovery.property.ServiceKey; public interface ServiceMonitorIf { /** Called when the monitored service has changed. * @return true if the monitor is to be refreshed */ boolean onChange(); + ServiceKey getKey(); - /** Register this monitor with the given registry. - * It is preferred to use {@link ServiceRegistryIf}'s - * registerMonitor function. - * */ - void register(ServiceRegistryIf registry) throws Exception; } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java deleted file mode 100644 index 42e04e89..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.service.discovery.monitor; - -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; - -public abstract class ServiceNodeChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; - public final int node; - public ServiceNodeChangeMonitor(ServiceId serviceId, int node) { - this.serviceId = serviceId; - this.node = node; - } - - public abstract boolean onChange(); - - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java deleted file mode 100644 index 1e8c8564..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.service.discovery.monitor; - -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; - -public abstract class ServiceRestEndpointChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; - public final int node; - public ServiceRestEndpointChangeMonitor(ServiceId serviceId, int node) { - this.serviceId = serviceId; - this.node = node; - } - - public abstract boolean onChange(); - - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java deleted file mode 100644 index dc8300af..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java +++ /dev/null @@ -1,6 +0,0 @@ -package nu.marginalia.service.discovery.property; - -public enum ApiSchema { - REST, - GRPC -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java new file mode 100644 index 00000000..64436bdd --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java @@ -0,0 +1,8 @@ +package nu.marginalia.service.discovery.property; + +public interface PartitionTraits { + interface Grpc {}; + interface Unicast {}; + interface Multicast {}; + interface NoGrpc {}; +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java index 7763eca0..c52d335c 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -1,17 +1,23 @@ package nu.marginalia.service.discovery.property; - -import lombok.SneakyThrows; - import java.net.*; import java.util.UUID; -public sealed interface ServiceEndpoint { - String host(); - int port(); +public record ServiceEndpoint(String host, int port) { - URL toURL(String endpoint, String query); - default InetSocketAddress toInetSocketAddress() { + public static ServiceEndpoint parse(String hostAndPort) { + var parts = hostAndPort.split(":"); + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid host:port string: " + hostAndPort); + } + return new ServiceEndpoint(parts[0], Integer.parseInt(parts[1])); + } + + public URL toURL(String endpoint, String query) throws URISyntaxException, MalformedURLException { + return new URI("http", null, host, port, endpoint, query, null) + .toURL(); + } + public InetSocketAddress toInetSocketAddress() { return new InetSocketAddress(host(), port()); } @@ -19,7 +25,7 @@ public sealed interface ServiceEndpoint { * * @return true if the host is a valid */ - default boolean validateHost() { + public boolean validateHost() { try { // Throws UnknownHostException if the host is not a valid IP address or hostname // (this should not be slow since the DNS lookup should be local, and if it isn't; @@ -31,63 +37,11 @@ public sealed interface ServiceEndpoint { } } - static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) { - return switch (schema) { - case REST -> new RestEndpoint(host, port); - case GRPC -> new GrpcEndpoint(host, port); - }; + public InstanceAddress asInstance(UUID instance, long cxTime) { + return new InstanceAddress(this, instance, cxTime); } - record RestEndpoint(String host, int port) implements ServiceEndpoint { - public static RestEndpoint parse(String hostColonPort) { - String[] parts = hostColonPort.split(":"); - - if (parts.length != 2) { - throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'"); - } - - return new RestEndpoint( - parts[0], - Integer.parseInt(parts[1]) - ); - } - - @SneakyThrows - public URL toURL(String endpoint, String query) { - return new URI("http", null, host, port, endpoint, query, null) - .toURL(); - } - - public InstanceAddress asInstance(UUID uuid) { - return new InstanceAddress<>(this, uuid); - } - } - - record GrpcEndpoint(String host, int port) implements ServiceEndpoint { - public static GrpcEndpoint parse(String hostColonPort) { - String[] parts = hostColonPort.split(":"); - - if (parts.length != 2) { - throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'"); - } - - return new GrpcEndpoint( - parts[0], - Integer.parseInt(parts[1]) - ); - } - - public InstanceAddress asInstance(UUID uuid) { - return new InstanceAddress<>(this, uuid); - } - - @Override - public URL toURL(String endpoint, String query) { - throw new UnsupportedOperationException(); - } - } - - record InstanceAddress(T endpoint, UUID instance) { + public record InstanceAddress(ServiceEndpoint endpoint, UUID instance, long cxTime) { public String host() { return endpoint.host(); } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java new file mode 100644 index 00000000..66ae5ded --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java @@ -0,0 +1,69 @@ +package nu.marginalia.service.discovery.property; + +import io.grpc.ServiceDescriptor; +import nu.marginalia.service.id.ServiceId; + +public sealed interface ServiceKey

{ + String toPath(); + + static ServiceKey forRest(ServiceId id) { + return new Rest(id.serviceName); + } + static ServiceKey forRest(ServiceId id, int node) { + if (node == 0) { + return forRest(id); + } + + return new Rest(id.serviceName + "-" + node); + } + + static Grpc forServiceDescriptor(ServiceDescriptor descriptor, ServicePartition partition) { + return new Grpc<>(descriptor.getName(), partition); + } + + static Grpc forGrpcApi(Class apiClass, P2 partition) { + try { + var name = apiClass.getField("SERVICE_NAME").get(null); + return new Grpc(name.toString(), partition); + } + catch (Exception e) { + throw new IllegalArgumentException("Could not get SERVICE_NAME from " + apiClass.getSimpleName(), e); + } + } + + + + Grpc forPartition(P2 partition); + + + record Rest(String name) implements ServiceKey { + public String toPath() { + return STR."/services/rest/\{name}"; + } + + @Override + public + + Grpc forPartition(P2 partition) + { + throw new UnsupportedOperationException(); + } + } + record Grpc

(String name, P partition) implements ServiceKey

{ + public String baseName() { + return STR."/services/grpc/\{name}"; + } + public String toPath() { + return STR."/services/grpc/\{name}/\{partition.identifier()}"; + } + + @Override + public + + Grpc forPartition(P2 partition) + { + return new Grpc<>(name, partition); + } + } + +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java new file mode 100644 index 00000000..32aa37fb --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java @@ -0,0 +1,29 @@ +package nu.marginalia.service.discovery.property; + +public sealed interface ServicePartition { + String identifier(); + + static Any any() { return new Any(); } + static Multi multi() { return new Multi(); } + static Partition partition(int node) { return new Partition(node); } + static None none() { return new None(); } + + record Any() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast { + public String identifier() { return "*"; } + + } + record Multi() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Multicast { + public String identifier() { return "*"; } + + } + record Partition(int node) implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast { + public String identifier() { + return Integer.toString(node); + } + + } + record None() implements ServicePartition, PartitionTraits.NoGrpc { + public String identifier() { return ""; } + + } +} diff --git a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java index 3684a75a..7963c857 100644 --- a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java +++ b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java @@ -1,6 +1,8 @@ package nu.marginalia.service.discovery; -import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.api.math.MathApiGrpc; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -13,7 +15,6 @@ import org.testcontainers.junit.jupiter.Testcontainers; import java.util.*; -import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; import static org.junit.jupiter.api.Assertions.*; @Testcontainers @@ -58,15 +59,18 @@ class ZkServiceRegistryTest { List ports = new ArrayList<>(); Set portsSet = new HashSet<>(); + + var key = ServiceKey.forRest(ServiceId.Search, 0); + for (int i = 0; i < 500; i++) { - int port = registry1.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0); + int port = registry1.requestPort("127.0.0.1", key); ports.add(port); // Ensure we get unique ports assertTrue(portsSet.add(port)); } for (int i = 0; i < 50; i++) { - int port = registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0); + int port = registry2.requestPort("127.0.0.1", key); ports.add(port); // Ensure we get unique ports @@ -75,39 +79,86 @@ class ZkServiceRegistryTest { registry1.shutDown(); for (int i = 0; i < 500; i++) { // Verify we can reclaim ports - ports.add(registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0)); + ports.add(registry2.requestPort("127.0.0.1", key)); } assertEquals(1050, ports.size()); } @Test - void getInstances() throws Exception { + void getInstancesRestgRPC() throws Exception { var uuid1 = UUID.randomUUID(); var uuid2 = UUID.randomUUID(); var registry1 = createRegistry(); var registry2 = createRegistry(); - var endpoint1 = (RestEndpoint) registry1.registerService(ApiSchema.REST, ServiceId.Search, 0, uuid1, "127.0.0.1"); - var endpoint2 = (GrpcEndpoint) registry2.registerService(ApiSchema.GRPC, ServiceId.Search, 0, uuid2, "127.0.0.2"); + var key1 = ServiceKey.forRest(ServiceId.Search, 0); + var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()); - registry1.announceInstance(ServiceId.Search, 0, uuid1); - registry2.announceInstance(ServiceId.Search, 0, uuid2); + var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1"); + var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2"); - assertEquals(Set.of(endpoint1.asInstance(uuid1)), - registry1.getRestEndpoints(ServiceId.Search, 0)); + registry1.announceInstance(uuid1); + registry2.announceInstance(uuid2); - assertEquals(Set.of(endpoint2.asInstance(uuid2)), - registry1.getGrpcEndpoints(ServiceId.Search, 0)); + assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), + registry1.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), + registry1.getEndpoints(key2)); registry1.shutDown(); Thread.sleep(100); - assertEquals(Set.of(), - registry2.getRestEndpoints(ServiceId.Search, 0)); - assertEquals(Set.of(endpoint2.asInstance(uuid2)), - registry2.getGrpcEndpoints(ServiceId.Search, 0)); + assertEquals(Set.of(), registry2.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key2)); + } + + @Test + void testInstancesTwoAny() throws Exception { + var uuid1 = UUID.randomUUID(); + var uuid2 = UUID.randomUUID(); + + var registry1 = createRegistry(); + var registry2 = createRegistry(); + + var key = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()); + + var endpoint1 = registry1.registerService(key, uuid1, "127.0.0.1"); + var endpoint2 = registry2.registerService(key, uuid2, "127.0.0.2"); + + registry1.announceInstance(uuid1); + registry2.announceInstance(uuid2); + + assertEquals(Set.of(endpoint1.asInstance(uuid1, 0), + endpoint2.asInstance(uuid2, 0)), + registry1.getEndpoints(key)); + + registry1.shutDown(); + Thread.sleep(100); + + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key)); + } + + @Test + void testInstancesTwoPartitions() throws Exception { + var uuid1 = UUID.randomUUID(); + var uuid2 = UUID.randomUUID(); + + var registry1 = createRegistry(); + var registry2 = createRegistry(); + + var key1 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(1)); + var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(2)); + + var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1"); + var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2"); + + registry1.announceInstance(uuid1); + registry2.announceInstance(uuid2); + + assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), registry1.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry1.getEndpoints(key2)); } @Test @@ -115,9 +166,9 @@ class ZkServiceRegistryTest { var registry1 = createRegistry(); var uuid1 = UUID.randomUUID(); - assertFalse(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1)); - registry1.announceInstance(ServiceId.Search, 0, uuid1); - assertTrue(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1)); + assertFalse(registry1.isInstanceRunning(uuid1)); + registry1.announceInstance(uuid1); + assertTrue(registry1.isInstanceRunning(uuid1)); registry1.shutDown(); } diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java index 800d6712..4a6f1c71 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java +++ b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java @@ -156,6 +156,8 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat { stmt.executeUpdate(); } } + + dataSource.close(); } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java index aa8b1203..a196872f 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java @@ -6,6 +6,7 @@ import com.google.inject.Singleton; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; +import nu.marginalia.WmsaHome; import nu.marginalia.service.ServiceHomeNotConfiguredException; import org.flywaydb.core.Flyway; import org.mariadb.jdbc.Driver; @@ -51,7 +52,7 @@ public class DatabaseModule extends AbstractModule { } private Properties loadDbProperties() { - Path propDir = getHomePath().resolve("conf/db.properties"); + Path propDir = WmsaHome.getHomePath().resolve("conf/db.properties"); if (!Files.isRegularFile(propDir)) { throw new IllegalStateException("Database properties file " + propDir + " does not exist"); } @@ -72,17 +73,6 @@ public class DatabaseModule extends AbstractModule { } - public static Path getHomePath() { - var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse("/var/lib/wmsa"); - - var ret = Path.of(retStr); - if (!Files.isDirectory(ret)) { - throw new ServiceHomeNotConfiguredException("Could not find WMSA_HOME, either set environment variable or ensure /var/lib/wmsa exists"); - } - return ret; - } - - @SneakyThrows @Singleton @Provides @@ -97,7 +87,6 @@ public class DatabaseModule extends AbstractModule { try { HikariConfig config = new HikariConfig(); - config.setJdbcUrl(connStr); config.setUsername(dbProperties.getProperty(DB_USER_KEY)); config.setPassword(dbProperties.getProperty(DB_PASS_KEY)); diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java index e64c88a8..cf152bf0 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java @@ -1,5 +1,6 @@ package nu.marginalia.service.module; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import java.util.UUID; diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java index 37ce99d5..8ed7f45c 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java @@ -65,7 +65,7 @@ public class ServiceConfigurationModule extends AbstractModule { } // If we're in docker, we'll use the hostname - if (isDocker()) { + if (Boolean.getBoolean("service.useDockerHostname")) { return System.getenv("HOSTNAME"); } @@ -82,14 +82,7 @@ public class ServiceConfigurationModule extends AbstractModule { return configuredValue; } - // If we're in docker, we'll bind to all interfaces - if (isDocker()) - return "0.0.0.0"; - else // If we're not in docker, we'll default to binding to localhost to avoid exposing services - return "127.0.0.1"; + return "127.0.0.1"; } - boolean isDocker() { - return System.getenv("WMSA_IN_DOCKER") != null; - } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java index b9bb082b..ba076ae5 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java @@ -5,8 +5,8 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.prometheus.client.Counter; import lombok.SneakyThrows; import nu.marginalia.mq.inbox.*; -import nu.marginalia.service.discovery.property.ApiSchema; -import nu.marginalia.service.discovery.property.ServiceEndpoint; +import nu.marginalia.service.discovery.property.*; +import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.server.mq.ServiceMqSubscription; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,6 +16,7 @@ import spark.Request; import spark.Response; import spark.Spark; +import java.net.InetSocketAddress; import java.util.List; import java.util.Optional; @@ -48,11 +49,24 @@ public class Service { @SneakyThrows public Service(BaseServiceParams params, Runnable configureStaticFiles, + ServicePartition partition, List grpcServices) { + this.initialization = params.initialization; var config = params.configuration; node = config.node(); + if (config.serviceId() == ServiceId.Control) { + // Special case for first boot, since the control service + // owns database migrations and so on, we need other processes + // to wait for this to be done before they start. This is + // only needed once. + params.serviceRegistry.declareFirstBoot(); + } + else { + params.serviceRegistry.waitForFirstBoot(); + } + String inboxName = config.serviceName(); logger.info("Inbox name: {}", inboxName); @@ -60,8 +74,7 @@ public class Service { var restEndpoint = serviceRegistry.registerService( - ApiSchema.REST, config.serviceId(), - config.node(), + ServiceKey.forRest(config.serviceId(), config.node()), config.instanceUuid(), config.externalAddress() ); @@ -75,7 +88,7 @@ public class Service { initialization.addCallback(params.heartbeat::start); initialization.addCallback(messageQueueInbox::start); initialization.addCallback(() -> params.eventLog.logEvent("SVC-INIT", serviceName + ":" + config.node())); - initialization.addCallback(() -> serviceRegistry.announceInstance(config.serviceId(), config.node(), config.instanceUuid())); + initialization.addCallback(() -> serviceRegistry.announceInstance(config.instanceUuid())); if (!initialization.isReady() && ! initialized ) { initialized = true; @@ -101,29 +114,39 @@ public class Service { Spark.get("/internal/started", this::isInitialized); Spark.get("/internal/ready", this::isReady); - ServiceEndpoint.GrpcEndpoint grpcEndpoint = (ServiceEndpoint.GrpcEndpoint) params.serviceRegistry.registerService( - ApiSchema.GRPC, config.serviceId(), - config.node(), - config.instanceUuid(), - config.externalAddress() - ); + int port = params.serviceRegistry.requestPort(config.externalAddress(), new ServiceKey.Grpc<>("-", partition)); // Start the gRPC server - var grpcServerBuilder = NettyServerBuilder.forAddress(grpcEndpoint.toInetSocketAddress()); + var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port)); for (var grpcService : grpcServices) { - grpcServerBuilder.addService(grpcService); + var svc = grpcService.bindService(); + + params.serviceRegistry.registerService( + ServiceKey.forServiceDescriptor(svc.getServiceDescriptor(), partition), + config.instanceUuid(), + config.externalAddress() + ); + + grpcServerBuilder.addService(svc); } grpcServerBuilder.build().start(); } } public Service(BaseServiceParams params, + ServicePartition partition, List grpcServices) { - this(params, Service::defaultSparkConfig, grpcServices); + this(params, + Service::defaultSparkConfig, + partition, + grpcServices); } public Service(BaseServiceParams params) { - this(params, Service::defaultSparkConfig, List.of()); + this(params, + Service::defaultSparkConfig, + ServicePartition.any(), + List.of()); } private static void defaultSparkConfig() { diff --git a/code/common/service/src/main/resources/log4j2-prod.xml b/code/common/service/src/main/resources/log4j2-prod.xml index 43150a2e..01c914ac 100644 --- a/code/common/service/src/main/resources/log4j2-prod.xml +++ b/code/common/service/src/main/resources/log4j2-prod.xml @@ -21,6 +21,8 @@ + + diff --git a/code/common/service/src/main/resources/log4j2-test.xml b/code/common/service/src/main/resources/log4j2-test.xml index 3158f632..8fd0b262 100644 --- a/code/common/service/src/main/resources/log4j2-test.xml +++ b/code/common/service/src/main/resources/log4j2-test.xml @@ -20,6 +20,8 @@ + + diff --git a/code/features-index/domain-ranking/build.gradle b/code/features-index/domain-ranking/build.gradle index fc8e8401..42af2c69 100644 --- a/code/features-index/domain-ranking/build.gradle +++ b/code/features-index/domain-ranking/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:model') implementation project(':code:common:service') - implementation project(':code:api:query-api') + implementation project(':code:functions:domain-links:api') implementation 'org.jgrapht:jgrapht-core:1.5.2' diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java index 9d5564d0..f4adf2b6 100644 --- a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java +++ b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java @@ -3,23 +3,20 @@ package nu.marginalia.ranking.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; /** A source for the inverted link graph, * which is the same as the regular graph except * the direction of the links have been inverted */ public class InvertedLinkGraphSource extends AbstractGraphSource { - private final QueryClient queryClient; + private final AggregateDomainLinksClient queryClient; @Inject - public InvertedLinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) { + public InvertedLinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient queryClient) { super(dataSource); this.queryClient = queryClient; } diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java index cc7f2b53..5367a570 100644 --- a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java +++ b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java @@ -3,19 +3,19 @@ package nu.marginalia.ranking.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; /** A source for the regular link graph. */ public class LinkGraphSource extends AbstractGraphSource { - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; @Inject - public LinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) { + public LinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) { super(dataSource); - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; } @SneakyThrows @@ -25,7 +25,7 @@ public class LinkGraphSource extends AbstractGraphSource { addVertices(graph); - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = domainLinksClient.getAllDomainLinks(); var iter = allLinks.iterator(); while (iter.advance()) { if (!graph.containsVertex(iter.dest())) { diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java b/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java index 7fdd2f82..b92543e2 100644 --- a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java +++ b/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java @@ -3,7 +3,7 @@ package nu.marginalia.ranking; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.ranking.data.InvertedLinkGraphSource; import nu.marginalia.ranking.data.LinkGraphSource; import nu.marginalia.ranking.data.SimilarityGraphSource; @@ -37,8 +37,9 @@ public class RankingAlgorithmsContainerTest { static HikariDataSource dataSource; - QueryClient queryClient; - QueryClient.AllLinks allLinks; + AggregateDomainLinksClient domainLinksClient; + AggregateDomainLinksClient.AllLinks allLinks; + @BeforeAll public static void setup() { HikariConfig config = new HikariConfig(); @@ -66,9 +67,9 @@ public class RankingAlgorithmsContainerTest { @BeforeEach public void setupQueryClient() { - queryClient = Mockito.mock(QueryClient.class); - allLinks = new QueryClient.AllLinks(); - when(queryClient.getAllDomainLinks()).thenReturn(allLinks); + domainLinksClient = Mockito.mock(AggregateDomainLinksClient.class); + allLinks = new AggregateDomainLinksClient.AllLinks(); + when(domainLinksClient.getAllDomainLinks()).thenReturn(allLinks); try (var conn = dataSource.getConnection(); var stmt = conn.createStatement()) { @@ -97,7 +98,7 @@ public class RankingAlgorithmsContainerTest { @Test public void testGetDomains() { // should all be the same, doesn't matter which one we use - var source = new LinkGraphSource(dataSource, queryClient); + var source = new LinkGraphSource(dataSource, domainLinksClient); Assertions.assertEquals(List.of(1), source.domainIds(List.of("memex.marginalia.nu"))); @@ -111,7 +112,7 @@ public class RankingAlgorithmsContainerTest { public void testLinkGraphSource() { allLinks.add(1, 3); - var graph = new LinkGraphSource(dataSource, queryClient).getGraph(); + var graph = new LinkGraphSource(dataSource, domainLinksClient).getGraph(); Assertions.assertTrue(graph.containsVertex(1)); Assertions.assertTrue(graph.containsVertex(2)); @@ -127,7 +128,7 @@ public class RankingAlgorithmsContainerTest { public void testInvertedLinkGraphSource() { allLinks.add(1, 3); - var graph = new InvertedLinkGraphSource(dataSource, queryClient).getGraph(); + var graph = new InvertedLinkGraphSource(dataSource, domainLinksClient).getGraph(); Assertions.assertTrue(graph.containsVertex(1)); Assertions.assertTrue(graph.containsVertex(2)); diff --git a/code/functions/domain-info/api/build.gradle b/code/functions/domain-info/api/build.gradle new file mode 100644 index 00000000..74c0548e --- /dev/null +++ b/code/functions/domain-info/api/build.gradle @@ -0,0 +1,45 @@ +plugins { + id 'java' + + id "com.google.protobuf" version "0.9.4" + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +jar.archiveBaseName = 'domain-info-api' + +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + } +} + +apply from: "$rootProject.projectDir/protobuf.gradle" + +dependencies { + implementation project(':code:common:model') + implementation project(':code:common:config') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.notnull + implementation libs.guice + implementation libs.gson + implementation libs.protobuf + implementation libs.javax.annotation + implementation libs.bundles.grpc + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + +} diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java new file mode 100644 index 00000000..ee305994 --- /dev/null +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java @@ -0,0 +1,56 @@ +package nu.marginalia.api.domains; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.api.domains.model.SimilarDomain; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.*; + +import nu.marginalia.api.domains.model.*; + +@Singleton +public class DomainInfoClient { + private static final Logger logger = LoggerFactory.getLogger(DomainInfoClient.class); + + private final GrpcSingleNodeChannelPool channelPool; + private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + + @Inject + public DomainInfoClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle( + ServiceKey.forGrpcApi(DomainInfoAPIGrpc.class, ServicePartition.any()), + DomainInfoAPIGrpc::newBlockingStub); + } + + public Future> similarDomains(int domainId, int count) { + return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getSimilarDomains) + .async(virtualExecutorService) + .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count)) + .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse); + } + + public Future> linkedDomains(int domainId, int count) { + return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getLinkingDomains) + .async(virtualExecutorService) + .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count)) + .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse); + } + + public Future domainInformation(int domainId) { + return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getDomainInfo) + .async(virtualExecutorService) + .run(DomainsProtobufCodec.DomainInfo.createRequest(domainId)) + .thenApply(DomainsProtobufCodec.DomainInfo::convertResponse); + } + + public boolean isAccepting() { + return channelPool.hasChannel(); + } +} diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java similarity index 51% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java rename to code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java index cd2d61fa..d3c90f9f 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java @@ -1,74 +1,14 @@ -package nu.marginalia.assistant.client; +package nu.marginalia.api.domains; import lombok.SneakyThrows; -import nu.marginalia.assistant.api.*; -import nu.marginalia.assistant.client.model.DictionaryEntry; -import nu.marginalia.assistant.client.model.DictionaryResponse; -import nu.marginalia.assistant.client.model.DomainInformation; -import nu.marginalia.assistant.client.model.SimilarDomain; import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.EdgeUrl; +import nu.marginalia.api.domains.model.*; import java.util.ArrayList; import java.util.List; -public class AssistantProtobufCodec { - - public static class DictionaryLookup { - public static RpcDictionaryLookupRequest createRequest(String word) { - return RpcDictionaryLookupRequest.newBuilder() - .setWord(word) - .build(); - } - public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) { - return new DictionaryResponse( - rsp.getWord(), - rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList() - ); - } - - private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) { - return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition()); - } - } - - public static class SpellCheck { - public static RpcSpellCheckRequest createRequest(String text) { - return RpcSpellCheckRequest.newBuilder() - .setText(text) - .build(); - } - - public static List convertResponse(RpcSpellCheckResponse rsp) { - return rsp.getSuggestionsList(); - } - } - - public static class UnitConversion { - public static RpcUnitConversionRequest createRequest(String from, String to, String unit) { - return RpcUnitConversionRequest.newBuilder() - .setFrom(from) - .setTo(to) - .setUnit(unit) - .build(); - } - - public static String convertResponse(RpcUnitConversionResponse rsp) { - return rsp.getResult(); - } - } - - public static class EvalMath { - public static RpcEvalMathRequest createRequest(String expression) { - return RpcEvalMathRequest.newBuilder() - .setExpression(expression) - .build(); - } - - public static String convertResponse(RpcEvalMathResponse rsp) { - return rsp.getResult(); - } - } +public class DomainsProtobufCodec { public static class DomainQueries { public static RpcDomainLinksRequest createRequest(int domainId, int count) { diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java similarity index 96% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java rename to code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java index 5be64a97..2e62af22 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.domains.model; import lombok.*; import nu.marginalia.model.EdgeDomain; diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java similarity index 97% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java rename to code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java index 1bdae22e..d15d0f3d 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.domains.model; import nu.marginalia.model.EdgeUrl; diff --git a/code/api/assistant-api/src/main/protobuf/assistant-api.proto b/code/functions/domain-info/api/src/main/protobuf/domain-info.proto similarity index 77% rename from code/api/assistant-api/src/main/protobuf/assistant-api.proto rename to code/functions/domain-info/api/src/main/protobuf/domain-info.proto index 12aa7ce1..8be12d11 100644 --- a/code/api/assistant-api/src/main/protobuf/assistant-api.proto +++ b/code/functions/domain-info/api/src/main/protobuf/domain-info.proto @@ -1,18 +1,10 @@ syntax="proto3"; -package assistantapi; +package marginalia.api.domain; -option java_package="nu.marginalia.assistant.api"; +option java_package="nu.marginalia.api.domains"; option java_multiple_files=true; -service AssistantApi { - /** Looks up a word in the dictionary. */ - rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {} - /** Checks the spelling of a text. */ - rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {} - /** Converts a unit from one to another. */ - rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {} - /** Evaluates a mathematical expression. */ - rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {} +service DomainInfoAPI { /** Fetches information about a domain. */ rpc getDomainInfo(RpcDomainId) returns (RpcDomainInfoResponse) {} diff --git a/code/functions/domain-info/build.gradle b/code/functions/domain-info/build.gradle new file mode 100644 index 00000000..c9f641d0 --- /dev/null +++ b/code/functions/domain-info/build.gradle @@ -0,0 +1,44 @@ +plugins { + id 'java' + + id 'application' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:functions:domain-info:api') + implementation project(':code:functions:domain-links:api') + + implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:model') + implementation project(':code:common:db') + implementation project(':code:common:service-discovery') + + implementation project(':code:libraries:geo-ip') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.spark + implementation libs.opencsv + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + +} diff --git a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java new file mode 100644 index 00000000..9bf8a1c2 --- /dev/null +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java @@ -0,0 +1,53 @@ +package nu.marginalia.functions.domains; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.api.domains.DomainInfoAPIGrpc; +import nu.marginalia.api.domains.*; + +public class DomainInfoGrpcService extends DomainInfoAPIGrpc.DomainInfoAPIImplBase { + + private final DomainInformationService domainInformationService; + private final SimilarDomainsService similarDomainsService; + @Inject + public DomainInfoGrpcService(DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService) + { + + this.domainInformationService = domainInformationService; + this.similarDomainsService = similarDomainsService; + } + + @Override + public void getDomainInfo(RpcDomainId request, StreamObserver responseObserver) { + var ret = domainInformationService.domainInfo(request.getDomainId()); + + ret.ifPresent(responseObserver::onNext); + + responseObserver.onCompleted(); + } + + @Override + public void getSimilarDomains(RpcDomainLinksRequest request, + StreamObserver responseObserver) { + var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount()); + + var responseBuilder = RpcSimilarDomains + .newBuilder() + .addAllDomains(ret); + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver responseObserver) { + var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount()); + + var responseBuilder = RpcSimilarDomains + .newBuilder() + .addAllDomains(ret); + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } +} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java similarity index 87% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java rename to code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java index f2590e7e..bf8ad9df 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java @@ -1,11 +1,11 @@ -package nu.marginalia.assistant.domains; +package nu.marginalia.functions.domains; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.assistant.api.RpcDomainInfoResponse; +import nu.marginalia.api.domains.RpcDomainInfoResponse; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.geoip.GeoIpDictionary; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; -import nu.marginalia.query.client.QueryClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +21,7 @@ public class DomainInformationService { private final GeoIpDictionary geoIpDictionary; private DbDomainQueries dbDomainQueries; - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; private HikariDataSource dataSource; private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -29,11 +29,11 @@ public class DomainInformationService { public DomainInformationService( DbDomainQueries dbDomainQueries, GeoIpDictionary geoIpDictionary, - QueryClient queryClient, + AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) { this.dbDomainQueries = dbDomainQueries; this.geoIpDictionary = geoIpDictionary; - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; this.dataSource = dataSource; } @@ -84,8 +84,8 @@ public class DomainInformationService { inCrawlQueue = rs.next(); builder.setInCrawlQueue(inCrawlQueue); - builder.setIncomingLinks(queryClient.countLinksToDomain(domainId)); - builder.setOutboundLinks(queryClient.countLinksFromDomain(domainId)); + builder.setIncomingLinks(domainLinksClient.countLinksToDomain(domainId)); + builder.setOutboundLinks(domainLinksClient.countLinksFromDomain(domainId)); rs = stmt.executeQuery(STR.""" SELECT KNOWN_URLS, GOOD_URLS, VISITED_URLS FROM DOMAIN_METADATA WHERE ID=\{domainId} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java similarity index 95% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java rename to code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java index 4d00864f..8ce2b55c 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.domains; +package nu.marginalia.functions.domains; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; @@ -8,10 +8,10 @@ import gnu.trove.map.hash.TIntDoubleHashMap; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.assistant.api.RpcSimilarDomain; -import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.api.domains.*; +import nu.marginalia.api.domains.model.SimilarDomain; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.model.EdgeDomain; -import nu.marginalia.query.client.QueryClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,7 +27,7 @@ public class SimilarDomainsService { private static final Logger logger = LoggerFactory.getLogger(SimilarDomainsService.class); private final HikariDataSource dataSource; - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; private volatile TIntIntHashMap domainIdToIdx = new TIntIntHashMap(100_000); private volatile int[] domainIdxToId; @@ -43,9 +43,9 @@ public class SimilarDomainsService { volatile boolean isReady = false; @Inject - public SimilarDomainsService(HikariDataSource dataSource, QueryClient queryClient) { + public SimilarDomainsService(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) { this.dataSource = dataSource; - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; Executors.newSingleThreadExecutor().submit(this::init); } @@ -256,7 +256,7 @@ public class SimilarDomainsService { private TIntSet getLinkingIdsDToS(int domainIdx) { var items = new TIntHashSet(); - for (int id : queryClient.getLinksFromDomain(domainIdxToId[domainIdx])) { + for (int id : domainLinksClient.getLinksFromDomain(domainIdxToId[domainIdx])) { items.add(domainIdToIdx.get(id)); } @@ -266,7 +266,7 @@ public class SimilarDomainsService { private TIntSet getLinkingIdsSToD(int domainIdx) { var items = new TIntHashSet(); - for (int id : queryClient.getLinksToDomain(domainIdxToId[domainIdx])) { + for (int id : domainLinksClient.getLinksToDomain(domainIdxToId[domainIdx])) { items.add(domainIdToIdx.get(id)); } diff --git a/code/functions/domain-links/aggregate/build.gradle b/code/functions/domain-links/aggregate/build.gradle new file mode 100644 index 00000000..b8702030 --- /dev/null +++ b/code/functions/domain-links/aggregate/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'java' + + id 'application' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:functions:domain-links:api') + + implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:model') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.fastutil + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + +} diff --git a/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java b/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java new file mode 100644 index 00000000..ff842075 --- /dev/null +++ b/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java @@ -0,0 +1,96 @@ +package nu.marginalia.functions.domainlinks; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.api.domainlink.*; +import nu.marginalia.api.indexdomainlinks.PartitionDomainLinksClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase { + private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksService.class); + private final PartitionDomainLinksClient client; + + @Inject + public AggregateDomainLinksService(PartitionDomainLinksClient client) { + this.client = client; + } + + @Override + public void getAllLinks(Empty request, + StreamObserver responseObserver) { + + client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks) + .run(Empty.getDefaultInstance()) + .forEach(iter -> iter.forEachRemaining(responseObserver::onNext)); + + responseObserver.onCompleted(); + } + + @Override + public void getLinksFromDomain(RpcDomainId request, + StreamObserver responseObserver) { + var rspBuilder = RpcDomainIdList.newBuilder(); + + client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain) + .run(request) + .stream() + .map(RpcDomainIdList::getDomainIdList) + .flatMap(List::stream) + .forEach(rspBuilder::addDomainId); + + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getLinksToDomain(RpcDomainId request, + StreamObserver responseObserver) { + var rspBuilder = RpcDomainIdList.newBuilder(); + + + client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain) + .run(request) + .stream() + .map(RpcDomainIdList::getDomainIdList) + .flatMap(List::stream) + .forEach(rspBuilder::addDomainId); + + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void countLinksFromDomain(RpcDomainId request, + StreamObserver responseObserver) { + int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain) + .run(request) + .stream() + .mapToInt(RpcDomainIdCount::getIdCount) + .sum(); + + var rspBuilder = RpcDomainIdCount.newBuilder(); + rspBuilder.setIdCount(sum); + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void countLinksToDomain(RpcDomainId request, + StreamObserver responseObserver) { + + int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain) + .run(request) + .stream() + .mapToInt(RpcDomainIdCount::getIdCount) + .sum(); + + var rspBuilder = RpcDomainIdCount.newBuilder(); + rspBuilder.setIdCount(sum); + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + +} diff --git a/code/functions/domain-links/api/build.gradle b/code/functions/domain-links/api/build.gradle new file mode 100644 index 00000000..081e3167 --- /dev/null +++ b/code/functions/domain-links/api/build.gradle @@ -0,0 +1,46 @@ +plugins { + id 'java' + + id "com.google.protobuf" version "0.9.4" + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +jar.archiveBaseName = 'index-domain-links-api' + +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + } +} + +apply from: "$rootProject.projectDir/protobuf.gradle" + +dependencies { + implementation project(':code:common:model') + implementation project(':code:common:config') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.notnull + implementation libs.guice + implementation libs.gson + implementation libs.protobuf + implementation libs.roaringbitmap + implementation libs.javax.annotation + implementation libs.bundles.grpc + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + +} diff --git a/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java new file mode 100644 index 00000000..c8f5c5ec --- /dev/null +++ b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java @@ -0,0 +1,138 @@ +package nu.marginalia.api.indexdomainlinks; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.api.domainlink.DomainLinksApiGrpc; +import nu.marginalia.api.domainlink.Empty; +import nu.marginalia.api.domainlink.RpcDomainId; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.roaringbitmap.longlong.PeekableLongIterator; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.List; + +@Singleton +public class AggregateDomainLinksClient { + private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksClient.class); + + private final GrpcSingleNodeChannelPool channelPool; + + @Inject + public AggregateDomainLinksClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle( + ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.any()), + DomainLinksApiGrpc::newBlockingStub); + } + + + public AllLinks getAllDomainLinks() { + AllLinks links = new AllLinks(); + + channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks) + .run(Empty.getDefaultInstance()) + .forEachRemaining(pairs -> { + for (int i = 0; i < pairs.getDestIdsCount(); i++) { + links.add(pairs.getSourceIds(i), pairs.getDestIds(i)); + } + }); + + return links; + } + + public List getLinksToDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getDomainIdList() + .stream() + .sorted() + .toList(); + } + catch (Exception e) { + logger.error("API Exception", e); + return List.of(); + } + } + + public List getLinksFromDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getDomainIdList() + .stream() + .sorted() + .toList(); + + } + catch (Exception e) { + logger.error("API Exception", e); + return List.of(); + } + } + + public int countLinksToDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getIdCount(); + + } + catch (Exception e) { + logger.error("API Exception", e); + return 0; + } + } + + public int countLinksFromDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getIdCount(); + } + catch (Exception e) { + logger.error("API Exception", e); + return 0; + } + } + + public boolean waitReady(Duration duration) throws InterruptedException { + return channelPool.awaitChannel(duration); + } + + public static class AllLinks { + private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap(); + + public void add(int source, int dest) { + sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest)); + } + + public Iterator iterator() { + return new Iterator(); + } + + public class Iterator { + private final PeekableLongIterator base = sourceToDest.getLongIterator(); + long val = Long.MIN_VALUE; + + public boolean advance() { + if (base.hasNext()) { + val = base.next(); + return true; + } + return false; + } + public int source() { + return (int) (val >>> 32); + } + public int dest() { + return (int) (val & 0xFFFF_FFFFL); + } + } + } +} diff --git a/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java new file mode 100644 index 00000000..6e561b12 --- /dev/null +++ b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java @@ -0,0 +1,30 @@ +package nu.marginalia.api.indexdomainlinks; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.api.domainlink.DomainLinksApiGrpc; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class PartitionDomainLinksClient { + private static final Logger logger = LoggerFactory.getLogger(PartitionDomainLinksClient.class); + + private final GrpcMultiNodeChannelPool channelPool; + + @Inject + public PartitionDomainLinksClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createMulti( + ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.multi()), + DomainLinksApiGrpc::newBlockingStub); + } + + public GrpcMultiNodeChannelPool getChannelPool() { + return channelPool; + } + +} diff --git a/code/functions/domain-links/api/src/main/protobuf/domain-links.proto b/code/functions/domain-links/api/src/main/protobuf/domain-links.proto new file mode 100644 index 00000000..2a31bbf7 --- /dev/null +++ b/code/functions/domain-links/api/src/main/protobuf/domain-links.proto @@ -0,0 +1,29 @@ +syntax="proto3"; +package nu.marginalia.api.domainlinks; + +option java_package="nu.marginalia.api.domainlink"; +option java_multiple_files=true; + +service DomainLinksApi { + rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {} + rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {} + rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {} + rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {} + rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {} +} + +message RpcDomainId { + int32 domainId = 1; +} +message RpcDomainIdList { + repeated int32 domainId = 1 [packed=true]; +} +message RpcDomainIdCount { + int32 idCount = 1; +} +message RpcDomainIdPairs { + repeated int32 sourceIds = 1 [packed=true]; + repeated int32 destIds = 2 [packed=true]; +} + +message Empty {} \ No newline at end of file diff --git a/code/functions/domain-links/partition/build.gradle b/code/functions/domain-links/partition/build.gradle new file mode 100644 index 00000000..1a5f9dde --- /dev/null +++ b/code/functions/domain-links/partition/build.gradle @@ -0,0 +1,42 @@ +plugins { + id 'java' + + id 'application' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:functions:domain-links:api') + + implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:model') + implementation project(':code:common:linkdb') + implementation project(':code:common:db') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.spark + implementation libs.opencsv + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + +} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java b/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java similarity index 88% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java rename to code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java index b368d289..323ec51d 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java +++ b/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java @@ -1,22 +1,23 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.domainlinks; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.index.api.*; +import nu.marginalia.api.domainlink.Empty; +import nu.marginalia.api.domainlink.*; import nu.marginalia.linkdb.dlinks.DomainLinkDb; /** GRPC service for interrogating domain links */ -public class IndexDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase { +public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase { private final DomainLinkDb domainLinkDb; @Inject - public IndexDomainLinksService(DomainLinkDb domainLinkDb) { + public PartitionDomainLinksService(DomainLinkDb domainLinkDb) { this.domainLinkDb = domainLinkDb; } - public void getAllLinks(nu.marginalia.index.api.Empty request, - io.grpc.stub.StreamObserver responseObserver) { + public void getAllLinks(Empty request, + io.grpc.stub.StreamObserver responseObserver) { try (var idsConverter = new AllIdsResponseConverter(responseObserver)) { domainLinkDb.forEach(idsConverter::accept); diff --git a/code/api/assistant-api/build.gradle b/code/functions/math/api/build.gradle similarity index 96% rename from code/api/assistant-api/build.gradle rename to code/functions/math/api/build.gradle index e87165d7..0ef23547 100644 --- a/code/api/assistant-api/build.gradle +++ b/code/functions/math/api/build.gradle @@ -11,6 +11,8 @@ java { } } +jar.archiveBaseName = 'math-api' + sourceSets { main { proto { diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java new file mode 100644 index 00000000..c851aee9 --- /dev/null +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java @@ -0,0 +1,90 @@ +package nu.marginalia.api.math; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import nu.marginalia.service.id.ServiceId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +import nu.marginalia.api.math.model.*; +import nu.marginalia.api.math.MathProtobufCodec.*; + + +@Singleton +public class MathClient { + private static final Logger logger = LoggerFactory.getLogger(MathClient.class); + + private final GrpcSingleNodeChannelPool channelPool; + private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + @Inject + public MathClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle( + ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()), + MathApiGrpc::newBlockingStub); + + } + + public Future dictionaryLookup(String word) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .async(virtualExecutorService) + .run(DictionaryLookup.createRequest(word)) + .thenApply(DictionaryLookup::convertResponse); + } + + @SuppressWarnings("unchecked") + public Future> spellCheck(String word) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck) + .async(virtualExecutorService) + .run(SpellCheck.createRequest(word)) + .thenApply(SpellCheck::convertResponse); + } + + public Map> spellCheck(List words, Duration timeout) throws InterruptedException { + List requests = words.stream().map(SpellCheck::createRequest).toList(); + + var future = channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck) + .async(virtualExecutorService) + .runFor(requests); + + try { + var results = future.get(); + Map> map = new HashMap<>(); + for (int i = 0; i < words.size(); i++) { + map.put(words.get(i), SpellCheck.convertResponse(results.get(i))); + } + return map; + } + catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + + public Future unitConversion(String value, String from, String to) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::unitConversion) + .async(virtualExecutorService) + .run(UnitConversion.createRequest(from, to, value)) + .thenApply(UnitConversion::convertResponse); + } + + public Future evalMath(String expression) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::evalMath) + .async(virtualExecutorService) + .run(EvalMath.createRequest(expression)) + .thenApply(EvalMath::convertResponse); + } + + public boolean isAccepting() { + return channelPool.hasChannel(); + } +} diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java new file mode 100644 index 00000000..2b865b21 --- /dev/null +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java @@ -0,0 +1,66 @@ +package nu.marginalia.api.math; + +import nu.marginalia.api.math.model.DictionaryEntry; +import nu.marginalia.api.math.model.DictionaryResponse; + +import java.util.List; + +public class MathProtobufCodec { + + public static class DictionaryLookup { + public static RpcDictionaryLookupRequest createRequest(String word) { + return RpcDictionaryLookupRequest.newBuilder() + .setWord(word) + .build(); + } + public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) { + return new DictionaryResponse( + rsp.getWord(), + rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList() + ); + } + + private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) { + return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition()); + } + } + + public static class SpellCheck { + public static RpcSpellCheckRequest createRequest(String text) { + return RpcSpellCheckRequest.newBuilder() + .setText(text) + .build(); + } + + public static List convertResponse(RpcSpellCheckResponse rsp) { + return rsp.getSuggestionsList(); + } + } + + public static class UnitConversion { + public static RpcUnitConversionRequest createRequest(String from, String to, String unit) { + return RpcUnitConversionRequest.newBuilder() + .setFrom(from) + .setTo(to) + .setUnit(unit) + .build(); + } + + public static String convertResponse(RpcUnitConversionResponse rsp) { + return rsp.getResult(); + } + } + + public static class EvalMath { + public static RpcEvalMathRequest createRequest(String expression) { + return RpcEvalMathRequest.newBuilder() + .setExpression(expression) + .build(); + } + + public static String convertResponse(RpcEvalMathResponse rsp) { + return rsp.getResult(); + } + } + +} diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java similarity index 84% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java rename to code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java index c40ea97f..084e2e47 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.math.model; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java similarity index 86% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java rename to code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java index 03fbd2e6..0668b6b2 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.math.model; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/code/functions/math/api/src/main/protobuf/math-api.proto b/code/functions/math/api/src/main/protobuf/math-api.proto new file mode 100644 index 00000000..b29732ee --- /dev/null +++ b/code/functions/math/api/src/main/protobuf/math-api.proto @@ -0,0 +1,57 @@ +syntax="proto3"; +package nu.marginalia.api.math; + +option java_package="nu.marginalia.api.math"; +option java_multiple_files=true; + +service MathApi { + /** Looks up a word in the dictionary. */ + rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {} + /** Checks the spelling of a text. */ + rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {} + /** Converts a unit from one to another. */ + rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {} + /** Evaluates a mathematical expression. */ + rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {} +} + +message RpcDictionaryLookupRequest { + string word = 1; +} + +message RpcDictionaryLookupResponse { + string word = 1; + repeated RpcDictionaryEntry entries = 2; +} + +message RpcDictionaryEntry { + string type = 1; + string word = 2; + string definition = 3; +} + +message RpcSpellCheckRequest { + string text = 1; +} + +message RpcSpellCheckResponse { + repeated string suggestions = 1; +} + +message RpcUnitConversionRequest { + string unit = 1; + string from = 2; + string to = 3; +} + +message RpcUnitConversionResponse { + string result = 1; +} + +message RpcEvalMathRequest { + string expression = 1; +} + +message RpcEvalMathResponse { + string result = 1; +} diff --git a/code/functions/math/build.gradle b/code/functions/math/build.gradle new file mode 100644 index 00000000..4ffbc11d --- /dev/null +++ b/code/functions/math/build.gradle @@ -0,0 +1,32 @@ +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':third-party:symspell') + implementation project(':code:functions:math:api') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.spark + implementation libs.opencsv + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito +} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java similarity index 53% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java index 7cf82cd1..66c0ffe5 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java @@ -1,34 +1,28 @@ -package nu.marginalia.assistant; +package nu.marginalia.functions.math; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.assistant.api.*; -import nu.marginalia.assistant.dict.DictionaryService; -import nu.marginalia.assistant.dict.SpellChecker; -import nu.marginalia.assistant.domains.DomainInformationService; -import nu.marginalia.assistant.domains.SimilarDomainsService; -import nu.marginalia.assistant.eval.MathParser; -import nu.marginalia.assistant.eval.Units; +import nu.marginalia.api.math.*; +import nu.marginalia.functions.math.dict.DictionaryService; +import nu.marginalia.functions.math.dict.SpellChecker; +import nu.marginalia.functions.math.eval.MathParser; +import nu.marginalia.functions.math.eval.Units; -public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase { +public class MathGrpcService extends MathApiGrpc.MathApiImplBase { private final DictionaryService dictionaryService; private final SpellChecker spellChecker; private final Units units; private final MathParser mathParser; - private final DomainInformationService domainInformationService; - private final SimilarDomainsService similarDomainsService; + @Inject - public AssistantGrpcService(DictionaryService dictionaryService, - SpellChecker spellChecker, Units units, MathParser mathParser, DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService) + public MathGrpcService(DictionaryService dictionaryService, SpellChecker spellChecker, Units units, MathParser mathParser) { this.dictionaryService = dictionaryService; this.spellChecker = spellChecker; this.units = units; this.mathParser = mathParser; - this.domainInformationService = domainInformationService; - this.similarDomainsService = similarDomainsService; } @Override @@ -95,37 +89,4 @@ public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase responseObserver.onCompleted(); } - @Override - public void getDomainInfo(RpcDomainId request, StreamObserver responseObserver) { - var ret = domainInformationService.domainInfo(request.getDomainId()); - - ret.ifPresent(responseObserver::onNext); - - responseObserver.onCompleted(); - } - - @Override - public void getSimilarDomains(RpcDomainLinksRequest request, - StreamObserver responseObserver) { - var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount()); - - var responseBuilder = RpcSimilarDomains - .newBuilder() - .addAllDomains(ret); - - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver responseObserver) { - var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount()); - - var responseBuilder = RpcSimilarDomains - .newBuilder() - .addAllDomains(ret); - - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } } diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java similarity index 89% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java index 40686f74..630e9035 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java @@ -1,10 +1,10 @@ -package nu.marginalia.assistant.dict; +package nu.marginalia.functions.math.dict; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.assistant.client.model.DictionaryEntry; -import nu.marginalia.assistant.client.model.DictionaryResponse; +import nu.marginalia.api.math.model.DictionaryEntry; +import nu.marginalia.api.math.model.DictionaryResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java similarity index 91% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java index d1710122..988a533d 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.dict; +package nu.marginalia.functions.math.dict; import com.google.inject.Singleton; import symspell.SymSpell; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java similarity index 99% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java index 1ec4bb0a..4e37f83d 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; import lombok.AllArgsConstructor; import lombok.SneakyThrows; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java similarity index 86% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java index cdc352c8..2216623b 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; public class Unit { diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java similarity index 98% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java index 4cbb141a..bc1cba7b 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; import com.opencsv.CSVReader; import lombok.SneakyThrows; diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java index 6e3dd46c..a3ddc17b 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java @@ -314,6 +314,10 @@ public class MqPersistence { */ public Collection pollInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException { + if (dataSource.isClosed()) { + return Collections.emptyList(); + } + // Mark new messages as claimed int expected = markInboxMessages(inboxName, instanceUUID, tick, n); if (expected == 0) { @@ -366,6 +370,10 @@ public class MqPersistence { */ public Collection pollReplyInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException { + if (dataSource.isClosed()) { + return Collections.emptyList(); + } + // Mark new messages as claimed int expected = markInboxMessages(inboxName, instanceUUID, tick, n); if (expected == 0) { diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java index 779e2573..27ad832c 100644 --- a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java +++ b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java @@ -76,6 +76,7 @@ public class IndexConstructorMain extends ProcessMainClass { // Grace period so we don't rug pull the logger or jdbc TimeUnit.SECONDS.sleep(5); + System.exit(0); } diff --git a/code/processes/website-adjacencies-calculator/build.gradle b/code/processes/website-adjacencies-calculator/build.gradle index a007c105..d262934e 100644 --- a/code/processes/website-adjacencies-calculator/build.gradle +++ b/code/processes/website-adjacencies-calculator/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation project(':code:common:process') implementation project(':code:common:service-discovery') implementation project(':code:common:service') - implementation project(':code:api:query-api') + implementation project(':code:functions:domain-links:api') implementation libs.bundles.slf4j diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java index 61c2ceee..d0886b9f 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java @@ -4,7 +4,7 @@ import gnu.trove.list.TIntList; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,14 +35,15 @@ public class AdjacenciesData { return ret; } - public AdjacenciesData(QueryClient queryClient, + public AdjacenciesData(AggregateDomainLinksClient linksClient, DomainAliases aliases) { logger.info("Loading adjacency data"); Map tmpMapDtoS = new HashMap<>(100_000); int count = 0; - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = linksClient.getAllDomainLinks(); + for (var iter = allLinks.iterator();;count++) { if (!iter.advance()) { break; diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index eef58571..cc720d5e 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -4,18 +4,20 @@ import com.google.inject.Guice; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.ProcessConfiguration; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.model.EdgeDomain; import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeatImpl; -import nu.marginalia.query.client.QueryClient; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ProcessMainClass; import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.module.DatabaseModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; +import java.time.Duration; import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; @@ -24,18 +26,18 @@ import java.util.stream.IntStream; import static nu.marginalia.adjacencies.SparseBitVector.*; -public class WebsiteAdjacenciesCalculator extends MainClass { +public class WebsiteAdjacenciesCalculator extends ProcessMainClass { private final HikariDataSource dataSource; public AdjacenciesData adjacenciesData; public DomainAliases domainAliases; private static final Logger logger = LoggerFactory.getLogger(WebsiteAdjacenciesCalculator.class); float[] weights; - public WebsiteAdjacenciesCalculator(QueryClient queryClient, HikariDataSource dataSource) throws SQLException { + public WebsiteAdjacenciesCalculator(AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) throws SQLException { this.dataSource = dataSource; domainAliases = new DomainAliases(dataSource); - adjacenciesData = new AdjacenciesData(queryClient, domainAliases); + adjacenciesData = new AdjacenciesData(domainLinksClient, domainAliases); weights = adjacenciesData.getWeights(); } @@ -146,16 +148,20 @@ public class WebsiteAdjacenciesCalculator extends MainClass { } - public static void main(String[] args) throws SQLException { + public static void main(String[] args) throws SQLException, InterruptedException { var injector = Guice.createInjector( new DatabaseModule(false), new ServiceDiscoveryModule()); var dataSource = injector.getInstance(HikariDataSource.class); - var qc = injector.getInstance(QueryClient.class); + var lc = injector.getInstance(AggregateDomainLinksClient.class); - var main = new WebsiteAdjacenciesCalculator(qc, dataSource); + if (!lc.waitReady(Duration.ofSeconds(30))) { + throw new IllegalStateException("Failed to connect to domain-links"); + } + + var main = new WebsiteAdjacenciesCalculator(lc, dataSource); if (args.length == 1 && "load".equals(args[0])) { var processHeartbeat = new ProcessHeartbeatImpl( diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index a3e402f7..798fb3a9 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -2,8 +2,8 @@ plugins { id 'java' id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } java { @@ -19,7 +19,21 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + dependencies { implementation project(':code:common:db') diff --git a/code/services-application/dating-service/build.gradle b/code/services-application/dating-service/build.gradle index a59476c8..b83eff9a 100644 --- a/code/services-application/dating-service/build.gradle +++ b/code/services-application/dating-service/build.gradle @@ -2,8 +2,8 @@ plugins { id 'java' id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { diff --git a/code/services-application/explorer-service/build.gradle b/code/services-application/explorer-service/build.gradle index 596afdf0..75607fae 100644 --- a/code/services-application/explorer-service/build.gradle +++ b/code/services-application/explorer-service/build.gradle @@ -2,8 +2,8 @@ plugins { id 'java' id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index 80936e48..e7d19834 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -2,9 +2,25 @@ plugins { id 'java' id 'io.freefair.sass-base' version '8.4' id 'io.freefair.sass-java' version '8.4' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + + id 'com.google.cloud.tools.jib' version '3.4.0' +} + +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } } application { @@ -14,7 +30,6 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" java { toolchain { @@ -26,6 +41,7 @@ sass { sourceMapEmbed = true outputStyle = EXPANDED } + dependencies { implementation project(':code:common:db') implementation project(':code:common:model') @@ -38,7 +54,8 @@ dependencies { implementation project(':code:libraries:braille-block-punch-cards') implementation project(':code:libraries:term-frequency-dict') - implementation project(':code:api:assistant-api') + implementation project(':code:functions:math:api') + implementation project(':code:functions:domain-info:api') implementation project(':code:api:query-api') implementation project(':code:api:index-api') implementation project(':code:common:service-discovery') diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java index 9c38ffef..28b21da8 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java @@ -4,7 +4,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; -import nu.marginalia.assistant.client.AssistantClient; +import nu.marginalia.api.math.MathClient; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.query.client.QueryClient; @@ -34,7 +34,7 @@ public class SearchOperator { // Marker for filtering out sensitive content from the persistent logs private final Marker queryMarker = MarkerFactory.getMarker("QUERY"); - private final AssistantClient assistantClient; + private final MathClient mathClient; private final DbDomainQueries domainQueries; private final QueryClient queryClient; private final SearchQueryIndexService searchQueryService; @@ -44,7 +44,7 @@ public class SearchOperator { @Inject - public SearchOperator(AssistantClient assistantClient, + public SearchOperator(MathClient mathClient, DbDomainQueries domainQueries, QueryClient queryClient, SearchQueryIndexService searchQueryService, @@ -53,7 +53,7 @@ public class SearchOperator { SearchUnitConversionService searchUnitConversionService) { - this.assistantClient = assistantClient; + this.mathClient = mathClient; this.domainQueries = domainQueries; this.queryClient = queryClient; @@ -162,7 +162,7 @@ public class SearchOperator { @SneakyThrows private void spellCheckTerms(QueryResponse response) { - var suggestions = assistantClient + var suggestions = mathClient .spellCheck(response.searchTermsHuman(), Duration.ofMillis(20)); suggestions.entrySet() diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java index 52b0dfdb..3025497f 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java @@ -2,11 +2,11 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; -import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.assistant.client.model.DictionaryResponse; +import nu.marginalia.api.math.MathClient; +import nu.marginalia.api.math.model.DictionaryResponse; +import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; -import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,18 +23,18 @@ public class DefinitionCommand implements SearchCommandInterface { private final Logger logger = LoggerFactory.getLogger(getClass()); private final MustacheRenderer dictionaryRenderer; - private final AssistantClient assistantClient; + private final MathClient mathClient; private final Predicate queryPatternPredicate = Pattern.compile("^define:[A-Za-z\\s-0-9]+$").asPredicate(); @Inject - public DefinitionCommand(RendererFactory rendererFactory, AssistantClient assistantClient) + public DefinitionCommand(RendererFactory rendererFactory, MathClient mathClient) throws IOException { dictionaryRenderer = rendererFactory.renderer("search/dictionary-results"); - this.assistantClient = assistantClient; + this.mathClient = mathClient; } @Override @@ -57,9 +57,9 @@ public class DefinitionCommand implements SearchCommandInterface { String word = humanQuery.substring(definePrefix.length()).toLowerCase(); try { - return assistantClient + return mathClient .dictionaryLookup(word) - .get(100, TimeUnit.MILLISECONDS); + .get(250, TimeUnit.MILLISECONDS); } catch (Exception e) { logger.error("Failed to lookup definition for word: " + word, e); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java index 76454538..323d744e 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java @@ -14,17 +14,13 @@ import java.io.IOException; import java.util.Optional; public class SearchCommand implements SearchCommandInterface { - private final DomainBlacklist blacklist; private final SearchOperator searchOperator; private final MustacheRenderer searchResultsRenderer; @Inject - public SearchCommand(DomainBlacklist blacklist, - SearchOperator searchOperator, - RendererFactory rendererFactory - ) throws IOException { - this.blacklist = blacklist; + public SearchCommand(SearchOperator searchOperator, + RendererFactory rendererFactory) throws IOException { this.searchOperator = searchOperator; searchResultsRenderer = rendererFactory.renderer("search/search-results"); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java index 2dcbbe96..6ba3fa49 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java @@ -1,8 +1,8 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.api.domains.DomainInfoClient; +import nu.marginalia.api.domains.model.SimilarDomain; import nu.marginalia.browse.DbBrowseDomainsRandom; import nu.marginalia.browse.model.BrowseResult; import nu.marginalia.browse.model.BrowseResultSet; @@ -25,20 +25,20 @@ public class SearchBrowseService { private final DbBrowseDomainsRandom randomDomains; private final DbDomainQueries domainQueries; private final DomainBlacklist blacklist; - private final AssistantClient assistantClient; + private final DomainInfoClient domainInfoClient; private final BrowseResultCleaner browseResultCleaner; @Inject public SearchBrowseService(DbBrowseDomainsRandom randomDomains, DbDomainQueries domainQueries, DomainBlacklist blacklist, - AssistantClient assistantClient, + DomainInfoClient domainInfoClient, BrowseResultCleaner browseResultCleaner) { this.randomDomains = randomDomains; this.domainQueries = domainQueries; this.blacklist = blacklist; - this.assistantClient = assistantClient; + this.domainInfoClient = domainInfoClient; this.browseResultCleaner = browseResultCleaner; } @@ -53,7 +53,7 @@ public class SearchBrowseService { public BrowseResultSet getRelatedEntries(String domainName) throws ExecutionException, InterruptedException, TimeoutException { var domain = domainQueries.getDomainId(new EdgeDomain(domainName)); - var neighbors = assistantClient.similarDomains(domain, 50) + var neighbors = domainInfoClient.similarDomains(domain, 50) .get(100, TimeUnit.MILLISECONDS); neighbors.removeIf(sd -> !sd.screenshot()); @@ -61,7 +61,7 @@ public class SearchBrowseService { // If the results are very few, supplement with the alternative shitty algorithm if (neighbors.size() < 25) { Set allNeighbors = new HashSet<>(neighbors); - allNeighbors.addAll(assistantClient + allNeighbors.addAll(domainInfoClient .linkedDomains(domain, 50) .get(100, TimeUnit.MILLISECONDS) ); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java index 7f228d7f..7cb5c809 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java @@ -1,8 +1,9 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.api.domains.DomainInfoClient; +import nu.marginalia.api.domains.model.DomainInformation; +import nu.marginalia.api.domains.model.SimilarDomain; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.feedlot.model.FeedItems; import nu.marginalia.model.EdgeDomain; @@ -10,7 +11,6 @@ import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import nu.marginalia.screenshot.ScreenshotService; import nu.marginalia.search.SearchOperator; -import nu.marginalia.assistant.client.model.DomainInformation; import nu.marginalia.feedlot.FeedlotClient; import nu.marginalia.search.model.UrlDetails; import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData; @@ -32,7 +32,7 @@ public class SearchSiteInfoService { private static final Logger logger = LoggerFactory.getLogger(SearchSiteInfoService.class); private final SearchOperator searchOperator; - private final AssistantClient assistantClient; + private final DomainInfoClient domainInfoClient; private final SearchFlagSiteService flagSiteService; private final DbDomainQueries domainQueries; private final MustacheRenderer renderer; @@ -41,7 +41,7 @@ public class SearchSiteInfoService { @Inject public SearchSiteInfoService(SearchOperator searchOperator, - AssistantClient assistantClient, + DomainInfoClient domainInfoClient, RendererFactory rendererFactory, SearchFlagSiteService flagSiteService, DbDomainQueries domainQueries, @@ -49,7 +49,7 @@ public class SearchSiteInfoService { ScreenshotService screenshotService) throws IOException { this.searchOperator = searchOperator; - this.assistantClient = assistantClient; + this.domainInfoClient = domainInfoClient; this.flagSiteService = flagSiteService; this.domainQueries = domainQueries; @@ -137,15 +137,20 @@ public class SearchSiteInfoService { boolean hasScreenshot = screenshotService.hasScreenshot(domainId); var feedItemsFuture = feedlotClient.getFeedItems(domainName); - if (domainId < 0 || !assistantClient.isAccepting()) { + if (domainId < 0) { + domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); + similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); + linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); + } + else if (!domainInfoClient.isAccepting()) { domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); } else { - domainInfoFuture = assistantClient.domainInformation(domainId); - similarSetFuture = assistantClient.similarDomains(domainId, 25); - linkingDomainsFuture = assistantClient.linkedDomains(domainId, 25); + domainInfoFuture = domainInfoClient.domainInformation(domainId); + similarSetFuture = domainInfoClient.similarDomains(domainId, 25); + linkingDomainsFuture = domainInfoClient.linkedDomains(domainId, 25); } List sampleResults = searchOperator.doSiteSearch(domainName, 5); @@ -174,7 +179,7 @@ public class SearchSiteInfoService { private T waitForFuture(Future future, Supplier fallback) { try { - return future.get(50, TimeUnit.MILLISECONDS); + return future.get(250, TimeUnit.MILLISECONDS); } catch (Exception e) { logger.info("Failed to get domain data: {}", e.getMessage()); return fallback.get(); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java index d581c674..1727878e 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java @@ -1,6 +1,6 @@ package nu.marginalia.search.svc; -import nu.marginalia.assistant.client.AssistantClient; +import nu.marginalia.api.math.MathClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,11 +21,11 @@ public class SearchUnitConversionService { private final Pattern conversionPattern = Pattern.compile("((\\d+|\\s+|[.()\\-^+%*/]|log[^a-z]|log2[^a-z]|sqrt[^a-z]|log10|cos[^a-z]|sin[^a-z]|tan[^a-z]|log2|pi[^a-z]|e[^a-z]|2pi[^a-z])+)\\s*([a-zA-Z][a-zA-Z^.0-9]*\\s?[a-zA-Z^.0-9]*)\\s+in\\s+([a-zA-Z^.0-9]+\\s?[a-zA-Z^.0-9]*)"); private final Predicate evalPredicate = Pattern.compile("(\\d+|\\s+|[.()\\-^+%*/]|log|log2|sqrt|log10|cos|sin|tan|pi|e|2pi)+").asMatchPredicate(); - private final AssistantClient assistantClient; + private final MathClient mathClient; @Inject - public SearchUnitConversionService(AssistantClient assistantClient) { - this.assistantClient = assistantClient; + public SearchUnitConversionService(MathClient mathClient) { + this.mathClient = mathClient; } public Optional tryConversion(String query) { @@ -40,9 +40,9 @@ public class SearchUnitConversionService { logger.info("{} -> '{}' '{}' '{}'", query, value, from, to); try { - var resultFuture = assistantClient.unitConversion(value, from, to); + var resultFuture = mathClient.unitConversion(value, from, to); return Optional.of( - resultFuture.get(100, TimeUnit.MILLISECONDS) + resultFuture.get(250, TimeUnit.MILLISECONDS) ); } catch (ExecutionException e) { logger.error("Error in unit conversion", e); @@ -68,6 +68,6 @@ public class SearchUnitConversionService { logger.info("eval({})", expr); - return assistantClient.evalMath(expr); + return mathClient.evalMath(expr); } } diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index b9f7625a..24a11e33 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -3,7 +3,7 @@ plugins { id 'application' id 'jvm-test-suite' - id 'com.palantir.docker' version '0.35.0' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { @@ -23,7 +38,12 @@ java { dependencies { implementation project(':third-party:symspell') - implementation project(':code:api:assistant-api') + + implementation project(':code:functions:math') + implementation project(':code:functions:math:api') + implementation project(':code:functions:domain-info') + implementation project(':code:functions:domain-info:api') + implementation project(':code:api:query-api') implementation project(':code:common:config') implementation project(':code:common:service') diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java index 51240c7a..741261b8 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java @@ -4,8 +4,11 @@ import com.google.gson.Gson; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.assistant.suggest.Suggestions; +import nu.marginalia.functions.domains.DomainInfoGrpcService; +import nu.marginalia.functions.math.MathGrpcService; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.screenshot.ScreenshotService; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,10 +27,13 @@ public class AssistantService extends Service { @Inject public AssistantService(BaseServiceParams params, ScreenshotService screenshotService, - AssistantGrpcService assistantGrpcService, + DomainInfoGrpcService domainInfoGrpcService, + MathGrpcService mathGrpcService, Suggestions suggestions) { - super(params, List.of(assistantGrpcService)); + super(params, + ServicePartition.any(), + List.of(domainInfoGrpcService, mathGrpcService)); this.suggestions = suggestions; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java index 7adf7921..2e7b28fb 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java @@ -2,9 +2,9 @@ package nu.marginalia.assistant.suggest; import com.google.inject.Inject; import com.google.inject.name.Named; +import nu.marginalia.functions.math.dict.SpellChecker; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.model.crawl.HtmlFeature; -import nu.marginalia.assistant.dict.SpellChecker; import org.apache.commons.collections4.trie.PatriciaTrie; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index f55610b7..af7e553f 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -1,9 +1,8 @@ plugins { id 'java' - id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } java { @@ -19,7 +18,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + dependencies { implementation libs.bundles.gson diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index 264174a5..8eb44279 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -60,6 +60,7 @@ public class ControlService extends Service { ) throws IOException { super(params); + this.monitors = monitors; this.heartbeatService = heartbeatService; this.eventLogService = eventLogService; diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 5ee7ef74..a4c0e973 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,24 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service-with-dist.gradle" +clean { + delete fileTree('build/dist-extra') +} + +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} java { toolchain { @@ -22,6 +39,15 @@ java { } dependencies { + // These look weird but they're needed to be able to spawn the processes + // from the executor service + + implementation project(':code:processes:website-adjacencies-calculator') + implementation project(':code:processes:crawling-process') + implementation project(':code:processes:loading-process') + implementation project(':code:processes:converting-process') + implementation project(':code:processes:index-constructor-process') + implementation project(':code:common:config') implementation project(':code:common:model') implementation project(':code:common:process') @@ -35,6 +61,8 @@ dependencies { implementation project(':code:libraries:message-queue') + implementation project(':code:functions:domain-links:api') + implementation project(':code:process-models:crawl-spec') implementation project(':code:process-models:crawling-model') implementation project(':code:features-crawl:link-parser') diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java index 537a0969..fa75f016 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java @@ -6,6 +6,7 @@ import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.prototype.RecordActorPrototype; import nu.marginalia.actor.state.ActorStep; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.query.client.QueryClient; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; @@ -32,7 +33,7 @@ public class ExportDataActor extends RecordActorPrototype { private final FileStorageService storageService; private final HikariDataSource dataSource; private final Logger logger = LoggerFactory.getLogger(getClass()); - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; public record Export() implements ActorStep {} public record ExportBlacklist(FileStorageId fid) implements ActorStep {} @@ -114,7 +115,7 @@ public class ExportDataActor extends RecordActorPrototype { var tmpFile = Files.createTempFile(storage.asPath(), "export", ".csv.gz", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--"))); - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = domainLinksClient.getAllDomainLinks(); try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))))) { @@ -154,12 +155,13 @@ public class ExportDataActor extends RecordActorPrototype { @Inject public ExportDataActor(Gson gson, FileStorageService storageService, - HikariDataSource dataSource, QueryClient queryClient) + HikariDataSource dataSource, + AggregateDomainLinksClient domainLinksClient) { super(gson); this.storageService = storageService; this.dataSource = dataSource; - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java index 61619823..0c9f08eb 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java @@ -2,12 +2,14 @@ package nu.marginalia.executor; import com.google.inject.AbstractModule; import com.google.inject.name.Names; +import nu.marginalia.WmsaHome; import java.nio.file.Path; public class ExecutorModule extends AbstractModule { public void configure() { - String dist = System.getProperty("distPath", System.getProperty("WMSA_HOME", "/var/lib/wmsa") + "/dist/current"); + + String dist = System.getProperty("distPath", WmsaHome.getHomePath().resolve("/dist").toString()); bind(Path.class).annotatedWith(Names.named("distPath")).toInstance(Path.of(dist)); } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java index 0091ac9f..61769f5b 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java @@ -1,10 +1,10 @@ package nu.marginalia.executor; -import com.google.gson.Gson; import com.google.inject.Inject; import nu.marginalia.actor.ExecutorActor; import nu.marginalia.actor.ExecutorActorControlService; import nu.marginalia.executor.svc.TransferService; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import nu.marginalia.service.server.mq.MqRequest; @@ -16,10 +16,7 @@ import java.util.List; // Weird name for this one to not have clashes with java.util.concurrent.ExecutorService public class ExecutorSvc extends Service { - private final BaseServiceParams params; - private final Gson gson; private final ExecutorActorControlService actorControlService; - private final TransferService transferService; private static final Logger logger = LoggerFactory.getLogger(ExecutorSvc.class); @@ -28,14 +25,12 @@ public class ExecutorSvc extends Service { public ExecutorSvc(BaseServiceParams params, ExecutorActorControlService actorControlService, ExecutorGrpcService executorGrpcService, - Gson gson, TransferService transferService) { - super(params, List.of(executorGrpcService)); - this.params = params; - this.gson = gson; + super(params, + ServicePartition.partition(params.configuration.node()), + List.of(executorGrpcService)); this.actorControlService = actorControlService; - this.transferService = transferService; Spark.get("/transfer/file/:fid", transferService::transferFile); } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java index 971b2ab9..93f89063 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java @@ -3,6 +3,14 @@ package nu.marginalia.process; import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; +import nu.marginalia.WmsaHome; +import nu.marginalia.adjacencies.WebsiteAdjacenciesCalculator; +import nu.marginalia.converting.ConverterMain; +import nu.marginalia.crawl.CrawlerMain; +import nu.marginalia.index.IndexConstructorMain; +import nu.marginalia.loading.LoaderMain; +import nu.marginalia.service.MainClass; +import nu.marginalia.service.ProcessMainClass; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.BaseServiceParams; import org.slf4j.Logger; @@ -43,16 +51,32 @@ public class ProcessService { } public enum ProcessId { - CRAWLER("crawler-process/bin/crawler-process"), - CONVERTER("converter-process/bin/converter-process"), - LOADER("loader-process/bin/loader-process"), - INDEX_CONSTRUCTOR("index-construction-process/bin/index-construction-process"), - ADJACENCIES_CALCULATOR("website-adjacencies-calculator/bin/website-adjacencies-calculator") + CRAWLER(CrawlerMain.class), + CONVERTER(ConverterMain.class), + LOADER(LoaderMain.class), + INDEX_CONSTRUCTOR(IndexConstructorMain.class), + ADJACENCIES_CALCULATOR(WebsiteAdjacenciesCalculator.class) ; - public final String path; - ProcessId(String path) { - this.path = path; + public final String mainClass; + ProcessId(Class mainClass) { + this.mainClass = mainClass.getName(); + } + + List envOpts() { + String variable = switch (this) { + case CRAWLER -> "CRAWLER_PROCESS_OPTS"; + case CONVERTER -> "CONVERTER_PROCESS_OPTS"; + case LOADER -> "LOADER_PROCESS_OPTS"; + case INDEX_CONSTRUCTOR -> "INDEX_CONSTRUCTION_PROCESS_OPTS"; + case ADJACENCIES_CALCULATOR -> "ADJACENCIES_CALCULATOR_PROCESS_OPTS"; + }; + String value = System.getenv(variable); + + if (value == null) + return List.of(); + else + return Arrays.asList(value.split("\\s+")); } } @@ -63,27 +87,27 @@ public class ProcessService { this.distPath = distPath; } - public boolean trigger(ProcessId processId) throws Exception { - return trigger(processId, new String[0]); - } - public boolean trigger(ProcessId processId, String... parameters) throws Exception { - final String processPath = distPath.resolve(processId.path).toString(); + public boolean trigger(ProcessId processId, String... extraArgs) throws Exception { final String[] env = createEnvironmentVariables(); - final String[] args = createCommandArguments(processPath, parameters); + List args = new ArrayList<>(); + String javaHome = System.getProperty("java.home"); + + args.add(STR."\{javaHome}/bin/java"); + args.add("-cp"); + args.add(System.getProperty("java.class.path")); + args.add("--enable-preview"); + args.addAll(processId.envOpts()); + args.add(processId.mainClass); + args.addAll(Arrays.asList(extraArgs)); Process process; - if (!Files.exists(Path.of(processPath))) { - logger.error("Process not found: {}", processPath); - return false; - } - - logger.info("Starting process: {}: {} // {}", processId, Arrays.toString(args), Arrays.toString(env)); + logger.info("Starting process: {} {}", processId, processId.envOpts()); synchronized (processes) { if (processes.containsKey(processId)) return false; - process = Runtime.getRuntime().exec(args, env); + process = Runtime.getRuntime().exec(args.toArray(String[]::new), env); processes.put(processId, process); } @@ -107,13 +131,6 @@ public class ProcessService { } - private String[] createCommandArguments(String processPath, String[] parameters) { - final String[] args = new String[parameters.length + 1]; - args[0] = processPath; - System.arraycopy(parameters, 0, args, 1, parameters.length); - return args; - } - public boolean isRunning(ProcessId processId) { return processes.containsKey(processId); } @@ -131,25 +148,14 @@ public class ProcessService { /** These environment variables are propagated from the parent process to the child process, * along with WMSA_HOME, but it has special logic */ private final List propagatedEnvironmentVariables = List.of( - "JAVA_HOME", "ZOOKEEPER_HOSTS", - "WMSA_SERVICE_NODE", - "CONVERTER_PROCESS_OPTS", - "LOADER_PROCESS_OPTS", - "INDEX_CONSTRUCTION_PROCESS_OPTS", - "CRAWLER_PROCESS_OPTS"); + "WMSA_SERVICE_NODE" + ); private String[] createEnvironmentVariables() { List opts = new ArrayList<>(); - String WMSA_HOME = System.getenv("WMSA_HOME"); - - if (WMSA_HOME == null || WMSA_HOME.isBlank()) { - WMSA_HOME = "/var/lib/wmsa"; - } - - opts.add(env2str("WMSA_HOME", WMSA_HOME)); - opts.add(env2str("JAVA_OPTS", "--enable-preview")); // + opts.add(env2str("WMSA_HOME", WmsaHome.getHomePath().toString())); for (String envKey : propagatedEnvironmentVariables) { String envValue = System.getenv(envKey); diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 160773ea..cfeeeb57 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { @@ -25,8 +40,13 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:linkdb') + + implementation project(':code:functions:domain-links:partition') + implementation project(':code:functions:domain-links:api') + implementation project(':code:common:service') implementation project(':code:api:index-api') + implementation project(':code:common:service-discovery') implementation project(':code:libraries:array') diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index 242c6cef..2dd90f78 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -4,8 +4,9 @@ import com.google.gson.Gson; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; -import nu.marginalia.index.svc.IndexDomainLinksService; +import nu.marginalia.functions.domainlinks.PartitionDomainLinksService; import nu.marginalia.linkdb.dlinks.DomainLinkDb; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.client.IndexMqEndpoints; import nu.marginalia.index.index.SearchIndex; @@ -51,10 +52,14 @@ public class IndexService extends Service { FileStorageService fileStorageService, DocumentDbReader documentDbReader, DomainLinkDb domainLinkDb, - IndexDomainLinksService indexDomainLinksService, + + PartitionDomainLinksService partitionDomainLinksService, + ServiceEventLog eventLog) { - super(params, List.of(indexQueryService, indexDomainLinksService)); + super(params, + ServicePartition.partition(params.configuration.node()), + List.of(indexQueryService, partitionDomainLinksService)); this.opsService = opsService; this.searchIndex = searchIndex; diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 4f87c11a..a60cdfec 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { @@ -35,6 +50,9 @@ dependencies { implementation project(':code:libraries:language-processing') implementation project(':code:libraries:term-frequency-dict') + implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:domain-links:aggregate') + implementation libs.bundles.slf4j implementation libs.spark diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java deleted file mode 100644 index 89de784c..00000000 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java +++ /dev/null @@ -1,90 +0,0 @@ -package nu.marginalia.query; - -import com.google.inject.Inject; -import io.grpc.stub.StreamObserver; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; -import nu.marginalia.index.api.IndexDomainLinksApiGrpc; -import nu.marginalia.index.api.RpcDomainIdCount; -import nu.marginalia.index.api.RpcDomainIdList; -import nu.marginalia.index.api.RpcDomainIdPairs; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.id.ServiceId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class QueryGRPCDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase { - private static final Logger logger = LoggerFactory.getLogger(QueryGRPCDomainLinksService.class); - private final GrpcMultiNodeChannelPool channelPool; - - @Inject - public QueryGRPCDomainLinksService(GrpcChannelPoolFactory channelPoolFactory) { - channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub); - } - - @Override - public void getAllLinks(nu.marginalia.index.api.Empty request, - StreamObserver responseObserver) { - channelPool.callEachSequential(stub -> stub.getAllLinks(request)) - .forEach( - iter -> iter.forEachRemaining(responseObserver::onNext) - ); - - responseObserver.onCompleted(); - } - - @Override - public void getLinksFromDomain(nu.marginalia.index.api.RpcDomainId request, - StreamObserver responseObserver) { - var rspBuilder = RpcDomainIdList.newBuilder(); - - channelPool.callEachSequential(stub -> stub.getLinksFromDomain(request)) - .map(RpcDomainIdList::getDomainIdList) - .forEach(rspBuilder::addAllDomainId); - - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void getLinksToDomain(nu.marginalia.index.api.RpcDomainId request, - StreamObserver responseObserver) { - var rspBuilder = RpcDomainIdList.newBuilder(); - - channelPool.callEachSequential(stub -> stub.getLinksToDomain(request)) - .map(RpcDomainIdList::getDomainIdList) - .forEach(rspBuilder::addAllDomainId); - - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void countLinksFromDomain(nu.marginalia.index.api.RpcDomainId request, - StreamObserver responseObserver) { - - int sum = channelPool.callEachSequential(stub -> stub.countLinksFromDomain(request)) - .mapToInt(RpcDomainIdCount::getIdCount) - .sum(); - - var rspBuilder = RpcDomainIdCount.newBuilder(); - rspBuilder.setIdCount(sum); - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void countLinksToDomain(nu.marginalia.index.api.RpcDomainId request, - io.grpc.stub.StreamObserver responseObserver) { - - int sum = channelPool.callEachSequential(stub -> stub.countLinksToDomain(request)) - .mapToInt(RpcDomainIdCount::getIdCount) - .sum(); - - var rspBuilder = RpcDomainIdCount.newBuilder(); - rspBuilder.setIdCount(sum); - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - -} diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java index b2bb9391..047a2c4c 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java @@ -10,7 +10,8 @@ import nu.marginalia.index.api.*; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.query.svc.QueryFactory; import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +34,7 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { private final QueryFactory queryFactory; private final DomainBlacklist blacklist; + private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); @Inject public QueryGRPCService(QueryFactory queryFactory, @@ -41,7 +43,9 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { { this.queryFactory = queryFactory; this.blacklist = blacklist; - this.channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexApiGrpc::newBlockingStub); + this.channelPool = channelPoolFactory.createMulti( + ServiceKey.forGrpcApi(IndexApiGrpc.class, ServicePartition.multi()), + IndexApiGrpc::newBlockingStub); } public void query(nu.marginalia.index.api.RpcQsQuery request, @@ -80,39 +84,22 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { @SneakyThrows List executeQueries(RpcIndexQuery indexRequest, int totalSize) { - return channelPool.invokeAll(stub -> new QueryTask(stub, indexRequest)) - .stream() - .filter(f -> f.state() == Future.State.SUCCESS) - .map(Future::resultNow) - .flatMap(List::stream) - .sorted(comparator) - .limit(totalSize) - .toList(); - } - - private class QueryTask implements Callable> { - private final IndexApiGrpc.IndexApiBlockingStub stub; - private final RpcIndexQuery indexRequest; - - public QueryTask(IndexApiGrpc.IndexApiBlockingStub stub, RpcIndexQuery indexRequest) { - this.stub = stub; - this.indexRequest = indexRequest; - } - - @Override - public List call() { - var rsp = stub.query(indexRequest); - List ret = new ArrayList<>(); - - while (rsp.hasNext()) { - RpcDecoratedResultItem next = rsp.next(); - if (isBlacklisted(next)) - continue; - ret.add(next); + var futures = + channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query) + .async(executor) + .runEach(indexRequest); + List results = new ArrayList<>(); + for (var future : futures) { + try { + future.get().forEachRemaining(results::add); + } + catch (Exception e) { + logger.error("Downstream exception", e); } - - return ret; } + results.sort(comparator); + results.removeIf(this::isBlacklisted); + return results.subList(0, Math.min(totalSize, results.size())); } private boolean isBlacklisted(RpcDecoratedResultItem item) { diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index 0c21558f..6d63be29 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -3,6 +3,8 @@ package nu.marginalia.query; import com.google.inject.Inject; import io.prometheus.client.Histogram; import lombok.SneakyThrows; +import nu.marginalia.functions.domainlinks.AggregateDomainLinksService; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import spark.Spark; @@ -22,12 +24,13 @@ public class QueryService extends Service { @SneakyThrows @Inject public QueryService(BaseServiceParams params, - QueryGRPCDomainLinksService domainLinksService, + AggregateDomainLinksService domainLinksService, QueryGRPCService queryGRPCService, QueryBasicInterface queryBasicInterface) { super(params, () -> Spark.staticFileLocation("/static/"), + ServicePartition.any(), List.of(queryGRPCService, domainLinksService)); diff --git a/code/tools/screenshot-capture-tool/build.gradle b/code/tools/screenshot-capture-tool/build.gradle index 6c1f7c67..b821f7b7 100644 --- a/code/tools/screenshot-capture-tool/build.gradle +++ b/code/tools/screenshot-capture-tool/build.gradle @@ -3,7 +3,7 @@ plugins { id 'application' id 'jvm-test-suite' - id 'com.palantir.docker' version '0.35.0' + id 'com.google.cloud.tools.jib' version '3.4.0' } java { @@ -17,7 +17,21 @@ application { applicationName = 'screenshot-capture-tool' } -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + tasks.distZip.enabled = false diff --git a/docker-service-with-dist.gradle b/docker-service-with-dist.gradle deleted file mode 100644 index c06bd4bd..00000000 --- a/docker-service-with-dist.gradle +++ /dev/null @@ -1,72 +0,0 @@ -ext { - dockerImage='openjdk:21-slim' -} - -tasks.register('dockerFile') { - buildDir.mkdir() - - var df = new File(buildDir, "Dockerfile") - doLast { - df.text = """# -# I'm auto-generated, please don't make changes to me or commit me to git -# -# The template exists in docker-service.gradle -# -FROM ${dockerImage} - -RUN apt-get update && apt-get install -y curl - -ADD ${application.applicationName}.tar / -ADD crawler-process.tar /dist -ADD loader-process.tar /dist -ADD converter-process.tar /dist -ADD website-adjacencies-calculator.tar /dist -ADD index-construction-process.tar /dist - -RUN mkdir /wmsa - -# This will make the service grab the hostname from the HOSTNAME variable -ENV WMSA_IN_DOCKER true - -ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} -""" - } - it.outputs.file(df) -} - -dockerPrepare { - dependsOn tasks.dockerFile - - dependsOn project(':code:processes:website-adjacencies-calculator').distTar - dependsOn project(':code:processes:crawling-process').distTar - dependsOn project(':code:processes:loading-process').distTar - dependsOn project(':code:processes:converting-process').distTar - dependsOn project(':code:processes:index-constructor-process').distTar -} - -dockerfileZip { - dependsOn tasks.dockerFile -} - -docker { - dockerfile = tasks.dockerFile.outputs.files.singleFile - - var registry = project.hasProperty('docker-registry') ? project.property('docker-registry') : 'marginalia' - var tagName = project.hasProperty('docker-tag') ? project.property('docker-tag') : 'latest' - - name = registry+'/'+application.applicationName+':'+tagName - tag 'test', (registry+'/'+application.applicationName+':'+tagName) - - files tasks.distTar.outputs, \ - project(':code:processes:crawling-process').distTar.outputs, \ - project(':code:processes:loading-process').distTar.outputs, \ - project(':code:processes:converting-process').distTar.outputs, \ - project(':code:processes:index-constructor-process').distTar.outputs, \ - project(':code:processes:website-adjacencies-calculator').distTar.outputs - - dependsOn project(':code:processes:crawling-process').distTar - dependsOn project(':code:processes:loading-process').distTar - dependsOn project(':code:processes:converting-process').distTar - dependsOn project(':code:processes:index-constructor-process').distTar - dependsOn project(':code:processes:website-adjacencies-calculator').distTar -} diff --git a/docker-service.gradle b/docker-service.gradle deleted file mode 100644 index 25dd0e17..00000000 --- a/docker-service.gradle +++ /dev/null @@ -1,50 +0,0 @@ -ext { - dockerImage='openjdk:21-slim' -} - -tasks.register('dockerFile') { - buildDir.mkdir() - - var df = new File(buildDir, "Dockerfile") - doLast { - df.text = """# -# I'm auto-generated, please don't make changes to me or commit me to git -# -# The template exists in docker-service.gradle -# -FROM ${dockerImage} - -RUN apt-get update && apt-get install -y curl -ADD ${application.applicationName}.tar / -RUN mkdir /wmsa - -# This will make the service grab the hostname from the HOSTNAME variable -ENV WMSA_IN_DOCKER true - -ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} -""" - } - it.outputs.file(df) -} - -dockerPrepare { - dependsOn tasks.dockerFile -} - -dockerfileZip { - dependsOn tasks.dockerFile -} - - -docker { - dockerfile = tasks.dockerFile.outputs.files.singleFile - - var registry = project.hasProperty('docker-registry') ? project.property('docker-registry') : 'marginalia' - var tagName = project.hasProperty('docker-tag') ? project.property('docker-tag') : 'latest' - - name = registry+'/'+application.applicationName+':'+tagName - tag 'test', (registry+'/'+application.applicationName+':'+tagName) - - files tasks.distTar.outputs - dependsOn tasks.distTar -} diff --git a/settings.gradle b/settings.gradle index 79aeace2..49462b94 100644 --- a/settings.gradle +++ b/settings.gradle @@ -11,6 +11,16 @@ include 'code:services-application:api-service' include 'code:services-application:dating-service' include 'code:services-application:explorer-service' +include 'code:functions:math' +include 'code:functions:math:api' + +include 'code:functions:domain-info' +include 'code:functions:domain-info:api' + +include 'code:functions:domain-links:partition' +include 'code:functions:domain-links:aggregate' +include 'code:functions:domain-links:api' + include 'code:libraries:array' include 'code:libraries:geo-ip' include 'code:libraries:btree' @@ -55,7 +65,6 @@ include 'code:features-index:domain-ranking' include 'code:api:query-api' include 'code:api:index-api' -include 'code:api:assistant-api' include 'code:api:process-mqapi' include 'code:api:executor-api' @@ -174,7 +183,7 @@ dependencyResolutionManagement { library('bucket4j','com.github.vladimir-bukhtoyarov','bucket4j-core').version('7.5.0') library('gson','com.google.code.gson','gson').version('2.10.1') - library('gson-type-adapter','com.github.Marcono1234','gson-record-type-adapter-factory').version('0.2.0') + library('gson-type-adapter','com.github.Marcono1234','gson-record-type-adapter-factory').version('0.3.0') library('zstd','com.github.luben','zstd-jni').version('1.5.2-2') library('lz4','org.lz4','lz4-java').version('1.8.0') @@ -221,8 +230,5 @@ dependencyResolutionManagement { bundle('curator', ['curator-framework', 'curator-x-discovery']) } - - } - } From 3fd2a83184a24a38befccc511bb3508fc3e207be Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 15:27:39 +0100 Subject: [PATCH 20/56] * Extract the search-query function --- code/api/index-api/build.gradle | 14 +- .../index/client/IndexProtobufCodecTest.java | 5 +- .../result-ranking/build.gradle | 1 + .../marginalia/ranking/ResultKeywordSet.java | 3 +- .../nu/marginalia/ranking/ResultValuator.java | 6 +- .../ranking/factors/Bm25Factor.java | 6 +- .../ranking/factors/PriorityTermBonus.java | 3 +- code/features-qs/query-parser/readme.md | 10 -- .../nu/marginalia/ngrams/DenseBitMapTest.java | 57 --------- .../query_parser/BodyQueryParserTest.java | 116 ----------------- .../query_parser/QueryParserTest.java | 93 -------------- .../query_parser/QueryVariantsTest.java | 75 ----------- .../transform_list/TransformListTest.java | 120 ------------------ .../marginalia/util/TestLanguageModels.java | 38 ------ code/features-qs/readme.md | 7 - code/functions/math/build.gradle | 1 - .../search-query/api}/build.gradle | 22 +++- .../api/searchquery}/IndexProtobufCodec.java | 10 +- .../api/searchquery}/QueryClient.java | 8 +- .../api/searchquery}/QueryProtobufCodec.java | 69 +++++----- .../model/query}/ProcessedQuery.java | 4 +- .../searchquery/model/query}/QueryParams.java | 5 +- .../model/query}/QueryResponse.java | 5 +- .../model/query/SearchSetIdentifier.java | 2 +- .../model/query/SearchSpecification.java | 4 +- .../model/query/SearchSubquery.java | 2 +- .../model/results/Bm25Parameters.java | 2 +- .../results/DecoratedSearchResultItem.java | 2 +- .../model/results/ResultRankingContext.java | 2 +- .../results/ResultRankingParameters.java | 2 +- .../model/results/SearchResultItem.java | 2 +- .../results/SearchResultKeywordScore.java | 2 +- .../results/SearchResultPreliminaryScore.java | 2 +- .../model/results/SearchResultSet.java | 2 +- .../api/src/main/protobuf/query-api.proto} | 14 +- .../search-query}/build.gradle | 34 +++-- .../searchquery}/QueryGRPCService.java | 24 +++- .../query_parser/QueryParser.java | 8 +- .../query_parser/QueryPermutation.java | 6 +- .../query_parser/QueryTokenizer.java | 6 +- .../query_parser/QueryVariants.java | 16 +-- .../query_parser/token/Token.java | 2 +- .../query_parser/token/TokenType.java | 2 +- .../query_parser/token/TokenVisitor.java | 2 +- .../searchquery}/svc/QueryFactory.java | 26 ++-- .../svc/QueryLimitsAccumulator.java | 8 +- .../svc/QuerySearchTermsAccumulator.java | 8 +- .../util}/language/EnglishDictionary.java | 2 +- .../marginalia/util}/ngrams/DenseBitMap.java | 2 +- .../util}/ngrams/NGramBloomFilter.java | 2 +- .../util}/transform_list/TransformList.java | 2 +- .../api-service/build.gradle | 3 +- .../nu/marginalia/api/ApiSearchOperator.java | 9 +- .../search-service/build.gradle | 4 +- .../nu/marginalia/search/SearchOperator.java | 4 +- .../search/SearchQueryParamFactory.java | 8 +- .../search/SearchResultClusterer.java | 2 +- .../search/command/SearchAdtechParameter.java | 2 +- .../search/command/SearchJsParameter.java | 2 +- .../search/command/SearchParameters.java | 2 +- .../search/model/SearchProfile.java | 4 +- .../marginalia/search/model/UrlDetails.java | 4 +- .../search/results/UrlDeduplicator.java | 2 +- .../search/svc/SearchQueryIndexService.java | 8 +- .../assistant-service/build.gradle | 1 - .../control-service/build.gradle | 4 +- .../control/app/svc/SearchToBanService.java | 4 +- .../executor-service/build.gradle | 1 - .../actor/task/ExportDataActor.java | 1 - code/services-core/index-service/build.gradle | 1 + .../index/results/IndexMetadataService.java | 2 +- .../index/results/IndexResultDecorator.java | 6 +- .../IndexResultDomainDeduplicator.java | 2 +- .../index/results/IndexResultValuator.java | 10 +- .../index/svc/IndexQueryService.java | 22 ++-- .../index/svc/SearchParameters.java | 10 +- .../index/svc/SearchTermsService.java | 2 +- .../index/svc/searchset/RankingSearchSet.java | 1 - code/services-core/query-service/build.gradle | 6 +- .../marginalia/query/QueryBasicInterface.java | 18 +-- .../nu/marginalia/query/QueryService.java | 1 + .../query/svc/QueryFactoryTest.java | 10 +- settings.gradle | 5 +- 83 files changed, 261 insertions(+), 766 deletions(-) delete mode 100644 code/features-qs/query-parser/readme.md delete mode 100644 code/features-qs/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java delete mode 100644 code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java delete mode 100644 code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java delete mode 100644 code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java delete mode 100644 code/features-qs/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java delete mode 100644 code/features-qs/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java delete mode 100644 code/features-qs/readme.md rename code/{api/query-api => functions/search-query/api}/build.gradle (70%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/IndexProtobufCodec.java (94%) rename code/{api/query-api/src/main/java/nu/marginalia/query/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/QueryClient.java (87%) rename code/{api/query-api/src/main/java/nu/marginalia/query => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/QueryProtobufCodec.java (76%) rename code/{api/query-api/src/main/java/nu/marginalia/query/model => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query}/ProcessedQuery.java (82%) rename code/{api/query-api/src/main/java/nu/marginalia/query/model => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query}/QueryParams.java (91%) rename code/{api/query-api/src/main/java/nu/marginalia/query/model => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query}/QueryResponse.java (76%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/query/SearchSetIdentifier.java (81%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/query/SearchSpecification.java (87%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/query/SearchSubquery.java (98%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/Bm25Parameters.java (83%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/DecoratedSearchResultItem.java (97%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/ResultRankingContext.java (95%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/ResultRankingParameters.java (97%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/SearchResultItem.java (97%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/SearchResultKeywordScore.java (98%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/SearchResultPreliminaryScore.java (92%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/search-query/api/src/main/java/nu/marginalia/api/searchquery}/model/results/SearchResultSet.java (94%) rename code/{api/index-api/src/main/protobuf/index-api.proto => functions/search-query/api/src/main/protobuf/query-api.proto} (94%) rename code/{features-qs/query-parser => functions/search-query}/build.gradle (54%) rename code/{services-core/query-service/src/main/java/nu/marginalia/query => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/QueryGRPCService.java (83%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/QueryParser.java (94%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/QueryPermutation.java (97%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/QueryTokenizer.java (90%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/QueryVariants.java (96%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/token/Token.java (95%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/token/TokenType.java (87%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/query_parser/token/TokenVisitor.java (87%) rename code/{services-core/query-service/src/main/java/nu/marginalia/query => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/svc/QueryFactory.java (87%) rename code/{services-core/query-service/src/main/java/nu/marginalia/query => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/svc/QueryLimitsAccumulator.java (91%) rename code/{services-core/query-service/src/main/java/nu/marginalia/query => functions/search-query/src/main/java/nu/marginalia/functions/searchquery}/svc/QuerySearchTermsAccumulator.java (92%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/util}/language/EnglishDictionary.java (99%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/util}/ngrams/DenseBitMap.java (98%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/util}/ngrams/NGramBloomFilter.java (98%) rename code/{features-qs/query-parser/src/main/java/nu/marginalia => functions/search-query/src/main/java/nu/marginalia/util}/transform_list/TransformList.java (98%) diff --git a/code/api/index-api/build.gradle b/code/api/index-api/build.gradle index a38fb9f0..3817c5de 100644 --- a/code/api/index-api/build.gradle +++ b/code/api/index-api/build.gradle @@ -1,7 +1,5 @@ plugins { id 'java' - - id "com.google.protobuf" version "0.9.4" id 'jvm-test-suite' } @@ -11,16 +9,6 @@ java { } } -sourceSets { - main { - proto { - srcDir 'src/main/protobuf' - } - } -} - -apply from: "$rootProject.projectDir/protobuf.gradle" - dependencies { implementation project(':code:common:model') implementation project(':code:common:config') @@ -28,6 +16,8 @@ dependencies { implementation project(':code:libraries:message-queue') implementation project(':code:features-index:index-query') + implementation project(':code:functions:search-query:api') + implementation libs.bundles.slf4j implementation libs.prometheus diff --git a/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java b/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java index 7178a4c9..1782765d 100644 --- a/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java +++ b/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java @@ -1,7 +1,8 @@ package nu.marginalia.index.client; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.IndexProtobufCodec; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.SpecificationLimit; import org.junit.jupiter.api.Test; diff --git a/code/features-index/result-ranking/build.gradle b/code/features-index/result-ranking/build.gradle index a8cb48a4..4dfabcaf 100644 --- a/code/features-index/result-ranking/build.gradle +++ b/code/features-index/result-ranking/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:service') implementation project(':code:api:index-api') + implementation project(':code:functions:search-query:api') implementation libs.bundles.slf4j implementation libs.guice diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java index af33281d..5377ce46 100644 --- a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java +++ b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java @@ -1,6 +1,7 @@ package nu.marginalia.ranking; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; + +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import java.util.List; diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java index 5698e64c..60fa2be1 100644 --- a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java +++ b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java @@ -1,8 +1,8 @@ package nu.marginalia.ranking; -import nu.marginalia.index.client.model.results.ResultRankingContext; -import nu.marginalia.index.client.model.results.ResultRankingParameters; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.model.crawl.HtmlFeature; import nu.marginalia.model.crawl.PubDate; import nu.marginalia.model.idx.DocumentFlags; diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java index d99df66f..98c58fd8 100644 --- a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java +++ b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java @@ -1,8 +1,8 @@ package nu.marginalia.ranking.factors; -import nu.marginalia.index.client.model.results.Bm25Parameters; -import nu.marginalia.index.client.model.results.ResultRankingContext; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.model.results.Bm25Parameters; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.ranking.ResultKeywordSet; diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java index 05579e47..5b942dc3 100644 --- a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java +++ b/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java @@ -1,6 +1,7 @@ package nu.marginalia.ranking.factors; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; + +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import java.util.List; diff --git a/code/features-qs/query-parser/readme.md b/code/features-qs/query-parser/readme.md deleted file mode 100644 index 1296be13..00000000 --- a/code/features-qs/query-parser/readme.md +++ /dev/null @@ -1,10 +0,0 @@ -# Query Parser - -End-user search query parsing tools used by the [query-service](../../services-core/query-service). - -## Central Classes - -* [QueryTokenizer](src/main/java/nu/marginalia/query_parser/QueryTokenizer.java) -* [QueryParser](src/main/java/nu/marginalia/query_parser/QueryParser.java) -* [QueryPermutations](src/main/java/nu/marginalia/query_parser/QueryVariants.java) - here be dragons -* [QueryVariants](src/main/java/nu/marginalia/query_parser/QueryVariants.java) - here be dragons \ No newline at end of file diff --git a/code/features-qs/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java deleted file mode 100644 index d2db16b6..00000000 --- a/code/features-qs/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package nu.marginalia.ngrams; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class DenseBitMapTest { - - @Test - public void testSetAll() { - var dbm = new DenseBitMap(129); - for (int i = 0; i < dbm.cardinality; i++) { - dbm.set(i); - } - - for (int i = 0; i < dbm.cardinality; i++) { - assertTrue(dbm.get(i)); - } - } - - @Test - public void testSetEven() { - var dbm = new DenseBitMap(131); - for (int i = 0; i < dbm.cardinality; i+=2) { - dbm.set(i); - } - - for (int i = 0; i < dbm.cardinality; i+=2) { - assertTrue(dbm.get(i)); - } - - for (int i = 1; i < dbm.cardinality; i+=2) { - assertFalse(dbm.get(i)); - } - } - - @Test - public void testSetAllClearSome() { - var dbm = new DenseBitMap(129); - - for (int i = 0; i < dbm.cardinality; i++) { - dbm.set(i); - } - for (int i = 1; i < dbm.cardinality; i+=2) { - dbm.clear(i); - } - - for (int i = 0; i < dbm.cardinality; i+=2) { - assertTrue(dbm.get(i), "Expected " + i + " to be set"); - } - - for (int i = 1; i < dbm.cardinality; i+=2) { - assertFalse(dbm.get(i), "Expected " + i + " to be clear"); - } - } -} \ No newline at end of file diff --git a/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java deleted file mode 100644 index 8cc38312..00000000 --- a/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package nu.marginalia.query_parser; - -import nu.marginalia.LanguageModels; -import nu.marginalia.language.EnglishDictionary; -import nu.marginalia.ngrams.NGramBloomFilter; -import nu.marginalia.term_frequency_dict.TermFrequencyDict; -import nu.marginalia.query_parser.token.TokenType; -import nu.marginalia.util.TestLanguageModels; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class BodyQueryParserTest { - private QueryParser parser; - private static TermFrequencyDict dict; - private static EnglishDictionary englishDictionary; - private static NGramBloomFilter nGramBloomFilter; - private static final LanguageModels lm = TestLanguageModels.getLanguageModels(); - private QueryPermutation permutation; - - @BeforeAll - public static void init() throws IOException { - dict = new TermFrequencyDict(lm); - nGramBloomFilter = new NGramBloomFilter(lm); - englishDictionary = new EnglishDictionary(dict); - } - - @BeforeEach - public void setUp() { - parser = new QueryParser(); - permutation = new QueryPermutation(new QueryVariants(lm, dict, nGramBloomFilter, englishDictionary)); - } - - @Test - public void testTitleMatcher() { - List terms = List.of("3d", "realms"); - assertEquals(2, terms.stream().map(String::toLowerCase).filter("3D Realms Site: Forums".toLowerCase()::contains).count()); - } - @Test - void parseSimple() { - var results = parser.parse("hello"); - results.forEach(System.out::println); - assertEquals(1, results.size()); - assertEquals(TokenType.LITERAL_TERM, results.get(0).type); - assertEquals("hello", results.get(0).str); - } - - @Test - void parseQuotes() { - var results = parser.parse("\u201Chello world\u201D"); - results.forEach(System.out::println); - assertEquals(TokenType.QUOT_TERM, results.get(0).type); - assertEquals("hello_world", results.get(0).str); - assertEquals("\"hello world\"", results.get(0).displayStr); - } - - @Test - void parseExclude() { - var results = parser.parse("-Hello"); - results.forEach(System.out::println); - assertEquals(TokenType.EXCLUDE_TERM, results.get(0).type); - assertEquals("hello", results.get(0).str); - assertEquals("-hello", results.get(0).displayStr); - } - - @Test - void parseNear() { - var results = parser.parse("near:memex.marginalia.nu"); - results.forEach(System.out::println); - assertEquals(TokenType.NEAR_TERM, results.get(0).type); - assertEquals("memex.marginalia.nu", results.get(0).str); - assertEquals("near:memex.marginalia.nu", results.get(0).displayStr); - } - - @Test - void parseCombined() { - for (var list : permutation.permuteQueries(parser.parse("dune 2 remake"))) { - for (var t: list) { - System.out.printf("%s ", t.str); - } - System.out.println(); - } - } - @Test - void parseCombinedDOS() { - for (var list : permutation.permuteQueries(parser.parse("ab ba baa abba baba ab ba"))) { - for (var t: list) { - System.out.printf("%s ", t.str); - } - System.out.println(); - } - } - - @Test - void parseCombinedSuperman() { - for (var list : permutation.permuteQueries(parser.parse("wizardry proving grounds of the mad overlord"))) { - for (var t: list) { - System.out.printf("%s ", t.str); - } - System.out.println(); - } - } - @Test - void testEdgeCases() { - parser.parse("site:localhost 3D").forEach(System.out::println); - parser.parse("-wolfenstein 3D").forEach(System.out::println); - parser.parse("-wolfenstein 3D \"").forEach(System.out::println); - } - - -} \ No newline at end of file diff --git a/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java deleted file mode 100644 index 8bc19545..00000000 --- a/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package nu.marginalia.query_parser; - -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenType; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class QueryParserTest { - private static QueryParser parser; - - @BeforeAll - public static void setUp() { - parser = new QueryParser(); - } - - @Test - public void testTabHandling() { - var query = " lorem\tipsum\ndolor sit"; - var ret = parser.parse(query); - assertEquals(4, ret.size()); - - var lorem = ret.get(0); - assertEquals("lorem", lorem.str); - assertEquals("lorem", lorem.displayStr); - - var ipsum = ret.get(1); - assertEquals("ipsum", ipsum.str); - assertEquals("ipsum", ipsum.displayStr); - - var dolor = ret.get(2); - assertEquals("dolor", dolor.str); - assertEquals("dolor", dolor.displayStr); - - var sit = ret.get(3); - assertEquals("sit", sit.str); - assertEquals("sit", sit.displayStr); - } - - @Test - public void testAdviceString() { - var ret = parser.parse("alcibiades (plato) \"my query\" -cars"); - assertEquals(4, ret.size()); - - var alcibiades = ret.get(0); - assertEquals(TokenType.LITERAL_TERM, alcibiades.type); - assertEquals("alcibiades", alcibiades.str); - assertEquals("alcibiades", alcibiades.displayStr); - - var plato = ret.get(1); - assertEquals(TokenType.ADVICE_TERM, plato.type); - assertEquals("plato", plato.str); - assertEquals("(plato)", plato.displayStr); - - var my_query = ret.get(2); - assertEquals(TokenType.QUOT_TERM, my_query.type); - assertEquals("my_query", my_query.str); - assertEquals("\"my query\"", my_query.displayStr); - - var not_cars = ret.get(3); - assertEquals(TokenType.EXCLUDE_TERM, not_cars.type); - assertEquals("cars", not_cars.str); - assertEquals("-cars", not_cars.displayStr); - } - - @Test - public void testParseYear() { - System.out.println(parser.parse("year>2000")); - System.out.println(parser.parse("year=2000")); - System.out.println(parser.parse("year<2000")); - } - - @Test - public void testNonAsciiNames() { - verifyParseResult("André the Giant", "andre", "the", "giant"); - verifyParseResult("Stanisław Lem", "stanislaw", "lem"); - verifyParseResult("Nicolae Ceaușescu", "nicolae", "ceausescu"); - verifyParseResult("Þorrablót", "thorrablot"); - verifyParseResult("Karolis Koncevičius", "karolis", "koncevicius"); - } - - private void verifyParseResult(String query, String... expectedTokens) { - assertArrayEquals(expectedTokens, getTokenStrings(parser.parse(query))); - } - private String[] getTokenStrings(List tokens) { - return tokens.stream().map(t -> t.str).toArray(String[]::new); - } - -} \ No newline at end of file diff --git a/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java deleted file mode 100644 index d82976e9..00000000 --- a/code/features-qs/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java +++ /dev/null @@ -1,75 +0,0 @@ -package nu.marginalia.query_parser; - -import nu.marginalia.LanguageModels; -import nu.marginalia.language.EnglishDictionary; -import nu.marginalia.ngrams.NGramBloomFilter; -import nu.marginalia.term_frequency_dict.TermFrequencyDict; -import nu.marginalia.util.TestLanguageModels; -import nu.marginalia.language.sentence.SentenceExtractor; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; - -class QueryVariantsTest { - static QueryVariants variants; - static QueryParser parser; - static SentenceExtractor se; - - @BeforeAll - public static void setUp() throws IOException { - LanguageModels lm = TestLanguageModels.getLanguageModels(); - - se = new SentenceExtractor(lm); - - var dict = new TermFrequencyDict(lm); - var ngrams = new NGramBloomFilter(lm); - variants = new QueryVariants(lm, dict, ngrams, new EnglishDictionary(dict)); - parser = new QueryParser(); - } - - @Test - void getQueryFood() { - System.out.println(se.extractSentence("we are alone")); - testCase("Omelet recipe"); - } - - @Test - void queryNegation() { - System.out.println(se.extractSentence("salt lake -city")); - testCase("salt lake -city"); - } - - - @Test - void getQueryVariants() { - System.out.println(se.extractSentence("we are alone")); - testCase("inside job reviews"); - testCase("plato apology"); - testCase("mechanical keyboard"); - testCase("DOS"); - testCase("dos"); - testCase("we are alone"); - testCase("3D Realms"); - testCase("I am alone"); - testCase("plato cave"); - testCase("The internet is dead"); - - testCase("TRS80"); - testCase("TRS-80"); - testCase("TRS-80"); - testCase("Raspberry Pi 2"); - testCase("Duke Nukem 3D"); - testCase("The Man of Tomorrow"); - testCase("Computer Manual"); - testCase("Knitting"); - testCase("capcom"); - testCase("the man of tomorrow"); - - } - - private void testCase(String input) { - var tokens = variants.getQueryVariants(parser.parse(input)); - System.out.println(tokens); - } -} \ No newline at end of file diff --git a/code/features-qs/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java deleted file mode 100644 index d34a86ea..00000000 --- a/code/features-qs/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package nu.marginalia.transform_list; - -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class TransformListTest { - - @Test - void transformEach() { - - List values = Stream.of(1,2,3,4).collect(Collectors.toList()); - new TransformList<>(values).transformEach(e -> { - int v = e.value(); - if (v == 1) e.remove(); - if (v == 2) e.replace(5); - if (v == 4) e.remove(); - }); - - assertEquals(List.of(5,3), values); - } - - @Test - void transformEachPairRemoveReplace() { - List values = Stream.of(1,2,3,4,5,6).collect(Collectors.toList()); - new TransformList<>(values).transformEachPair((a,b) -> { - System.out.println(a.value() + ":" + b.value()); - int v = a.value(); - if (v == 1 || v == 3 || v == 5) { - a.remove(); - b.replace(-b.value()); - } - - }); - - assertEquals(List.of(-2, -4, -6), values); - } - - @Test - void transformEachPairRemoveRemove() { - List values = Stream.of(1,2,3,4,5,6).collect(Collectors.toList()); - new TransformList<>(values).transformEachPair((a,b) -> { - System.out.println(a.value() + ":" + b.value()); - int v = a.value(); - if (v == 1 || v == 3 || v == 5) { - a.remove(); - b.remove(); - } - - }); - - assertEquals(List.of(), values); - } - - @Test - void transformEachPairReplaceRemove() { - List values = Stream.of(1,2,3,4,5,6).collect(Collectors.toList()); - new TransformList<>(values).transformEachPair((a,b) -> { - System.out.println(a.value() + ":" + b.value()); - int v = a.value(); - if (v == 1 || v == 3 || v == 5) { - a.replace(-a.value()); - b.remove(); - } - - }); - - assertEquals(List.of(-1, -3, -5), values); - } - - @Test - void transformEachPairReplaceReplace() { - List values = Stream.of(1,2,3,4,5,6).collect(Collectors.toList()); - new TransformList<>(values).transformEachPair((a,b) -> { - System.out.println(a.value() + ":" + b.value()); - int v = a.value(); - if (v == 1 || v == 3 || v == 5) { - a.replace(-a.value()); - b.replace(-b.value()); - } - - }); - - assertEquals(List.of(-1, -2, -3, -4, -5, -6), values); - } - - @Test - void scanAndTransform() { - List values = Stream.of(1,2,3,4,5,6,7,8,9,10).collect(Collectors.toList()); - new TransformList<>(values).scanAndTransform(Integer.valueOf(3)::equals, Integer.valueOf(7)::equals, entity -> { - entity.replace(entity.value() * 2); - }); - - assertEquals(List.of(1,2,6,8,10,12,14,8,9,10), values); - } - - @Test - void scanAndTransformEndsAtEnd() { - List values = Stream.of(1,2,3,4,5,6,7,8,9,10).collect(Collectors.toList()); - new TransformList<>(values).scanAndTransform(Integer.valueOf(3)::equals, Integer.valueOf(10)::equals, entity -> { - entity.replace(entity.value() * 2); - }); - - assertEquals(List.of(1,2,6,8,10,12,14,16,18,20), values); - } - - @Test - void scanAndTransformOverlap() { - List values = Stream.of(1,2,3,3,5,7,7,8,9,10).collect(Collectors.toList()); - new TransformList<>(values).scanAndTransform(Integer.valueOf(3)::equals, Integer.valueOf(7)::equals, entity -> { - entity.replace(entity.value() * 2); - }); - - assertEquals(List.of(1, 2, 6, 6, 10, 14, 7, 8, 9, 10), values); - } -} \ No newline at end of file diff --git a/code/features-qs/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java b/code/features-qs/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java deleted file mode 100644 index 5efd2025..00000000 --- a/code/features-qs/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java +++ /dev/null @@ -1,38 +0,0 @@ -package nu.marginalia.util; - -import nu.marginalia.LanguageModels; -import nu.marginalia.WmsaHome; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; - -public class TestLanguageModels { - private static final Path LANGUAGE_MODELS_DEFAULT = WmsaHome.getHomePath().resolve("model"); - - public static Path getLanguageModelsPath() { - final Path languageModelsHome = Optional.ofNullable(System.getenv("LANGUAGE_MODELS_HOME")) - .map(Path::of) - .orElse(LANGUAGE_MODELS_DEFAULT); - - if (!Files.isDirectory(languageModelsHome)) { - throw new IllegalStateException("Could not find $LANGUAGE_MODELS_HOME, see doc/language-models.md"); - } - return languageModelsHome; - } - - public static LanguageModels getLanguageModels() { - - var languageModelsHome = getLanguageModelsPath(); - - return new LanguageModels( - languageModelsHome.resolve("ngrams.bin"), - languageModelsHome.resolve("tfreq-new-algo3.bin"), - languageModelsHome.resolve("opennlp-sentence.bin"), - languageModelsHome.resolve("English.RDR"), - languageModelsHome.resolve("English.DICT"), - languageModelsHome.resolve("opennlp-tokens.bin"), - languageModelsHome.resolve("lid.176.ftz") - ); - } -} diff --git a/code/features-qs/readme.md b/code/features-qs/readme.md deleted file mode 100644 index c0d77dec..00000000 --- a/code/features-qs/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -**Note**: This package is called `features-qs` rather than `features-query` because the latter, -though more consistent with other packages like features-index, would be very confusing -as there are other packages elsewhere with the 'query' name (e.g. features-index/index-query). - -## Contents - -* [query-parser](query-parser/) contains code for parsing the user-facing query grammar. diff --git a/code/functions/math/build.gradle b/code/functions/math/build.gradle index 4ffbc11d..2a843de8 100644 --- a/code/functions/math/build.gradle +++ b/code/functions/math/build.gradle @@ -19,7 +19,6 @@ dependencies { implementation libs.bundles.grpc implementation libs.notnull implementation libs.guice - implementation libs.spark implementation libs.opencsv implementation libs.trove implementation libs.fastutil diff --git a/code/api/query-api/build.gradle b/code/functions/search-query/api/build.gradle similarity index 70% rename from code/api/query-api/build.gradle rename to code/functions/search-query/api/build.gradle index 81b8a55e..955675df 100644 --- a/code/api/query-api/build.gradle +++ b/code/functions/search-query/api/build.gradle @@ -1,5 +1,7 @@ plugins { id 'java' + + id "com.google.protobuf" version "0.9.4" id 'jvm-test-suite' } @@ -9,24 +11,34 @@ java { } } +jar.archiveBaseName = 'search-query-api' + +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + } +} + +apply from: "$rootProject.projectDir/protobuf.gradle" + dependencies { implementation project(':code:common:model') - implementation project(':code:api:index-api') implementation project(':code:common:config') - implementation project(':code:libraries:message-queue') implementation project(':code:features-index:index-query') implementation project(':code:common:service-discovery') implementation libs.bundles.slf4j - implementation libs.roaringbitmap implementation libs.prometheus implementation libs.notnull - implementation libs.trove implementation libs.guice implementation libs.gson - implementation libs.bundles.grpc implementation libs.protobuf + implementation libs.javax.annotation + implementation libs.bundles.grpc + implementation libs.fastutil testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java similarity index 94% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java index 3b8650ad..4b2f0032 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexProtobufCodec.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java @@ -1,9 +1,9 @@ -package nu.marginalia.index.client; +package nu.marginalia.api.searchquery; -import nu.marginalia.index.api.*; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.Bm25Parameters; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.*; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.Bm25Parameters; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.index.query.limit.SpecificationLimitType; diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryClient.java similarity index 87% rename from code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryClient.java index 616b84c2..76bdae5f 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryClient.java @@ -1,12 +1,10 @@ -package nu.marginalia.query.client; +package nu.marginalia.api.searchquery; import com.google.inject.Inject; import com.google.inject.Singleton; import io.prometheus.client.Summary; -import nu.marginalia.index.api.*; -import nu.marginalia.query.QueryProtobufCodec; -import nu.marginalia.query.model.QueryParams; -import nu.marginalia.query.model.QueryResponse; +import nu.marginalia.api.searchquery.model.query.QueryParams; +import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.client.GrpcSingleNodeChannelPool; import nu.marginalia.service.discovery.property.ServiceKey; diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java similarity index 76% rename from code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java index f1b9b6f4..735c4034 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/QueryProtobufCodec.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java @@ -1,25 +1,21 @@ -package nu.marginalia.query; +package nu.marginalia.api.searchquery; import lombok.SneakyThrows; -import nu.marginalia.index.api.*; -import nu.marginalia.index.client.IndexProtobufCodec; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; -import nu.marginalia.index.client.model.results.ResultRankingParameters; -import nu.marginalia.index.client.model.results.SearchResultItem; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.model.EdgeUrl; -import nu.marginalia.query.model.ProcessedQuery; -import nu.marginalia.query.model.QueryParams; -import nu.marginalia.query.model.QueryResponse; +import nu.marginalia.api.searchquery.model.query.ProcessedQuery; +import nu.marginalia.api.searchquery.model.query.QueryParams; +import nu.marginalia.api.searchquery.model.query.QueryResponse; import java.util.ArrayList; import java.util.List; -import static nu.marginalia.index.client.IndexProtobufCodec.*; - public class QueryProtobufCodec { public static RpcIndexQuery convertQuery(RpcQsQuery request, ProcessedQuery query) { @@ -34,11 +30,11 @@ public class QueryProtobufCodec { builder.setSearchSetIdentifier(query.specs.searchSetIdentifier); builder.setHumanQuery(request.getHumanQuery()); - builder.setQuality(convertSpecLimit(query.specs.quality)); - builder.setYear(convertSpecLimit(query.specs.year)); - builder.setSize(convertSpecLimit(query.specs.size)); - builder.setRank(convertSpecLimit(query.specs.rank)); - builder.setDomainCount(convertSpecLimit(query.specs.domainCount)); + builder.setQuality(IndexProtobufCodec.convertSpecLimit(query.specs.quality)); + builder.setYear(IndexProtobufCodec.convertSpecLimit(query.specs.year)); + builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size)); + builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank)); + builder.setDomainCount(IndexProtobufCodec.convertSpecLimit(query.specs.domainCount)); builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); @@ -63,11 +59,11 @@ public class QueryProtobufCodec { builder.setSearchSetIdentifier(query.specs.searchSetIdentifier); builder.setHumanQuery(humanQuery); - builder.setQuality(convertSpecLimit(query.specs.quality)); - builder.setYear(convertSpecLimit(query.specs.year)); - builder.setSize(convertSpecLimit(query.specs.size)); - builder.setRank(convertSpecLimit(query.specs.rank)); - builder.setDomainCount(convertSpecLimit(query.specs.domainCount)); + builder.setQuality(IndexProtobufCodec.convertSpecLimit(query.specs.quality)); + builder.setYear(IndexProtobufCodec.convertSpecLimit(query.specs.year)); + builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size)); + builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank)); + builder.setDomainCount(IndexProtobufCodec.convertSpecLimit(query.specs.domainCount)); builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); @@ -92,11 +88,11 @@ public class QueryProtobufCodec { request.getTacitExcludesList(), request.getTacitPriorityList(), request.getTacitAdviceList(), - convertSpecLimit(request.getQuality()), - convertSpecLimit(request.getYear()), - convertSpecLimit(request.getSize()), - convertSpecLimit(request.getRank()), - convertSpecLimit(request.getDomainCount()), + IndexProtobufCodec.convertSpecLimit(request.getQuality()), + IndexProtobufCodec.convertSpecLimit(request.getYear()), + IndexProtobufCodec.convertSpecLimit(request.getSize()), + IndexProtobufCodec.convertSpecLimit(request.getRank()), + IndexProtobufCodec.convertSpecLimit(request.getDomainCount()), request.getDomainIdsList(), IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()), request.getSearchSetIdentifier(), @@ -167,7 +163,7 @@ public class QueryProtobufCodec { List subqueries = new ArrayList<>(specs.getSubqueriesCount()); for (int i = 0; i < specs.getSubqueriesCount(); i++) { - subqueries.add(convertSearchSubquery(specs.getSubqueries(i))); + subqueries.add(IndexProtobufCodec.convertSearchSubquery(specs.getSubqueries(i))); } return new SearchSpecification( @@ -182,7 +178,7 @@ public class QueryProtobufCodec { IndexProtobufCodec.convertSpecLimit(specs.getDomainCount()), IndexProtobufCodec.convertQueryLimits(specs.getQueryLimits()), QueryStrategy.valueOf(specs.getQueryStrategy()), - convertRankingParameterss(specs.getParameters()) + IndexProtobufCodec.convertRankingParameterss(specs.getParameters()) ); } @@ -194,11 +190,11 @@ public class QueryProtobufCodec { .addAllTacitIncludes(params.tacitIncludes()) .addAllTacitPriority(params.tacitPriority()) .setHumanQuery(params.humanQuery()) - .setQueryLimits(convertQueryLimits(params.limits())) - .setQuality(convertSpecLimit(params.quality())) - .setYear(convertSpecLimit(params.year())) - .setSize(convertSpecLimit(params.size())) - .setRank(convertSpecLimit(params.rank())) + .setQueryLimits(IndexProtobufCodec.convertQueryLimits(params.limits())) + .setQuality(IndexProtobufCodec.convertSpecLimit(params.quality())) + .setYear(IndexProtobufCodec.convertSpecLimit(params.year())) + .setSize(IndexProtobufCodec.convertSpecLimit(params.size())) + .setRank(IndexProtobufCodec.convertSpecLimit(params.rank())) .setSearchSetIdentifier(params.identifier()) .setQueryStrategy(params.queryStrategy().name()) .setTemporalBias(RpcTemporalBias.newBuilder() @@ -227,4 +223,5 @@ public class QueryProtobufCodec { rpcDecoratedResultItem.getRankingScore() ); } + } diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/model/ProcessedQuery.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java similarity index 82% rename from code/api/query-api/src/main/java/nu/marginalia/query/model/ProcessedQuery.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java index b5f7e323..09ebdcbd 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/model/ProcessedQuery.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java @@ -1,6 +1,4 @@ -package nu.marginalia.query.model; - -import nu.marginalia.index.client.model.query.SearchSpecification; +package nu.marginalia.api.searchquery.model.query; import java.util.*; diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/model/QueryParams.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryParams.java similarity index 91% rename from code/api/query-api/src/main/java/nu/marginalia/query/model/QueryParams.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryParams.java index d588ac2f..b713217b 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/model/QueryParams.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryParams.java @@ -1,10 +1,9 @@ -package nu.marginalia.query.model; +package nu.marginalia.api.searchquery.model.query; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; - import javax.annotation.Nullable; import java.util.List; diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/model/QueryResponse.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java similarity index 76% rename from code/api/query-api/src/main/java/nu/marginalia/query/model/QueryResponse.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java index 2adadd83..80e5b61a 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/model/QueryResponse.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java @@ -1,7 +1,6 @@ -package nu.marginalia.query.model; +package nu.marginalia.api.searchquery.model.query; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import java.util.HashSet; import java.util.List; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSetIdentifier.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java similarity index 81% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSetIdentifier.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java index 4eb0eae0..43cc13ac 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSetIdentifier.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.query; +package nu.marginalia.api.searchquery.model.query; /** Identifies a RankingSearchSet, associated with an EdgeSearchProfile * diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSpecification.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java similarity index 87% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSpecification.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java index 2ea743cf..2b5d1867 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSpecification.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java @@ -1,7 +1,7 @@ -package nu.marginalia.index.client.model.query; +package nu.marginalia.api.searchquery.model.query; import lombok.*; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSubquery.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java similarity index 98% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSubquery.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java index f401008e..3798ae89 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSubquery.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.query; +package nu.marginalia.api.searchquery.model.query; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/Bm25Parameters.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java similarity index 83% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/Bm25Parameters.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java index 9c8b9209..48608624 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/Bm25Parameters.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; /** Tuning parameters for BM25. * diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/DecoratedSearchResultItem.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java similarity index 97% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/DecoratedSearchResultItem.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java index 92504672..0a3f1f3f 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/DecoratedSearchResultItem.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import lombok.Getter; import lombok.ToString; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingContext.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java similarity index 95% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingContext.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java index 5837a543..f0ad172f 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingContext.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; import lombok.ToString; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java similarity index 97% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java index 112db234..a16ccf8b 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/ResultRankingParameters.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultItem.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java similarity index 97% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultItem.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java index 7393e3b4..523d2fb5 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultItem.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultKeywordScore.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java similarity index 98% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultKeywordScore.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java index 77266112..e4402ce0 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultKeywordScore.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordMetadata; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultPreliminaryScore.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java similarity index 92% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultPreliminaryScore.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java index a46e14b1..287db40a 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultPreliminaryScore.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import org.jetbrains.annotations.NotNull; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultSet.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java similarity index 94% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultSet.java rename to code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java index f4159f3a..c7e76fd9 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/model/results/SearchResultSet.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client.model.results; +package nu.marginalia.api.searchquery.model.results; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/code/api/index-api/src/main/protobuf/index-api.proto b/code/functions/search-query/api/src/main/protobuf/query-api.proto similarity index 94% rename from code/api/index-api/src/main/protobuf/index-api.proto rename to code/functions/search-query/api/src/main/protobuf/query-api.proto index 521362f5..e508ba34 100644 --- a/code/api/index-api/src/main/protobuf/index-api.proto +++ b/code/functions/search-query/api/src/main/protobuf/query-api.proto @@ -1,7 +1,7 @@ syntax="proto3"; -package actorapi; +package nu.marginalia.api.searchquery; -option java_package="nu.marginalia.index.api"; +option java_package="nu.marginalia.api.searchquery"; option java_multiple_files=true; service QueryApi { @@ -35,11 +35,11 @@ message RpcQsQuery { /* Query service query response */ message RpcQsResponse { - RpcIndexQuery specs = 1; - repeated RpcDecoratedResultItem results = 2; - repeated string searchTermsHuman = 3; - repeated string problems = 4; - string domain = 5; + RpcIndexQuery specs = 1; + repeated RpcDecoratedResultItem results = 2; + repeated string searchTermsHuman = 3; + repeated string problems = 4; + string domain = 5; } message RpcTemporalBias { diff --git a/code/features-qs/query-parser/build.gradle b/code/functions/search-query/build.gradle similarity index 54% rename from code/features-qs/query-parser/build.gradle rename to code/functions/search-query/build.gradle index 8fd14810..4af1b6f6 100644 --- a/code/features-qs/query-parser/build.gradle +++ b/code/functions/search-query/build.gradle @@ -1,34 +1,46 @@ plugins { id 'java' - - id 'jvm-test-suite' } - java { toolchain { languageVersion.set(JavaLanguageVersion.of(21)) } } + dependencies { + implementation project(':code:api:index-api') + + implementation project(':code:common:db') + implementation project(':code:common:config') + implementation project(':code:common:model') + implementation project(':code:common:service-discovery') + + implementation project(':code:functions:search-query:api') + + implementation project(':code:features-index:index-query') + + implementation project(':code:libraries:language-processing') + implementation project(':code:libraries:term-frequency-dict') + + implementation project(':third-party:porterstemmer') implementation project(':code:libraries:language-processing') implementation project(':code:libraries:term-frequency-dict') implementation project(':code:features-convert:keyword-extraction') - implementation project(':code:common:config') - implementation project(':code:common:model') - - implementation project(':third-party:porterstemmer') implementation libs.bundles.slf4j - implementation libs.bundles.nlp - implementation libs.bundles.handlebars - implementation libs.trove + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull implementation libs.guice + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit testImplementation libs.mockito } - diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java similarity index 83% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java index 047a2c4c..177fd979 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java @@ -1,15 +1,18 @@ -package nu.marginalia.query; +package nu.marginalia.functions.searchquery; import com.google.inject.Inject; import com.google.inject.Singleton; +import io.grpc.stub.StreamObserver; import io.prometheus.client.Histogram; import lombok.SneakyThrows; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.api.searchquery.*; +import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.db.DomainBlacklist; -import nu.marginalia.index.api.*; +import nu.marginalia.functions.searchquery.svc.QueryFactory; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.model.id.UrlIdCodec; -import nu.marginalia.query.svc.QueryFactory; import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; import nu.marginalia.service.discovery.property.ServiceKey; import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; @@ -48,8 +51,7 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { IndexApiGrpc::newBlockingStub); } - public void query(nu.marginalia.index.api.RpcQsQuery request, - io.grpc.stub.StreamObserver responseObserver) + public void query(RpcQsQuery request, StreamObserver responseObserver) { try { wmsa_qs_query_time_grpc @@ -105,4 +107,14 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { private boolean isBlacklisted(RpcDecoratedResultItem item) { return blacklist.isBlacklisted(UrlIdCodec.getDomainId(item.getRawItem().getCombinedId())); } + + public List executeDirect(String originalQuery, QueryParams params, int count) { + var query = queryFactory.createQuery(params); + + return executeQueries( + QueryProtobufCodec.convertQuery(originalQuery, query), + count) + .stream().map(QueryProtobufCodec::convertQueryResult) + .toList(); + } } diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java similarity index 94% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java index 2abda918..27ad1cbc 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java @@ -1,9 +1,9 @@ -package nu.marginalia.query_parser; +package nu.marginalia.functions.searchquery.query_parser; import nu.marginalia.language.WordPatterns; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenType; -import nu.marginalia.transform_list.TransformList; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenType; +import nu.marginalia.util.transform_list.TransformList; import java.util.List; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java similarity index 97% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java index 62774449..417ceda3 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java @@ -1,8 +1,8 @@ -package nu.marginalia.query_parser; +package nu.marginalia.functions.searchquery.query_parser; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenType; import nu.marginalia.language.WordPatterns; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java similarity index 90% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java index 992e3ee5..b7b0a2b7 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java @@ -1,8 +1,8 @@ -package nu.marginalia.query_parser; +package nu.marginalia.functions.searchquery.query_parser; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenType; import nu.marginalia.language.encoding.AsciiFlattener; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenType; import java.util.ArrayList; import java.util.List; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java similarity index 96% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java index eb4abd79..9732e53f 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java @@ -1,20 +1,20 @@ -package nu.marginalia.query_parser; +package nu.marginalia.functions.searchquery.query_parser; +import ca.rmen.porterstemmer.PorterStemmer; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenType; +import nu.marginalia.util.language.EnglishDictionary; import nu.marginalia.LanguageModels; import nu.marginalia.keyword.KeywordExtractor; -import nu.marginalia.language.EnglishDictionary; import nu.marginalia.language.sentence.SentenceExtractor; -import nu.marginalia.ngrams.NGramBloomFilter; +import nu.marginalia.util.ngrams.NGramBloomFilter; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.language.model.DocumentSentence; import nu.marginalia.language.model.WordSpan; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenType; -import opennlp.tools.stemmer.PorterStemmer; import java.util.*; import java.util.regex.Pattern; @@ -251,7 +251,7 @@ public class QueryVariants { for (var span : ls) { var matcher = dashBoundary.matcher(span.word); - if (matcher.find() && nGramBloomFilter.isKnownNGram(ps.stem(dashBoundary.matcher(span.word).replaceAll("")))) { + if (matcher.find() && nGramBloomFilter.isKnownNGram(ps.stemWord(dashBoundary.matcher(span.word).replaceAll("")))) { dash = true; String combined = dashBoundary.matcher(span.word).replaceAll(""); asTokens2.add(combined); @@ -332,7 +332,7 @@ public class QueryVariants { var a = span.get(i); var b = span.get(i+1); - var stemmed = ps.stem(a.word + b.word); + var stemmed = ps.stemWord(a.word + b.word); double scoreCombo = dict.getTermFreqStemmed(stemmed); if (scoreCombo > 10000) { diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java similarity index 95% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java index 31aa4e22..b973d777 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java @@ -1,4 +1,4 @@ -package nu.marginalia.query_parser.token; +package nu.marginalia.functions.searchquery.query_parser.token; import lombok.EqualsAndHashCode; import lombok.ToString; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java similarity index 87% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java index d929665f..8c542e5a 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java @@ -1,4 +1,4 @@ -package nu.marginalia.query_parser.token; +package nu.marginalia.functions.searchquery.query_parser.token; import java.util.function.Predicate; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java similarity index 87% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java index a5c53e4e..f6a48d4d 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenVisitor.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java @@ -1,4 +1,4 @@ -package nu.marginalia.query_parser.token; +package nu.marginalia.functions.searchquery.query_parser.token; public interface TokenVisitor { void onLiteralTerm(Token token); diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QueryFactory.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java similarity index 87% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QueryFactory.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java index d0234326..4ed97afb 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QueryFactory.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java @@ -1,21 +1,21 @@ -package nu.marginalia.query.svc; +package nu.marginalia.functions.searchquery.svc; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.LanguageModels; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; -import nu.marginalia.language.EnglishDictionary; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.util.language.EnglishDictionary; import nu.marginalia.language.WordPatterns; -import nu.marginalia.ngrams.NGramBloomFilter; -import nu.marginalia.query.model.QueryParams; -import nu.marginalia.query.model.ProcessedQuery; -import nu.marginalia.query_parser.QueryParser; -import nu.marginalia.query_parser.QueryPermutation; -import nu.marginalia.query_parser.QueryVariants; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenType; +import nu.marginalia.util.ngrams.NGramBloomFilter; +import nu.marginalia.api.searchquery.model.query.QueryParams; +import nu.marginalia.api.searchquery.model.query.ProcessedQuery; +import nu.marginalia.functions.searchquery.query_parser.QueryParser; +import nu.marginalia.functions.searchquery.query_parser.QueryPermutation; +import nu.marginalia.functions.searchquery.query_parser.QueryVariants; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenType; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QueryLimitsAccumulator.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java similarity index 91% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QueryLimitsAccumulator.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java index f1f17bed..4d230650 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QueryLimitsAccumulator.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java @@ -1,10 +1,10 @@ -package nu.marginalia.query.svc; +package nu.marginalia.functions.searchquery.svc; +import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; -import nu.marginalia.query.model.QueryParams; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenVisitor; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenVisitor; public class QueryLimitsAccumulator implements TokenVisitor { public SpecificationLimit qualityLimit; diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QuerySearchTermsAccumulator.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java similarity index 92% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QuerySearchTermsAccumulator.java rename to code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java index 1d5588bc..00991eac 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/QuerySearchTermsAccumulator.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java @@ -1,9 +1,9 @@ -package nu.marginalia.query.svc; +package nu.marginalia.functions.searchquery.svc; -import nu.marginalia.index.client.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.language.WordPatterns; -import nu.marginalia.query_parser.token.Token; -import nu.marginalia.query_parser.token.TokenVisitor; +import nu.marginalia.functions.searchquery.query_parser.token.Token; +import nu.marginalia.functions.searchquery.query_parser.token.TokenVisitor; import java.util.ArrayList; import java.util.Arrays; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java b/code/functions/search-query/src/main/java/nu/marginalia/util/language/EnglishDictionary.java similarity index 99% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java rename to code/functions/search-query/src/main/java/nu/marginalia/util/language/EnglishDictionary.java index 0afd3625..56e90701 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/util/language/EnglishDictionary.java @@ -1,4 +1,4 @@ -package nu.marginalia.language; +package nu.marginalia.util.language; import com.google.inject.Inject; import nu.marginalia.term_frequency_dict.TermFrequencyDict; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java b/code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/DenseBitMap.java similarity index 98% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java rename to code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/DenseBitMap.java index ca5666b1..5234b62d 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/DenseBitMap.java @@ -1,4 +1,4 @@ -package nu.marginalia.ngrams; +package nu.marginalia.util.ngrams; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java b/code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/NGramBloomFilter.java similarity index 98% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java rename to code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/NGramBloomFilter.java index 89a6d9cf..3326956d 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/NGramBloomFilter.java @@ -1,4 +1,4 @@ -package nu.marginalia.ngrams; +package nu.marginalia.util.ngrams; import ca.rmen.porterstemmer.PorterStemmer; import com.google.common.hash.HashFunction; diff --git a/code/features-qs/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java b/code/functions/search-query/src/main/java/nu/marginalia/util/transform_list/TransformList.java similarity index 98% rename from code/features-qs/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java rename to code/functions/search-query/src/main/java/nu/marginalia/util/transform_list/TransformList.java index 20b2b5ea..08bc428e 100644 --- a/code/features-qs/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/util/transform_list/TransformList.java @@ -1,4 +1,4 @@ -package nu.marginalia.transform_list; +package nu.marginalia.util.transform_list; import java.util.List; import java.util.function.BiConsumer; diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index 798fb3a9..4e61dff0 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -41,8 +41,7 @@ dependencies { implementation project(':code:common:service') implementation project(':code:common:config') implementation project(':code:common:service-discovery') - implementation project(':code:api:query-api') - implementation project(':code:api:index-api') + implementation project(':code:functions:search-query:api') implementation project(':code:features-index:index-query') implementation libs.bundles.slf4j diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java index 0938dc34..25ba4945 100644 --- a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java +++ b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java @@ -5,13 +5,12 @@ import com.google.inject.Singleton; import nu.marginalia.api.model.ApiSearchResult; import nu.marginalia.api.model.ApiSearchResultQueryDetails; import nu.marginalia.api.model.ApiSearchResults; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.QueryClient; +import nu.marginalia.api.searchquery.model.query.QueryParams; +import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier; +import nu.marginalia.api.searchquery.model.results.*; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.query.client.QueryClient; -import nu.marginalia.query.model.QueryParams; import java.util.ArrayList; import java.util.Comparator; diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index e7d19834..702780dd 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -56,7 +56,9 @@ dependencies { implementation project(':code:functions:math:api') implementation project(':code:functions:domain-info:api') - implementation project(':code:api:query-api') + implementation project(':code:functions:search-query:api') + + implementation project(':code:api:index-api') implementation project(':code:common:service-discovery') implementation project(':code:common:renderer') diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java index 28b21da8..d3141159 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java @@ -5,10 +5,10 @@ import com.google.inject.Singleton; import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; import nu.marginalia.api.math.MathClient; +import nu.marginalia.api.searchquery.QueryClient; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; -import nu.marginalia.query.client.QueryClient; -import nu.marginalia.query.model.QueryResponse; +import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.search.command.SearchParameters; import nu.marginalia.search.model.*; import nu.marginalia.search.svc.SearchQueryIndexService; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchQueryParamFactory.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchQueryParamFactory.java index 59cd1088..100e62b6 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchQueryParamFactory.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchQueryParamFactory.java @@ -1,12 +1,12 @@ package nu.marginalia.search; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; -import nu.marginalia.query.model.QueryParams; +import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.search.command.SearchParameters; import java.util.List; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchResultClusterer.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchResultClusterer.java index e5b61788..4e4cd086 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchResultClusterer.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchResultClusterer.java @@ -1,6 +1,6 @@ package nu.marginalia.search; -import nu.marginalia.query.model.QueryResponse; +import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.search.model.ClusteredUrlDetails; import nu.marginalia.search.model.UrlDetails; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchAdtechParameter.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchAdtechParameter.java index 0551533e..9e8383f3 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchAdtechParameter.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchAdtechParameter.java @@ -1,6 +1,6 @@ package nu.marginalia.search.command; -import nu.marginalia.index.client.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; import javax.annotation.Nullable; import java.util.Arrays; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchJsParameter.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchJsParameter.java index ab67486c..6c8634ac 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchJsParameter.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchJsParameter.java @@ -1,6 +1,6 @@ package nu.marginalia.search.command; -import nu.marginalia.index.client.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; import javax.annotation.Nullable; import java.util.Arrays; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchParameters.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchParameters.java index eed5b451..fcb90f13 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchParameters.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchParameters.java @@ -1,7 +1,7 @@ package nu.marginalia.search.command; import nu.marginalia.WebsiteUrl; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.search.model.SearchProfile; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchProfile.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchProfile.java index 85fafa8f..27d9f4aa 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchProfile.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchProfile.java @@ -2,8 +2,8 @@ package nu.marginalia.search.model; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.model.crawl.HtmlFeature; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier; import java.util.Objects; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/UrlDetails.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/model/UrlDetails.java index 4968a876..cc2afed6 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/UrlDetails.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/model/UrlDetails.java @@ -1,8 +1,8 @@ package nu.marginalia.search.model; import lombok.*; -import nu.marginalia.index.client.model.results.SearchResultItem; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.model.EdgeUrl; import nu.marginalia.model.crawl.DomainIndexingState; import nu.marginalia.model.crawl.HtmlFeature; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java index 16fe8aa3..ccddb8d9 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java @@ -4,7 +4,7 @@ import gnu.trove.list.TLongList; import gnu.trove.list.array.TLongArrayList; import gnu.trove.map.hash.TObjectIntHashMap; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.lsh.EasyLSH; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java index 87194275..785c8952 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java @@ -5,11 +5,11 @@ import com.google.inject.Singleton; import it.unimi.dsi.fastutil.ints.Int2LongArrayMap; import lombok.SneakyThrows; import nu.marginalia.bbpc.BrailleBlockPunchCards; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; -import nu.marginalia.index.client.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.model.crawl.DomainIndexingState; -import nu.marginalia.query.model.QueryResponse; +import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.search.model.UrlDetails; import nu.marginalia.search.results.UrlDeduplicator; import org.slf4j.Logger; diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index 24a11e33..ce7e8843 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -44,7 +44,6 @@ dependencies { implementation project(':code:functions:domain-info') implementation project(':code:functions:domain-info:api') - implementation project(':code:api:query-api') implementation project(':code:common:config') implementation project(':code:common:service') implementation project(':code:common:model') diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index af7e553f..498d2b76 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -45,9 +45,9 @@ dependencies { implementation project(':code:common:renderer') implementation project(':code:libraries:message-queue') implementation project(':code:common:service-discovery') - implementation project(':code:api:index-api') - implementation project(':code:api:query-api') + implementation project(':code:functions:search-query:api') implementation project(':code:api:executor-api') + implementation project(':code:api:index-api') implementation project(':code:api:process-mqapi') implementation project(':code:features-search:screenshots') implementation project(':code:features-index:index-journal') diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java index 69537a37..f9487e22 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java @@ -1,12 +1,12 @@ package nu.marginalia.control.app.svc; import com.google.inject.Inject; +import nu.marginalia.api.searchquery.QueryClient; +import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.control.ControlRendererFactory; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.EdgeUrl; import nu.marginalia.nodecfg.NodeConfigurationService; -import nu.marginalia.query.client.QueryClient; -import nu.marginalia.query.model.QueryParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index a4c0e973..1ad1b541 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -71,7 +71,6 @@ dependencies { implementation project(':code:features-convert:reddit-json') implementation project(':code:features-index:index-journal') implementation project(':code:api:index-api') - implementation project(':code:api:query-api') implementation project(':code:api:process-mqapi') implementation project(':code:api:executor-api') implementation project(':third-party:encyclopedia-marginalia-nu') diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java index fa75f016..b3d9b0a2 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java @@ -7,7 +7,6 @@ import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.prototype.RecordActorPrototype; import nu.marginalia.actor.state.ActorStep; import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; -import nu.marginalia.query.client.QueryClient; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; import nu.marginalia.storage.model.FileStorageType; diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index cfeeeb57..4548710c 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -43,6 +43,7 @@ dependencies { implementation project(':code:functions:domain-links:partition') implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:search-query:api') implementation project(':code:common:service') implementation project(':code:api:index-api') diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java index 6be5f1df..bc847978 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java @@ -6,7 +6,7 @@ import gnu.trove.set.hash.TLongHashSet; import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap; import it.unimi.dsi.fastutil.longs.LongArrayList; -import nu.marginalia.index.client.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.svc.SearchTermsService; import nu.marginalia.model.idx.WordMetadata; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java index 6b4d062b..24d9da48 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java @@ -4,9 +4,9 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import gnu.trove.list.TLongList; import gnu.trove.list.array.TLongArrayList; -import nu.marginalia.index.client.model.results.DecoratedSearchResultItem; -import nu.marginalia.index.client.model.results.ResultRankingContext; -import nu.marginalia.index.client.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.linkdb.docs.DocumentDbReader; import nu.marginalia.linkdb.model.DocdbUrlDetail; import nu.marginalia.ranking.ResultValuator; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java index 4c29886a..fac47a47 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java @@ -2,7 +2,7 @@ package nu.marginalia.index.results; import gnu.trove.map.TLongIntMap; import gnu.trove.map.hash.TLongIntHashMap; -import nu.marginalia.index.client.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; public class IndexResultDomainDeduplicator { final TLongIntMap resultsByDomainId = CachedObjects.getMap(); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java index 1e51fbd6..414d4306 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java @@ -2,15 +2,15 @@ package nu.marginalia.index.results; import gnu.trove.list.TLongList; import gnu.trove.set.hash.TLongHashSet; -import nu.marginalia.index.client.model.results.SearchResultPreliminaryScore; -import nu.marginalia.index.client.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.model.results.SearchResultPreliminaryScore; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.index.client.model.results.SearchResultItem; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; -import nu.marginalia.index.client.model.query.SearchSubquery; import nu.marginalia.index.query.IndexQueryParams; import nu.marginalia.ranking.ResultValuator; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java index c3f088f6..98f1d988 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java @@ -4,18 +4,18 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import gnu.trove.list.TLongList; import gnu.trove.list.array.TLongArrayList; +import io.grpc.stub.StreamObserver; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import io.prometheus.client.Histogram; import lombok.SneakyThrows; -import nu.marginalia.index.api.*; -import nu.marginalia.index.api.IndexApiGrpc.IndexApiImplBase; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; -import nu.marginalia.index.client.model.results.SearchResultItem; -import nu.marginalia.index.client.model.results.ResultRankingContext; -import nu.marginalia.index.client.model.results.SearchResultSet; -import nu.marginalia.index.client.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.*; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultSet; import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.index.SearchIndexSearchTerms; import nu.marginalia.index.query.IndexQueryPriority; @@ -41,7 +41,7 @@ import java.util.*; import java.util.stream.Collectors; @Singleton -public class IndexQueryService extends IndexApiImplBase { +public class IndexQueryService extends IndexApiGrpc.IndexApiImplBase { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -122,8 +122,8 @@ public class IndexQueryService extends IndexApiImplBase { // GRPC endpoint @SneakyThrows - public void query(nu.marginalia.index.api.RpcIndexQuery request, - io.grpc.stub.StreamObserver responseObserver) { + public void query(RpcIndexQuery request, + StreamObserver responseObserver) { try { var params = new SearchParameters(request, getSearchSet(request)); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java index 7403979c..9824f462 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java @@ -1,11 +1,11 @@ package nu.marginalia.index.svc; import gnu.trove.set.hash.TLongHashSet; -import nu.marginalia.index.api.RpcIndexQuery; -import nu.marginalia.index.client.IndexProtobufCodec; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.RpcIndexQuery; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.IndexProtobufCodec; import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.index.SearchIndexSearchTerms; import nu.marginalia.index.query.IndexQuery; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java index 668731b0..e4247d43 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java @@ -3,8 +3,8 @@ package nu.marginalia.index.svc; import com.google.inject.Singleton; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongList; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.hash.MurmurHash3_128; -import nu.marginalia.index.client.model.query.SearchSubquery; import nu.marginalia.index.index.SearchIndexSearchTerms; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java index 3437bf57..e9e1653d 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java @@ -1,7 +1,6 @@ package nu.marginalia.index.svc.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; -import nu.marginalia.index.client.model.query.SearchSetIdentifier; import nu.marginalia.index.searchset.SearchSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index a60cdfec..58fa802a 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -43,13 +43,11 @@ dependencies { implementation project(':code:common:service') implementation project(':code:common:renderer') implementation project(':code:api:index-api') - implementation project(':code:api:query-api') implementation project(':code:common:service-discovery') - implementation project(':code:features-qs:query-parser') implementation project(':code:features-index:index-query') - implementation project(':code:libraries:language-processing') - implementation project(':code:libraries:term-frequency-dict') + implementation project(':code:functions:search-query') + implementation project(':code:functions:search-query:api') implementation project(':code:functions:domain-links:api') implementation project(':code:functions:domain-links:aggregate') diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java index 2db3dcab..dc0ae2a0 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java @@ -2,12 +2,12 @@ package nu.marginalia.query; import com.google.gson.Gson; import com.google.inject.Inject; +import nu.marginalia.functions.searchquery.QueryGRPCService; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.gson.GsonFactory; -import nu.marginalia.query.model.QueryParams; +import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; -import nu.marginalia.query.svc.QueryFactory; import spark.Request; import spark.Response; @@ -16,20 +16,16 @@ import java.util.Map; public class QueryBasicInterface { private final MustacheRenderer renderer; - private final QueryFactory queryFactory; private final Gson gson = GsonFactory.get(); private final QueryGRPCService queryGRPCService; @Inject public QueryBasicInterface(RendererFactory rendererFactory, - QueryFactory queryFactory, QueryGRPCService queryGRPCService ) throws IOException { this.renderer = rendererFactory.renderer("search"); - - this.queryFactory = queryFactory; this.queryGRPCService = queryGRPCService; } @@ -43,17 +39,15 @@ public class QueryBasicInterface { int domainCount = request.queryParams("domainCount") == null ? 5 : Integer.parseInt(request.queryParams("domainCount")); String set = request.queryParams("set") == null ? "" : request.queryParams("set"); - var query = queryFactory.createQuery(new QueryParams(queryParam, new QueryLimits( + var params = new QueryParams(queryParam, new QueryLimits( domainCount, count, 250, 8192 - ), set)); + ), set); - var rsp = queryGRPCService.executeQueries(QueryProtobufCodec.convertQuery(queryParam, query), count); - - var results = rsp.stream().map(QueryProtobufCodec::convertQueryResult).toList(); + var results = queryGRPCService.executeDirect(queryParam, params, count); if (request.headers("Accept").contains("application/json")) { response.type("application/json"); - return gson.toJson(rsp); + return gson.toJson(results); } else { return renderer.render( diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index 6d63be29..7a6cc25e 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -4,6 +4,7 @@ import com.google.inject.Inject; import io.prometheus.client.Histogram; import lombok.SneakyThrows; import nu.marginalia.functions.domainlinks.AggregateDomainLinksService; +import nu.marginalia.functions.searchquery.QueryGRPCService; import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; diff --git a/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java b/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java index 7ce9635f..d3f17b52 100644 --- a/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java +++ b/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java @@ -1,15 +1,15 @@ package nu.marginalia.query.svc; import nu.marginalia.WmsaHome; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.index.query.limit.SpecificationLimitType; -import nu.marginalia.language.EnglishDictionary; -import nu.marginalia.ngrams.NGramBloomFilter; -import nu.marginalia.query.model.QueryParams; +import nu.marginalia.util.language.EnglishDictionary; +import nu.marginalia.util.ngrams.NGramBloomFilter; +import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/settings.gradle b/settings.gradle index 49462b94..8306baf3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -21,6 +21,9 @@ include 'code:functions:domain-links:partition' include 'code:functions:domain-links:aggregate' include 'code:functions:domain-links:api' +include 'code:functions:search-query' +include 'code:functions:search-query:api' + include 'code:libraries:array' include 'code:libraries:geo-ip' include 'code:libraries:btree' @@ -40,7 +43,6 @@ include 'code:libraries:message-queue' include 'code:features-search:screenshots' include 'code:features-search:random-websites' include 'code:features-search:feedlot-client' -include 'code:features-qs:query-parser' include 'code:features-index:result-ranking' include 'code:features-convert:adblock' @@ -63,7 +65,6 @@ include 'code:features-index:index-forward' include 'code:features-index:index-reverse' include 'code:features-index:domain-ranking' -include 'code:api:query-api' include 'code:api:index-api' include 'code:api:process-mqapi' include 'code:api:executor-api' From 085137ca63541b0d872c59eb57660c673fb482c6 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 17:31:25 +0100 Subject: [PATCH 21/56] * Extract the index functionality --- code/api/executor-api/build.gradle | 2 +- code/api/index-api/readme.md | 8 - .../src/main/java/nu/marginalia/WmsaHome.java | 22 +- .../client/GrpcSingleNodeChannelPool.java | 8 +- .../service/discovery/ZkServiceRegistry.java | 6 +- .../discovery/property/ServiceEndpoint.java | 6 +- .../discovery/ZkServiceRegistryTest.java | 25 +- .../forward/ParamMatchingQueryFilter.java | 100 ----- .../index/api}/build.gradle | 2 - .../functions/index/api/IndexClient.java | 52 +++ .../functions/index/api/IndexMqClient.java} | 8 +- .../index/api}/IndexMqEndpoints.java | 2 +- code/functions/index/build.gradle | 53 +++ .../index}/domain-ranking/build.gradle | 0 .../index}/domain-ranking/readme.md | 0 .../nu/marginalia/ranking/DomainRankings.java | 0 .../ranking/PageRankDomainRanker.java | 0 .../marginalia/ranking/RankingAlgorithm.java | 0 .../accumulator/RankingResultAccumulator.java | 0 .../RankingResultBitSetAccumulator.java | 0 .../RankingResultHashMapAccumulator.java | 0 .../RankingResultHashSetAccumulator.java | 0 .../RankingResultListAccumulator.java | 0 .../ranking/data/AbstractGraphSource.java | 0 .../marginalia/ranking/data/GraphSource.java | 0 .../ranking/data/InvertedLinkGraphSource.java | 0 .../ranking/data/LinkGraphSource.java | 0 .../ranking/data/SimilarityGraphSource.java | 0 .../ranking/jgrapht/PersonalizedPageRank.java | 0 .../RankingAlgorithmWithRealDataTest.java | 0 .../RankingAlgorithmsContainerTest.java | 0 .../TestGraphSourceForInvertedLinkData.java | 0 .../ranking/TestGraphSourceForLinkData.java | 0 .../TestGraphSourceForSimilarityData.java | 0 .../index/forward-index}/build.gradle | 6 +- .../index/forward-index}/readme.md | 0 .../index/forward/ForwardIndexConverter.java | 0 .../index/forward/ForwardIndexFileNames.java | 0 .../index/forward/ForwardIndexParameters.java | 0 .../index/forward/ForwardIndexReader.java | 0 .../forward/ForwardIndexConverterTest.java | 0 .../java/nu/marginalia/test/TestUtil.java | 0 .../index}/index-journal/build.gradle | 0 .../index}/index-journal/readme.md | 0 .../index/journal/IndexJournalFileNames.java | 0 .../journal/model/IndexJournalEntry.java | 0 .../model/IndexJournalEntryBuilder.java | 0 .../journal/model/IndexJournalEntryData.java | 0 .../model/IndexJournalEntryHeader.java | 0 .../journal/model/IndexJournalFileHeader.java | 0 .../journal/reader/IndexJournalReadEntry.java | 0 .../journal/reader/IndexJournalReader.java | 0 .../reader/IndexJournalReaderPagingImpl.java | 0 .../reader/IndexJournalReaderSingleFile.java | 0 .../reader/pointer/IndexJournalPointer.java | 0 .../journal/writer/IndexJournalWriter.java | 0 .../writer/IndexJournalWriterPagingImpl.java | 0 .../IndexJournalWriterSingleFileImpl.java | 0 .../index/journal/IndexJournalTest.java | 0 .../pointer/IndexJournalPointerTest.java | 0 .../index/query}/build.gradle | 0 .../index/query}/readme.md | 0 .../index/query/EmptyEntrySource.java | 0 .../marginalia/index/query/EntrySource.java | 0 .../nu/marginalia/index/query/IndexQuery.java | 0 .../index/query/IndexQueryBuilder.java | 0 .../index/query/IndexQueryPriority.java | 0 .../index/query/IndexSearchBudget.java | 0 .../index/query/filter/QueryFilterAnyOf.java | 0 .../query/filter/QueryFilterLetThrough.java | 0 .../index/query/filter/QueryFilterNoPass.java | 0 .../QueryFilterStepExcludeFromPredicate.java | 0 .../filter/QueryFilterStepFromPredicate.java | 0 .../index/query/filter/QueryFilterStepIf.java | 0 .../index/query/limit/QueryLimits.java | 0 .../index/query/limit/QueryStrategy.java | 0 .../index/query/limit/SpecificationLimit.java | 0 .../query/limit/SpecificationLimitType.java | 0 .../query/filter/QueryFilterStepIfTest.java | 0 .../index}/readme.md | 4 +- .../index}/result-ranking/build.gradle | 2 +- .../index}/result-ranking/readme.md | 0 .../marginalia/ranking/ResultKeywordSet.java | 0 .../nu/marginalia/ranking/ResultValuator.java | 0 .../ranking/factors/Bm25Factor.java | 0 .../ranking/factors/PriorityTermBonus.java | 0 .../ranking/factors/TermCoherenceFactor.java | 0 .../ranking/ResultValuatorTest.java | 6 +- .../factors/TermCoherenceFactorTest.java | 2 +- .../index/reverse-index}/build.gradle | 6 +- .../index/reverse-index}/index.svg | 0 .../index/reverse-index}/merging.svg | 0 .../index/reverse-index}/preindex.svg | 0 .../index/reverse-index}/readme.md | 0 .../index/ReverseIndexEntrySource.java | 0 .../index/ReverseIndexFullFileNames.java | 0 .../index/ReverseIndexParameters.java | 0 .../index/ReverseIndexPrioFileNames.java | 0 .../marginalia/index/ReverseIndexReader.java | 0 .../index/ReverseIndexSelfTest.java | 0 .../CountToOffsetTransformer.java | 0 .../index/construction/DocIdRewriter.java | 0 .../construction/IndexSizeEstimator.java | 0 .../construction/JournalReaderSource.java | 0 .../ReverseIndexBTreeTransformer.java | 0 .../construction/ReverseIndexConstructor.java | 0 .../index/construction/ReversePreindex.java | 0 .../ReversePreindexDocuments.java | 0 .../ReversePreindexReference.java | 0 .../ReversePreindexWordSegments.java | 0 .../index/query/ReverseIndexRejectFilter.java | 0 .../index/query/ReverseIndexRetainFilter.java | 0 .../index/ReverseIndexDebugTest.java | 0 .../index/ReverseIndexReaderTest.java | 0 .../construction/ReversePreindexDocsTest.java | 0 .../ReversePreindexFinalizeTest.java | 0 .../ReversePreindexMergeTest.java | 0 .../ReversePreindexWordSegmentsTest.java | 0 .../construction/TestJournalFactory.java | 0 .../index/construction/TestSegmentData.java | 0 .../java/nu/marginalia/test/TestUtil.java | 0 .../index/IndexQueryGrpcService.java | 208 +++++++++ .../index/IndexServicesFactory.java | 18 +- .../functions/index/SearchTermsUtil.java} | 37 +- .../index/index/CombinedIndexReader.java} | 46 +- .../index/index/IndexQueryBuilderImpl.java | 168 ++++++++ .../index/index/IndexQueryService.java | 157 +++++++ .../functions/index/index/StatefulIndex.java} | 62 +-- .../index/model}/IndexQueryParams.java | 4 +- .../index/model/IndexSearchParameters.java} | 38 +- .../index/model/IndexSearchTerms.java} | 6 +- .../index/results/IndexMetadataService.java | 22 +- .../IndexResultDomainDeduplicator.java | 2 +- .../results/IndexResultValuationContext.java} | 19 +- .../results/IndexResultValuatorService.java | 152 +++++++ .../index/searchset}/DbUpdateRanks.java | 2 +- .../index}/searchset/RankingSearchSet.java | 3 +- .../functions}/index/searchset/SearchSet.java | 2 +- .../index}/searchset/SearchSetAny.java | 4 +- .../index/searchset/SearchSetsService.java} | 32 +- .../index}/searchset/SmallSearchSet.java | 3 +- ...IndexQueryServiceIntegrationSmokeTest.java | 27 +- .../IndexQueryServiceIntegrationTest.java | 19 +- ...ndexQueryServiceIntegrationTestModule.java | 11 +- .../IndexResultDomainDeduplicatorTest.java | 4 +- .../searchset/RankingSearchSetTest.java | 2 +- .../functions}/index/util/TestUtil.java | 2 +- .../math}/src/main/resources/units.csv | 0 .../functions/math}/eval/MathParserTest.java | 3 +- .../functions/math}/eval/UnitsTest.java | 4 +- code/functions/search-query/api/build.gradle | 2 +- .../index/client/IndexProtobufCodecTest.java | 0 code/functions/search-query/build.gradle | 4 +- .../searchquery/QueryGRPCService.java | 42 +- .../query/svc/QueryFactoryTest.java | 1 + .../crawling-model/build.gradle | 2 +- .../processes/converting-process/build.gradle | 2 +- code/processes/crawling-process/build.gradle | 2 +- .../index-constructor-process/build.gradle | 8 +- code/processes/loading-process/build.gradle | 4 +- .../api-service/build.gradle | 2 +- .../search-service/build.gradle | 4 +- .../search/svc/SearchErrorPageService.java | 8 +- .../control-service/build.gradle | 6 +- .../actor/precession/ReindexAllActor.java | 10 +- .../node/svc/ControlNodeActionsService.java | 10 +- .../executor-service/build.gradle | 4 +- .../actor/task/ConvertAndLoadActor.java | 8 +- code/services-core/index-service/build.gradle | 17 +- .../index/{svc => }/IndexOpsService.java | 36 +- .../nu/marginalia/index/IndexService.java | 35 +- .../index/index/SearchIndexQueryBuilder.java | 69 --- .../index/results/IndexResultDecorator.java | 90 ---- .../index/svc/IndexQueryExecutor.java | 40 -- .../index/svc/IndexQueryService.java | 407 ------------------ code/services-core/query-service/build.gradle | 4 +- .../nu/marginalia/query/QueryService.java | 7 - settings.gradle | 15 +- 178 files changed, 1121 insertions(+), 1095 deletions(-) delete mode 100644 code/api/index-api/readme.md delete mode 100644 code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java rename code/{api/index-api => functions/index/api}/build.gradle (93%) create mode 100644 code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java rename code/{api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java => functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java} (86%) rename code/{api/index-api/src/main/java/nu/marginalia/index/client => functions/index/api/src/main/java/nu/marginalia/functions/index/api}/IndexMqEndpoints.java (89%) create mode 100644 code/functions/index/build.gradle rename code/{features-index => functions/index}/domain-ranking/build.gradle (100%) rename code/{features-index => functions/index}/domain-ranking/readme.md (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java (100%) rename code/{features-index => functions/index}/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java (100%) rename code/{features-index/index-forward => functions/index/forward-index}/build.gradle (77%) rename code/{features-index/index-forward => functions/index/forward-index}/readme.md (100%) rename code/{features-index/index-forward => functions/index/forward-index}/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java (100%) rename code/{features-index/index-forward => functions/index/forward-index}/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java (100%) rename code/{features-index/index-forward => functions/index/forward-index}/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java (100%) rename code/{features-index/index-forward => functions/index/forward-index}/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java (100%) rename code/{features-index/index-forward => functions/index/forward-index}/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java (100%) rename code/{features-index/index-forward => functions/index/forward-index}/src/test/java/nu/marginalia/test/TestUtil.java (100%) rename code/{features-index => functions/index}/index-journal/build.gradle (100%) rename code/{features-index => functions/index}/index-journal/readme.md (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java (100%) rename code/{features-index => functions/index}/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java (100%) rename code/{features-index => functions/index}/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java (100%) rename code/{features-index => functions/index}/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java (100%) rename code/{features-index/index-query => functions/index/query}/build.gradle (100%) rename code/{features-index/index-query => functions/index/query}/readme.md (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/EntrySource.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/IndexQuery.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java (100%) rename code/{features-index/index-query => functions/index/query}/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java (100%) rename code/{features-index/index-query => functions/index/query}/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java (100%) rename code/{features-index => functions/index}/readme.md (86%) rename code/{features-index => functions/index}/result-ranking/build.gradle (93%) rename code/{features-index => functions/index}/result-ranking/readme.md (100%) rename code/{features-index => functions/index}/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java (100%) rename code/{features-index => functions/index}/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java (100%) rename code/{features-index => functions/index}/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java (100%) rename code/{features-index => functions/index}/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java (100%) rename code/{features-index => functions/index}/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java (100%) rename code/{features-index => functions/index}/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java (93%) rename code/{features-index => functions/index}/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java (97%) rename code/{features-index/index-reverse => functions/index/reverse-index}/build.gradle (76%) rename code/{features-index/index-reverse => functions/index/reverse-index}/index.svg (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/merging.svg (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/preindex.svg (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/readme.md (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/ReverseIndexParameters.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/ReverseIndexReader.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/ReversePreindex.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/index/construction/TestSegmentData.java (100%) rename code/{features-index/index-reverse => functions/index/reverse-index}/src/test/java/nu/marginalia/test/TestUtil.java (100%) create mode 100644 code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java rename code/{services-core/index-service/src/main/java/nu/marginalia => functions/index/src/main/java/nu/marginalia/functions}/index/IndexServicesFactory.java (89%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java => functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java} (63%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java => functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java} (63%) create mode 100644 code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java create mode 100644 code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java rename code/{services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java => functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java} (75%) rename code/{features-index/index-query/src/main/java/nu/marginalia/index/query => functions/index/src/main/java/nu/marginalia/functions/index/model}/IndexQueryParams.java (92%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java => functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java} (78%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexSearchTerms.java => functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java} (89%) rename code/{services-core/index-service/src/main/java/nu/marginalia => functions/index/src/main/java/nu/marginalia/functions}/index/results/IndexMetadataService.java (89%) rename code/{services-core/index-service/src/main/java/nu/marginalia => functions/index/src/main/java/nu/marginalia/functions}/index/results/IndexResultDomainDeduplicator.java (96%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java => functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java} (90%) create mode 100644 code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java rename code/{services-core/index-service/src/main/java/nu/marginalia/index/db => functions/index/src/main/java/nu/marginalia/functions/index/searchset}/DbUpdateRanks.java (96%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc => functions/index/src/main/java/nu/marginalia/functions/index}/searchset/RankingSearchSet.java (96%) rename code/{features-index/index-query/src/main/java/nu/marginalia => functions/index/src/main/java/nu/marginalia/functions}/index/searchset/SearchSet.java (82%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc => functions/index/src/main/java/nu/marginalia/functions/index}/searchset/SearchSetAny.java (72%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexSearchSetsService.java => functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java} (87%) rename code/{services-core/index-service/src/main/java/nu/marginalia/index/svc => functions/index/src/main/java/nu/marginalia/functions/index}/searchset/SmallSearchSet.java (87%) rename code/{services-core/index-service/src/test/java/nu/marginalia/index/svc => functions/index/src/test/java/nu/marginalia/functions/index}/IndexQueryServiceIntegrationSmokeTest.java (96%) rename code/{services-core/index-service/src/test/java/nu/marginalia/index/svc => functions/index/src/test/java/nu/marginalia/functions/index}/IndexQueryServiceIntegrationTest.java (98%) rename code/{services-core/index-service/src/test/java/nu/marginalia/index/svc => functions/index/src/test/java/nu/marginalia/functions/index}/IndexQueryServiceIntegrationTestModule.java (90%) rename code/{services-core/index-service/src/test/java/nu/marginalia => functions/index/src/test/java/nu/marginalia/functions}/index/results/IndexResultDomainDeduplicatorTest.java (88%) rename code/{services-core/index-service/src/test/java/nu/marginalia/index/svc => functions/index/src/test/java/nu/marginalia/functions/index}/searchset/RankingSearchSetTest.java (94%) rename code/{services-core/index-service/src/test/java/nu/marginalia => functions/index/src/test/java/nu/marginalia/functions}/index/util/TestUtil.java (96%) rename code/{services-core/assistant-service => functions/math}/src/main/resources/units.csv (100%) rename code/{services-core/assistant-service/src/test/java/nu/marginalia/assistant => functions/math/src/test/java/nu/marginalia/functions/math}/eval/MathParserTest.java (93%) rename code/{services-core/assistant-service/src/test/java/nu/marginalia/assistant => functions/math/src/test/java/nu/marginalia/functions/math}/eval/UnitsTest.java (90%) rename code/{api/index-api => functions/search-query/api}/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java (100%) rename code/{services-core/query-service => functions/search-query}/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java (98%) rename code/services-core/index-service/src/main/java/nu/marginalia/index/{svc => }/IndexOpsService.java (62%) delete mode 100644 code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java delete mode 100644 code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java delete mode 100644 code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryExecutor.java delete mode 100644 code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java diff --git a/code/api/executor-api/build.gradle b/code/api/executor-api/build.gradle index aca172ac..0103df0f 100644 --- a/code/api/executor-api/build.gradle +++ b/code/api/executor-api/build.gradle @@ -24,7 +24,7 @@ sourceSets { dependencies { implementation project(':code:common:model') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:config') implementation project(':code:common:db') implementation project(':code:libraries:message-queue') diff --git a/code/api/index-api/readme.md b/code/api/index-api/readme.md deleted file mode 100644 index 796debf5..00000000 --- a/code/api/index-api/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# Index API - -Client and models for talking to the [index-service](../../services-core/index-service), -implemented with the base client from [service-client](../../common/service-client). - -## Central Classes - -* [IndexClient](src/main/java/nu/marginalia/index/client/IndexClient.java) \ No newline at end of file diff --git a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java index b520f6e2..122f6211 100644 --- a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java +++ b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java @@ -5,7 +5,9 @@ import nu.marginalia.service.ServiceHomeNotConfiguredException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Objects; +import java.util.Optional; import java.util.stream.Stream; public class WmsaHome { @@ -32,20 +34,30 @@ public class WmsaHome { "/wmsa" }; - String retStr = Stream.of(possibleLocations) + Optional retStr = Stream.of(possibleLocations) .filter(Objects::nonNull) .map(Path::of) .filter(Files::isDirectory) .map(Path::toString) - .findFirst() - .orElseThrow(() -> - new ServiceHomeNotConfiguredException(""" + .findFirst(); + + if (retStr.isEmpty()) { + // Check if we are running in a test environment + + var testRoot = Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent) + .filter(p -> Files.exists(p.resolve("run/env"))) + .filter(p -> Files.exists(p.resolve("run/setup.sh"))) + .map(p -> p.resolve("run")) + .findAny(); + + return testRoot.orElseThrow(() -> new ServiceHomeNotConfiguredException(""" Could not find $WMSA_HOME, either set environment variable, the 'system.homePath' property, or ensure either /wmssa or /var/lib/wmsa exists """)); + } - var ret = Path.of(retStr); + var ret = Path.of(retStr.get()); if (!Files.isDirectory(ret.resolve("model"))) { throw new ServiceHomeNotConfiguredException("You need to run 'run/setup.sh' to download models to run/ before this will work!"); diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index 68c33277..3e724183 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -74,7 +74,7 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { return true; } - private class ConnectionHolder implements Comparable { + private class ConnectionHolder { private final AtomicReference channel = new AtomicReference<>(); private final InstanceAddress address; @@ -126,12 +126,6 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { public int hashCode() { return Objects.hash(address); } - - @Override - public int compareTo(@NotNull GrpcSingleNodeChannelPool.ConnectionHolder o) { - - return -Long.compare(address.cxTime(), o.address.cxTime()); // Reverse order - } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java index 1af51d75..e4eca465 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -191,12 +191,10 @@ public class ZkServiceRegistry implements ServiceRegistryIf { .getData() .forPath(path); - long cxTime = curatorFramework.checkExists().forPath(path).getMzxid(); - String hostAndPort = new String(data); var address = ServiceEndpoint .parse(hostAndPort) - .asInstance(UUID.fromString(uuid), cxTime); + .asInstance(UUID.fromString(uuid)); ret.add(address); } @@ -242,7 +240,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf { /* Exposed for tests */ public synchronized void shutDown() { - if (!stopped) + if (stopped) return; stopped = true; diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java index c52d335c..0e25ce19 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -37,11 +37,11 @@ public record ServiceEndpoint(String host, int port) { } } - public InstanceAddress asInstance(UUID instance, long cxTime) { - return new InstanceAddress(this, instance, cxTime); + public InstanceAddress asInstance(UUID instance) { + return new InstanceAddress(this, instance); } - public record InstanceAddress(ServiceEndpoint endpoint, UUID instance, long cxTime) { + public record InstanceAddress(ServiceEndpoint endpoint, UUID instance) { public String host() { return endpoint.host(); } diff --git a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java index 7963c857..3ea107e1 100644 --- a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java +++ b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java @@ -6,10 +6,9 @@ import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.junit.jupiter.Testcontainers; @@ -18,6 +17,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.*; @Testcontainers +@Execution(ExecutionMode.SAME_THREAD) @Tag("slow") class ZkServiceRegistryTest { private static final int ZOOKEEPER_PORT = 2181; @@ -51,6 +51,7 @@ class ZkServiceRegistryTest { } @Test + @Disabled // flaky on CI void getPort() { System.setProperty("service.random-port", "true"); @@ -101,17 +102,17 @@ class ZkServiceRegistryTest { registry1.announceInstance(uuid1); registry2.announceInstance(uuid2); - assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), + assertEquals(Set.of(endpoint1.asInstance(uuid1)), registry1.getEndpoints(key1)); - assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), + assertEquals(Set.of(endpoint2.asInstance(uuid2)), registry1.getEndpoints(key2)); registry1.shutDown(); Thread.sleep(100); assertEquals(Set.of(), registry2.getEndpoints(key1)); - assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key2)); + assertEquals(Set.of(endpoint2.asInstance(uuid2)), registry2.getEndpoints(key2)); } @Test @@ -130,14 +131,14 @@ class ZkServiceRegistryTest { registry1.announceInstance(uuid1); registry2.announceInstance(uuid2); - assertEquals(Set.of(endpoint1.asInstance(uuid1, 0), - endpoint2.asInstance(uuid2, 0)), + assertEquals(Set.of(endpoint1.asInstance(uuid1), + endpoint2.asInstance(uuid2)), registry1.getEndpoints(key)); registry1.shutDown(); Thread.sleep(100); - assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key)); + assertEquals(Set.of(endpoint2.asInstance(uuid2)), registry2.getEndpoints(key)); } @Test @@ -157,8 +158,8 @@ class ZkServiceRegistryTest { registry1.announceInstance(uuid1); registry2.announceInstance(uuid2); - assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), registry1.getEndpoints(key1)); - assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry1.getEndpoints(key2)); + assertEquals(Set.of(endpoint1.asInstance(uuid1)), registry1.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2)), registry1.getEndpoints(key2)); } @Test diff --git a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java deleted file mode 100644 index d7e6a9b3..00000000 --- a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java +++ /dev/null @@ -1,100 +0,0 @@ -package nu.marginalia.index.forward; - -import nu.marginalia.model.id.UrlIdCodec; -import nu.marginalia.model.idx.DocumentMetadata; -import nu.marginalia.index.query.limit.SpecificationLimitType; -import nu.marginalia.index.query.IndexQueryParams; -import nu.marginalia.index.query.filter.QueryFilterStepIf; - -public class ParamMatchingQueryFilter implements QueryFilterStepIf { - private final IndexQueryParams params; - private final ForwardIndexReader forwardIndexReader; - - public ParamMatchingQueryFilter(IndexQueryParams params, ForwardIndexReader forwardIndexReader) { - this.params = params; - this.forwardIndexReader = forwardIndexReader; - } - - @Override - public boolean test(long combinedId) { - long docId = UrlIdCodec.removeRank(combinedId); - int domainId = UrlIdCodec.getDomainId(docId); - - long meta = forwardIndexReader.getDocMeta(docId); - - if (!validateDomain(domainId, meta)) { - return false; - } - - if (!validateQuality(meta)) { - return false; - } - - if (!validateYear(meta)) { - return false; - } - - if (!validateSize(meta)) { - return false; - } - - if (!validateRank(meta)) { - return false; - } - - return true; - } - - private boolean validateDomain(int domainId, long meta) { - return params.searchSet().contains(domainId, meta); - } - - private boolean validateQuality(long meta) { - final var limit = params.qualityLimit(); - - if (limit.type() == SpecificationLimitType.NONE) { - return true; - } - - final int quality = DocumentMetadata.decodeQuality(meta); - - return limit.test(quality); - } - - private boolean validateYear(long meta) { - if (params.year().type() == SpecificationLimitType.NONE) - return true; - - int postVal = DocumentMetadata.decodeYear(meta); - - return params.year().test(postVal); - } - - private boolean validateSize(long meta) { - if (params.size().type() == SpecificationLimitType.NONE) - return true; - - int postVal = DocumentMetadata.decodeSize(meta); - - return params.size().test(postVal); - } - - private boolean validateRank(long meta) { - if (params.rank().type() == SpecificationLimitType.NONE) - return true; - - int postVal = DocumentMetadata.decodeRank(meta); - - return params.rank().test(postVal); - } - - @Override - public double cost() { - return 32; - } - - @Override - public String describe() { - return getClass().getSimpleName(); - } -} diff --git a/code/api/index-api/build.gradle b/code/functions/index/api/build.gradle similarity index 93% rename from code/api/index-api/build.gradle rename to code/functions/index/api/build.gradle index 3817c5de..69c5afff 100644 --- a/code/api/index-api/build.gradle +++ b/code/functions/index/api/build.gradle @@ -14,8 +14,6 @@ dependencies { implementation project(':code:common:config') implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') - implementation project(':code:features-index:index-query') - implementation project(':code:functions:search-query:api') implementation libs.bundles.slf4j diff --git a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java b/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java new file mode 100644 index 00000000..9cea5afd --- /dev/null +++ b/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java @@ -0,0 +1,52 @@ +package nu.marginalia.functions.index.api; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import lombok.SneakyThrows; +import nu.marginalia.api.searchquery.IndexApiGrpc; +import nu.marginalia.api.searchquery.RpcDecoratedResultItem; +import nu.marginalia.api.searchquery.RpcIndexQuery; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Singleton +public class IndexClient { + private static final Logger logger = LoggerFactory.getLogger(IndexClient.class); + private final GrpcMultiNodeChannelPool channelPool; + private static final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); + @Inject + public IndexClient(GrpcChannelPoolFactory channelPoolFactory) { + this.channelPool = channelPoolFactory.createMulti( + ServiceKey.forGrpcApi(IndexApiGrpc.class, ServicePartition.multi()), + IndexApiGrpc::newBlockingStub); + } + + @SneakyThrows + public List executeQueries(RpcIndexQuery indexRequest) { + var futures = + channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query) + .async(executor) + .runEach(indexRequest); + List results = new ArrayList<>(); + for (var future : futures) { + try { + future.get().forEachRemaining(results::add); + } + catch (Exception e) { + logger.error("Downstream exception", e); + } + } + + return results; + } + +} diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java b/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java similarity index 86% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java rename to code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java index 20a8891a..3a9b4314 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java +++ b/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client; +package nu.marginalia.functions.index.api; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -10,15 +10,15 @@ import nu.marginalia.service.id.ServiceId; import java.util.UUID; @Singleton -public class IndexClient { +public class IndexMqClient { private final MessageQueueFactory messageQueueFactory; MqOutbox outbox; @Inject - public IndexClient(MessageQueueFactory messageQueueFactory, - @Named("wmsa-system-node") Integer nodeId) + public IndexMqClient(MessageQueueFactory messageQueueFactory, + @Named("wmsa-system-node") Integer nodeId) { this.messageQueueFactory = messageQueueFactory; diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java b/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqEndpoints.java similarity index 89% rename from code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java rename to code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqEndpoints.java index 0a4635eb..118221ca 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java +++ b/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqEndpoints.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.client; +package nu.marginalia.functions.index.api; public class IndexMqEndpoints { public static final String INDEX_IS_BLOCKED = "INDEX-IS-BLOCKED"; diff --git a/code/functions/index/build.gradle b/code/functions/index/build.gradle new file mode 100644 index 00000000..e1223f15 --- /dev/null +++ b/code/functions/index/build.gradle @@ -0,0 +1,53 @@ +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':third-party:commons-codec') + + implementation project(':code:functions:index:api') + + implementation project(':code:libraries:array') + implementation project(':code:libraries:btree') + + implementation project(':code:common:db') + implementation project(':code:common:config') + implementation project(':code:common:model') + implementation project(':code:common:linkdb') + implementation project(':code:common:service') + implementation project(':code:common:service-discovery') + + implementation project(':code:functions:search-query:api') + + implementation project(':code:functions:index:domain-ranking') + implementation project(':code:functions:index:forward-index') + implementation project(':code:functions:index:reverse-index') + implementation project(':code:functions:index:query') + implementation project(':code:functions:index:index-journal') + implementation project(':code:functions:index:result-ranking') + + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + testImplementation project(':code:common:process') + +} diff --git a/code/features-index/domain-ranking/build.gradle b/code/functions/index/domain-ranking/build.gradle similarity index 100% rename from code/features-index/domain-ranking/build.gradle rename to code/functions/index/domain-ranking/build.gradle diff --git a/code/features-index/domain-ranking/readme.md b/code/functions/index/domain-ranking/readme.md similarity index 100% rename from code/features-index/domain-ranking/readme.md rename to code/functions/index/domain-ranking/readme.md diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java b/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java similarity index 100% rename from code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java rename to code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java b/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java similarity index 100% rename from code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java rename to code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java b/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java similarity index 100% rename from code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java rename to code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java b/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java similarity index 100% rename from code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java rename to code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java b/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java similarity index 100% rename from code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java rename to code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java b/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java similarity index 100% rename from code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java rename to code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java diff --git a/code/features-index/index-forward/build.gradle b/code/functions/index/forward-index/build.gradle similarity index 77% rename from code/features-index/index-forward/build.gradle rename to code/functions/index/forward-index/build.gradle index 8ceb710e..7ba989e2 100644 --- a/code/features-index/index-forward/build.gradle +++ b/code/functions/index/forward-index/build.gradle @@ -13,9 +13,9 @@ java { dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') - implementation project(':code:features-index:domain-ranking') - implementation project(':code:features-index:index-query') - implementation project(':code:features-index:index-journal') + implementation project(':code:functions:index:domain-ranking') + implementation project(':code:functions:index:query') + implementation project(':code:functions:index:index-journal') implementation project(':code:common:model') implementation project(':code:common:process') diff --git a/code/features-index/index-forward/readme.md b/code/functions/index/forward-index/readme.md similarity index 100% rename from code/features-index/index-forward/readme.md rename to code/functions/index/forward-index/readme.md diff --git a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java b/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java similarity index 100% rename from code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java rename to code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java diff --git a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java b/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java similarity index 100% rename from code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java rename to code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java diff --git a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java b/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java similarity index 100% rename from code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java rename to code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java diff --git a/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java b/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java similarity index 100% rename from code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java rename to code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java diff --git a/code/features-index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java b/code/functions/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java similarity index 100% rename from code/features-index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java rename to code/functions/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java diff --git a/code/features-index/index-forward/src/test/java/nu/marginalia/test/TestUtil.java b/code/functions/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/features-index/index-forward/src/test/java/nu/marginalia/test/TestUtil.java rename to code/functions/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java diff --git a/code/features-index/index-journal/build.gradle b/code/functions/index/index-journal/build.gradle similarity index 100% rename from code/features-index/index-journal/build.gradle rename to code/functions/index/index-journal/build.gradle diff --git a/code/features-index/index-journal/readme.md b/code/functions/index/index-journal/readme.md similarity index 100% rename from code/features-index/index-journal/readme.md rename to code/functions/index/index-journal/readme.md diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java diff --git a/code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java b/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java similarity index 100% rename from code/features-index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java rename to code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java diff --git a/code/features-index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java b/code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java similarity index 100% rename from code/features-index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java rename to code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java diff --git a/code/features-index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java b/code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java similarity index 100% rename from code/features-index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java rename to code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java diff --git a/code/features-index/index-query/build.gradle b/code/functions/index/query/build.gradle similarity index 100% rename from code/features-index/index-query/build.gradle rename to code/functions/index/query/build.gradle diff --git a/code/features-index/index-query/readme.md b/code/functions/index/query/readme.md similarity index 100% rename from code/features-index/index-query/readme.md rename to code/functions/index/query/readme.md diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java b/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java similarity index 100% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java rename to code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java diff --git a/code/features-index/index-query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java b/code/functions/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java similarity index 100% rename from code/features-index/index-query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java rename to code/functions/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java diff --git a/code/features-index/readme.md b/code/functions/index/readme.md similarity index 86% rename from code/features-index/readme.md rename to code/functions/index/readme.md index 28fe56b2..da5df4c1 100644 --- a/code/features-index/readme.md +++ b/code/functions/index/readme.md @@ -1,6 +1,6 @@ # Index -These are components that offer functionality for the [index-service](../services-core/index-service). +These are components that offer functionality for the [index-service](../../services-core/index-service). ## Indexes @@ -10,7 +10,7 @@ There are two indexes with accompanying tools for constructing them. * [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. -These indices rely heavily on the [libraries/btree](../libraries/btree) and [libraries/array](../libraries/array) components. +These indices rely heavily on the [libraries/btree](../../libraries/btree) and [libraries/array](../../libraries/array) components. ## Algorithms diff --git a/code/features-index/result-ranking/build.gradle b/code/functions/index/result-ranking/build.gradle similarity index 93% rename from code/features-index/result-ranking/build.gradle rename to code/functions/index/result-ranking/build.gradle index 4dfabcaf..64417a17 100644 --- a/code/features-index/result-ranking/build.gradle +++ b/code/functions/index/result-ranking/build.gradle @@ -13,7 +13,7 @@ java { dependencies { implementation project(':code:common:model') implementation project(':code:common:service') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:functions:search-query:api') implementation libs.bundles.slf4j diff --git a/code/features-index/result-ranking/readme.md b/code/functions/index/result-ranking/readme.md similarity index 100% rename from code/features-index/result-ranking/readme.md rename to code/functions/index/result-ranking/readme.md diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java b/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java similarity index 100% rename from code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java rename to code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java b/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java similarity index 100% rename from code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java rename to code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java b/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java similarity index 100% rename from code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java rename to code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java b/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java similarity index 100% rename from code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java rename to code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java diff --git a/code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java b/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java similarity index 100% rename from code/features-index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java rename to code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java diff --git a/code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java b/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java similarity index 93% rename from code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java rename to code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java index a4100e79..e5eda34c 100644 --- a/code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java +++ b/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java @@ -1,8 +1,8 @@ package nu.marginalia.ranking; -import nu.marginalia.index.client.model.results.ResultRankingContext; -import nu.marginalia.index.client.model.results.ResultRankingParameters; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.model.idx.DocumentFlags; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.crawl.PubDate; diff --git a/code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java b/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java similarity index 97% rename from code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java rename to code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java index b4f455f4..8718dc83 100644 --- a/code/features-index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java +++ b/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java @@ -1,7 +1,7 @@ package nu.marginalia.ranking.factors; +import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.bbpc.BrailleBlockPunchCards; -import nu.marginalia.index.client.model.results.SearchResultKeywordScore; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.ranking.ResultKeywordSet; import org.junit.jupiter.api.Test; diff --git a/code/features-index/index-reverse/build.gradle b/code/functions/index/reverse-index/build.gradle similarity index 76% rename from code/features-index/index-reverse/build.gradle rename to code/functions/index/reverse-index/build.gradle index afa0136f..0025c3fd 100644 --- a/code/features-index/index-reverse/build.gradle +++ b/code/functions/index/reverse-index/build.gradle @@ -15,9 +15,9 @@ dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') implementation project(':code:libraries:random-write-funnel') - implementation project(':code:features-index:domain-ranking') - implementation project(':code:features-index:index-query') - implementation project(':code:features-index:index-journal') + implementation project(':code:functions:index:domain-ranking') + implementation project(':code:functions:index:query') + implementation project(':code:functions:index:index-journal') implementation project(':code:common:model') implementation project(':code:common:process') diff --git a/code/features-index/index-reverse/index.svg b/code/functions/index/reverse-index/index.svg similarity index 100% rename from code/features-index/index-reverse/index.svg rename to code/functions/index/reverse-index/index.svg diff --git a/code/features-index/index-reverse/merging.svg b/code/functions/index/reverse-index/merging.svg similarity index 100% rename from code/features-index/index-reverse/merging.svg rename to code/functions/index/reverse-index/merging.svg diff --git a/code/features-index/index-reverse/preindex.svg b/code/functions/index/reverse-index/preindex.svg similarity index 100% rename from code/features-index/index-reverse/preindex.svg rename to code/functions/index/reverse-index/preindex.svg diff --git a/code/features-index/index-reverse/readme.md b/code/functions/index/reverse-index/readme.md similarity index 100% rename from code/features-index/index-reverse/readme.md rename to code/functions/index/reverse-index/readme.md diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexParameters.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexParameters.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexReader.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexReader.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindex.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindex.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java b/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java similarity index 100% rename from code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java rename to code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/TestSegmentData.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/TestSegmentData.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java b/code/functions/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/features-index/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java rename to code/functions/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java new file mode 100644 index 00000000..20e208e4 --- /dev/null +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java @@ -0,0 +1,208 @@ +package nu.marginalia.functions.index; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.grpc.stub.StreamObserver; +import io.prometheus.client.Counter; +import io.prometheus.client.Gauge; +import io.prometheus.client.Histogram; +import lombok.SneakyThrows; +import nu.marginalia.api.searchquery.*; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.SearchResultSet; +import nu.marginalia.functions.index.index.IndexQueryService; +import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.functions.index.model.IndexSearchParameters; +import nu.marginalia.functions.index.results.IndexResultValuatorService; +import nu.marginalia.functions.index.searchset.SearchSetsService; +import nu.marginalia.functions.index.searchset.SmallSearchSet; +import nu.marginalia.functions.index.searchset.SearchSet; +import nu.marginalia.service.module.ServiceConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; + +import java.sql.SQLException; +import java.util.*; + +@Singleton +public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + // This marker is used to mark sensitive log messages that are related to queries + // so that they can be filtered out in the production logging configuration + private final Marker queryMarker = MarkerFactory.getMarker("QUERY"); + + private static final Counter wmsa_query_timeouts = Counter.build() + .name("wmsa_index_query_timeouts") + .help("Query timeout counter") + .labelNames("node", "api") + .register(); + private static final Gauge wmsa_query_cost = Gauge.build() + .name("wmsa_index_query_cost") + .help("Computational cost of query") + .labelNames("node", "api") + .register(); + private static final Histogram wmsa_query_time = Histogram.build() + .name("wmsa_index_query_time") + .linearBuckets(0.05, 0.05, 15) + .labelNames("node", "api") + .help("Index-side query time") + .register(); + + + private final StatefulIndex index; + private final SearchSetsService searchSetsService; + + private final IndexQueryService indexQueryService; + private final IndexResultValuatorService resultValuator; + + private final int nodeId; + + + @Inject + public IndexQueryGrpcService(ServiceConfiguration serviceConfiguration, + StatefulIndex index, + SearchSetsService searchSetsService, + IndexQueryService indexQueryService, + IndexResultValuatorService resultValuator) + { + this.nodeId = serviceConfiguration.node(); + this.index = index; + this.searchSetsService = searchSetsService; + this.resultValuator = resultValuator; + this.indexQueryService = indexQueryService; + } + + // GRPC endpoint + @SneakyThrows + public void query(RpcIndexQuery request, + StreamObserver responseObserver) { + + try { + var params = new IndexSearchParameters(request, getSearchSet(request)); + + final String nodeName = Integer.toString(nodeId); + + SearchResultSet results = wmsa_query_time + .labels(nodeName, "GRPC") + .time(() -> executeSearch(params)); + + wmsa_query_cost + .labels(nodeName, "GRPC") + .set(params.getDataCost()); + + if (!params.hasTimeLeft()) { + wmsa_query_timeouts + .labels(nodeName, "GRPC") + .inc(); + } + + for (var result : results.results) { + + var rawResult = result.rawIndexResult; + + var rawItem = RpcRawResultItem.newBuilder(); + rawItem.setCombinedId(rawResult.combinedId); + rawItem.setResultsFromDomain(rawResult.resultsFromDomain); + + for (var score : rawResult.keywordScores) { + rawItem.addKeywordScores( + RpcResultKeywordScore.newBuilder() + .setEncodedDocMetadata(score.encodedDocMetadata()) + .setEncodedWordMetadata(score.encodedWordMetadata()) + .setKeyword(score.keyword) + .setHtmlFeatures(score.htmlFeatures()) + .setHasPriorityTerms(score.hasPriorityTerms()) + .setSubquery(score.subquery) + ); + } + + var decoratedBuilder = RpcDecoratedResultItem.newBuilder() + .setDataHash(result.dataHash) + .setDescription(result.description) + .setFeatures(result.features) + .setFormat(result.format) + .setRankingScore(result.rankingScore) + .setTitle(result.title) + .setUrl(result.url.toString()) + .setWordsTotal(result.wordsTotal) + .setRawItem(rawItem); + + if (result.pubYear != null) { + decoratedBuilder.setPubYear(result.pubYear); + } + responseObserver.onNext(decoratedBuilder.build()); + } + + responseObserver.onCompleted(); + } + catch (Exception ex) { + logger.error("Error in handling request", ex); + responseObserver.onError(ex); + } + } + + // exists for test access + @SneakyThrows + SearchResultSet justQuery(SearchSpecification specsSet) { + return executeSearch(new IndexSearchParameters(specsSet, getSearchSet(specsSet))); + } + + private SearchSet getSearchSet(SearchSpecification specsSet) { + + if (specsSet.domains != null && !specsSet.domains.isEmpty()) { + return new SmallSearchSet(specsSet.domains); + } + + return searchSetsService.getSearchSetByName(specsSet.searchSetIdentifier); + } + + private SearchSet getSearchSet(RpcIndexQuery request) { + + if (request.getDomainsCount() > 0) { + return new SmallSearchSet(request.getDomainsList()); + } + + return searchSetsService.getSearchSetByName(request.getSearchSetIdentifier()); + } + private SearchResultSet executeSearch(IndexSearchParameters params) throws SQLException { + + if (!index.isLoaded()) { + // Short-circuit if the index is not loaded, as we trivially know that there can be no results + return new SearchResultSet(List.of()); + } + + var rankingContext = createRankingContext(params.rankingParams, params.subqueries); + + logger.info(queryMarker, "{}", params.queryParams); + + var resultIds = indexQueryService.evaluateSubqueries(params); + var resultItems = resultValuator.findBestResults(params, + rankingContext, + resultIds); + + return new SearchResultSet(resultItems); + } + + private ResultRankingContext createRankingContext(ResultRankingParameters rankingParams, List subqueries) { + final var termToId = SearchTermsUtil.getAllIncludeTerms(subqueries); + final Map termFrequencies = new HashMap<>(termToId.size()); + final Map prioFrequencies = new HashMap<>(termToId.size()); + + termToId.forEach((key, id) -> termFrequencies.put(key, index.getTermFrequency(id))); + termToId.forEach((key, id) -> prioFrequencies.put(key, index.getTermFrequencyPrio(id))); + + return new ResultRankingContext(index.getTotalDocCount(), + rankingParams, + termFrequencies, + prioFrequencies); + } + +} + diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexServicesFactory.java similarity index 89% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/IndexServicesFactory.java index bcf041ca..6a0bfd79 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexServicesFactory.java @@ -1,13 +1,15 @@ -package nu.marginalia.index; +package nu.marginalia.functions.index; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.IndexLocations; +import nu.marginalia.functions.index.index.CombinedIndexReader; +import nu.marginalia.index.ReverseIndexFullFileNames; +import nu.marginalia.index.ReverseIndexPrioFileNames; +import nu.marginalia.index.ReverseIndexReader; import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.forward.ForwardIndexFileNames; import nu.marginalia.index.forward.ForwardIndexReader; -import nu.marginalia.index.index.SearchIndexReader; -import nu.marginalia.service.control.ServiceHeartbeat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,7 +17,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.sql.SQLException; @Singleton public class IndexServicesFactory { @@ -24,10 +25,7 @@ public class IndexServicesFactory { private final Path liveStorage; @Inject - public IndexServicesFactory( - ServiceHeartbeat heartbeat, - FileStorageService fileStorageService - ) throws SQLException { + public IndexServicesFactory(FileStorageService fileStorageService) { this.fileStorageService = fileStorageService; this.liveStorage = IndexLocations.getCurrentIndex(fileStorageService); @@ -87,8 +85,8 @@ public class IndexServicesFactory { } } - public SearchIndexReader getSearchIndexReader() throws IOException { - return new SearchIndexReader( + public CombinedIndexReader getSearchIndexReader() throws IOException { + return new CombinedIndexReader( getForwardIndexReader(), getReverseIndexReader(), getReverseIndexPrioReader() diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java similarity index 63% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java index e4247d43..3573d283 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchTermsService.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java @@ -1,44 +1,40 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.index; -import com.google.inject.Singleton; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.functions.index.model.IndexSearchTerms; import nu.marginalia.hash.MurmurHash3_128; -import nu.marginalia.index.index.SearchIndexSearchTerms; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -@Singleton -public class SearchTermsService { - private final Logger logger = LoggerFactory.getLogger(getClass()); +public class SearchTermsUtil { - public SearchIndexSearchTerms getSearchTerms(SearchSubquery request) { + /** Extract the search terms from the specified subquery. */ + public static IndexSearchTerms extractSearchTerms(SearchSubquery request) { final LongList excludes = new LongArrayList(); final LongList includes = new LongArrayList(); final LongList priority = new LongArrayList(); final List coherences = new ArrayList<>(); if (!addEachTerm(includes, request.searchTermsInclude)) { - return new SearchIndexSearchTerms(); + return new IndexSearchTerms(); } // This looks like a bug, but it's not // v--- ----v if (!addEachTerm(includes, request.searchTermsAdvice)) { - return new SearchIndexSearchTerms(); + return new IndexSearchTerms(); } for (var coherence : request.searchTermCoherences) { LongList parts = new LongArrayList(coherence.size()); if (!addEachTerm(parts, coherence)) { - return new SearchIndexSearchTerms(); + return new IndexSearchTerms(); } coherences.add(parts); @@ -48,10 +44,10 @@ public class SearchTermsService { addEachNonMandatoryTerm(excludes, request.searchTermsExclude); addEachNonMandatoryTerm(priority, request.searchTermsPriority); - return new SearchIndexSearchTerms(includes, excludes, priority, coherences); + return new IndexSearchTerms(includes, excludes, priority, coherences); } - private boolean addEachTerm(LongList ret, List words) { + private static boolean addEachTerm(LongList ret, List words) { boolean success = true; for (var word : words) { @@ -61,14 +57,17 @@ public class SearchTermsService { return success; } - private void addEachNonMandatoryTerm(LongList ret, List words) { + private static void addEachNonMandatoryTerm(LongList ret, List words) { for (var word : words) { ret.add(getWordId(word)); } } - public Map getAllIncludeTerms(List subqueries) { + /** Extract all include-terms from the specified subqueries, + * and a return a map of the terms and their termIds. + */ + public static Map getAllIncludeTerms(List subqueries) { Map ret = new HashMap<>(); for (var subquery : subqueries) { @@ -80,8 +79,10 @@ public class SearchTermsService { return ret; } - static MurmurHash3_128 hasher = new MurmurHash3_128(); - public long getWordId(String s) { + private static final MurmurHash3_128 hasher = new MurmurHash3_128(); + + /** Translate the word to a unique id. */ + public static long getWordId(String s) { return hasher.hashKeyword(s); } } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java similarity index 63% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java index 23b9d954..6e3e17cf 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java @@ -1,8 +1,8 @@ -package nu.marginalia.index.index; +package nu.marginalia.functions.index.index; +import nu.marginalia.functions.index.model.IndexQueryParams; import nu.marginalia.index.ReverseIndexReader; import nu.marginalia.index.forward.ForwardIndexReader; -import nu.marginalia.index.forward.ParamMatchingQueryFilter; import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepIf; import org.slf4j.Logger; @@ -12,7 +12,8 @@ import java.time.Duration; import java.util.List; import java.util.concurrent.TimeUnit; -public class SearchIndexReader { +/** A reader for the combined forward and reverse indexes */ +public class CombinedIndexReader { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -20,55 +21,70 @@ public class SearchIndexReader { private final ReverseIndexReader reverseIndexFullReader; private final ReverseIndexReader reverseIndexPriorityReader; - public SearchIndexReader(ForwardIndexReader forwardIndexReader, - ReverseIndexReader reverseIndexFullReader, - ReverseIndexReader reverseIndexPriorityReader) { + public CombinedIndexReader(ForwardIndexReader forwardIndexReader, + ReverseIndexReader reverseIndexFullReader, + ReverseIndexReader reverseIndexPriorityReader) { this.forwardIndexReader = forwardIndexReader; this.reverseIndexFullReader = reverseIndexFullReader; this.reverseIndexPriorityReader = reverseIndexPriorityReader; } + + /** Creates a query builder for terms in the priority index */ public IndexQueryBuilder findPriorityWord(IndexQueryPriority priority, long wordId, int fetchSizeMultiplier) { - var sources = List.of(reverseIndexPriorityReader.documents(wordId)); - - return new SearchIndexQueryBuilder(reverseIndexFullReader, reverseIndexPriorityReader, - new IndexQuery(sources, priority, fetchSizeMultiplier), wordId); + return new IndexQueryBuilderImpl(reverseIndexFullReader, reverseIndexPriorityReader, + new IndexQuery( + List.of(reverseIndexPriorityReader.documents(wordId)), + priority, + fetchSizeMultiplier), wordId); } + /** Creates a query builder for terms in the full index */ public IndexQueryBuilder findFullWord(IndexQueryPriority priority, long wordId, int fetchSizeMultiplier) { - var sources = List.of(reverseIndexFullReader.documents(wordId)); - - return new SearchIndexQueryBuilder(reverseIndexFullReader, reverseIndexPriorityReader, - new IndexQuery(sources, priority, fetchSizeMultiplier), wordId); + return new IndexQueryBuilderImpl(reverseIndexFullReader, reverseIndexPriorityReader, + new IndexQuery(List.of(reverseIndexFullReader.documents(wordId)), + priority, + fetchSizeMultiplier), + wordId); } + /** Creates a parameter matching filter step for the provided parameters */ public QueryFilterStepIf filterForParams(IndexQueryParams params) { - return new ParamMatchingQueryFilter(params, forwardIndexReader); + return new IndexQueryBuilderImpl.ParamMatchingQueryFilter(params, forwardIndexReader); } + /** Returns the number of occurrences of the word in the full index */ public long numHits(long word) { return reverseIndexFullReader.numDocuments(word); } + + /** Returns the number of occurrences of the word in the priority index */ public long numHitsPrio(long word) { return reverseIndexPriorityReader.numDocuments(word); } + /** Retrieves the term metadata for the specified word for the provided documents */ public long[] getMetadata(long wordId, long[] docIds) { return reverseIndexFullReader.getTermMeta(wordId, docIds); } + /** Retrieves the document metadata for the specified document */ public long getDocumentMetadata(long docId) { return forwardIndexReader.getDocMeta(docId); } + /** Returns the total number of documents in the index */ public int totalDocCount() { return forwardIndexReader.totalDocCount(); } + /** Retrieves the HTML features for the specified document */ public int getHtmlFeatures(long docId) { return forwardIndexReader.getHtmlFeatures(docId); } + /** Close the indexes (this is not done immediately) + * */ public void close() throws InterruptedException { /* Delay the invocation of close method to allow for a clean shutdown of the service. * diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java new file mode 100644 index 00000000..e91b14d1 --- /dev/null +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java @@ -0,0 +1,168 @@ +package nu.marginalia.functions.index.index; + +import gnu.trove.set.hash.TLongHashSet; +import nu.marginalia.functions.index.model.IndexQueryParams; +import nu.marginalia.index.ReverseIndexReader; +import nu.marginalia.index.forward.ForwardIndexReader; +import nu.marginalia.index.query.IndexQuery; +import nu.marginalia.index.query.IndexQueryBuilder; +import nu.marginalia.index.query.filter.QueryFilterStepIf; +import nu.marginalia.index.query.limit.SpecificationLimitType; +import nu.marginalia.model.id.UrlIdCodec; +import nu.marginalia.model.idx.DocumentMetadata; + +public class IndexQueryBuilderImpl implements IndexQueryBuilder { + private final IndexQuery query; + private final ReverseIndexReader reverseIndexFullReader; + private final ReverseIndexReader reverseIndexPrioReader; + + /* Keep track of already added include terms to avoid redundant checks. + * + * Warning: This may cause unexpected behavior if for example attempting to + * first check one index and then another for the same term. At the moment, that + * makes no sense, but in the future, that might be a thing one might want to do. + * */ + private final TLongHashSet alreadyConsideredTerms = new TLongHashSet(); + + IndexQueryBuilderImpl(ReverseIndexReader reverseIndexFullReader, + ReverseIndexReader reverseIndexPrioReader, + IndexQuery query, long... sourceTerms) + { + this.query = query; + this.reverseIndexFullReader = reverseIndexFullReader; + this.reverseIndexPrioReader = reverseIndexPrioReader; + + alreadyConsideredTerms.addAll(sourceTerms); + } + + public IndexQueryBuilder alsoFull(long termId) { + + if (alreadyConsideredTerms.add(termId)) { + query.addInclusionFilter(reverseIndexFullReader.also(termId)); + } + + return this; + } + + public IndexQueryBuilder alsoPrio(long termId) { + + if (alreadyConsideredTerms.add(termId)) { + query.addInclusionFilter(reverseIndexPrioReader.also(termId)); + } + + return this; + } + + public IndexQueryBuilder notFull(long termId) { + + query.addInclusionFilter(reverseIndexFullReader.not(termId)); + + return this; + } + + public IndexQueryBuilder addInclusionFilter(QueryFilterStepIf filterStep) { + + query.addInclusionFilter(filterStep); + + return this; + } + + public IndexQuery build() { + return query; + } + + public static class ParamMatchingQueryFilter implements QueryFilterStepIf { + private final IndexQueryParams params; + private final ForwardIndexReader forwardIndexReader; + + public ParamMatchingQueryFilter(IndexQueryParams params, + ForwardIndexReader forwardIndexReader) + { + this.params = params; + this.forwardIndexReader = forwardIndexReader; + } + + @Override + public boolean test(long combinedId) { + long docId = UrlIdCodec.removeRank(combinedId); + int domainId = UrlIdCodec.getDomainId(docId); + + long meta = forwardIndexReader.getDocMeta(docId); + + if (!validateDomain(domainId, meta)) { + return false; + } + + if (!validateQuality(meta)) { + return false; + } + + if (!validateYear(meta)) { + return false; + } + + if (!validateSize(meta)) { + return false; + } + + if (!validateRank(meta)) { + return false; + } + + return true; + } + + private boolean validateDomain(int domainId, long meta) { + return params.searchSet().contains(domainId, meta); + } + + private boolean validateQuality(long meta) { + final var limit = params.qualityLimit(); + + if (limit.type() == SpecificationLimitType.NONE) { + return true; + } + + final int quality = DocumentMetadata.decodeQuality(meta); + + return limit.test(quality); + } + + private boolean validateYear(long meta) { + if (params.year().type() == SpecificationLimitType.NONE) + return true; + + int postVal = DocumentMetadata.decodeYear(meta); + + return params.year().test(postVal); + } + + private boolean validateSize(long meta) { + if (params.size().type() == SpecificationLimitType.NONE) + return true; + + int postVal = DocumentMetadata.decodeSize(meta); + + return params.size().test(postVal); + } + + private boolean validateRank(long meta) { + if (params.rank().type() == SpecificationLimitType.NONE) + return true; + + int postVal = DocumentMetadata.decodeRank(meta); + + return params.rank().test(postVal); + } + + @Override + public double cost() { + return 32; + } + + @Override + public String describe() { + return getClass().getSimpleName(); + } + } +} diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java new file mode 100644 index 00000000..64c9fa16 --- /dev/null +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java @@ -0,0 +1,157 @@ +package nu.marginalia.functions.index.index; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import gnu.trove.list.TLongList; +import gnu.trove.list.array.TLongArrayList; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.functions.index.model.IndexSearchTerms; +import nu.marginalia.functions.index.model.IndexSearchParameters; +import nu.marginalia.functions.index.SearchTermsUtil; +import nu.marginalia.index.query.IndexQuery; +import nu.marginalia.index.query.IndexQueryPriority; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; + +import java.util.List; + +@Singleton +public class IndexQueryService { + private final Marker queryMarker = MarkerFactory.getMarker("QUERY"); + + /** Execute subqueries and return a list of document ids. The index is queried for each subquery, + * at different priorty depths until timeout is reached or the results are all visited. + *
+ * Then the results are combined. + * */ + private final ThreadLocal resultsArrayListPool = ThreadLocal.withInitial(TLongArrayList::new); + private static final Logger logger = LoggerFactory.getLogger(IndexQueryService.class); + private final StatefulIndex index; + + @Inject + public IndexQueryService(StatefulIndex index) { + this.index = index; + } + + /** Execute subqueries and return a list of document ids. The index is queried for each subquery, + * at different priorty depths until timeout is reached or the results are all visited. + * Then the results are combined. + * */ + public TLongList evaluateSubqueries(IndexSearchParameters params) { + final TLongArrayList results = resultsArrayListPool.get(); + results.resetQuick(); + results.ensureCapacity(params.fetchSize); + + // These queries are various term combinations + for (var subquery : params.subqueries) { + + if (!params.hasTimeLeft()) { + logger.info("Query timed out {}, ({}), -{}", + subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); + break; + } + + logger.info(queryMarker, "{}", subquery); + + final IndexSearchTerms searchTerms = SearchTermsUtil.extractSearchTerms(subquery); + + if (searchTerms.isEmpty()) { + logger.info(queryMarker, "empty"); + continue; + } + + // logSearchTerms(subquery, searchTerms); + + // These queries are different indices for one subquery + List queries = params.createIndexQueries(index, searchTerms); + for (var query : queries) { + + if (!params.hasTimeLeft()) + break; + + if (shouldOmitQuery(params, query, results.size())) { + logger.info(queryMarker, "Omitting {}", query); + continue; + } + + final int fetchSize = params.fetchSize * query.fetchSizeMultiplier; + final LongQueryBuffer buffer = new LongQueryBuffer(fetchSize); + + int cnt = 0; + + while (query.hasMore() + && results.size() < fetchSize + && params.budget.hasTimeLeft()) + { + buffer.reset(); + query.getMoreResults(buffer); + + for (int i = 0; i < buffer.size() && results.size() < fetchSize; i++) { + results.add(buffer.data[i]); + cnt++; + } + } + + params.dataCost += query.dataCost(); + + + logger.info(queryMarker, "{} from {}", cnt, query); + } + } + + return results; + } + + /** @see IndexQueryPriority */ + private boolean shouldOmitQuery(IndexSearchParameters params, IndexQuery query, int resultCount) { + + var priority = query.queryPriority; + + return switch (priority) { + case IndexQueryPriority.BEST -> false; + case IndexQueryPriority.GOOD -> resultCount > params.fetchSize / 4; + case IndexQueryPriority.FALLBACK -> resultCount > params.fetchSize / 8; + }; + } + + private void logSearchTerms(SearchSubquery subquery, IndexSearchTerms searchTerms) { + + // This logging should only be enabled in testing, as it is very verbose + // and contains sensitive information + + if (!logger.isInfoEnabled(queryMarker)) { + return; + } + + var includes = subquery.searchTermsInclude; + var advice = subquery.searchTermsAdvice; + var excludes = subquery.searchTermsExclude; + var priority = subquery.searchTermsPriority; + + for (int i = 0; i < includes.size(); i++) { + logger.info(queryMarker, "{} -> {} I", includes.get(i), + Long.toHexString(searchTerms.includes().getLong(i)) + ); + } + for (int i = 0; i < advice.size(); i++) { + logger.info(queryMarker, "{} -> {} A", advice.get(i), + Long.toHexString(searchTerms.includes().getLong(includes.size() + i)) + ); + } + for (int i = 0; i < excludes.size(); i++) { + logger.info(queryMarker, "{} -> {} E", excludes.get(i), + Long.toHexString(searchTerms.excludes().getLong(i)) + ); + } + for (int i = 0; i < priority.size(); i++) { + logger.info(queryMarker, "{} -> {} P", priority.get(i), + Long.toHexString(searchTerms.priority().getLong(i)) + ); + } + } + + +} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java similarity index 75% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java index b21a3e72..4a81693f 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java @@ -1,8 +1,10 @@ -package nu.marginalia.index.index; +package nu.marginalia.functions.index.index; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.index.IndexServicesFactory; +import nu.marginalia.functions.index.model.IndexQueryParams; +import nu.marginalia.functions.index.IndexServicesFactory; +import nu.marginalia.functions.index.model.IndexSearchTerms; import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepFromPredicate; import nu.marginalia.service.control.ServiceEventLog; @@ -24,22 +26,20 @@ import java.util.function.LongPredicate; * */ @Singleton -public class SearchIndex { +public class StatefulIndex { private final Logger logger = LoggerFactory.getLogger(getClass()); - private volatile SearchIndexReader indexReader; - private final ReadWriteLock indexReplacementLock = new ReentrantReadWriteLock(); - @NotNull private final IndexServicesFactory servicesFactory; - private final ServiceEventLog eventLog; + private volatile CombinedIndexReader combinedIndexReader; + @Inject - public SearchIndex(@NotNull IndexServicesFactory servicesFactory, - ServiceEventLog eventLog) { + public StatefulIndex(@NotNull IndexServicesFactory servicesFactory, + ServiceEventLog eventLog) { this.servicesFactory = servicesFactory; this.eventLog = eventLog; } @@ -51,8 +51,8 @@ public class SearchIndex { lock.lock(); logger.info("Initializing index"); - if (indexReader == null) { - indexReader = servicesFactory.getSearchIndexReader(); + if (combinedIndexReader == null) { + combinedIndexReader = servicesFactory.getSearchIndexReader(); eventLog.logEvent("INDEX-INIT", "Index loaded"); } else { @@ -73,12 +73,12 @@ public class SearchIndex { try { lock.lock(); - if (indexReader != null) - indexReader.close(); + if (combinedIndexReader != null) + combinedIndexReader.close(); servicesFactory.switchFiles(); - indexReader = servicesFactory.getSearchIndexReader(); + combinedIndexReader = servicesFactory.getSearchIndexReader(); eventLog.logEvent("INDEX-SWITCH-OK", ""); } @@ -97,16 +97,16 @@ public class SearchIndex { /** Returns true if the service has initialized */ public boolean isAvailable() { - return indexReader != null; + return combinedIndexReader != null; } /** Stronger version of isAvailable() that also checks that the index is loaded */ public boolean isLoaded() { - return indexReader != null && indexReader.isLoaded(); + return combinedIndexReader != null && combinedIndexReader.isLoaded(); } - public List createQueries(SearchIndexSearchTerms terms, IndexQueryParams params, LongPredicate includePred) { + public List createQueries(IndexSearchTerms terms, IndexQueryParams params, LongPredicate includePred) { if (!isLoaded()) { logger.warn("Index reader not ready"); @@ -130,7 +130,7 @@ public class SearchIndex { if (orderedIncludesPrio.length > 1) { for (int i = 0; i + 1 < orderedIncludesPrio.length; i++) { for (int j = i + 1; j < orderedIncludesPrio.length; j++) { - var entrySource = indexReader + var entrySource = combinedIndexReader .findPriorityWord(IndexQueryPriority.BEST, orderedIncludesPrio[i], fetchSizeMultiplier) .alsoPrio(orderedIncludesPrio[j]); queryHeads.add(entrySource); @@ -140,12 +140,12 @@ public class SearchIndex { // Next consider entries that appear only once in the priority index for (var wordId : orderedIncludesPrio) { - queryHeads.add(indexReader.findPriorityWord(IndexQueryPriority.GOOD, wordId, fetchSizeMultiplier)); + queryHeads.add(combinedIndexReader.findPriorityWord(IndexQueryPriority.GOOD, wordId, fetchSizeMultiplier)); } // Finally consider terms in the full index, but only do this for sufficiently long queries // as short queries tend to be too underspecified to produce anything other than CPU warmth - queryHeads.add(indexReader.findFullWord(IndexQueryPriority.FALLBACK, orderedIncludes[0], fetchSizeMultiplier)); + queryHeads.add(combinedIndexReader.findFullWord(IndexQueryPriority.FALLBACK, orderedIncludes[0], fetchSizeMultiplier)); for (var query : queryHeads) { if (query == null) { @@ -166,7 +166,7 @@ public class SearchIndex { // Run these last, as they'll worst-case cause as many page faults as there are // items in the buffer - queries.add(query.addInclusionFilter(indexReader.filterForParams(params)).build()); + queries.add(query.addInclusionFilter(combinedIndexReader.filterForParams(params)).build()); } return queries; @@ -174,15 +174,15 @@ public class SearchIndex { private int compareKeywords(long a, long b) { return Long.compare( - indexReader.numHits(a), - indexReader.numHits(b) + combinedIndexReader.numHits(a), + combinedIndexReader.numHits(b) ); } private int compareKeywordsPrio(long a, long b) { return Long.compare( - indexReader.numHitsPrio(a), - indexReader.numHitsPrio(b) + combinedIndexReader.numHitsPrio(a), + combinedIndexReader.numHitsPrio(b) ); } @@ -191,24 +191,24 @@ public class SearchIndex { * docs[] *must* be sorted. */ public long[] getTermMetadata(long termId, long[] docs) { - return indexReader.getMetadata(termId, docs); + return combinedIndexReader.getMetadata(termId, docs); } public long getDocumentMetadata(long docId) { - return indexReader.getDocumentMetadata(docId); + return combinedIndexReader.getDocumentMetadata(docId); } public int getHtmlFeatures(long docId) { - return indexReader.getHtmlFeatures(docId); + return combinedIndexReader.getHtmlFeatures(docId); } public int getTotalDocCount() { - return indexReader.totalDocCount(); + return combinedIndexReader.totalDocCount(); } public int getTermFrequency(long id) { - return (int) indexReader.numHits(id); + return (int) combinedIndexReader.numHits(id); } public int getTermFrequencyPrio(long id) { - return (int) indexReader.numHitsPrio(id); + return (int) combinedIndexReader.numHitsPrio(id); } } diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexQueryParams.java similarity index 92% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexQueryParams.java index 46224638..f8e64e0b 100644 --- a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexQueryParams.java @@ -1,7 +1,7 @@ -package nu.marginalia.index.query; +package nu.marginalia.functions.index.model; +import nu.marginalia.functions.index.searchset.SearchSet; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.index.searchset.SearchSet; import nu.marginalia.index.query.limit.SpecificationLimit; /** IndexQueryParams is a set of parameters for a query. diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java similarity index 78% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java index 9824f462..61442ecf 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/SearchParameters.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.index.model; import gnu.trove.set.hash.TLongHashSet; import nu.marginalia.api.searchquery.RpcIndexQuery; @@ -6,45 +6,43 @@ import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.api.searchquery.IndexProtobufCodec; -import nu.marginalia.index.index.SearchIndex; -import nu.marginalia.index.index.SearchIndexSearchTerms; +import nu.marginalia.functions.index.index.StatefulIndex; import nu.marginalia.index.query.IndexQuery; -import nu.marginalia.index.query.IndexQueryParams; import nu.marginalia.index.query.IndexSearchBudget; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.index.searchset.SearchSet; +import nu.marginalia.functions.index.searchset.SearchSet; import java.util.ArrayList; import java.util.List; -public class SearchParameters { +public class IndexSearchParameters { /** * This is how many results matching the keywords we'll try to get * before evaluating them for the best result. */ - final int fetchSize; - final IndexSearchBudget budget; - final List subqueries; - final IndexQueryParams queryParams; - final ResultRankingParameters rankingParams; + public final int fetchSize; + public final IndexSearchBudget budget; + public final List subqueries; + public final IndexQueryParams queryParams; + public final ResultRankingParameters rankingParams; - final int limitByDomain; - final int limitTotal; + public final int limitByDomain; + public final int limitTotal; // mutable: /** * An estimate of how much data has been read */ - long dataCost = 0; + public long dataCost = 0; /** * A set of id:s considered during each subquery, * for deduplication */ - final TLongHashSet consideredUrlIds; + public final TLongHashSet consideredUrlIds; - public SearchParameters(SearchSpecification specsSet, SearchSet searchSet) { + public IndexSearchParameters(SearchSpecification specsSet, SearchSet searchSet) { var limits = specsSet.queryLimits; this.fetchSize = limits.fetchSize(); @@ -67,7 +65,7 @@ public class SearchParameters { rankingParams = specsSet.rankingParams; } - public SearchParameters(RpcIndexQuery request, SearchSet searchSet) { + public IndexSearchParameters(RpcIndexQuery request, SearchSet searchSet) { var limits = IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()); this.fetchSize = limits.fetchSize(); @@ -93,15 +91,15 @@ public class SearchParameters { rankingParams = IndexProtobufCodec.convertRankingParameterss(request.getParameters()); } - List createIndexQueries(SearchIndex index, SearchIndexSearchTerms terms) { + public List createIndexQueries(StatefulIndex index, IndexSearchTerms terms) { return index.createQueries(terms, queryParams, consideredUrlIds::add); } - boolean hasTimeLeft() { + public boolean hasTimeLeft() { return budget.hasTimeLeft(); } - long getDataCost() { + public long getDataCost() { return dataCost; } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexSearchTerms.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java similarity index 89% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexSearchTerms.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java index 833778df..4d4edd75 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexSearchTerms.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.index; +package nu.marginalia.functions.index.model; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongComparator; @@ -8,14 +8,14 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import java.util.Collections; import java.util.List; -public record SearchIndexSearchTerms( +public record IndexSearchTerms( LongList includes, LongList excludes, LongList priority, List coherences ) { - public SearchIndexSearchTerms() { + public IndexSearchTerms() { this(LongList.of(), LongList.of(), LongList.of(), Collections.emptyList()); } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexMetadataService.java similarity index 89% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexMetadataService.java index bc847978..c96d47ab 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexMetadataService.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexMetadataService.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.results; +package nu.marginalia.functions.index.results; import com.google.inject.Inject; import gnu.trove.map.hash.TObjectLongHashMap; @@ -7,8 +7,8 @@ import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap; import it.unimi.dsi.fastutil.longs.LongArrayList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; -import nu.marginalia.index.index.SearchIndex; -import nu.marginalia.index.svc.SearchTermsService; +import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.functions.index.SearchTermsUtil; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.ranking.ResultValuator; import org.slf4j.Logger; @@ -18,17 +18,17 @@ import java.util.ArrayList; import java.util.List; public class IndexMetadataService { - private final SearchIndex index; - private final SearchTermsService searchTermsService; + private final StatefulIndex index; + private final SearchTermsUtil searchTermsUtil; private final ResultValuator searchResultValuator; private static final Logger logger = LoggerFactory.getLogger(IndexMetadataService.class); @Inject - public IndexMetadataService(SearchIndex index, - SearchTermsService searchTermsService, + public IndexMetadataService(StatefulIndex index, + SearchTermsUtil searchTermsUtil, ResultValuator searchResultValuator) { this.index = index; - this.searchTermsService = searchTermsService; + this.searchTermsUtil = searchTermsUtil; this.searchResultValuator = searchResultValuator; } @@ -56,7 +56,7 @@ public class IndexMetadataService { continue; } - long id = searchTermsService.getWordId(term); + long id = searchTermsUtil.getWordId(term); termIdsList.add(id); termToId.put(term, id); } @@ -73,7 +73,7 @@ public class IndexMetadataService { for (var subquery : searchTermVariants) { for (var coh : subquery.searchTermCoherences) { - long[] ids = coh.stream().mapToLong(searchTermsService::getWordId).toArray(); + long[] ids = coh.stream().mapToLong(SearchTermsUtil::getWordId).toArray(); coherences.add(ids); } @@ -89,7 +89,7 @@ public class IndexMetadataService { subqueries.stream() .flatMap(sq -> sq.searchTermsPriority.stream()) .distinct() - .mapToLong(searchTermsService::getWordId) + .mapToLong(SearchTermsUtil::getWordId) .toArray(); var ret = new TLongHashSet(resultsArray.length); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicator.java similarity index 96% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicator.java index fac47a47..8d59f81f 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicator.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.results; +package nu.marginalia.functions.index.results; import gnu.trove.map.TLongIntMap; import gnu.trove.map.hash.TLongIntHashMap; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java similarity index 90% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java index 414d4306..ac297e2e 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultValuator.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.results; +package nu.marginalia.functions.index.results; import gnu.trove.list.TLongList; import gnu.trove.set.hash.TLongHashSet; @@ -11,14 +11,17 @@ import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.index.query.IndexQueryParams; +import nu.marginalia.functions.index.model.IndexQueryParams; import nu.marginalia.ranking.ResultValuator; import javax.annotation.Nullable; import java.util.Arrays; import java.util.List; -public class IndexResultValuator { +/** This class is responsible for calculating the score of a search result. + * It holds the data required to perform the scoring, as there is strong + * reasons to cache this data, and performs the calculations */ +public class IndexResultValuationContext { private final IndexMetadataService metadataService; private final List> searchTermVariants; private final IndexQueryParams queryParams; @@ -30,11 +33,11 @@ public class IndexResultValuator { private final ResultRankingContext rankingContext; private final ResultValuator searchResultValuator; - public IndexResultValuator(IndexMetadataService metadataService, - TLongList results, - ResultRankingContext rankingContext, - List subqueries, - IndexQueryParams queryParams + public IndexResultValuationContext(IndexMetadataService metadataService, + TLongList results, + ResultRankingContext rankingContext, + List subqueries, + IndexQueryParams queryParams ) { this.rankingContext = rankingContext; this.searchResultValuator = metadataService.getSearchResultValuator(); diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java new file mode 100644 index 00000000..4b9efac4 --- /dev/null +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java @@ -0,0 +1,152 @@ +package nu.marginalia.functions.index.results; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import gnu.trove.list.TLongList; +import gnu.trove.list.array.TLongArrayList; +import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; +import nu.marginalia.api.searchquery.model.results.ResultRankingContext; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; +import nu.marginalia.functions.index.model.IndexSearchParameters; +import nu.marginalia.linkdb.docs.DocumentDbReader; +import nu.marginalia.linkdb.model.DocdbUrlDetail; +import nu.marginalia.ranking.ResultValuator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +public class IndexResultValuatorService { + private static final Logger logger = LoggerFactory.getLogger(IndexResultValuatorService.class); + + private final IndexMetadataService metadataService; + private final DocumentDbReader documentDbReader; + private final ResultValuator resultValuator; + + @Inject + public IndexResultValuatorService(IndexMetadataService metadataService, + DocumentDbReader documentDbReader, + ResultValuator resultValuator) { + this.metadataService = metadataService; + this.documentDbReader = documentDbReader; + this.resultValuator = resultValuator; + } + + + /** From a list of candidate document IDs, find the best results and decorate them with additional information */ + public List findBestResults(IndexSearchParameters params, ResultRankingContext rankingContext, TLongList resultIds) throws SQLException { + + final var evaluator = new IndexResultValuationContext(metadataService, + resultIds, + rankingContext, + params.subqueries, + params.queryParams); + + // Sort the ids for more favorable access patterns on disk + resultIds.sort(); + + // Parallel stream to calculate scores is a minor performance boost + var results = Arrays.stream(resultIds.toArray()) + .parallel() + .mapToObj(evaluator::calculatePreliminaryScore) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + results = selectBestResults(params, results); + + return decorateAndRerank(results, rankingContext); + } + + private List selectBestResults(IndexSearchParameters params, List results) { + + var domainCountFilter = new IndexResultDomainDeduplicator(params.limitByDomain); + + results.sort(Comparator.naturalOrder()); + + List resultsList = new ArrayList<>(results.size()); + + for (var item : results) { + if (domainCountFilter.test(item)) { + resultsList.add(item); + } + } + + if (!params.queryParams.domainCount().isNone()) { + // Remove items that don't meet the domain count requirement + // This isn't perfect because the domain count is calculated + // after the results are sorted + resultsList.removeIf(item -> !params.queryParams.domainCount().test(domainCountFilter.getCount(item))); + } + + if (resultsList.size() > params.limitTotal) { + // This can't be made a stream limit() operation because we need domainCountFilter + // to run over the entire list to provide accurate statistics + + resultsList.subList(params.limitTotal, resultsList.size()).clear(); + } + + // populate results with the total number of results encountered from + // the same domain so this information can be presented to the user + for (var result : resultsList) { + result.resultsFromDomain = domainCountFilter.getCount(result); + } + + return resultsList; + } + + /** Decorate the result items with additional information from the link database + * and calculate an updated ranking with the additional information */ + public List decorateAndRerank(List rawResults, + ResultRankingContext rankingContext) + throws SQLException + { + TLongList idsList = new TLongArrayList(rawResults.size()); + + for (var result : rawResults) + idsList.add(result.getDocumentId()); + + Map urlDetailsById = new HashMap<>(rawResults.size()); + + for (var item : documentDbReader.getUrlDetails(idsList)) + urlDetailsById.put(item.urlId(), item); + + List decoratedItems = new ArrayList<>(); + for (var result : rawResults) { + var docData = urlDetailsById.get(result.getDocumentId()); + + if (null == docData) { + logger.warn("No data for document id {}", result.getDocumentId()); + continue; + } + + decoratedItems.add(createCombinedItem(result, docData, rankingContext)); + } + + if (decoratedItems.size() != rawResults.size()) + logger.warn("Result list shrunk during decoration?"); + + return decoratedItems; + } + + private DecoratedSearchResultItem createCombinedItem(SearchResultItem result, + DocdbUrlDetail docData, + ResultRankingContext rankingContext) { + return new DecoratedSearchResultItem( + result, + docData.url(), + docData.title(), + docData.description(), + docData.urlQuality(), + docData.format(), + docData.features(), + docData.pubYear(), + docData.dataHash(), + docData.wordsTotal(), + resultValuator.calculateSearchResultValue(result.keywordScores, docData.wordsTotal(), rankingContext) + ); + + } +} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/db/DbUpdateRanks.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/DbUpdateRanks.java similarity index 96% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/db/DbUpdateRanks.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/DbUpdateRanks.java index d6696f59..ac04de51 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/db/DbUpdateRanks.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/DbUpdateRanks.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.db; +package nu.marginalia.functions.index.searchset; import com.zaxxer.hikari.HikariDataSource; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/RankingSearchSet.java similarity index 96% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/RankingSearchSet.java index e9e1653d..632e3504 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/RankingSearchSet.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/RankingSearchSet.java @@ -1,7 +1,6 @@ -package nu.marginalia.index.svc.searchset; +package nu.marginalia.functions.index.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; -import nu.marginalia.index.searchset.SearchSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSet.java similarity index 82% rename from code/features-index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSet.java index 8fba1801..85c3a418 100644 --- a/code/features-index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSet.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.searchset; +package nu.marginalia.functions.index.searchset; public interface SearchSet { diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/SearchSetAny.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetAny.java similarity index 72% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/SearchSetAny.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetAny.java index b0ee4e39..f6a99443 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/SearchSetAny.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetAny.java @@ -1,6 +1,4 @@ -package nu.marginalia.index.svc.searchset; - -import nu.marginalia.index.searchset.SearchSet; +package nu.marginalia.functions.index.searchset; public class SearchSetAny implements SearchSet { @Override diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexSearchSetsService.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java similarity index 87% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexSearchSetsService.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java index df597a4d..df274814 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexSearchSetsService.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.index.searchset; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -6,14 +6,10 @@ import gnu.trove.list.TIntList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import nu.marginalia.db.DomainRankingSetsService; import nu.marginalia.db.DomainTypes; -import nu.marginalia.index.IndexServicesFactory; -import nu.marginalia.index.searchset.SearchSet; -import nu.marginalia.ranking.*; -import nu.marginalia.ranking.accumulator.RankingResultHashMapAccumulator; -import nu.marginalia.ranking.accumulator.RankingResultHashSetAccumulator; -import nu.marginalia.index.svc.searchset.RankingSearchSet; -import nu.marginalia.index.svc.searchset.SearchSetAny; -import nu.marginalia.index.db.DbUpdateRanks; +import nu.marginalia.functions.index.IndexServicesFactory; +import nu.marginalia.ranking.DomainRankings; +import nu.marginalia.ranking.PageRankDomainRanker; +import nu.marginalia.ranking.accumulator.*; import nu.marginalia.ranking.data.GraphSource; import nu.marginalia.ranking.data.LinkGraphSource; import nu.marginalia.ranking.data.SimilarityGraphSource; @@ -29,7 +25,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @Singleton -public class IndexSearchSetsService { +public class SearchSetsService { private final Logger logger = LoggerFactory.getLogger(getClass()); private final DomainTypes domainTypes; private final IndexServicesFactory indexServicesFactory; @@ -50,14 +46,14 @@ public class IndexSearchSetsService { private static final String primaryRankingSet = "RANK"; @Inject - public IndexSearchSetsService(DomainTypes domainTypes, - ServiceConfiguration serviceConfiguration, - LinkGraphSource rankingDomains, - SimilarityGraphSource similarityDomains, - IndexServicesFactory indexServicesFactory, - ServiceEventLog eventLog, - DomainRankingSetsService domainRankingSetsService, - DbUpdateRanks dbUpdateRanks) throws IOException { + public SearchSetsService(DomainTypes domainTypes, + ServiceConfiguration serviceConfiguration, + LinkGraphSource rankingDomains, + SimilarityGraphSource similarityDomains, + IndexServicesFactory indexServicesFactory, + ServiceEventLog eventLog, + DomainRankingSetsService domainRankingSetsService, + DbUpdateRanks dbUpdateRanks) throws IOException { this.nodeId = serviceConfiguration.node(); this.domainTypes = domainTypes; this.indexServicesFactory = indexServicesFactory; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/SmallSearchSet.java b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SmallSearchSet.java similarity index 87% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/SmallSearchSet.java rename to code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SmallSearchSet.java index 37cc07e0..7fca5ac4 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/searchset/SmallSearchSet.java +++ b/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SmallSearchSet.java @@ -1,7 +1,6 @@ -package nu.marginalia.index.svc.searchset; +package nu.marginalia.functions.index.searchset; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.index.searchset.SearchSet; import java.util.Arrays; import java.util.Collection; diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationSmokeTest.java b/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationSmokeTest.java similarity index 96% rename from code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationSmokeTest.java rename to code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationSmokeTest.java index 68e3437f..ea977821 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationSmokeTest.java +++ b/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationSmokeTest.java @@ -1,21 +1,23 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.index; import com.google.inject.Guice; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.process.control.FakeProcessHeartbeat; +import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.storage.FileStorageService; import nu.marginalia.hash.MurmurHash3_128; import nu.marginalia.index.ReverseIndexFullFileNames; import nu.marginalia.index.ReverseIndexPrioFileNames; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.ReverseIndexConstructor; import nu.marginalia.index.forward.ForwardIndexConverter; import nu.marginalia.index.forward.ForwardIndexFileNames; -import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.journal.model.IndexJournalEntryData; import nu.marginalia.index.journal.model.IndexJournalEntryHeader; import nu.marginalia.index.journal.reader.IndexJournalReader; @@ -31,8 +33,6 @@ import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.DocumentMetadata; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.process.control.FakeProcessHeartbeat; -import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.ranking.DomainRankings; import nu.marginalia.service.control.ServiceHeartbeat; import nu.marginalia.service.server.Initialization; @@ -41,7 +41,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; -import spark.Spark; import java.io.IOException; import java.nio.file.Files; @@ -63,9 +62,9 @@ public class IndexQueryServiceIntegrationSmokeTest { IndexQueryServiceIntegrationTestModule testModule; @Inject - IndexQueryService queryService; + IndexQueryGrpcService queryService; @Inject - SearchIndex searchIndex; + StatefulIndex statefulIndex; @Inject ServiceHeartbeat heartbeat; @@ -97,8 +96,6 @@ public class IndexQueryServiceIntegrationSmokeTest { @AfterEach public void tearDown() throws IOException { testModule.cleanUp(); - - Spark.stop(); } @Test @@ -115,7 +112,7 @@ public class IndexQueryServiceIntegrationSmokeTest { indexJournalWriter.close(); constructIndex(); - searchIndex.switchIndex(); + statefulIndex.switchIndex(); var rsp = queryService.justQuery( SearchSpecification.builder() @@ -158,7 +155,7 @@ public class IndexQueryServiceIntegrationSmokeTest { indexJournalWriter.close(); constructIndex(); - searchIndex.switchIndex(); + statefulIndex.switchIndex(); var rsp = queryService.justQuery( SearchSpecification.builder() @@ -195,7 +192,7 @@ public class IndexQueryServiceIntegrationSmokeTest { indexJournalWriter.close(); constructIndex(); - searchIndex.switchIndex(); + statefulIndex.switchIndex(); var rsp = queryService.justQuery( SearchSpecification.builder() diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java b/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTest.java similarity index 98% rename from code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java rename to code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTest.java index 845b643b..f88da805 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java +++ b/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTest.java @@ -1,20 +1,20 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.index; import com.google.inject.Guice; import com.google.inject.Inject; import nu.marginalia.IndexLocations; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.functions.index.index.StatefulIndex; import nu.marginalia.storage.FileStorageService; import nu.marginalia.hash.MurmurHash3_128; import nu.marginalia.index.ReverseIndexFullFileNames; import nu.marginalia.index.ReverseIndexPrioFileNames; -import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.index.client.model.query.SearchSubquery; -import nu.marginalia.index.client.model.results.ResultRankingParameters; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.ReverseIndexConstructor; import nu.marginalia.index.forward.ForwardIndexConverter; import nu.marginalia.index.forward.ForwardIndexFileNames; -import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.journal.model.IndexJournalEntryData; import nu.marginalia.index.journal.model.IndexJournalEntryHeader; import nu.marginalia.index.journal.reader.IndexJournalReader; @@ -41,7 +41,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; -import spark.Spark; import javax.annotation.CheckReturnValue; import java.io.IOException; @@ -65,9 +64,9 @@ public class IndexQueryServiceIntegrationTest { IndexQueryServiceIntegrationTestModule testModule; @Inject - IndexQueryService queryService; + IndexQueryGrpcService queryService; @Inject - SearchIndex searchIndex; + StatefulIndex statefulIndex; @Inject ServiceHeartbeat heartbeat; @@ -98,8 +97,6 @@ public class IndexQueryServiceIntegrationTest { @AfterEach public void tearDown() throws IOException { testModule.cleanUp(); - - Spark.stop(); } @Test @@ -588,7 +585,7 @@ public class IndexQueryServiceIntegrationTest { indexJournalWriter.close(); constructIndex(); documentDbReader.reconnect(); - searchIndex.switchIndex(); + statefulIndex.switchIndex(); } } diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java b/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTestModule.java similarity index 90% rename from code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java rename to code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTestModule.java index 438b3f0f..60498fca 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java +++ b/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTestModule.java @@ -1,7 +1,9 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.index; import com.google.inject.AbstractModule; import nu.marginalia.IndexLocations; +import nu.marginalia.functions.index.searchset.SearchSetAny; +import nu.marginalia.functions.index.searchset.SearchSetsService; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageBase; import nu.marginalia.storage.model.FileStorageBaseType; @@ -11,8 +13,7 @@ import nu.marginalia.linkdb.docs.DocumentDbReader; import nu.marginalia.process.control.FakeProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.ranking.DomainRankings; -import nu.marginalia.index.svc.searchset.SearchSetAny; -import nu.marginalia.index.util.TestUtil; +import nu.marginalia.functions.index.util.TestUtil; import nu.marginalia.service.control.*; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfiguration; @@ -67,10 +68,10 @@ public class IndexQueryServiceIntegrationTestModule extends AbstractModule { bind(ServiceHeartbeat.class).toInstance(new FakeServiceHeartbeat()); bind(ProcessHeartbeat.class).toInstance(new FakeProcessHeartbeat()); - IndexSearchSetsService setsServiceMock = Mockito.mock(IndexSearchSetsService.class); + SearchSetsService setsServiceMock = Mockito.mock(SearchSetsService.class); when(setsServiceMock.getSearchSetByName("NONE")).thenReturn(new SearchSetAny()); when(setsServiceMock.getDomainRankings()).thenReturn(new DomainRankings()); - bind(IndexSearchSetsService.class).toInstance(setsServiceMock); + bind(SearchSetsService.class).toInstance(setsServiceMock); bind(ServiceEventLog.class).toInstance(Mockito.mock(ServiceEventLog.class)); diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java b/code/functions/index/src/test/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicatorTest.java similarity index 88% rename from code/services-core/index-service/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java rename to code/functions/index/src/test/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicatorTest.java index 891a5ff0..8ed5be16 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java +++ b/code/functions/index/src/test/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicatorTest.java @@ -1,6 +1,6 @@ -package nu.marginalia.index.results; +package nu.marginalia.functions.index.results; -import nu.marginalia.index.client.model.results.SearchResultItem; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.model.id.UrlIdCodec; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java b/code/functions/index/src/test/java/nu/marginalia/functions/index/searchset/RankingSearchSetTest.java similarity index 94% rename from code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java rename to code/functions/index/src/test/java/nu/marginalia/functions/index/searchset/RankingSearchSetTest.java index 01837bd2..ad8460d1 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/svc/searchset/RankingSearchSetTest.java +++ b/code/functions/index/src/test/java/nu/marginalia/functions/index/searchset/RankingSearchSetTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.svc.searchset; +package nu.marginalia.functions.index.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import org.junit.jupiter.api.Test; diff --git a/code/services-core/index-service/src/test/java/nu/marginalia/index/util/TestUtil.java b/code/functions/index/src/test/java/nu/marginalia/functions/index/util/TestUtil.java similarity index 96% rename from code/services-core/index-service/src/test/java/nu/marginalia/index/util/TestUtil.java rename to code/functions/index/src/test/java/nu/marginalia/functions/index/util/TestUtil.java index 651dd316..a9ae0780 100644 --- a/code/services-core/index-service/src/test/java/nu/marginalia/index/util/TestUtil.java +++ b/code/functions/index/src/test/java/nu/marginalia/functions/index/util/TestUtil.java @@ -1,4 +1,4 @@ -package nu.marginalia.index.util; +package nu.marginalia.functions.index.util; import java.io.File; diff --git a/code/services-core/assistant-service/src/main/resources/units.csv b/code/functions/math/src/main/resources/units.csv similarity index 100% rename from code/services-core/assistant-service/src/main/resources/units.csv rename to code/functions/math/src/main/resources/units.csv diff --git a/code/services-core/assistant-service/src/test/java/nu/marginalia/assistant/eval/MathParserTest.java b/code/functions/math/src/test/java/nu/marginalia/functions/math/eval/MathParserTest.java similarity index 93% rename from code/services-core/assistant-service/src/test/java/nu/marginalia/assistant/eval/MathParserTest.java rename to code/functions/math/src/test/java/nu/marginalia/functions/math/eval/MathParserTest.java index 4fdfdc71..e9db457e 100644 --- a/code/services-core/assistant-service/src/test/java/nu/marginalia/assistant/eval/MathParserTest.java +++ b/code/functions/math/src/test/java/nu/marginalia/functions/math/eval/MathParserTest.java @@ -1,5 +1,6 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; +import nu.marginalia.functions.math.eval.MathParser; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/assistant-service/src/test/java/nu/marginalia/assistant/eval/UnitsTest.java b/code/functions/math/src/test/java/nu/marginalia/functions/math/eval/UnitsTest.java similarity index 90% rename from code/services-core/assistant-service/src/test/java/nu/marginalia/assistant/eval/UnitsTest.java rename to code/functions/math/src/test/java/nu/marginalia/functions/math/eval/UnitsTest.java index de2b709b..7f608b5b 100644 --- a/code/services-core/assistant-service/src/test/java/nu/marginalia/assistant/eval/UnitsTest.java +++ b/code/functions/math/src/test/java/nu/marginalia/functions/math/eval/UnitsTest.java @@ -1,5 +1,7 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; +import nu.marginalia.functions.math.eval.MathParser; +import nu.marginalia.functions.math.eval.Units; import org.junit.jupiter.api.Test; class UnitsTest { diff --git a/code/functions/search-query/api/build.gradle b/code/functions/search-query/api/build.gradle index 955675df..9aa0d000 100644 --- a/code/functions/search-query/api/build.gradle +++ b/code/functions/search-query/api/build.gradle @@ -26,7 +26,7 @@ apply from: "$rootProject.projectDir/protobuf.gradle" dependencies { implementation project(':code:common:model') implementation project(':code:common:config') - implementation project(':code:features-index:index-query') + implementation project(':code:functions:index:query') implementation project(':code:common:service-discovery') implementation libs.bundles.slf4j diff --git a/code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java b/code/functions/search-query/api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java similarity index 100% rename from code/api/index-api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java rename to code/functions/search-query/api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java diff --git a/code/functions/search-query/build.gradle b/code/functions/search-query/build.gradle index 4af1b6f6..9a4c381d 100644 --- a/code/functions/search-query/build.gradle +++ b/code/functions/search-query/build.gradle @@ -10,7 +10,7 @@ java { } dependencies { - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:db') implementation project(':code:common:config') @@ -19,7 +19,7 @@ dependencies { implementation project(':code:functions:search-query:api') - implementation project(':code:features-index:index-query') + implementation project(':code:functions:index:query') implementation project(':code:libraries:language-processing') implementation project(':code:libraries:term-frequency-dict') diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java index 177fd979..edd65bde 100644 --- a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java @@ -8,6 +8,7 @@ import lombok.SneakyThrows; import nu.marginalia.api.searchquery.*; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.db.DomainBlacklist; +import nu.marginalia.functions.index.api.IndexClient; import nu.marginalia.functions.searchquery.svc.QueryFactory; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.model.id.UrlIdCodec; @@ -33,22 +34,18 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { .help("QS-side query time (GRPC endpoint)") .register(); - private final GrpcMultiNodeChannelPool channelPool; private final QueryFactory queryFactory; private final DomainBlacklist blacklist; - private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); - + private final IndexClient indexClient; @Inject public QueryGRPCService(QueryFactory queryFactory, - GrpcChannelPoolFactory channelPoolFactory, - DomainBlacklist blacklist) + DomainBlacklist blacklist, + IndexClient indexClient) { this.queryFactory = queryFactory; this.blacklist = blacklist; - this.channelPool = channelPoolFactory.createMulti( - ServiceKey.forGrpcApi(IndexApiGrpc.class, ServicePartition.multi()), - IndexApiGrpc::newBlockingStub); + this.indexClient = indexClient; } public void query(RpcQsQuery request, StreamObserver responseObserver) @@ -84,25 +81,6 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { private static final Comparator comparator = Comparator.comparing(RpcDecoratedResultItem::getRankingScore); - @SneakyThrows - List executeQueries(RpcIndexQuery indexRequest, int totalSize) { - var futures = - channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query) - .async(executor) - .runEach(indexRequest); - List results = new ArrayList<>(); - for (var future : futures) { - try { - future.get().forEachRemaining(results::add); - } - catch (Exception e) { - logger.error("Downstream exception", e); - } - } - results.sort(comparator); - results.removeIf(this::isBlacklisted); - return results.subList(0, Math.min(totalSize, results.size())); - } private boolean isBlacklisted(RpcDecoratedResultItem item) { return blacklist.isBlacklisted(UrlIdCodec.getDomainId(item.getRawItem().getCombinedId())); @@ -117,4 +95,14 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { .stream().map(QueryProtobufCodec::convertQueryResult) .toList(); } + + @SneakyThrows + List executeQueries(RpcIndexQuery indexRequest, int totalSize) { + var results = indexClient.executeQueries(indexRequest); + + results.sort(comparator); + results.removeIf(this::isBlacklisted); + return results.subList(0, Math.min(totalSize, results.size())); + } + } diff --git a/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java b/code/functions/search-query/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java similarity index 98% rename from code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java rename to code/functions/search-query/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java index d3f17b52..bafb929d 100644 --- a/code/services-core/query-service/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java +++ b/code/functions/search-query/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java @@ -3,6 +3,7 @@ package nu.marginalia.query.svc; import nu.marginalia.WmsaHome; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.functions.searchquery.svc.QueryFactory; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; diff --git a/code/process-models/crawling-model/build.gradle b/code/process-models/crawling-model/build.gradle index 62ac447f..7a894e04 100644 --- a/code/process-models/crawling-model/build.gradle +++ b/code/process-models/crawling-model/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation project(':code:common:config') implementation project(':code:common:process') implementation project(':code:libraries:big-string') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:service-discovery') implementation project(':code:features-crawl:content-type') implementation project(':code:libraries:language-processing') diff --git a/code/processes/converting-process/build.gradle b/code/processes/converting-process/build.gradle index 4def16db..be64de5b 100644 --- a/code/processes/converting-process/build.gradle +++ b/code/processes/converting-process/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation project(':third-party:porterstemmer') implementation project(':third-party:count-min-sketch') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:common:model') diff --git a/code/processes/crawling-process/build.gradle b/code/processes/crawling-process/build.gradle index f9cf5029..f98229cd 100644 --- a/code/processes/crawling-process/build.gradle +++ b/code/processes/crawling-process/build.gradle @@ -27,7 +27,7 @@ dependencies { implementation project(':code:common:service') implementation project(':code:libraries:big-string') implementation project(':code:libraries:blocking-thread-pool') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') diff --git a/code/processes/index-constructor-process/build.gradle b/code/processes/index-constructor-process/build.gradle index e92db1b6..bdf7869e 100644 --- a/code/processes/index-constructor-process/build.gradle +++ b/code/processes/index-constructor-process/build.gradle @@ -27,10 +27,10 @@ dependencies { implementation project(':code:common:model') implementation project(':code:libraries:message-queue') - implementation project(':code:features-index:index-forward') - implementation project(':code:features-index:index-reverse') - implementation project(':code:features-index:index-journal') - implementation project(':code:features-index:domain-ranking') + implementation project(':code:functions:index:forward-index') + implementation project(':code:functions:index:reverse-index') + implementation project(':code:functions:index:index-journal') + implementation project(':code:functions:index:domain-ranking') implementation libs.bundles.slf4j implementation libs.guice diff --git a/code/processes/loading-process/build.gradle b/code/processes/loading-process/build.gradle index 030e6e28..aa6da441 100644 --- a/code/processes/loading-process/build.gradle +++ b/code/processes/loading-process/build.gradle @@ -20,14 +20,14 @@ tasks.distZip.enabled = false dependencies { implementation project(':code:common:process') implementation project(':code:api:process-mqapi') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:config') implementation project(':code:common:service') implementation project(':code:common:service-discovery') implementation project(':code:common:linkdb') - implementation project(':code:features-index:index-journal') + implementation project(':code:functions:index:index-journal') implementation project(':code:libraries:message-queue') implementation project(':code:libraries:language-processing') implementation project(':third-party:commons-codec') diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index 4e61dff0..5c2e3abc 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -42,7 +42,7 @@ dependencies { implementation project(':code:common:config') implementation project(':code:common:service-discovery') implementation project(':code:functions:search-query:api') - implementation project(':code:features-index:index-query') + implementation project(':code:functions:index:query') implementation libs.bundles.slf4j diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index 702780dd..4ca22863 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -47,7 +47,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:service') implementation project(':code:common:config') - implementation project(':code:features-index:index-query') + implementation project(':code:functions:index:query') implementation project(':code:libraries:easy-lsh') implementation project(':code:libraries:language-processing') @@ -59,7 +59,7 @@ dependencies { implementation project(':code:functions:search-query:api') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:service-discovery') implementation project(':code:common:renderer') diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java index 7af3cabe..ecdcdf22 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java @@ -1,7 +1,7 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.index.client.IndexClient; +import nu.marginalia.functions.index.api.IndexMqClient; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import org.slf4j.Logger; @@ -13,17 +13,17 @@ import java.io.IOException; import java.util.Map; public class SearchErrorPageService { - private final IndexClient indexClient; + private final IndexMqClient indexMqClient; private final Logger logger = LoggerFactory.getLogger(getClass()); private final MustacheRenderer renderer; @Inject - public SearchErrorPageService(IndexClient indexClient, + public SearchErrorPageService(IndexMqClient indexMqClient, RendererFactory rendererFactory) throws IOException { renderer = rendererFactory.renderer("search/error-page-search"); - this.indexClient = indexClient; + this.indexMqClient = indexMqClient; } public void serveError(Request request, Response rsp) { diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 498d2b76..ee683a7c 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -47,11 +47,11 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:functions:search-query:api') implementation project(':code:api:executor-api') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:features-search:screenshots') - implementation project(':code:features-index:index-journal') - implementation project(':code:features-index:index-query') + implementation project(':code:functions:index:index-journal') + implementation project(':code:functions:index:query') implementation project(':code:process-models:crawl-spec') diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java index b07df2c0..ecc7d422 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java @@ -8,7 +8,7 @@ import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.actor.state.ActorStep; import nu.marginalia.actor.state.Resume; import nu.marginalia.control.actor.PrecessionNodes; -import nu.marginalia.index.client.IndexClient; +import nu.marginalia.functions.index.api.IndexMqClient; import nu.marginalia.mq.persistence.MqPersistence; import java.sql.SQLException; @@ -18,7 +18,7 @@ import java.util.concurrent.TimeUnit; public class ReindexAllActor extends RecordActorPrototype { private final MqPersistence persistence; - private final IndexClient indexClient; + private final IndexMqClient indexMqClient; private final PrecessionNodes precessionNodes; @@ -42,7 +42,7 @@ public class ReindexAllActor extends RecordActorPrototype { if (first.isEmpty()) yield new End(); else yield new ReindexNode(first.getAsInt()); } - case ReindexNode(int node, long msgId) when msgId < 0 -> new ReindexNode(node, indexClient.triggerRepartition(node)); + case ReindexNode(int node, long msgId) when msgId < 0 -> new ReindexNode(node, indexMqClient.triggerRepartition(node)); case ReindexNode(int node, long msgId) -> { while (!isMessageTerminal(msgId)) { TimeUnit.SECONDS.sleep(10); @@ -66,12 +66,12 @@ public class ReindexAllActor extends RecordActorPrototype { @Inject public ReindexAllActor(Gson gson, MqPersistence persistence, - IndexClient indexClient, + IndexMqClient indexMqClient, PrecessionNodes precessionNodes) { super(gson); this.persistence = persistence; - this.indexClient = indexClient; + this.indexMqClient = indexMqClient; this.precessionNodes = precessionNodes; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java index 05de9b07..2344f36f 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java @@ -5,7 +5,7 @@ import com.google.inject.Singleton; import nu.marginalia.control.ControlValidationError; import nu.marginalia.control.RedirectControl; import nu.marginalia.executor.client.ExecutorClient; -import nu.marginalia.index.client.IndexClient; +import nu.marginalia.functions.index.api.IndexMqClient; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; @@ -26,7 +26,7 @@ import java.util.Set; @Singleton public class ControlNodeActionsService { private static final Logger logger = LoggerFactory.getLogger(ControlNodeActionsService.class); - private final IndexClient indexClient; + private final IndexMqClient indexMqClient; private final RedirectControl redirectControl; private final FileStorageService fileStorageService; private final ServiceEventLog eventLog; @@ -34,14 +34,14 @@ public class ControlNodeActionsService { @Inject public ControlNodeActionsService(ExecutorClient executorClient, - IndexClient indexClient, + IndexMqClient indexMqClient, RedirectControl redirectControl, FileStorageService fileStorageService, ServiceEventLog eventLog) { this.executorClient = executorClient; - this.indexClient = indexClient; + this.indexMqClient = indexMqClient; this.redirectControl = redirectControl; this.fileStorageService = fileStorageService; this.eventLog = eventLog; @@ -184,7 +184,7 @@ public class ControlNodeActionsService { } public Object triggerRepartition(Request request, Response response) throws Exception { - indexClient.triggerRepartition(Integer.parseInt(request.params("node"))); + indexMqClient.triggerRepartition(Integer.parseInt(request.params("node"))); return ""; } diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 1ad1b541..2a3bb0a7 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -69,8 +69,8 @@ dependencies { implementation project(':code:features-convert:data-extractors') implementation project(':code:features-convert:stackexchange-xml') implementation project(':code:features-convert:reddit-json') - implementation project(':code:features-index:index-journal') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:index-journal') + implementation project(':code:functions:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:api:executor-api') implementation project(':third-party:encyclopedia-marginalia-nu') diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java index 698c8c27..a70d977b 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java @@ -19,8 +19,8 @@ import nu.marginalia.svc.BackupService; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; import nu.marginalia.storage.model.FileStorageType; -import nu.marginalia.index.client.IndexClient; -import nu.marginalia.index.client.IndexMqEndpoints; +import nu.marginalia.functions.index.api.IndexMqClient; +import nu.marginalia.functions.index.api.IndexMqEndpoints; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mqapi.converting.ConvertRequest; @@ -196,7 +196,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype { public ConvertAndLoadActor(ActorProcessWatcher processWatcher, ProcessOutboxes processOutboxes, FileStorageService storageService, - IndexClient indexClient, + IndexMqClient indexMqClient, BackupService backupService, Gson gson, NodeConfigurationService nodeConfigurationService, @@ -204,7 +204,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype { { super(gson); this.processWatcher = processWatcher; - this.indexOutbox = indexClient.outbox(); + this.indexOutbox = indexMqClient.outbox(); this.mqConverterOutbox = processOutboxes.getConverterOutbox(); this.mqLoaderOutbox = processOutboxes.getLoaderOutbox(); this.mqIndexConstructorOutbox = processOutboxes.getIndexConstructorOutbox(); diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 4548710c..89e778c6 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -37,29 +37,20 @@ java { } dependencies { implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:service-discovery') implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:linkdb') + implementation project(':code:functions:index') implementation project(':code:functions:domain-links:partition') implementation project(':code:functions:domain-links:api') implementation project(':code:functions:search-query:api') - - implementation project(':code:common:service') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:service-discovery') - implementation project(':code:libraries:array') - implementation project(':code:libraries:btree') - - implementation project(':code:features-index:index-journal') - implementation project(':code:features-index:index-query') - implementation project(':code:features-index:index-forward') - implementation project(':code:features-index:index-reverse') - implementation project(':code:features-index:domain-ranking') - implementation project(':code:features-index:result-ranking') - implementation project(':third-party:commons-codec') testImplementation project(path: ':code:services-core:control-service') testImplementation project(':code:common:process') diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java similarity index 62% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java rename to code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java index 632621b0..fb2a4a85 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java @@ -1,11 +1,9 @@ -package nu.marginalia.index.svc; +package nu.marginalia.index; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.index.index.SearchIndex; -import spark.Request; -import spark.Response; -import spark.Spark; +import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.functions.index.searchset.SearchSetsService; import javax.annotation.CheckReturnValue; import java.util.Optional; @@ -16,12 +14,12 @@ import java.util.concurrent.locks.ReentrantLock; public class IndexOpsService { private final ReentrantLock opsLock = new ReentrantLock(); - private final SearchIndex index; - private final IndexSearchSetsService searchSetService; + private final StatefulIndex index; + private final SearchSetsService searchSetService; @Inject - public IndexOpsService(SearchIndex index, - IndexSearchSetsService searchSetService) { + public IndexOpsService(StatefulIndex index, + SearchSetsService searchSetService) { this.index = index; this.searchSetService = searchSetService; } @@ -43,26 +41,6 @@ public class IndexOpsService { } - public Object repartitionEndpoint(Request request, Response response) throws Exception { - - if (!run(searchSetService::recalculateSecondary)) { - Spark.halt(503, "Operations busy"); - } - - return "OK"; - } - - public Object reindexEndpoint(Request request, Response response) throws Exception { - - if (!run(index::switchIndex).isPresent()) { - Spark.halt(503, "Operations busy"); - } - - return "OK"; - } - - - @CheckReturnValue public Optional run(Callable c) throws Exception { if (!opsLock.tryLock()) diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index 2dd90f78..48749650 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -1,26 +1,22 @@ package nu.marginalia.index; -import com.google.gson.Gson; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; import nu.marginalia.functions.domainlinks.PartitionDomainLinksService; +import nu.marginalia.functions.index.IndexQueryGrpcService; +import nu.marginalia.functions.index.index.StatefulIndex; import nu.marginalia.linkdb.dlinks.DomainLinkDb; import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.storage.FileStorageService; -import nu.marginalia.index.client.IndexMqEndpoints; -import nu.marginalia.index.index.SearchIndex; -import nu.marginalia.index.svc.IndexOpsService; -import nu.marginalia.index.svc.IndexQueryService; +import nu.marginalia.functions.index.api.IndexMqEndpoints; import nu.marginalia.linkdb.docs.DocumentDbReader; -import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.*; import nu.marginalia.service.server.mq.MqRequest; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import spark.Spark; import java.nio.file.Files; import java.nio.file.Path; @@ -35,7 +31,7 @@ public class IndexService extends Service { @NotNull private final Initialization init; private final IndexOpsService opsService; - private final SearchIndex searchIndex; + private final StatefulIndex statefulIndex; private final FileStorageService fileStorageService; private final DocumentDbReader documentDbReader; @@ -47,38 +43,29 @@ public class IndexService extends Service { @Inject public IndexService(BaseServiceParams params, IndexOpsService opsService, - IndexQueryService indexQueryService, - SearchIndex searchIndex, + IndexQueryGrpcService indexQueryService, + StatefulIndex statefulIndex, FileStorageService fileStorageService, DocumentDbReader documentDbReader, DomainLinkDb domainLinkDb, - PartitionDomainLinksService partitionDomainLinksService, - ServiceEventLog eventLog) { super(params, ServicePartition.partition(params.configuration.node()), - List.of(indexQueryService, partitionDomainLinksService)); + List.of(indexQueryService, + partitionDomainLinksService) + ); this.opsService = opsService; - this.searchIndex = searchIndex; + this.statefulIndex = statefulIndex; this.fileStorageService = fileStorageService; this.documentDbReader = documentDbReader; this.domainLinkDb = domainLinkDb; this.eventLog = eventLog; - final Gson gson = GsonFactory.get(); - this.init = params.initialization; - Spark.get("/public/debug/docmeta", indexQueryService::debugEndpointDocMetadata, gson::toJson); - Spark.get("/public/debug/wordmeta", indexQueryService::debugEndpointWordMetadata, gson::toJson); - Spark.get("/public/debug/word", indexQueryService::debugEndpointWordEncoding, gson::toJson); - - Spark.post("/ops/repartition", opsService::repartitionEndpoint); - Spark.post("/ops/reindex", opsService::reindexEndpoint); - Thread.ofPlatform().name("initialize-index").start(this::initialize); } @@ -141,7 +128,7 @@ public class IndexService extends Service { public void initialize() { if (!initialized) { init.waitReady(); - searchIndex.init(); + statefulIndex.init(); initialized = true; } } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java deleted file mode 100644 index 1d1396f9..00000000 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java +++ /dev/null @@ -1,69 +0,0 @@ -package nu.marginalia.index.index; - -import gnu.trove.set.hash.TLongHashSet; -import nu.marginalia.index.ReverseIndexReader; -import nu.marginalia.index.query.IndexQuery; -import nu.marginalia.index.query.IndexQueryBuilder; -import nu.marginalia.index.query.filter.QueryFilterStepIf; - -public class SearchIndexQueryBuilder implements IndexQueryBuilder { - private final IndexQuery query; - private final ReverseIndexReader reverseIndexFullReader; - private final ReverseIndexReader reverseIndexPrioReader; - - /* Keep track of already added include terms to avoid redundant checks. - * - * Warning: This may cause unexpected behavior if for example attempting to - * first check one index and then another for the same term. At the moment, that - * makes no sense, but in the future, that might be a thing one might want to do. - * */ - private final TLongHashSet alreadyConsideredTerms = new TLongHashSet(); - - SearchIndexQueryBuilder(ReverseIndexReader reverseIndexFullReader, - ReverseIndexReader reverseIndexPrioReader, - IndexQuery query, long... sourceTerms) - { - this.query = query; - this.reverseIndexFullReader = reverseIndexFullReader; - this.reverseIndexPrioReader = reverseIndexPrioReader; - - alreadyConsideredTerms.addAll(sourceTerms); - } - - public IndexQueryBuilder alsoFull(long termId) { - - if (alreadyConsideredTerms.add(termId)) { - query.addInclusionFilter(reverseIndexFullReader.also(termId)); - } - - return this; - } - - public IndexQueryBuilder alsoPrio(long termId) { - - if (alreadyConsideredTerms.add(termId)) { - query.addInclusionFilter(reverseIndexPrioReader.also(termId)); - } - - return this; - } - - public IndexQueryBuilder notFull(long termId) { - - query.addInclusionFilter(reverseIndexFullReader.not(termId)); - - return this; - } - - public IndexQueryBuilder addInclusionFilter(QueryFilterStepIf filterStep) { - - query.addInclusionFilter(filterStep); - - return this; - } - - public IndexQuery build() { - return query; - } - -} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java deleted file mode 100644 index 24d9da48..00000000 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/results/IndexResultDecorator.java +++ /dev/null @@ -1,90 +0,0 @@ -package nu.marginalia.index.results; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import gnu.trove.list.TLongList; -import gnu.trove.list.array.TLongArrayList; -import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; -import nu.marginalia.api.searchquery.model.results.ResultRankingContext; -import nu.marginalia.api.searchquery.model.results.SearchResultItem; -import nu.marginalia.linkdb.docs.DocumentDbReader; -import nu.marginalia.linkdb.model.DocdbUrlDetail; -import nu.marginalia.ranking.ResultValuator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Joins the index view of a set of search results with data from the linkdb */ -@Singleton -public class IndexResultDecorator { - - private static final Logger logger = LoggerFactory.getLogger(IndexResultDecorator.class); - - private final DocumentDbReader documentDbReader; - private final ResultValuator valuator; - - @Inject - public IndexResultDecorator(DocumentDbReader documentDbReader, - ResultValuator valuator) { - this.documentDbReader = documentDbReader; - this.valuator = valuator; - } - - /** Decorate the result items with additional information from the link database - * and calculate an updated ranking with the additional information */ - public List decorateAndRerank(List rawResults, - ResultRankingContext rankingContext) - throws SQLException - { - TLongList idsList = new TLongArrayList(rawResults.size()); - - for (var result : rawResults) - idsList.add(result.getDocumentId()); - - Map urlDetailsById = new HashMap<>(rawResults.size()); - - for (var item : documentDbReader.getUrlDetails(idsList)) - urlDetailsById.put(item.urlId(), item); - - List decoratedItems = new ArrayList<>(); - for (var result : rawResults) { - var docData = urlDetailsById.get(result.getDocumentId()); - - if (null == docData) { - logger.warn("No data for document id {}", result.getDocumentId()); - continue; - } - - decoratedItems.add(createCombinedItem(result, docData, rankingContext)); - } - - if (decoratedItems.size() != rawResults.size()) - logger.warn("Result list shrunk during decoration?"); - - return decoratedItems; - } - - private DecoratedSearchResultItem createCombinedItem(SearchResultItem result, - DocdbUrlDetail docData, - ResultRankingContext rankingContext) { - return new DecoratedSearchResultItem( - result, - docData.url(), - docData.title(), - docData.description(), - docData.urlQuality(), - docData.format(), - docData.features(), - docData.pubYear(), - docData.dataHash(), - docData.wordsTotal(), - valuator.calculateSearchResultValue(result.keywordScores, docData.wordsTotal(), rankingContext) - ); - - } -} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryExecutor.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryExecutor.java deleted file mode 100644 index 11268ae4..00000000 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryExecutor.java +++ /dev/null @@ -1,40 +0,0 @@ -package nu.marginalia.index.svc; - -import com.google.inject.Singleton; -import gnu.trove.list.array.TLongArrayList; -import nu.marginalia.array.buffer.LongQueryBuffer; -import nu.marginalia.index.query.IndexQuery; - -@Singleton -public class IndexQueryExecutor { - - /* Re-use these buffers as they contribute to a large amount of memory churn */ - private static final ThreadLocal bufferTL = ThreadLocal.withInitial(() -> new LongQueryBuffer(4096)); - - public int executeQuery(IndexQuery query, TLongArrayList results, SearchParameters params) - { - final int fetchSize = params.fetchSize * query.fetchSizeMultiplier; - - final LongQueryBuffer buffer = bufferTL.get(); - - int cnt = 0; - - while (query.hasMore() - && results.size() < fetchSize - && params.budget.hasTimeLeft()) - { - buffer.reset(); - query.getMoreResults(buffer); - - for (int i = 0; i < buffer.size() && results.size() < fetchSize; i++) { - results.add(buffer.data[i]); - cnt++; - } - } - - params.dataCost += query.dataCost(); - - return cnt; - } - -} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java deleted file mode 100644 index 98f1d988..00000000 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java +++ /dev/null @@ -1,407 +0,0 @@ -package nu.marginalia.index.svc; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import gnu.trove.list.TLongList; -import gnu.trove.list.array.TLongArrayList; -import io.grpc.stub.StreamObserver; -import io.prometheus.client.Counter; -import io.prometheus.client.Gauge; -import io.prometheus.client.Histogram; -import lombok.SneakyThrows; -import nu.marginalia.api.searchquery.*; -import nu.marginalia.api.searchquery.model.query.SearchSpecification; -import nu.marginalia.api.searchquery.model.query.SearchSubquery; -import nu.marginalia.api.searchquery.model.results.ResultRankingContext; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.api.searchquery.model.results.SearchResultItem; -import nu.marginalia.api.searchquery.model.results.SearchResultSet; -import nu.marginalia.index.index.SearchIndex; -import nu.marginalia.index.index.SearchIndexSearchTerms; -import nu.marginalia.index.query.IndexQueryPriority; -import nu.marginalia.index.results.IndexMetadataService; -import nu.marginalia.index.results.IndexResultDecorator; -import nu.marginalia.index.searchset.SearchSet; -import nu.marginalia.index.results.IndexResultValuator; -import nu.marginalia.index.query.IndexQuery; -import nu.marginalia.index.results.IndexResultDomainDeduplicator; -import nu.marginalia.index.svc.searchset.SmallSearchSet; -import nu.marginalia.model.idx.DocumentMetadata; -import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.service.module.ServiceConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.Marker; -import org.slf4j.MarkerFactory; -import spark.Request; -import spark.Response; - -import java.sql.SQLException; -import java.util.*; -import java.util.stream.Collectors; - -@Singleton -public class IndexQueryService extends IndexApiGrpc.IndexApiImplBase { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - // This marker is used to mark sensitive log messages that are related to queries - // so that they can be filtered out in the production logging configuration - private final Marker queryMarker = MarkerFactory.getMarker("QUERY"); - - private static final Counter wmsa_query_timeouts = Counter.build() - .name("wmsa_index_query_timeouts") - .help("Query timeout counter") - .labelNames("node", "api") - .register(); - private static final Gauge wmsa_query_cost = Gauge.build() - .name("wmsa_index_query_cost") - .help("Computational cost of query") - .labelNames("node", "api") - .register(); - private static final Histogram wmsa_query_time = Histogram.build() - .name("wmsa_index_query_time") - .linearBuckets(0.05, 0.05, 15) - .labelNames("node", "api") - .help("Index-side query time") - .register(); - - private final IndexQueryExecutor queryExecutor; - - private final SearchIndex index; - private final IndexResultDecorator resultDecorator; - private final IndexSearchSetsService searchSetsService; - - private final IndexMetadataService metadataService; - private final SearchTermsService searchTermsSvc; - private final int nodeId; - - - @Inject - public IndexQueryService(IndexQueryExecutor queryExecutor, - ServiceConfiguration serviceConfiguration, - SearchIndex index, - IndexResultDecorator resultDecorator, - IndexSearchSetsService searchSetsService, - IndexMetadataService metadataService, - SearchTermsService searchTerms) - { - this.nodeId = serviceConfiguration.node(); - this.queryExecutor = queryExecutor; - this.index = index; - this.resultDecorator = resultDecorator; - this.searchSetsService = searchSetsService; - this.metadataService = metadataService; - this.searchTermsSvc = searchTerms; - } - - public DocumentMetadata debugEndpointDocMetadata(Request request, Response response) { - String docId = request.queryParams("docId"); - response.type("application/json"); - - return new DocumentMetadata(index.getDocumentMetadata(Long.parseLong(docId))); - } - - public WordMetadata debugEndpointWordMetadata(Request request, Response response) { - String word = request.queryParams("word"); - String docId = request.queryParams("docId"); - response.type("application/json"); - - return new WordMetadata(index.getTermMetadata( - searchTermsSvc.getWordId(word), - new long[] { Long.parseLong(docId) } - )[0]); - } - - public String debugEndpointWordEncoding(Request request, Response response) { - String word = request.queryParams("word"); - response.type("application/json"); - - return Long.toHexString(searchTermsSvc.getWordId(word)); - } - - // GRPC endpoint - @SneakyThrows - public void query(RpcIndexQuery request, - StreamObserver responseObserver) { - - try { - var params = new SearchParameters(request, getSearchSet(request)); - - final String nodeName = Integer.toString(nodeId); - - SearchResultSet results = wmsa_query_time - .labels(nodeName, "GRPC") - .time(() -> executeSearch(params)); - - wmsa_query_cost - .labels(nodeName, "GRPC") - .set(params.getDataCost()); - - if (!params.hasTimeLeft()) { - wmsa_query_timeouts - .labels(nodeName, "GRPC") - .inc(); - } - - for (var result : results.results) { - - var rawResult = result.rawIndexResult; - - var rawItem = RpcRawResultItem.newBuilder(); - rawItem.setCombinedId(rawResult.combinedId); - rawItem.setResultsFromDomain(rawResult.resultsFromDomain); - - for (var score : rawResult.keywordScores) { - rawItem.addKeywordScores( - RpcResultKeywordScore.newBuilder() - .setEncodedDocMetadata(score.encodedDocMetadata()) - .setEncodedWordMetadata(score.encodedWordMetadata()) - .setKeyword(score.keyword) - .setHtmlFeatures(score.htmlFeatures()) - .setHasPriorityTerms(score.hasPriorityTerms()) - .setSubquery(score.subquery) - ); - } - - var decoratedBuilder = RpcDecoratedResultItem.newBuilder() - .setDataHash(result.dataHash) - .setDescription(result.description) - .setFeatures(result.features) - .setFormat(result.format) - .setRankingScore(result.rankingScore) - .setTitle(result.title) - .setUrl(result.url.toString()) - .setWordsTotal(result.wordsTotal) - .setRawItem(rawItem); - - if (result.pubYear != null) { - decoratedBuilder.setPubYear(result.pubYear); - } - responseObserver.onNext(decoratedBuilder.build()); - } - - responseObserver.onCompleted(); - } - catch (Exception ex) { - logger.error("Error in handling request", ex); - responseObserver.onError(ex); - } - } - - // exists for test access - @SneakyThrows - SearchResultSet justQuery(SearchSpecification specsSet) { - return executeSearch(new SearchParameters(specsSet, getSearchSet(specsSet))); - } - - private SearchSet getSearchSet(SearchSpecification specsSet) { - - if (specsSet.domains != null && !specsSet.domains.isEmpty()) { - return new SmallSearchSet(specsSet.domains); - } - - return searchSetsService.getSearchSetByName(specsSet.searchSetIdentifier); - } - private SearchSet getSearchSet(RpcIndexQuery request) { - - if (request.getDomainsCount() > 0) { - return new SmallSearchSet(request.getDomainsList()); - } - - return searchSetsService.getSearchSetByName(request.getSearchSetIdentifier()); - } - private SearchResultSet executeSearch(SearchParameters params) throws SQLException { - - if (!index.isLoaded()) { - // Short-circuit if the index is not loaded, as we trivially know that there can be no results - return new SearchResultSet(List.of()); - } - - var rankingContext = createRankingContext(params.rankingParams, params.subqueries); - - logger.info(queryMarker, "{}", params.queryParams); - - var resultIds = evaluateSubqueries(params); - var resultItems = calculateResultScores(params, rankingContext, resultIds); - - logger.info(queryMarker, "After filtering: {} -> {}", resultIds.size(), resultItems.size()); - - var bestResults = selectBestResults(params, resultItems); - - return new SearchResultSet(resultDecorator.decorateAndRerank(bestResults, rankingContext)); - } - - private ResultRankingContext createRankingContext(ResultRankingParameters rankingParams, List subqueries) { - final var termToId = searchTermsSvc.getAllIncludeTerms(subqueries); - final Map termFrequencies = new HashMap<>(termToId.size()); - final Map prioFrequencies = new HashMap<>(termToId.size()); - - termToId.forEach((key, id) -> termFrequencies.put(key, index.getTermFrequency(id))); - termToId.forEach((key, id) -> prioFrequencies.put(key, index.getTermFrequencyPrio(id))); - - return new ResultRankingContext(index.getTotalDocCount(), - rankingParams, - termFrequencies, - prioFrequencies); - } - - /** Execute subqueries and return a list of document ids. The index is queried for each subquery, - * at different priorty depths until timeout is reached or the results are all visited. - *
- * Then the results are combined. - * */ - private final ThreadLocal resultsArrayListPool = ThreadLocal.withInitial(TLongArrayList::new); - - private TLongList evaluateSubqueries(SearchParameters params) { - final TLongArrayList results = resultsArrayListPool.get(); - results.resetQuick(); - results.ensureCapacity(params.fetchSize); - - // These queries are various term combinations - for (var subquery : params.subqueries) { - - if (!params.hasTimeLeft()) { - logger.info("Query timed out {}, ({}), -{}", - subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); - break; - } - - logger.info(queryMarker, "{}", subquery); - - final SearchIndexSearchTerms searchTerms = searchTermsSvc.getSearchTerms(subquery); - - if (searchTerms.isEmpty()) { - logger.info(queryMarker, "empty"); - continue; - } - - logSearchTerms(subquery, searchTerms); - - // These queries are different indices for one subquery - List queries = params.createIndexQueries(index, searchTerms); - for (var query : queries) { - - if (!params.hasTimeLeft()) - break; - - if (shouldOmitQuery(params, query, results.size())) { - logger.info(queryMarker, "Omitting {}", query); - continue; - } - - int cnt = queryExecutor.executeQuery(query, results, params); - - logger.info(queryMarker, "{} from {}", cnt, query); - } - } - - return results; - } - - /** @see IndexQueryPriority */ - private boolean shouldOmitQuery(SearchParameters params, IndexQuery query, int resultCount) { - - var priority = query.queryPriority; - - return switch (priority) { - case BEST -> false; - case GOOD -> resultCount > params.fetchSize / 4; - case FALLBACK -> resultCount > params.fetchSize / 8; - }; - } - - private void logSearchTerms(SearchSubquery subquery, SearchIndexSearchTerms searchTerms) { - - // This logging should only be enabled in testing, as it is very verbose - // and contains sensitive information - - if (!logger.isInfoEnabled(queryMarker)) { - return; - } - - var includes = subquery.searchTermsInclude; - var advice = subquery.searchTermsAdvice; - var excludes = subquery.searchTermsExclude; - var priority = subquery.searchTermsPriority; - - for (int i = 0; i < includes.size(); i++) { - logger.info(queryMarker, "{} -> {} I", includes.get(i), - Long.toHexString(searchTerms.includes().getLong(i)) - ); - } - for (int i = 0; i < advice.size(); i++) { - logger.info(queryMarker, "{} -> {} A", advice.get(i), - Long.toHexString(searchTerms.includes().getLong(includes.size() + i)) - ); - } - for (int i = 0; i < excludes.size(); i++) { - logger.info(queryMarker, "{} -> {} E", excludes.get(i), - Long.toHexString(searchTerms.excludes().getLong(i)) - ); - } - for (int i = 0; i < priority.size(); i++) { - logger.info(queryMarker, "{} -> {} P", priority.get(i), - Long.toHexString(searchTerms.priority().getLong(i)) - ); - } - } - - private List calculateResultScores(SearchParameters params, ResultRankingContext rankingContext, TLongList resultIds) { - - final var evaluator = new IndexResultValuator(metadataService, - resultIds, - rankingContext, - params.subqueries, - params.queryParams); - - // Sort the ids for more favorable access patterns on disk - resultIds.sort(); - - // Parallel stream to calculate scores is a minor performance boost - return Arrays.stream(resultIds.toArray()) - .parallel() - .mapToObj(evaluator::calculatePreliminaryScore) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private List selectBestResults(SearchParameters params, List results) { - - var domainCountFilter = new IndexResultDomainDeduplicator(params.limitByDomain); - - results.sort(Comparator.naturalOrder()); - - List resultsList = new ArrayList<>(results.size()); - - for (var item : results) { - if (domainCountFilter.test(item)) { - resultsList.add(item); - } - } - - if (!params.queryParams.domainCount().isNone()) { - // Remove items that don't meet the domain count requirement - // This isn't perfect because the domain count is calculated - // after the results are sorted - resultsList.removeIf(item -> !params.queryParams.domainCount().test(domainCountFilter.getCount(item))); - } - - if (resultsList.size() > params.limitTotal) { - // This can't be made a stream limit() operation because we need domainCountFilter - // to run over the entire list to provide accurate statistics - - resultsList.subList(params.limitTotal, resultsList.size()).clear(); - } - - // populate results with the total number of results encountered from - // the same domain so this information can be presented to the user - for (var result : resultsList) { - result.resultsFromDomain = domainCountFilter.getCount(result); - } - - return resultsList; - } - -} - diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 58fa802a..fcd5e625 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -42,9 +42,9 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:service') implementation project(':code:common:renderer') - implementation project(':code:api:index-api') + implementation project(':code:functions:index:api') implementation project(':code:common:service-discovery') - implementation project(':code:features-index:index-query') + implementation project(':code:functions:index:query') implementation project(':code:functions:search-query') implementation project(':code:functions:search-query:api') diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index 7a6cc25e..9ccb3f95 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -15,13 +15,6 @@ import java.util.List; public class QueryService extends Service { - private static final Histogram wmsa_qs_query_time_rest = Histogram.build() - .name("wmsa_qs_query_time_rest") - .linearBuckets(0.05, 0.05, 15) - .help("QS-side query time (REST endpoint)") - .register(); - - @SneakyThrows @Inject public QueryService(BaseServiceParams params, diff --git a/settings.gradle b/settings.gradle index 8306baf3..e88308e9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,15 @@ include 'code:functions:domain-links:api' include 'code:functions:search-query' include 'code:functions:search-query:api' +include 'code:functions:index' +include 'code:functions:index:api' +include 'code:functions:index:index-journal' +include 'code:functions:index:query' +include 'code:functions:index:forward-index' +include 'code:functions:index:reverse-index' +include 'code:functions:index:domain-ranking' +include 'code:functions:index:result-ranking' + include 'code:libraries:array' include 'code:libraries:geo-ip' include 'code:libraries:btree' @@ -43,7 +52,6 @@ include 'code:libraries:message-queue' include 'code:features-search:screenshots' include 'code:features-search:random-websites' include 'code:features-search:feedlot-client' -include 'code:features-index:result-ranking' include 'code:features-convert:adblock' include 'code:features-convert:anchor-keywords' @@ -59,11 +67,6 @@ include 'code:features-crawl:crawl-blocklist' include 'code:features-crawl:link-parser' include 'code:features-crawl:content-type' -include 'code:features-index:index-journal' -include 'code:features-index:index-query' -include 'code:features-index:index-forward' -include 'code:features-index:index-reverse' -include 'code:features-index:domain-ranking' include 'code:api:index-api' include 'code:api:process-mqapi' From f8e7f7583127e2fa5ee4445700a07aded18a238f Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 18:01:35 +0100 Subject: [PATCH 22/56] Move index to top level of code --- code/api/executor-api/build.gradle | 2 +- .../index/domain-ranking/build.gradle | 44 ------------ code/functions/index/domain-ranking/readme.md | 39 ----------- code/functions/index/readme.md | 24 ------- .../index/result-ranking/build.gradle | 31 -------- code/functions/index/result-ranking/readme.md | 14 ---- code/functions/search-query/api/build.gradle | 2 +- .../model/results/Bm25Parameters.java | 2 +- code/functions/search-query/build.gradle | 4 +- .../searchquery/QueryGRPCService.java | 7 +- code/{functions => }/index/api/build.gradle | 0 .../nu/marginalia}/index/api/IndexClient.java | 2 +- .../marginalia}/index/api/IndexMqClient.java | 2 +- .../index/api/IndexMqEndpoints.java | 2 +- code/{functions => }/index/build.gradle | 24 +++++-- .../index/forward-index/build.gradle | 5 +- .../index/forward-index/readme.md | 0 .../index/forward/ForwardIndexConverter.java | 2 +- .../index/forward/ForwardIndexFileNames.java | 0 .../index/forward/ForwardIndexParameters.java | 0 .../index/forward/ForwardIndexReader.java | 0 .../forward/ForwardIndexConverterTest.java | 2 +- .../java/nu/marginalia/test/TestUtil.java | 0 .../index/index-journal/build.gradle | 0 .../index/index-journal/readme.md | 0 .../index/journal/IndexJournalFileNames.java | 0 .../journal/model/IndexJournalEntry.java | 0 .../model/IndexJournalEntryBuilder.java | 0 .../journal/model/IndexJournalEntryData.java | 0 .../model/IndexJournalEntryHeader.java | 0 .../journal/model/IndexJournalFileHeader.java | 0 .../journal/reader/IndexJournalReadEntry.java | 0 .../journal/reader/IndexJournalReader.java | 0 .../reader/IndexJournalReaderPagingImpl.java | 0 .../reader/IndexJournalReaderSingleFile.java | 0 .../reader/pointer/IndexJournalPointer.java | 0 .../journal/writer/IndexJournalWriter.java | 0 .../writer/IndexJournalWriterPagingImpl.java | 0 .../IndexJournalWriterSingleFileImpl.java | 0 .../index/journal/IndexJournalTest.java | 0 .../pointer/IndexJournalPointerTest.java | 0 code/{functions => }/index/query/build.gradle | 3 + code/{functions => }/index/query/readme.md | 0 .../index/domainrankings}/DomainRankings.java | 2 +- .../index/query/EmptyEntrySource.java | 0 .../marginalia/index/query/EntrySource.java | 0 .../nu/marginalia/index/query/IndexQuery.java | 0 .../index/query/IndexQueryBuilder.java | 0 .../index/query/IndexQueryPriority.java | 0 .../index/query/IndexSearchBudget.java | 0 .../index/query/filter/QueryFilterAnyOf.java | 0 .../query/filter/QueryFilterLetThrough.java | 0 .../index/query/filter/QueryFilterNoPass.java | 0 .../QueryFilterStepExcludeFromPredicate.java | 0 .../filter/QueryFilterStepFromPredicate.java | 0 .../index/query/filter/QueryFilterStepIf.java | 0 .../index/query/limit/QueryLimits.java | 0 .../index/query/limit/QueryStrategy.java | 0 .../index/query/limit/SpecificationLimit.java | 0 .../query/limit/SpecificationLimitType.java | 0 .../query/filter/QueryFilterStepIfTest.java | 0 code/index/readme.md | 70 +++++++++++++++++++ .../index/reverse-index/build.gradle | 5 +- .../index/reverse-index/index.svg | 0 .../index/reverse-index/merging.svg | 0 .../index/reverse-index/preindex.svg | 0 .../index/reverse-index/readme.md | 0 .../index/ReverseIndexEntrySource.java | 0 .../index/ReverseIndexFullFileNames.java | 0 .../index/ReverseIndexParameters.java | 0 .../index/ReverseIndexPrioFileNames.java | 0 .../marginalia/index/ReverseIndexReader.java | 0 .../index/ReverseIndexSelfTest.java | 0 .../CountToOffsetTransformer.java | 0 .../index/construction/DocIdRewriter.java | 0 .../construction/IndexSizeEstimator.java | 0 .../construction/JournalReaderSource.java | 0 .../ReverseIndexBTreeTransformer.java | 0 .../construction/ReverseIndexConstructor.java | 0 .../index/construction/ReversePreindex.java | 0 .../ReversePreindexDocuments.java | 0 .../ReversePreindexReference.java | 0 .../ReversePreindexWordSegments.java | 0 .../index/query/ReverseIndexRejectFilter.java | 0 .../index/query/ReverseIndexRetainFilter.java | 0 .../index/ReverseIndexDebugTest.java | 0 .../index/ReverseIndexReaderTest.java | 0 .../construction/ReversePreindexDocsTest.java | 0 .../ReversePreindexFinalizeTest.java | 0 .../ReversePreindexMergeTest.java | 0 .../ReversePreindexWordSegmentsTest.java | 0 .../construction/TestJournalFactory.java | 0 .../index/construction/TestSegmentData.java | 0 .../java/nu/marginalia/test/TestUtil.java | 0 .../index/IndexQueryGrpcService.java | 16 ++--- .../index/IndexServicesFactory.java | 7 +- .../nu/marginalia}/index/SearchTermsUtil.java | 4 +- .../index/index/CombinedIndexReader.java | 4 +- .../index/index/IndexQueryBuilderImpl.java | 4 +- .../index/index/IndexQueryService.java | 8 +-- .../index/index/StatefulIndex.java | 8 +-- .../index/model/IndexQueryParams.java | 4 +- .../index/model/IndexSearchParameters.java | 6 +- .../index/model/IndexSearchTerms.java | 2 +- .../index/results/IndexMetadataService.java | 8 +-- .../IndexResultDomainDeduplicator.java | 2 +- .../results/IndexResultValuationContext.java | 6 +- .../results/IndexResultValuatorService.java | 6 +- .../index/searchset/DbUpdateRanks.java | 2 +- .../index/searchset/RankingSearchSet.java | 2 +- .../index/searchset/SearchSet.java | 2 +- .../index/searchset/SearchSetAny.java | 2 +- .../index/searchset/SearchSetsService.java | 17 ++--- .../index/searchset/SmallSearchSet.java | 2 +- .../domains}/PageRankDomainRanker.java | 8 +-- .../ranking/domains}/RankingAlgorithm.java | 4 +- .../accumulator/RankingResultAccumulator.java | 2 +- .../RankingResultBitSetAccumulator.java | 2 +- .../RankingResultHashMapAccumulator.java | 2 +- .../RankingResultHashSetAccumulator.java | 2 +- .../RankingResultListAccumulator.java | 2 +- .../domains}/data/AbstractGraphSource.java | 2 +- .../ranking/domains}/data/GraphSource.java | 2 +- .../data/InvertedLinkGraphSource.java | 2 +- .../domains}/data/LinkGraphSource.java | 2 +- .../domains}/data/SimilarityGraphSource.java | 2 +- .../jgrapht/PersonalizedPageRank.java | 2 +- .../ranking/results}/ResultKeywordSet.java | 2 +- .../ranking/results}/ResultValuator.java | 4 +- .../ranking/results}/factors/Bm25Factor.java | 4 +- .../results}/factors/PriorityTermBonus.java | 2 +- .../results}/factors/TermCoherenceFactor.java | 4 +- ...IndexQueryServiceIntegrationSmokeTest.java | 8 +-- .../IndexQueryServiceIntegrationTest.java | 8 +-- ...ndexQueryServiceIntegrationTestModule.java | 10 +-- .../IndexResultDomainDeduplicatorTest.java | 2 +- .../index/searchset/RankingSearchSetTest.java | 2 +- .../nu/marginalia}/index/util/TestUtil.java | 2 +- .../RankingAlgorithmWithRealDataTest.java | 4 +- .../RankingAlgorithmsContainerTest.java | 8 +-- .../TestGraphSourceForInvertedLinkData.java | 4 +- .../domains}/TestGraphSourceForLinkData.java | 4 +- .../TestGraphSourceForSimilarityData.java | 7 +- .../ranking/results}/ResultValuatorTest.java | 4 +- .../factors/TermCoherenceFactorTest.java | 4 +- .../crawling-model/build.gradle | 2 +- .../processes/converting-process/build.gradle | 2 +- code/processes/crawling-process/build.gradle | 2 +- .../index-constructor-process/build.gradle | 8 +-- .../index/IndexConstructorMain.java | 2 +- .../index/IndexConstructorModule.java | 2 +- code/processes/loading-process/build.gradle | 4 +- .../WebsiteAdjacenciesCalculator.java | 1 - .../api-service/build.gradle | 2 +- .../search-service/build.gradle | 4 +- .../search/svc/SearchErrorPageService.java | 2 +- .../control-service/build.gradle | 6 +- code/services-core/control-service/readme.md | 2 +- .../actor/precession/ReindexAllActor.java | 2 +- .../node/svc/ControlNodeActionsService.java | 2 +- .../executor-service/build.gradle | 4 +- .../actor/task/ConvertAndLoadActor.java | 4 +- code/services-core/index-service/build.gradle | 4 +- .../nu/marginalia/index/IndexOpsService.java | 4 +- .../nu/marginalia/index/IndexService.java | 5 +- code/services-core/query-service/build.gradle | 4 +- settings.gradle | 14 ++-- 167 files changed, 255 insertions(+), 344 deletions(-) delete mode 100644 code/functions/index/domain-ranking/build.gradle delete mode 100644 code/functions/index/domain-ranking/readme.md delete mode 100644 code/functions/index/readme.md delete mode 100644 code/functions/index/result-ranking/build.gradle delete mode 100644 code/functions/index/result-ranking/readme.md rename code/{functions => }/index/api/build.gradle (100%) rename code/{functions/index/api/src/main/java/nu/marginalia/functions => index/api/src/main/java/nu/marginalia}/index/api/IndexClient.java (97%) rename code/{functions/index/api/src/main/java/nu/marginalia/functions => index/api/src/main/java/nu/marginalia}/index/api/IndexMqClient.java (97%) rename code/{functions/index/api/src/main/java/nu/marginalia/functions => index/api/src/main/java/nu/marginalia}/index/api/IndexMqEndpoints.java (89%) rename code/{functions => }/index/build.gradle (56%) rename code/{functions => }/index/forward-index/build.gradle (77%) rename code/{functions => }/index/forward-index/readme.md (100%) rename code/{functions => }/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java (98%) rename code/{functions => }/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java (100%) rename code/{functions => }/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java (100%) rename code/{functions => }/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java (100%) rename code/{functions => }/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java (98%) rename code/{functions => }/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java (100%) rename code/{functions => }/index/index-journal/build.gradle (100%) rename code/{functions => }/index/index-journal/readme.md (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java (100%) rename code/{functions => }/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java (100%) rename code/{functions => }/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java (100%) rename code/{functions => }/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java (100%) rename code/{functions => }/index/query/build.gradle (77%) rename code/{functions => }/index/query/readme.md (100%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/query/src/main/java/nu/marginalia/index/domainrankings}/DomainRankings.java (98%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java (100%) rename code/{functions => }/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java (100%) rename code/{functions => }/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java (100%) create mode 100644 code/index/readme.md rename code/{functions => }/index/reverse-index/build.gradle (76%) rename code/{functions => }/index/reverse-index/index.svg (100%) rename code/{functions => }/index/reverse-index/merging.svg (100%) rename code/{functions => }/index/reverse-index/preindex.svg (100%) rename code/{functions => }/index/reverse-index/readme.md (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java (100%) rename code/{functions => }/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java (100%) rename code/{functions => }/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java (100%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/IndexQueryGrpcService.java (94%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/IndexServicesFactory.java (93%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/SearchTermsUtil.java (96%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/index/CombinedIndexReader.java (97%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/index/IndexQueryBuilderImpl.java (97%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/index/IndexQueryService.java (96%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/index/StatefulIndex.java (97%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/model/IndexQueryParams.java (92%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/model/IndexSearchParameters.java (96%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/model/IndexSearchTerms.java (95%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/results/IndexMetadataService.java (96%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/results/IndexResultDomainDeduplicator.java (96%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/results/IndexResultValuationContext.java (98%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/results/IndexResultValuatorService.java (97%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/searchset/DbUpdateRanks.java (96%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/searchset/RankingSearchSet.java (97%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/searchset/SearchSet.java (82%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/searchset/SearchSetAny.java (83%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/searchset/SearchSetsService.java (93%) rename code/{functions/index/src/main/java/nu/marginalia/functions => index/src/main/java/nu/marginalia}/index/searchset/SmallSearchSet.java (92%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/PageRankDomainRanker.java (88%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/RankingAlgorithm.java (75%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/accumulator/RankingResultAccumulator.java (66%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/accumulator/RankingResultBitSetAccumulator.java (88%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/accumulator/RankingResultHashMapAccumulator.java (90%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/accumulator/RankingResultHashSetAccumulator.java (88%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/accumulator/RankingResultListAccumulator.java (91%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/data/AbstractGraphSource.java (97%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/data/GraphSource.java (94%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/data/InvertedLinkGraphSource.java (96%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/data/LinkGraphSource.java (96%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/data/SimilarityGraphSource.java (97%) rename code/{functions/index/domain-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/domains}/jgrapht/PersonalizedPageRank.java (99%) rename code/{functions/index/result-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/results}/ResultKeywordSet.java (94%) rename code/{functions/index/result-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/results}/ResultValuator.java (99%) rename code/{functions/index/result-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/results}/factors/Bm25Factor.java (97%) rename code/{functions/index/result-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/results}/factors/PriorityTermBonus.java (89%) rename code/{functions/index/result-ranking/src/main/java/nu/marginalia/ranking => index/src/main/java/nu/marginalia/ranking/results}/factors/TermCoherenceFactor.java (87%) rename code/{functions/index/src/test/java/nu/marginalia/functions => index/src/test/java/nu/marginalia}/index/IndexQueryServiceIntegrationSmokeTest.java (98%) rename code/{functions/index/src/test/java/nu/marginalia/functions => index/src/test/java/nu/marginalia}/index/IndexQueryServiceIntegrationTest.java (98%) rename code/{functions/index/src/test/java/nu/marginalia/functions => index/src/test/java/nu/marginalia}/index/IndexQueryServiceIntegrationTestModule.java (93%) rename code/{functions/index/src/test/java/nu/marginalia/functions => index/src/test/java/nu/marginalia}/index/results/IndexResultDomainDeduplicatorTest.java (95%) rename code/{functions/index/src/test/java/nu/marginalia/functions => index/src/test/java/nu/marginalia}/index/searchset/RankingSearchSetTest.java (94%) rename code/{functions/index/src/test/java/nu/marginalia/functions => index/src/test/java/nu/marginalia}/index/util/TestUtil.java (96%) rename code/{functions/index/domain-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/domains}/RankingAlgorithmWithRealDataTest.java (95%) rename code/{functions/index/domain-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/domains}/RankingAlgorithmsContainerTest.java (96%) rename code/{functions/index/domain-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/domains}/TestGraphSourceForInvertedLinkData.java (97%) rename code/{functions/index/domain-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/domains}/TestGraphSourceForLinkData.java (97%) rename code/{functions/index/domain-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/domains}/TestGraphSourceForSimilarityData.java (92%) rename code/{functions/index/result-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/results}/ResultValuatorTest.java (97%) rename code/{functions/index/result-ranking/src/test/java/nu/marginalia/ranking => index/src/test/java/nu/marginalia/ranking/results}/factors/TermCoherenceFactorTest.java (96%) diff --git a/code/api/executor-api/build.gradle b/code/api/executor-api/build.gradle index 0103df0f..3ea8f63c 100644 --- a/code/api/executor-api/build.gradle +++ b/code/api/executor-api/build.gradle @@ -24,7 +24,7 @@ sourceSets { dependencies { implementation project(':code:common:model') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:config') implementation project(':code:common:db') implementation project(':code:libraries:message-queue') diff --git a/code/functions/index/domain-ranking/build.gradle b/code/functions/index/domain-ranking/build.gradle deleted file mode 100644 index 42af2c69..00000000 --- a/code/functions/index/domain-ranking/build.gradle +++ /dev/null @@ -1,44 +0,0 @@ -plugins { - id 'java' - - - id "de.undercouch.download" version "5.1.0" - - id 'jvm-test-suite' -} - -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(21)) - } -} - -dependencies { - implementation project(':code:common:db') - implementation project(':code:common:model') - implementation project(':code:common:service') - implementation project(':code:functions:domain-links:api') - - implementation 'org.jgrapht:jgrapht-core:1.5.2' - - implementation libs.bundles.slf4j - implementation libs.bundles.mariadb - implementation libs.guice - implementation libs.notnull - implementation libs.roaringbitmap - implementation libs.trove - implementation libs.fastutil - implementation libs.hll - - testImplementation project(':code:libraries:array') - - testImplementation libs.commons.lang3 - testImplementation libs.bundles.slf4j.test - testImplementation libs.bundles.junit - testImplementation libs.mockito - - testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') - testImplementation 'org.testcontainers:mariadb:1.17.4' - testImplementation 'org.testcontainers:junit-jupiter:1.17.4' - testImplementation project(':code:libraries:test-helpers') -} \ No newline at end of file diff --git a/code/functions/index/domain-ranking/readme.md b/code/functions/index/domain-ranking/readme.md deleted file mode 100644 index 5dc4ec2c..00000000 --- a/code/functions/index/domain-ranking/readme.md +++ /dev/null @@ -1,39 +0,0 @@ -# Domain Ranking - -Contains domain ranking algorithms. The domain ranking algorithms are based on -the JGraphT library. - -Two principal algorithms are available, the standard PageRank algorithm, -and personalized pagerank; each are available for two graphs, the link graph -and a similarity graph where each edge corresponds to the similarity between -the sets of incident links to two domains, their cosine similarity acting as -the weight of the links. - -With the standard PageRank algorithm, the similarity graph does not produce -anything useful, but something magical happens when you apply Personalized PageRank -to this graph. It turns into a very good "vibe"-sensitive ranking algorithm. - -It's unclear if this is a well known result, but it's a very interesting one -for creating a ranking algorithm that is focused on a particular segment of the web. - -## Central Classes - -* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java) - Ranks domains using the - PageRank or Personalized PageRank algorithm depending on whether a list of influence domains is provided. - -### Data sources - -* [LinkGraphSource](src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java) - fetches the link graph -* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java) - fetches the inverted link graph -* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java) - fetches the similarity graph from the database - -Note that the similarity graph needs to be precomputed and stored in the database for -the similarity graph source to be available. - -## See Also - -* [result-ranking](../result-ranking) - Ranks search results - -## Useful Resources - -* [The PageRank Citation Ranking: Bringing Order to the Web](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf) diff --git a/code/functions/index/readme.md b/code/functions/index/readme.md deleted file mode 100644 index da5df4c1..00000000 --- a/code/functions/index/readme.md +++ /dev/null @@ -1,24 +0,0 @@ -# Index - -These are components that offer functionality for the [index-service](../../services-core/index-service). - -## Indexes - -There are two indexes with accompanying tools for constructing them. - -* [index-reverse](index-reverse/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. - -* [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. - -These indices rely heavily on the [libraries/btree](../../libraries/btree) and [libraries/array](../../libraries/array) components. - -## Algorithms - -* [domain-ranking](domain-ranking/) contains domain ranking algorithms. -* [result-ranking](result-ranking/) contains logic for ranking search results by relevance. - -# Libraries - -* [index-query](index-query/) contains structures for evaluating search queries. -* [index-journal](index-journal/) contains tools for writing and reading index data. - diff --git a/code/functions/index/result-ranking/build.gradle b/code/functions/index/result-ranking/build.gradle deleted file mode 100644 index 64417a17..00000000 --- a/code/functions/index/result-ranking/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id 'java' - id "de.undercouch.download" version "5.1.0" - id 'jvm-test-suite' -} - -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(21)) - } -} - -dependencies { - implementation project(':code:common:model') - implementation project(':code:common:service') - implementation project(':code:functions:index:api') - implementation project(':code:functions:search-query:api') - - implementation libs.bundles.slf4j - implementation libs.guice - implementation libs.notnull - implementation libs.trove - implementation libs.fastutil - - testImplementation libs.bundles.slf4j.test - testImplementation libs.bundles.junit - testImplementation libs.mockito - testImplementation project(':code:libraries:term-frequency-dict') - testImplementation project(':code:libraries:braille-block-punch-cards') -} - diff --git a/code/functions/index/result-ranking/readme.md b/code/functions/index/result-ranking/readme.md deleted file mode 100644 index 99ba8647..00000000 --- a/code/functions/index/result-ranking/readme.md +++ /dev/null @@ -1,14 +0,0 @@ -# Result Ranking - -Contains various heuristics for deciding which search results are important -with regard to a query. In broad strokes [BM-25](https://nlp.stanford.edu/IR-book/html/htmledition/okapi-bm25-a-non-binary-model-1.html) -is used, with a number of additional bonuses and penalties to rank the appropriate search -results higher. - -## Central Classes - -* [ResultValuator](src/main/java/nu/marginalia/ranking/ResultValuator.java) - -## See Also - -* [features-index/domain-ranking](../domain-ranking) - Ranks domains \ No newline at end of file diff --git a/code/functions/search-query/api/build.gradle b/code/functions/search-query/api/build.gradle index 9aa0d000..5605473c 100644 --- a/code/functions/search-query/api/build.gradle +++ b/code/functions/search-query/api/build.gradle @@ -26,7 +26,7 @@ apply from: "$rootProject.projectDir/protobuf.gradle" dependencies { implementation project(':code:common:model') implementation project(':code:common:config') - implementation project(':code:functions:index:query') + implementation project(':code:index:query') implementation project(':code:common:service-discovery') implementation libs.bundles.slf4j diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java index 48608624..8a21f177 100644 --- a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java +++ b/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java @@ -5,7 +5,7 @@ package nu.marginalia.api.searchquery.model.results; * @param k determines the size of the impact of a single term * @param b determines the magnitude of the length normalization * - * @see nu.marginalia.ranking.factors.Bm25Factor + * @see nu.marginalia.ranking.results.factors.Bm25Factor */ public record Bm25Parameters(double k, double b) { } diff --git a/code/functions/search-query/build.gradle b/code/functions/search-query/build.gradle index 9a4c381d..d75504fe 100644 --- a/code/functions/search-query/build.gradle +++ b/code/functions/search-query/build.gradle @@ -10,7 +10,7 @@ java { } dependencies { - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:db') implementation project(':code:common:config') @@ -19,7 +19,7 @@ dependencies { implementation project(':code:functions:search-query:api') - implementation project(':code:functions:index:query') + implementation project(':code:index:query') implementation project(':code:libraries:language-processing') implementation project(':code:libraries:term-frequency-dict') diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java index edd65bde..d2cdd27d 100644 --- a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java +++ b/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java @@ -8,19 +8,14 @@ import lombok.SneakyThrows; import nu.marginalia.api.searchquery.*; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.db.DomainBlacklist; -import nu.marginalia.functions.index.api.IndexClient; +import nu.marginalia.index.api.IndexClient; import nu.marginalia.functions.searchquery.svc.QueryFactory; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.model.id.UrlIdCodec; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; -import nu.marginalia.service.discovery.property.ServiceKey; -import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.*; @Singleton public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { diff --git a/code/functions/index/api/build.gradle b/code/index/api/build.gradle similarity index 100% rename from code/functions/index/api/build.gradle rename to code/index/api/build.gradle diff --git a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java b/code/index/api/src/main/java/nu/marginalia/index/api/IndexClient.java similarity index 97% rename from code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java rename to code/index/api/src/main/java/nu/marginalia/index/api/IndexClient.java index 9cea5afd..4d8b79d2 100644 --- a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexClient.java +++ b/code/index/api/src/main/java/nu/marginalia/index/api/IndexClient.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.api; +package nu.marginalia.index.api; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java b/code/index/api/src/main/java/nu/marginalia/index/api/IndexMqClient.java similarity index 97% rename from code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java rename to code/index/api/src/main/java/nu/marginalia/index/api/IndexMqClient.java index 3a9b4314..27465f6e 100644 --- a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqClient.java +++ b/code/index/api/src/main/java/nu/marginalia/index/api/IndexMqClient.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.api; +package nu.marginalia.index.api; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqEndpoints.java b/code/index/api/src/main/java/nu/marginalia/index/api/IndexMqEndpoints.java similarity index 89% rename from code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqEndpoints.java rename to code/index/api/src/main/java/nu/marginalia/index/api/IndexMqEndpoints.java index 118221ca..ec618912 100644 --- a/code/functions/index/api/src/main/java/nu/marginalia/functions/index/api/IndexMqEndpoints.java +++ b/code/index/api/src/main/java/nu/marginalia/index/api/IndexMqEndpoints.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.api; +package nu.marginalia.index.api; public class IndexMqEndpoints { public static final String INDEX_IS_BLOCKED = "INDEX-IS-BLOCKED"; diff --git a/code/functions/index/build.gradle b/code/index/build.gradle similarity index 56% rename from code/functions/index/build.gradle rename to code/index/build.gradle index e1223f15..cfeb66d8 100644 --- a/code/functions/index/build.gradle +++ b/code/index/build.gradle @@ -10,9 +10,12 @@ java { } dependencies { + implementation 'org.jgrapht:jgrapht-core:1.5.2' + implementation project(':third-party:commons-codec') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') + implementation project(':code:functions:domain-links:api') implementation project(':code:libraries:array') implementation project(':code:libraries:btree') @@ -26,17 +29,16 @@ dependencies { implementation project(':code:functions:search-query:api') - implementation project(':code:functions:index:domain-ranking') - implementation project(':code:functions:index:forward-index') - implementation project(':code:functions:index:reverse-index') - implementation project(':code:functions:index:query') - implementation project(':code:functions:index:index-journal') - implementation project(':code:functions:index:result-ranking') + implementation project(':code:index:forward-index') + implementation project(':code:index:reverse-index') + implementation project(':code:index:query') + implementation project(':code:index:index-journal') implementation libs.bundles.slf4j implementation libs.prometheus + implementation libs.roaringbitmap implementation libs.bundles.grpc implementation libs.notnull implementation libs.guice @@ -48,6 +50,14 @@ dependencies { testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit testImplementation libs.mockito + testImplementation libs.commons.lang3 testImplementation project(':code:common:process') + testImplementation project(':code:libraries:array') + testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') + testImplementation 'org.testcontainers:mariadb:1.17.4' + testImplementation 'org.testcontainers:junit-jupiter:1.17.4' + testImplementation project(':code:libraries:test-helpers') + testImplementation project(':code:libraries:term-frequency-dict') + testImplementation project(':code:libraries:braille-block-punch-cards') } diff --git a/code/functions/index/forward-index/build.gradle b/code/index/forward-index/build.gradle similarity index 77% rename from code/functions/index/forward-index/build.gradle rename to code/index/forward-index/build.gradle index 7ba989e2..35e0146e 100644 --- a/code/functions/index/forward-index/build.gradle +++ b/code/index/forward-index/build.gradle @@ -13,9 +13,8 @@ java { dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') - implementation project(':code:functions:index:domain-ranking') - implementation project(':code:functions:index:query') - implementation project(':code:functions:index:index-journal') + implementation project(':code:index:query') + implementation project(':code:index:index-journal') implementation project(':code:common:model') implementation project(':code:common:process') diff --git a/code/functions/index/forward-index/readme.md b/code/index/forward-index/readme.md similarity index 100% rename from code/functions/index/forward-index/readme.md rename to code/index/forward-index/readme.md diff --git a/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java b/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java similarity index 98% rename from code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java rename to code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java index e7e11e9b..80cf502b 100644 --- a/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java +++ b/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java @@ -2,12 +2,12 @@ package nu.marginalia.index.forward; import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; import nu.marginalia.array.LongArrayFactory; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.index.journal.reader.IndexJournalReader; import nu.marginalia.array.LongArray; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.DocumentMetadata; import nu.marginalia.process.control.ProcessHeartbeat; -import nu.marginalia.ranking.DomainRankings; import org.roaringbitmap.longlong.LongConsumer; import org.roaringbitmap.longlong.Roaring64Bitmap; import org.slf4j.Logger; diff --git a/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java b/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java similarity index 100% rename from code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java rename to code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java diff --git a/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java b/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java similarity index 100% rename from code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java rename to code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java diff --git a/code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java b/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java similarity index 100% rename from code/functions/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java rename to code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java diff --git a/code/functions/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java b/code/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java similarity index 98% rename from code/functions/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java rename to code/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java index 062d3716..de571664 100644 --- a/code/functions/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java +++ b/code/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java @@ -1,13 +1,13 @@ package nu.marginalia.index.forward; import lombok.SneakyThrows; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.index.journal.model.IndexJournalEntry; import nu.marginalia.index.journal.reader.IndexJournalReaderSingleFile; import nu.marginalia.index.journal.writer.IndexJournalWriter; import nu.marginalia.index.journal.writer.IndexJournalWriterSingleFileImpl; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.process.control.FakeProcessHeartbeat; -import nu.marginalia.ranking.DomainRankings; import nu.marginalia.test.TestUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/code/functions/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java b/code/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/functions/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java rename to code/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java diff --git a/code/functions/index/index-journal/build.gradle b/code/index/index-journal/build.gradle similarity index 100% rename from code/functions/index/index-journal/build.gradle rename to code/index/index-journal/build.gradle diff --git a/code/functions/index/index-journal/readme.md b/code/index/index-journal/readme.md similarity index 100% rename from code/functions/index/index-journal/readme.md rename to code/index/index-journal/readme.md diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java diff --git a/code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java b/code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java similarity index 100% rename from code/functions/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java rename to code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java diff --git a/code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java b/code/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java similarity index 100% rename from code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java rename to code/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java diff --git a/code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java b/code/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java similarity index 100% rename from code/functions/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java rename to code/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java diff --git a/code/functions/index/query/build.gradle b/code/index/query/build.gradle similarity index 77% rename from code/functions/index/query/build.gradle rename to code/index/query/build.gradle index 71b925e8..c54b7fc4 100644 --- a/code/functions/index/query/build.gradle +++ b/code/index/query/build.gradle @@ -13,10 +13,13 @@ java { dependencies { implementation project(':code:libraries:array') + implementation project(':code:common:model') implementation libs.bundles.slf4j implementation libs.prometheus + implementation libs.fastutil + implementation libs.bundles.mariadb testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit diff --git a/code/functions/index/query/readme.md b/code/index/query/readme.md similarity index 100% rename from code/functions/index/query/readme.md rename to code/index/query/readme.md diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java b/code/index/query/src/main/java/nu/marginalia/index/domainrankings/DomainRankings.java similarity index 98% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java rename to code/index/query/src/main/java/nu/marginalia/index/domainrankings/DomainRankings.java index db5321b1..5d79a0f9 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java +++ b/code/index/query/src/main/java/nu/marginalia/index/domainrankings/DomainRankings.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking; +package nu.marginalia.index.domainrankings; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2ShortOpenHashMap; diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java b/code/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java rename to code/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java b/code/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java rename to code/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java b/code/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java rename to code/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java b/code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java rename to code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java b/code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java rename to code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java b/code/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java rename to code/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java b/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java rename to code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java b/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java rename to code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java b/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java rename to code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java b/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java rename to code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java b/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java rename to code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java b/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java rename to code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java b/code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java rename to code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java b/code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java rename to code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java b/code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java rename to code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java diff --git a/code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java b/code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java similarity index 100% rename from code/functions/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java rename to code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java diff --git a/code/functions/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java b/code/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java similarity index 100% rename from code/functions/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java rename to code/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java diff --git a/code/index/readme.md b/code/index/readme.md new file mode 100644 index 00000000..0ba4961c --- /dev/null +++ b/code/index/readme.md @@ -0,0 +1,70 @@ +# Index + +These are components that offer functionality for the [index-service](../../services-core/index-service). + +## Indexes + +There are two indexes with accompanying tools for constructing them. + +* [index-reverse](index-reverse/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. + +* [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. + +These indices rely heavily on the [libraries/btree](../../libraries/btree) and [libraries/array](../../libraries/array) components. + +## Algorithms + +* [domain-ranking](domain-ranking/) contains domain ranking algorithms. +* [result-ranking](result-ranking/) contains logic for ranking search results by relevance. + +# Libraries + +* [index-query](index-query/) contains structures for evaluating search queries. +* [index-journal](index-journal/) contains tools for writing and reading index data. + +# Domain Ranking + +Contains domain ranking algorithms. The domain ranking algorithms are based on +the JGraphT library. + +Two principal algorithms are available, the standard PageRank algorithm, +and personalized pagerank; each are available for two graphs, the link graph +and a similarity graph where each edge corresponds to the similarity between +the sets of incident links to two domains, their cosine similarity acting as +the weight of the links. + +With the standard PageRank algorithm, the similarity graph does not produce +anything useful, but something magical happens when you apply Personalized PageRank +to this graph. It turns into a very good "vibe"-sensitive ranking algorithm. + +It's unclear if this is a well known result, but it's a very interesting one +for creating a ranking algorithm that is focused on a particular segment of the web. + +## Central Classes + +* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java) - Ranks domains using the + PageRank or Personalized PageRank algorithm depending on whether a list of influence domains is provided. + +### Data sources + +* [LinkGraphSource](src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java) - fetches the link graph +* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java) - fetches the inverted link graph +* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java) - fetches the similarity graph from the database + +Note that the similarity graph needs to be precomputed and stored in the database for +the similarity graph source to be available. + +## Useful Resources + +* [The PageRank Citation Ranking: Bringing Order to the Web](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf) + +# Result Ranking + +Contains various heuristics for deciding which search results are important +with regard to a query. In broad strokes [BM-25](https://nlp.stanford.edu/IR-book/html/htmledition/okapi-bm25-a-non-binary-model-1.html) +is used, with a number of additional bonuses and penalties to rank the appropriate search +results higher. + +## Central Classes + +* [ResultValuator](src/main/java/nu/marginalia/ranking/ResultValuator.java) diff --git a/code/functions/index/reverse-index/build.gradle b/code/index/reverse-index/build.gradle similarity index 76% rename from code/functions/index/reverse-index/build.gradle rename to code/index/reverse-index/build.gradle index 0025c3fd..216d1142 100644 --- a/code/functions/index/reverse-index/build.gradle +++ b/code/index/reverse-index/build.gradle @@ -15,9 +15,8 @@ dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') implementation project(':code:libraries:random-write-funnel') - implementation project(':code:functions:index:domain-ranking') - implementation project(':code:functions:index:query') - implementation project(':code:functions:index:index-journal') + implementation project(':code:index:query') + implementation project(':code:index:index-journal') implementation project(':code:common:model') implementation project(':code:common:process') diff --git a/code/functions/index/reverse-index/index.svg b/code/index/reverse-index/index.svg similarity index 100% rename from code/functions/index/reverse-index/index.svg rename to code/index/reverse-index/index.svg diff --git a/code/functions/index/reverse-index/merging.svg b/code/index/reverse-index/merging.svg similarity index 100% rename from code/functions/index/reverse-index/merging.svg rename to code/index/reverse-index/merging.svg diff --git a/code/functions/index/reverse-index/preindex.svg b/code/index/reverse-index/preindex.svg similarity index 100% rename from code/functions/index/reverse-index/preindex.svg rename to code/index/reverse-index/preindex.svg diff --git a/code/functions/index/reverse-index/readme.md b/code/index/reverse-index/readme.md similarity index 100% rename from code/functions/index/reverse-index/readme.md rename to code/index/reverse-index/readme.md diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java diff --git a/code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java b/code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java similarity index 100% rename from code/functions/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java rename to code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java b/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java rename to code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java diff --git a/code/functions/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java b/code/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/functions/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java rename to code/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java b/code/index/src/main/java/nu/marginalia/index/IndexQueryGrpcService.java similarity index 94% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java rename to code/index/src/main/java/nu/marginalia/index/IndexQueryGrpcService.java index 20e208e4..8524d5b4 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexQueryGrpcService.java +++ b/code/index/src/main/java/nu/marginalia/index/IndexQueryGrpcService.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index; +package nu.marginalia.index; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -13,13 +13,13 @@ import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.api.searchquery.model.results.SearchResultSet; -import nu.marginalia.functions.index.index.IndexQueryService; -import nu.marginalia.functions.index.index.StatefulIndex; -import nu.marginalia.functions.index.model.IndexSearchParameters; -import nu.marginalia.functions.index.results.IndexResultValuatorService; -import nu.marginalia.functions.index.searchset.SearchSetsService; -import nu.marginalia.functions.index.searchset.SmallSearchSet; -import nu.marginalia.functions.index.searchset.SearchSet; +import nu.marginalia.index.index.IndexQueryService; +import nu.marginalia.index.index.StatefulIndex; +import nu.marginalia.index.model.IndexSearchParameters; +import nu.marginalia.index.results.IndexResultValuatorService; +import nu.marginalia.index.searchset.SearchSetsService; +import nu.marginalia.index.searchset.SmallSearchSet; +import nu.marginalia.index.searchset.SearchSet; import nu.marginalia.service.module.ServiceConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexServicesFactory.java b/code/index/src/main/java/nu/marginalia/index/IndexServicesFactory.java similarity index 93% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/IndexServicesFactory.java rename to code/index/src/main/java/nu/marginalia/index/IndexServicesFactory.java index 6a0bfd79..b59e7960 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/IndexServicesFactory.java +++ b/code/index/src/main/java/nu/marginalia/index/IndexServicesFactory.java @@ -1,12 +1,9 @@ -package nu.marginalia.functions.index; +package nu.marginalia.index; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.IndexLocations; -import nu.marginalia.functions.index.index.CombinedIndexReader; -import nu.marginalia.index.ReverseIndexFullFileNames; -import nu.marginalia.index.ReverseIndexPrioFileNames; -import nu.marginalia.index.ReverseIndexReader; +import nu.marginalia.index.index.CombinedIndexReader; import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.forward.ForwardIndexFileNames; import nu.marginalia.index.forward.ForwardIndexReader; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java b/code/index/src/main/java/nu/marginalia/index/SearchTermsUtil.java similarity index 96% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java rename to code/index/src/main/java/nu/marginalia/index/SearchTermsUtil.java index 3573d283..6aa4592e 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/SearchTermsUtil.java +++ b/code/index/src/main/java/nu/marginalia/index/SearchTermsUtil.java @@ -1,9 +1,9 @@ -package nu.marginalia.functions.index; +package nu.marginalia.index; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; -import nu.marginalia.functions.index.model.IndexSearchTerms; +import nu.marginalia.index.model.IndexSearchTerms; import nu.marginalia.hash.MurmurHash3_128; import java.util.ArrayList; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java b/code/index/src/main/java/nu/marginalia/index/index/CombinedIndexReader.java similarity index 97% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java rename to code/index/src/main/java/nu/marginalia/index/index/CombinedIndexReader.java index 6e3e17cf..aaed1269 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/CombinedIndexReader.java +++ b/code/index/src/main/java/nu/marginalia/index/index/CombinedIndexReader.java @@ -1,6 +1,6 @@ -package nu.marginalia.functions.index.index; +package nu.marginalia.index.index; -import nu.marginalia.functions.index.model.IndexQueryParams; +import nu.marginalia.index.model.IndexQueryParams; import nu.marginalia.index.ReverseIndexReader; import nu.marginalia.index.forward.ForwardIndexReader; import nu.marginalia.index.query.*; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java b/code/index/src/main/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java similarity index 97% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java rename to code/index/src/main/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java index e91b14d1..e159adc2 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryBuilderImpl.java +++ b/code/index/src/main/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java @@ -1,7 +1,7 @@ -package nu.marginalia.functions.index.index; +package nu.marginalia.index.index; import gnu.trove.set.hash.TLongHashSet; -import nu.marginalia.functions.index.model.IndexQueryParams; +import nu.marginalia.index.model.IndexQueryParams; import nu.marginalia.index.ReverseIndexReader; import nu.marginalia.index.forward.ForwardIndexReader; import nu.marginalia.index.query.IndexQuery; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java b/code/index/src/main/java/nu/marginalia/index/index/IndexQueryService.java similarity index 96% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java rename to code/index/src/main/java/nu/marginalia/index/index/IndexQueryService.java index 64c9fa16..49417539 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/IndexQueryService.java +++ b/code/index/src/main/java/nu/marginalia/index/index/IndexQueryService.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.index; +package nu.marginalia.index.index; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -6,9 +6,9 @@ import gnu.trove.list.TLongList; import gnu.trove.list.array.TLongArrayList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.array.buffer.LongQueryBuffer; -import nu.marginalia.functions.index.model.IndexSearchTerms; -import nu.marginalia.functions.index.model.IndexSearchParameters; -import nu.marginalia.functions.index.SearchTermsUtil; +import nu.marginalia.index.model.IndexSearchTerms; +import nu.marginalia.index.model.IndexSearchParameters; +import nu.marginalia.index.SearchTermsUtil; import nu.marginalia.index.query.IndexQuery; import nu.marginalia.index.query.IndexQueryPriority; import org.slf4j.Logger; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java b/code/index/src/main/java/nu/marginalia/index/index/StatefulIndex.java similarity index 97% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java rename to code/index/src/main/java/nu/marginalia/index/index/StatefulIndex.java index 4a81693f..f2e006d0 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/index/StatefulIndex.java +++ b/code/index/src/main/java/nu/marginalia/index/index/StatefulIndex.java @@ -1,10 +1,10 @@ -package nu.marginalia.functions.index.index; +package nu.marginalia.index.index; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.functions.index.model.IndexQueryParams; -import nu.marginalia.functions.index.IndexServicesFactory; -import nu.marginalia.functions.index.model.IndexSearchTerms; +import nu.marginalia.index.model.IndexQueryParams; +import nu.marginalia.index.IndexServicesFactory; +import nu.marginalia.index.model.IndexSearchTerms; import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepFromPredicate; import nu.marginalia.service.control.ServiceEventLog; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexQueryParams.java b/code/index/src/main/java/nu/marginalia/index/model/IndexQueryParams.java similarity index 92% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexQueryParams.java rename to code/index/src/main/java/nu/marginalia/index/model/IndexQueryParams.java index f8e64e0b..a8bd5026 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexQueryParams.java +++ b/code/index/src/main/java/nu/marginalia/index/model/IndexQueryParams.java @@ -1,6 +1,6 @@ -package nu.marginalia.functions.index.model; +package nu.marginalia.index.model; -import nu.marginalia.functions.index.searchset.SearchSet; +import nu.marginalia.index.searchset.SearchSet; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java b/code/index/src/main/java/nu/marginalia/index/model/IndexSearchParameters.java similarity index 96% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java rename to code/index/src/main/java/nu/marginalia/index/model/IndexSearchParameters.java index 61442ecf..b4fe8cb3 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchParameters.java +++ b/code/index/src/main/java/nu/marginalia/index/model/IndexSearchParameters.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.model; +package nu.marginalia.index.model; import gnu.trove.set.hash.TLongHashSet; import nu.marginalia.api.searchquery.RpcIndexQuery; @@ -6,11 +6,11 @@ import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.api.searchquery.IndexProtobufCodec; -import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.query.IndexQuery; import nu.marginalia.index.query.IndexSearchBudget; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.functions.index.searchset.SearchSet; +import nu.marginalia.index.searchset.SearchSet; import java.util.ArrayList; import java.util.List; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java b/code/index/src/main/java/nu/marginalia/index/model/IndexSearchTerms.java similarity index 95% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java rename to code/index/src/main/java/nu/marginalia/index/model/IndexSearchTerms.java index 4d4edd75..51110333 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/model/IndexSearchTerms.java +++ b/code/index/src/main/java/nu/marginalia/index/model/IndexSearchTerms.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.model; +package nu.marginalia.index.model; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongComparator; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexMetadataService.java b/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java similarity index 96% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexMetadataService.java rename to code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java index c96d47ab..5b082efe 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexMetadataService.java +++ b/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.results; +package nu.marginalia.index.results; import com.google.inject.Inject; import gnu.trove.map.hash.TObjectLongHashMap; @@ -7,10 +7,10 @@ import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap; import it.unimi.dsi.fastutil.longs.LongArrayList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; -import nu.marginalia.functions.index.index.StatefulIndex; -import nu.marginalia.functions.index.SearchTermsUtil; +import nu.marginalia.index.index.StatefulIndex; +import nu.marginalia.index.SearchTermsUtil; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.ranking.ResultValuator; +import nu.marginalia.ranking.results.ResultValuator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicator.java b/code/index/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java similarity index 96% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicator.java rename to code/index/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java index 8d59f81f..fac47a47 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicator.java +++ b/code/index/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.results; +package nu.marginalia.index.results; import gnu.trove.map.TLongIntMap; import gnu.trove.map.hash.TLongIntHashMap; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java b/code/index/src/main/java/nu/marginalia/index/results/IndexResultValuationContext.java similarity index 98% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java rename to code/index/src/main/java/nu/marginalia/index/results/IndexResultValuationContext.java index ac297e2e..58689d69 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuationContext.java +++ b/code/index/src/main/java/nu/marginalia/index/results/IndexResultValuationContext.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.results; +package nu.marginalia.index.results; import gnu.trove.list.TLongList; import gnu.trove.set.hash.TLongHashSet; @@ -7,12 +7,12 @@ import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.api.searchquery.model.results.SearchResultPreliminaryScore; +import nu.marginalia.index.model.IndexQueryParams; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.index.query.limit.QueryStrategy; -import nu.marginalia.functions.index.model.IndexQueryParams; -import nu.marginalia.ranking.ResultValuator; +import nu.marginalia.ranking.results.ResultValuator; import javax.annotation.Nullable; import java.util.Arrays; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java b/code/index/src/main/java/nu/marginalia/index/results/IndexResultValuatorService.java similarity index 97% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java rename to code/index/src/main/java/nu/marginalia/index/results/IndexResultValuatorService.java index 4b9efac4..0a1d6cf9 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/results/IndexResultValuatorService.java +++ b/code/index/src/main/java/nu/marginalia/index/results/IndexResultValuatorService.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.results; +package nu.marginalia.index.results; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -7,10 +7,10 @@ import gnu.trove.list.array.TLongArrayList; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.SearchResultItem; -import nu.marginalia.functions.index.model.IndexSearchParameters; +import nu.marginalia.index.model.IndexSearchParameters; import nu.marginalia.linkdb.docs.DocumentDbReader; import nu.marginalia.linkdb.model.DocdbUrlDetail; -import nu.marginalia.ranking.ResultValuator; +import nu.marginalia.ranking.results.ResultValuator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/DbUpdateRanks.java b/code/index/src/main/java/nu/marginalia/index/searchset/DbUpdateRanks.java similarity index 96% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/DbUpdateRanks.java rename to code/index/src/main/java/nu/marginalia/index/searchset/DbUpdateRanks.java index ac04de51..ce8d8af0 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/DbUpdateRanks.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/DbUpdateRanks.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; import com.zaxxer.hikari.HikariDataSource; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/RankingSearchSet.java b/code/index/src/main/java/nu/marginalia/index/searchset/RankingSearchSet.java similarity index 97% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/RankingSearchSet.java rename to code/index/src/main/java/nu/marginalia/index/searchset/RankingSearchSet.java index 632e3504..1734cf01 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/RankingSearchSet.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/RankingSearchSet.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import org.slf4j.Logger; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSet.java b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSet.java similarity index 82% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSet.java rename to code/index/src/main/java/nu/marginalia/index/searchset/SearchSet.java index 85c3a418..8fba1801 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSet.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSet.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; public interface SearchSet { diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetAny.java b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetAny.java similarity index 83% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetAny.java rename to code/index/src/main/java/nu/marginalia/index/searchset/SearchSetAny.java index f6a99443..c00bb435 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetAny.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetAny.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; public class SearchSetAny implements SearchSet { @Override diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java similarity index 93% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java rename to code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java index df274814..317bc239 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SearchSetsService.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -6,13 +6,14 @@ import gnu.trove.list.TIntList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import nu.marginalia.db.DomainRankingSetsService; import nu.marginalia.db.DomainTypes; -import nu.marginalia.functions.index.IndexServicesFactory; -import nu.marginalia.ranking.DomainRankings; -import nu.marginalia.ranking.PageRankDomainRanker; -import nu.marginalia.ranking.accumulator.*; -import nu.marginalia.ranking.data.GraphSource; -import nu.marginalia.ranking.data.LinkGraphSource; -import nu.marginalia.ranking.data.SimilarityGraphSource; +import nu.marginalia.index.IndexServicesFactory; +import nu.marginalia.index.domainrankings.DomainRankings; +import nu.marginalia.ranking.domains.PageRankDomainRanker; +import nu.marginalia.ranking.domains.accumulator.RankingResultHashMapAccumulator; +import nu.marginalia.ranking.domains.accumulator.RankingResultHashSetAccumulator; +import nu.marginalia.ranking.domains.data.GraphSource; +import nu.marginalia.ranking.domains.data.LinkGraphSource; +import nu.marginalia.ranking.domains.data.SimilarityGraphSource; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.module.ServiceConfiguration; import org.slf4j.Logger; diff --git a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SmallSearchSet.java b/code/index/src/main/java/nu/marginalia/index/searchset/SmallSearchSet.java similarity index 92% rename from code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SmallSearchSet.java rename to code/index/src/main/java/nu/marginalia/index/searchset/SmallSearchSet.java index 7fca5ac4..0c4c0862 100644 --- a/code/functions/index/src/main/java/nu/marginalia/functions/index/searchset/SmallSearchSet.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/SmallSearchSet.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; import gnu.trove.set.hash.TIntHashSet; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java b/code/index/src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java similarity index 88% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java index e87352b0..354ebb31 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java @@ -1,10 +1,10 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; import gnu.trove.list.TIntList; import gnu.trove.list.array.TIntArrayList; -import nu.marginalia.ranking.accumulator.RankingResultAccumulator; -import nu.marginalia.ranking.data.GraphSource; -import nu.marginalia.ranking.jgrapht.PersonalizedPageRank; +import nu.marginalia.ranking.domains.accumulator.RankingResultAccumulator; +import nu.marginalia.ranking.domains.data.GraphSource; +import nu.marginalia.ranking.domains.jgrapht.PersonalizedPageRank; import org.jgrapht.Graph; import org.jgrapht.alg.interfaces.VertexScoringAlgorithm; import org.jgrapht.alg.scoring.PageRank; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java b/code/index/src/main/java/nu/marginalia/ranking/domains/RankingAlgorithm.java similarity index 75% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/RankingAlgorithm.java index f67d47be..9df110b5 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/RankingAlgorithm.java @@ -1,6 +1,6 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; -import nu.marginalia.ranking.accumulator.RankingResultAccumulator; +import nu.marginalia.ranking.domains.accumulator.RankingResultAccumulator; import java.util.function.Supplier; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java similarity index 66% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java index e9055f6e..8fcbfea2 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.accumulator; +package nu.marginalia.ranking.domains.accumulator; public interface RankingResultAccumulator { void add(int domainId, int rank); diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java similarity index 88% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java index 3a806d95..aa12d378 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.accumulator; +package nu.marginalia.ranking.domains.accumulator; import org.roaringbitmap.RoaringBitmap; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java similarity index 90% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java index 15365466..f5cdd410 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.accumulator; +package nu.marginalia.ranking.domains.accumulator; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java similarity index 88% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java index 54ee465d..36777478 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.accumulator; +package nu.marginalia.ranking.domains.accumulator; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java similarity index 91% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java index ecfab27c..5722b22e 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.accumulator; +package nu.marginalia.ranking.domains.accumulator; import gnu.trove.list.array.TIntArrayList; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java b/code/index/src/main/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java similarity index 97% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java index 26b4f3d3..57e10323 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/AbstractGraphSource.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.data; +package nu.marginalia.ranking.domains.data; import com.zaxxer.hikari.HikariDataSource; import org.jgrapht.Graph; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java b/code/index/src/main/java/nu/marginalia/ranking/domains/data/GraphSource.java similarity index 94% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/data/GraphSource.java index b6e1daeb..7c137245 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/GraphSource.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/data/GraphSource.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.data; +package nu.marginalia.ranking.domains.data; import org.jgrapht.Graph; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java b/code/index/src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java similarity index 96% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java index f4adf2b6..a3863f6b 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.data; +package nu.marginalia.ranking.domains.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java b/code/index/src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java similarity index 96% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java index 5367a570..7c4bcb55 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.data; +package nu.marginalia.ranking.domains.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java b/code/index/src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java similarity index 97% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java index 359f76fd..84f9730e 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.data; +package nu.marginalia.ranking.domains.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; diff --git a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java b/code/index/src/main/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java similarity index 99% rename from code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java rename to code/index/src/main/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java index 0fd6a194..4eedb684 100644 --- a/code/functions/index/domain-ranking/src/main/java/nu/marginalia/ranking/jgrapht/PersonalizedPageRank.java +++ b/code/index/src/main/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.jgrapht; +package nu.marginalia.ranking.domains.jgrapht; /* * (C) Copyright 2016-2023, by Dimitrios Michail and Contributors. diff --git a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java b/code/index/src/main/java/nu/marginalia/ranking/results/ResultKeywordSet.java similarity index 94% rename from code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java rename to code/index/src/main/java/nu/marginalia/ranking/results/ResultKeywordSet.java index 5377ce46..19405dcb 100644 --- a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultKeywordSet.java +++ b/code/index/src/main/java/nu/marginalia/ranking/results/ResultKeywordSet.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.results; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; diff --git a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java b/code/index/src/main/java/nu/marginalia/ranking/results/ResultValuator.java similarity index 99% rename from code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java rename to code/index/src/main/java/nu/marginalia/ranking/results/ResultValuator.java index 60fa2be1..332f5326 100644 --- a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/ResultValuator.java +++ b/code/index/src/main/java/nu/marginalia/ranking/results/ResultValuator.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.results; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; @@ -7,7 +7,7 @@ import nu.marginalia.model.crawl.HtmlFeature; import nu.marginalia.model.crawl.PubDate; import nu.marginalia.model.idx.DocumentFlags; import nu.marginalia.model.idx.DocumentMetadata; -import nu.marginalia.ranking.factors.*; +import nu.marginalia.ranking.results.factors.*; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java b/code/index/src/main/java/nu/marginalia/ranking/results/factors/Bm25Factor.java similarity index 97% rename from code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java rename to code/index/src/main/java/nu/marginalia/ranking/results/factors/Bm25Factor.java index 98c58fd8..335b5fa8 100644 --- a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/Bm25Factor.java +++ b/code/index/src/main/java/nu/marginalia/ranking/results/factors/Bm25Factor.java @@ -1,10 +1,10 @@ -package nu.marginalia.ranking.factors; +package nu.marginalia.ranking.results.factors; import nu.marginalia.api.searchquery.model.results.Bm25Parameters; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.model.idx.WordFlags; -import nu.marginalia.ranking.ResultKeywordSet; +import nu.marginalia.ranking.results.ResultKeywordSet; public class Bm25Factor { private static final int AVG_LENGTH = 5000; diff --git a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java b/code/index/src/main/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java similarity index 89% rename from code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java rename to code/index/src/main/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java index 5b942dc3..18d044c3 100644 --- a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/PriorityTermBonus.java +++ b/code/index/src/main/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking.factors; +package nu.marginalia.ranking.results.factors; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; diff --git a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java b/code/index/src/main/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java similarity index 87% rename from code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java rename to code/index/src/main/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java index 54964dc1..f956ce88 100644 --- a/code/functions/index/result-ranking/src/main/java/nu/marginalia/ranking/factors/TermCoherenceFactor.java +++ b/code/index/src/main/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java @@ -1,7 +1,7 @@ -package nu.marginalia.ranking.factors; +package nu.marginalia.ranking.results.factors; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.ranking.ResultKeywordSet; +import nu.marginalia.ranking.results.ResultKeywordSet; /** Rewards documents where terms appear frequently within the same sentences */ diff --git a/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationSmokeTest.java b/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java similarity index 98% rename from code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationSmokeTest.java rename to code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java index ea977821..758c1a3d 100644 --- a/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationSmokeTest.java +++ b/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index; +package nu.marginalia.index; import com.google.inject.Guice; import com.google.inject.Inject; @@ -7,13 +7,11 @@ import nu.marginalia.IndexLocations; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.process.control.FakeProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.storage.FileStorageService; import nu.marginalia.hash.MurmurHash3_128; -import nu.marginalia.index.ReverseIndexFullFileNames; -import nu.marginalia.index.ReverseIndexPrioFileNames; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.ReverseIndexConstructor; import nu.marginalia.index.forward.ForwardIndexConverter; @@ -33,7 +31,7 @@ import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.DocumentMetadata; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.ranking.DomainRankings; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.service.control.ServiceHeartbeat; import nu.marginalia.service.server.Initialization; import org.junit.jupiter.api.AfterEach; diff --git a/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTest.java b/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTest.java similarity index 98% rename from code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTest.java rename to code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTest.java index f88da805..b9207437 100644 --- a/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTest.java +++ b/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index; +package nu.marginalia.index; import com.google.inject.Guice; import com.google.inject.Inject; @@ -6,11 +6,9 @@ import nu.marginalia.IndexLocations; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.storage.FileStorageService; import nu.marginalia.hash.MurmurHash3_128; -import nu.marginalia.index.ReverseIndexFullFileNames; -import nu.marginalia.index.ReverseIndexPrioFileNames; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.ReverseIndexConstructor; import nu.marginalia.index.forward.ForwardIndexConverter; @@ -34,7 +32,7 @@ import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordMetadata; import nu.marginalia.process.control.FakeProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeat; -import nu.marginalia.ranking.DomainRankings; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.service.control.ServiceHeartbeat; import nu.marginalia.service.server.Initialization; import org.junit.jupiter.api.AfterEach; diff --git a/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTestModule.java b/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java similarity index 93% rename from code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTestModule.java rename to code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java index 60498fca..d04b458c 100644 --- a/code/functions/index/src/test/java/nu/marginalia/functions/index/IndexQueryServiceIntegrationTestModule.java +++ b/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java @@ -1,9 +1,10 @@ -package nu.marginalia.functions.index; +package nu.marginalia.index; import com.google.inject.AbstractModule; import nu.marginalia.IndexLocations; -import nu.marginalia.functions.index.searchset.SearchSetAny; -import nu.marginalia.functions.index.searchset.SearchSetsService; +import nu.marginalia.index.searchset.SearchSetAny; +import nu.marginalia.index.searchset.SearchSetsService; +import nu.marginalia.index.util.TestUtil; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageBase; import nu.marginalia.storage.model.FileStorageBaseType; @@ -12,8 +13,7 @@ import nu.marginalia.index.journal.writer.IndexJournalWriterPagingImpl; import nu.marginalia.linkdb.docs.DocumentDbReader; import nu.marginalia.process.control.FakeProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeat; -import nu.marginalia.ranking.DomainRankings; -import nu.marginalia.functions.index.util.TestUtil; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.service.control.*; import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfiguration; diff --git a/code/functions/index/src/test/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicatorTest.java b/code/index/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java similarity index 95% rename from code/functions/index/src/test/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicatorTest.java rename to code/index/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java index 8ed5be16..7bc88ce4 100644 --- a/code/functions/index/src/test/java/nu/marginalia/functions/index/results/IndexResultDomainDeduplicatorTest.java +++ b/code/index/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.results; +package nu.marginalia.index.results; import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.model.id.UrlIdCodec; diff --git a/code/functions/index/src/test/java/nu/marginalia/functions/index/searchset/RankingSearchSetTest.java b/code/index/src/test/java/nu/marginalia/index/searchset/RankingSearchSetTest.java similarity index 94% rename from code/functions/index/src/test/java/nu/marginalia/functions/index/searchset/RankingSearchSetTest.java rename to code/index/src/test/java/nu/marginalia/index/searchset/RankingSearchSetTest.java index ad8460d1..30f3ded4 100644 --- a/code/functions/index/src/test/java/nu/marginalia/functions/index/searchset/RankingSearchSetTest.java +++ b/code/index/src/test/java/nu/marginalia/index/searchset/RankingSearchSetTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.searchset; +package nu.marginalia.index.searchset; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import org.junit.jupiter.api.Test; diff --git a/code/functions/index/src/test/java/nu/marginalia/functions/index/util/TestUtil.java b/code/index/src/test/java/nu/marginalia/index/util/TestUtil.java similarity index 96% rename from code/functions/index/src/test/java/nu/marginalia/functions/index/util/TestUtil.java rename to code/index/src/test/java/nu/marginalia/index/util/TestUtil.java index a9ae0780..651dd316 100644 --- a/code/functions/index/src/test/java/nu/marginalia/functions/index/util/TestUtil.java +++ b/code/index/src/test/java/nu/marginalia/index/util/TestUtil.java @@ -1,4 +1,4 @@ -package nu.marginalia.functions.index.util; +package nu.marginalia.index.util; import java.io.File; diff --git a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java b/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java similarity index 95% rename from code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java rename to code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java index 88a8b5e8..10195f92 100644 --- a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmWithRealDataTest.java +++ b/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java @@ -1,6 +1,6 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; -import nu.marginalia.ranking.accumulator.RankingResultListAccumulator; +import nu.marginalia.ranking.domains.accumulator.RankingResultListAccumulator; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java b/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java similarity index 96% rename from code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java rename to code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java index b92543e2..e27d881c 100644 --- a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java +++ b/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java @@ -1,12 +1,12 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; -import nu.marginalia.ranking.data.InvertedLinkGraphSource; -import nu.marginalia.ranking.data.LinkGraphSource; -import nu.marginalia.ranking.data.SimilarityGraphSource; +import nu.marginalia.ranking.domains.data.InvertedLinkGraphSource; +import nu.marginalia.ranking.domains.data.LinkGraphSource; +import nu.marginalia.ranking.domains.data.SimilarityGraphSource; import nu.marginalia.test.TestMigrationLoader; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultWeightedEdge; diff --git a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java b/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java similarity index 97% rename from code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java rename to code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java index e07cd176..a9a51fe4 100644 --- a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForInvertedLinkData.java +++ b/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java @@ -1,8 +1,8 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; import lombok.SneakyThrows; import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.ranking.data.GraphSource; +import nu.marginalia.ranking.domains.data.GraphSource; import org.apache.commons.lang3.StringUtils; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; diff --git a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java b/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java similarity index 97% rename from code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java rename to code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java index e009f628..03fcdb14 100644 --- a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForLinkData.java +++ b/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java @@ -1,8 +1,8 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; import lombok.SneakyThrows; import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.ranking.data.GraphSource; +import nu.marginalia.ranking.domains.data.GraphSource; import org.apache.commons.lang3.StringUtils; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; diff --git a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java b/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java similarity index 92% rename from code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java rename to code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java index 47fd6e40..4aa48fbc 100644 --- a/code/functions/index/domain-ranking/src/test/java/nu/marginalia/ranking/TestGraphSourceForSimilarityData.java +++ b/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java @@ -1,12 +1,9 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.domains; import lombok.SneakyThrows; -import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.ranking.data.GraphSource; +import nu.marginalia.ranking.domains.data.GraphSource; import org.apache.commons.lang3.StringUtils; import org.jgrapht.Graph; -import org.jgrapht.graph.DefaultDirectedGraph; -import org.jgrapht.graph.DefaultEdge; import org.jgrapht.graph.DefaultUndirectedWeightedGraph; import org.jgrapht.graph.DefaultWeightedEdge; diff --git a/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java b/code/index/src/test/java/nu/marginalia/ranking/results/ResultValuatorTest.java similarity index 97% rename from code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java rename to code/index/src/test/java/nu/marginalia/ranking/results/ResultValuatorTest.java index e5eda34c..60beead2 100644 --- a/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/ResultValuatorTest.java +++ b/code/index/src/test/java/nu/marginalia/ranking/results/ResultValuatorTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.ranking; +package nu.marginalia.ranking.results; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; @@ -8,7 +8,7 @@ import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.crawl.PubDate; import nu.marginalia.model.idx.DocumentMetadata; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.ranking.factors.*; +import nu.marginalia.ranking.results.factors.*; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java b/code/index/src/test/java/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java similarity index 96% rename from code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java rename to code/index/src/test/java/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java index 8718dc83..6ae9c448 100644 --- a/code/functions/index/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java +++ b/code/index/src/test/java/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java @@ -1,9 +1,9 @@ -package nu.marginalia.ranking.factors; +package nu.marginalia.ranking.results.factors; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.bbpc.BrailleBlockPunchCards; import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.ranking.ResultKeywordSet; +import nu.marginalia.ranking.results.ResultKeywordSet; import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/code/process-models/crawling-model/build.gradle b/code/process-models/crawling-model/build.gradle index 7a894e04..f7b5cf49 100644 --- a/code/process-models/crawling-model/build.gradle +++ b/code/process-models/crawling-model/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation project(':code:common:config') implementation project(':code:common:process') implementation project(':code:libraries:big-string') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:service-discovery') implementation project(':code:features-crawl:content-type') implementation project(':code:libraries:language-processing') diff --git a/code/processes/converting-process/build.gradle b/code/processes/converting-process/build.gradle index be64de5b..caa7e4a8 100644 --- a/code/processes/converting-process/build.gradle +++ b/code/processes/converting-process/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation project(':third-party:porterstemmer') implementation project(':third-party:count-min-sketch') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:common:model') diff --git a/code/processes/crawling-process/build.gradle b/code/processes/crawling-process/build.gradle index f98229cd..6a1a3823 100644 --- a/code/processes/crawling-process/build.gradle +++ b/code/processes/crawling-process/build.gradle @@ -27,7 +27,7 @@ dependencies { implementation project(':code:common:service') implementation project(':code:libraries:big-string') implementation project(':code:libraries:blocking-thread-pool') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') diff --git a/code/processes/index-constructor-process/build.gradle b/code/processes/index-constructor-process/build.gradle index bdf7869e..6b6ae660 100644 --- a/code/processes/index-constructor-process/build.gradle +++ b/code/processes/index-constructor-process/build.gradle @@ -27,10 +27,10 @@ dependencies { implementation project(':code:common:model') implementation project(':code:libraries:message-queue') - implementation project(':code:functions:index:forward-index') - implementation project(':code:functions:index:reverse-index') - implementation project(':code:functions:index:index-journal') - implementation project(':code:functions:index:domain-ranking') + implementation project(':code:index:query') + implementation project(':code:index:forward-index') + implementation project(':code:index:reverse-index') + implementation project(':code:index:index-journal') implementation libs.bundles.slf4j implementation libs.guice diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java index 27ad832c..47d3fba2 100644 --- a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java +++ b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java @@ -6,6 +6,7 @@ import com.google.inject.Inject; import nu.marginalia.IndexLocations; import nu.marginalia.ProcessConfiguration; import nu.marginalia.ProcessConfigurationModule; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.service.ProcessMainClass; import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.construction.ReverseIndexConstructor; @@ -23,7 +24,6 @@ import nu.marginalia.mq.inbox.MqSingleShotInbox; import nu.marginalia.mqapi.index.CreateIndexRequest; import nu.marginalia.mqapi.index.IndexName; import nu.marginalia.process.control.ProcessHeartbeatImpl; -import nu.marginalia.ranking.DomainRankings; import nu.marginalia.service.module.DatabaseModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorModule.java b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorModule.java index 025e90eb..0fa3ac6d 100644 --- a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorModule.java +++ b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorModule.java @@ -4,7 +4,7 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; import nu.marginalia.IndexLocations; -import nu.marginalia.ranking.DomainRankings; +import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.storage.FileStorageService; public class IndexConstructorModule extends AbstractModule { diff --git a/code/processes/loading-process/build.gradle b/code/processes/loading-process/build.gradle index aa6da441..db139f36 100644 --- a/code/processes/loading-process/build.gradle +++ b/code/processes/loading-process/build.gradle @@ -20,14 +20,14 @@ tasks.distZip.enabled = false dependencies { implementation project(':code:common:process') implementation project(':code:api:process-mqapi') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:config') implementation project(':code:common:service') implementation project(':code:common:service-discovery') implementation project(':code:common:linkdb') - implementation project(':code:functions:index:index-journal') + implementation project(':code:index:index-journal') implementation project(':code:libraries:message-queue') implementation project(':code:libraries:language-processing') implementation project(':third-party:commons-codec') diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index cc720d5e..1a40c354 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -9,7 +9,6 @@ import nu.marginalia.db.DbDomainQueries; import nu.marginalia.model.EdgeDomain; import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeatImpl; -import nu.marginalia.service.MainClass; import nu.marginalia.service.ProcessMainClass; import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.module.DatabaseModule; diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index 5c2e3abc..24293012 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -42,7 +42,7 @@ dependencies { implementation project(':code:common:config') implementation project(':code:common:service-discovery') implementation project(':code:functions:search-query:api') - implementation project(':code:functions:index:query') + implementation project(':code:index:query') implementation libs.bundles.slf4j diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index 4ca22863..a32dc71a 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -47,7 +47,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:service') implementation project(':code:common:config') - implementation project(':code:functions:index:query') + implementation project(':code:index:query') implementation project(':code:libraries:easy-lsh') implementation project(':code:libraries:language-processing') @@ -59,7 +59,7 @@ dependencies { implementation project(':code:functions:search-query:api') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:service-discovery') implementation project(':code:common:renderer') diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java index ecdcdf22..346506e7 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java @@ -1,7 +1,7 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.functions.index.api.IndexMqClient; +import nu.marginalia.index.api.IndexMqClient; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import org.slf4j.Logger; diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index ee683a7c..99390c27 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -47,11 +47,11 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:functions:search-query:api') implementation project(':code:api:executor-api') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:features-search:screenshots') - implementation project(':code:functions:index:index-journal') - implementation project(':code:functions:index:query') + implementation project(':code:index:index-journal') + implementation project(':code:index:query') implementation project(':code:process-models:crawl-spec') diff --git a/code/services-core/control-service/readme.md b/code/services-core/control-service/readme.md index df5e2c5a..73d2742e 100644 --- a/code/services-core/control-service/readme.md +++ b/code/services-core/control-service/readme.md @@ -1,6 +1,6 @@ # Control Service -The control service provides an operator's user interface. By default this interface is +The control service provides an operator's user interface. By default, this interface is exposed on port 8081. It does not offer any sort of access control or authentication. The control service will itself execute tasks that affect the entire system, but delegate diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java index ecc7d422..48eaf851 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java @@ -8,7 +8,7 @@ import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.actor.state.ActorStep; import nu.marginalia.actor.state.Resume; import nu.marginalia.control.actor.PrecessionNodes; -import nu.marginalia.functions.index.api.IndexMqClient; +import nu.marginalia.index.api.IndexMqClient; import nu.marginalia.mq.persistence.MqPersistence; import java.sql.SQLException; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java index 2344f36f..fbcf07f5 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java @@ -5,7 +5,7 @@ import com.google.inject.Singleton; import nu.marginalia.control.ControlValidationError; import nu.marginalia.control.RedirectControl; import nu.marginalia.executor.client.ExecutorClient; -import nu.marginalia.functions.index.api.IndexMqClient; +import nu.marginalia.index.api.IndexMqClient; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 2a3bb0a7..5139c4eb 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -69,8 +69,8 @@ dependencies { implementation project(':code:features-convert:data-extractors') implementation project(':code:features-convert:stackexchange-xml') implementation project(':code:features-convert:reddit-json') - implementation project(':code:functions:index:index-journal') - implementation project(':code:functions:index:api') + implementation project(':code:index:index-journal') + implementation project(':code:index:api') implementation project(':code:api:process-mqapi') implementation project(':code:api:executor-api') implementation project(':third-party:encyclopedia-marginalia-nu') diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java index a70d977b..085dffed 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java @@ -19,8 +19,8 @@ import nu.marginalia.svc.BackupService; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; import nu.marginalia.storage.model.FileStorageType; -import nu.marginalia.functions.index.api.IndexMqClient; -import nu.marginalia.functions.index.api.IndexMqEndpoints; +import nu.marginalia.index.api.IndexMqClient; +import nu.marginalia.index.api.IndexMqEndpoints; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mqapi.converting.ConvertRequest; diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 89e778c6..1c7f81fe 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -43,11 +43,11 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:linkdb') - implementation project(':code:functions:index') + implementation project(':code:index') implementation project(':code:functions:domain-links:partition') implementation project(':code:functions:domain-links:api') implementation project(':code:functions:search-query:api') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:service-discovery') diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java index fb2a4a85..e7ed68b2 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java @@ -2,8 +2,8 @@ package nu.marginalia.index; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.functions.index.index.StatefulIndex; -import nu.marginalia.functions.index.searchset.SearchSetsService; +import nu.marginalia.index.index.StatefulIndex; +import nu.marginalia.index.searchset.SearchSetsService; import javax.annotation.CheckReturnValue; import java.util.Optional; diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index 48749650..9278f699 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -4,12 +4,11 @@ import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; import nu.marginalia.functions.domainlinks.PartitionDomainLinksService; -import nu.marginalia.functions.index.IndexQueryGrpcService; -import nu.marginalia.functions.index.index.StatefulIndex; +import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.linkdb.dlinks.DomainLinkDb; import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.storage.FileStorageService; -import nu.marginalia.functions.index.api.IndexMqEndpoints; +import nu.marginalia.index.api.IndexMqEndpoints; import nu.marginalia.linkdb.docs.DocumentDbReader; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.*; diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index fcd5e625..a013e86d 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -42,9 +42,9 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:service') implementation project(':code:common:renderer') - implementation project(':code:functions:index:api') + implementation project(':code:index:api') implementation project(':code:common:service-discovery') - implementation project(':code:functions:index:query') + implementation project(':code:index:query') implementation project(':code:functions:search-query') implementation project(':code:functions:search-query:api') diff --git a/settings.gradle b/settings.gradle index e88308e9..ac6f98d0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,14 +24,12 @@ include 'code:functions:domain-links:api' include 'code:functions:search-query' include 'code:functions:search-query:api' -include 'code:functions:index' -include 'code:functions:index:api' -include 'code:functions:index:index-journal' -include 'code:functions:index:query' -include 'code:functions:index:forward-index' -include 'code:functions:index:reverse-index' -include 'code:functions:index:domain-ranking' -include 'code:functions:index:result-ranking' +include 'code:index' +include 'code:index:api' +include 'code:index:index-journal' +include 'code:index:query' +include 'code:index:forward-index' +include 'code:index:reverse-index' include 'code:libraries:array' include 'code:libraries:geo-ip' From 4740156cfa063865bf46687915e3bfcd06b220e6 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 18:18:58 +0100 Subject: [PATCH 23/56] Clean up docs --- .../client/GrpcSingleNodeChannelPool.java | 1 - code/index/readme.md | 52 +++++++++---------- code/services-core/index-service/readme.md | 15 +----- 3 files changed, 28 insertions(+), 40 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index 3e724183..ec3bb38a 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -8,7 +8,6 @@ import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor; import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; import nu.marginalia.service.discovery.property.ServiceKey; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/index/readme.md b/code/index/readme.md index 0ba4961c..a16c8515 100644 --- a/code/index/readme.md +++ b/code/index/readme.md @@ -1,30 +1,41 @@ # Index -These are components that offer functionality for the [index-service](../../services-core/index-service). +This module contains the components that make up the search index. + +It exposes an API for querying the index, and contains the logic +for ranking search results. It does not parse the query, that is +the responsibility of the [search-query](../functions/search-query) module. ## Indexes There are two indexes with accompanying tools for constructing them. -* [index-reverse](index-reverse/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. +* [index-reverse](reverse-index/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. -* [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. +* [index-forward](forward-index/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. -These indices rely heavily on the [libraries/btree](../../libraries/btree) and [libraries/array](../../libraries/array) components. +Additionally, the [index-journal](index-journal/) contains code for constructing a journal of the index, which is used to keep the index up to date. -## Algorithms +These indices rely heavily on the [libraries/btree](../libraries/btree) and [libraries/array](../libraries/array) components. -* [domain-ranking](domain-ranking/) contains domain ranking algorithms. -* [result-ranking](result-ranking/) contains logic for ranking search results by relevance. +--- -# Libraries +# Result Ranking -* [index-query](index-query/) contains structures for evaluating search queries. -* [index-journal](index-journal/) contains tools for writing and reading index data. +The module is also responsible for ranking search results, and contains various heuristics +for deciding which search results are important with regard to a query. In broad strokes [BM-25](https://nlp.stanford.edu/IR-book/html/htmledition/okapi-bm25-a-non-binary-model-1.html) +is used, with a number of additional bonuses and penalties to rank the appropriate search +results higher. + +## Central Classes + +* [ResultValuator](src/main/java/nu/marginalia/ranking/results/ResultValuator.java) + +--- # Domain Ranking -Contains domain ranking algorithms. The domain ranking algorithms are based on +The module contains domain ranking algorithms. The domain ranking algorithms are based on the JGraphT library. Two principal algorithms are available, the standard PageRank algorithm, @@ -42,14 +53,14 @@ for creating a ranking algorithm that is focused on a particular segment of the ## Central Classes -* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/PageRankDomainRanker.java) - Ranks domains using the +* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java) - Ranks domains using the PageRank or Personalized PageRank algorithm depending on whether a list of influence domains is provided. ### Data sources -* [LinkGraphSource](src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java) - fetches the link graph -* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java) - fetches the inverted link graph -* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/data/SimilarityGraphSource.java) - fetches the similarity graph from the database +* [LinkGraphSource](src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java) - fetches the link graph +* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java) - fetches the inverted link graph +* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java) - fetches the similarity graph from the database Note that the similarity graph needs to be precomputed and stored in the database for the similarity graph source to be available. @@ -57,14 +68,3 @@ the similarity graph source to be available. ## Useful Resources * [The PageRank Citation Ranking: Bringing Order to the Web](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf) - -# Result Ranking - -Contains various heuristics for deciding which search results are important -with regard to a query. In broad strokes [BM-25](https://nlp.stanford.edu/IR-book/html/htmledition/okapi-bm25-a-non-binary-model-1.html) -is used, with a number of additional bonuses and penalties to rank the appropriate search -results higher. - -## Central Classes - -* [ResultValuator](src/main/java/nu/marginalia/ranking/ResultValuator.java) diff --git a/code/services-core/index-service/readme.md b/code/services-core/index-service/readme.md index 4ce81002..b7bbcfb0 100644 --- a/code/services-core/index-service/readme.md +++ b/code/services-core/index-service/readme.md @@ -6,17 +6,6 @@ It is the service that most directly executes a search query. It does this by evaluating a low-level query, and then using the index to find the documents that match the query, finally ranking the results and picking the best matches. -## Central Classes +This module only contains service boilerplate. The guts of this service are +in the [index](../../index) module. -* [IndexService](src/main/java/nu/marginalia/index/IndexService.java) is the REST entry point that the internal API talks to. -* [IndexQueryService](src/main/java/nu/marginalia/index/svc/IndexQueryService.java) executes queries. -* [SearchIndex](src/main/java/nu/marginalia/index/index/SearchIndex.java) owns the state of the index and helps with building a query strategy from parameters. -* [IndexResultValuator](src/main/java/nu/marginalia/index/results/IndexResultValuator.java) determines the best results. - -## See Also - -The index service relies heavily on the primitives in [features-index](../../features-index): - -* [features-index/index-forward](../../features-index/index-forward/) -* [features-index/index-reverse](../../features-index/index-reverse/) -* [features-index/index-query](../../features-index/index-query) From 8d4ef982d0f057c27b743a9488b75e1dce3d2d39 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 19:37:59 +0100 Subject: [PATCH 24/56] Clean up docs --- code/common/service-discovery/readme.md | 38 ++++++++++++++-------- code/services-core/query-service/readme.md | 13 +++----- settings.gradle | 2 -- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/code/common/service-discovery/readme.md b/code/common/service-discovery/readme.md index 3436db6a..764a271e 100644 --- a/code/common/service-discovery/readme.md +++ b/code/common/service-discovery/readme.md @@ -28,11 +28,22 @@ services: ```java // Register one or more services -registry.registerService(ApiSchema.GRPC, - ServiceId.Test, - nodeId, - instanceUUID, //must be unique to the runtime - "127.0.0.1"); // bind-address +serviceRegistry.registerService( + ServiceKey.forRest(serviceId, nodeId), + instanceUuid, // unique + externalAddress); // bind-address + +// Non-partitioned GRPC service +serviceRegistry.registerService( + ServiceKey.forServiceDescriptor(descriptor, ServicePartition.any()), + instanceUuid, + externalAddress); + +// Partitioned GRPC service +serviceRegistry.registerService( + ServiceKey.forServiceDescriptor(descriptor, ServicePartition.partition(5)), + instanceUuid, + externalAddress); // (+ any other services) ``` @@ -40,9 +51,7 @@ Then, the caller must announce their instance. Before this is done, the service is not discoverable. ```java -registry.announceInstance(ServiceId.Test, - nodeId, - instanceUUID); +registry.announceInstance(instanceUUID); ``` All of this is done automatically by the `Service` base class @@ -51,16 +60,19 @@ in the [service](../service/) module. To discover a service, the caller can query the registry: ```java -Set> endpoints = registry.getEndpoints(ApiSchema.GRPC, ServiceId.Test, nodeId); - -for (var endpoint : endpoints) { - System.out.println(endpoint.getHost() + ":" + endpoint.getPort()); -} +Set endpoints = registry.getEndpoints(serviceKey); ``` It's also possible to subscribe to changes in the registry, so that the caller can be notified when a service comes or goes, with `registry.registerMonitor()`. +However the `GrpcChannelPoolFactory` is a more convenient way to access the services, +it will let the caller create a pool of channels to the services, and manage their +lifecycle, listen to lifecycle notifications and so on. + +The ChannelPools exist in two flavors, one for partitioned services, and one for non-partitioned services. + + ### Central Classes * [ServiceRegistryIf](src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java) diff --git a/code/services-core/query-service/readme.md b/code/services-core/query-service/readme.md index 1c9d2b66..d2ba1961 100644 --- a/code/services-core/query-service/readme.md +++ b/code/services-core/query-service/readme.md @@ -13,13 +13,10 @@ specific stuff. This mode of operations is available through a `barebones` inst The web interface also offers a JSON API for machine-based queries. -## Main Classes +## Central Classes -* [QueryService](src/main/java/nu/marginalia/query/QueryService.java) - The REST service implementation -* [QueryGRPCService](src/main/java/nu/marginalia/query/QueryGRPCService.java) - The GRPC service implementation +This module is almost entirely boilerplate, except the [QueryBasicInterface](src/main/java/nu/marginalia/query/QueryBasicInterface.java) +class, which offers a REST API for querying the index. -## See Also - -* [api/query-api](../../api/query-api) -* [features-qs/query-parser](../../features-qs/query-parser) -* [features-index/index-query](../../features-index/index-query) \ No newline at end of file +Much of the guts of the query service are in the [query-service](../../functions/search-query) +module; which offers query parsing and an interface to the index service partitions. diff --git a/settings.gradle b/settings.gradle index ac6f98d0..b6f2f9c0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -65,8 +65,6 @@ include 'code:features-crawl:crawl-blocklist' include 'code:features-crawl:link-parser' include 'code:features-crawl:content-type' - -include 'code:api:index-api' include 'code:api:process-mqapi' include 'code:api:executor-api' From 6357d30ea02759aa384401440cadfb5b3bfa6177 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 22 Feb 2024 19:53:20 +0100 Subject: [PATCH 25/56] Clean up docs --- code/common/service-discovery/readme.md | 130 +++++++++++++----- .../client/GrpcMultiNodeChannelPool.java | 2 + 2 files changed, 95 insertions(+), 37 deletions(-) diff --git a/code/common/service-discovery/readme.md b/code/common/service-discovery/readme.md index 764a271e..7e6ab016 100644 --- a/code/common/service-discovery/readme.md +++ b/code/common/service-discovery/readme.md @@ -6,22 +6,20 @@ and managing connections between them. ## Service Registry The service registry is a class that keeps track of the services -that are currently running, and their connection information. +that are currently running, and their connection information. -There are two implementations: +The service register implementation is based on [Zookeeper](https://zookeeper.apache.org/), +which is a distributed coordination service. This lets services register +themselves and announce their liveness, and then discover each other. -* A simple implementation that effectively hard-codes the services. -This is sufficient when running in docker, although loses some -smart features. It is fundamentally incompatible with running -the system bare-metal, and does not permit multiple instances -of a service to run. +It supports multiple instances of a service running, and +supports running the system bare-metal, where it will assign +ports to the services from a range. -* A more advanced implementation that is based on [Zookeeper](https://zookeeper.apache.org/), -which is a distributed coordination service. This implementation -lets services register themselves and announce their liveness, -and then discover each other. It supports multiple instances of -a service running, and supports running the system bare-metal, -where it will assign ports to the services from a range. +* REST services are registered on a per-node basis, and are always non-partitioned. +* gRPC services are registered on a per-api basis, and can be partitioned + or non-partitioned. This means that if a gRPC api is moved between nodes, + the clients will not need to be reconfigured. To be discoverable, the caller must first register their services: @@ -30,24 +28,25 @@ services: // Register one or more services serviceRegistry.registerService( ServiceKey.forRest(serviceId, nodeId), - instanceUuid, // unique - externalAddress); // bind-address +instanceUuid, // unique +externalAddress); // bind-address // Non-partitioned GRPC service -serviceRegistry.registerService( + serviceRegistry.registerService( ServiceKey.forServiceDescriptor(descriptor, ServicePartition.any()), - instanceUuid, - externalAddress); +instanceUuid, +externalAddress); // Partitioned GRPC service -serviceRegistry.registerService( + serviceRegistry.registerService( ServiceKey.forServiceDescriptor(descriptor, ServicePartition.partition(5)), - instanceUuid, - externalAddress); +instanceUuid, +externalAddress); + // (+ any other services) ``` -Then, the caller must announce their instance. Before this is done, +Then, the caller must announce their instance. Before this is done, the service is not discoverable. ```java @@ -55,7 +54,7 @@ registry.announceInstance(instanceUUID); ``` All of this is done automatically by the `Service` base class -in the [service](../service/) module. +in the [service](../service/) module. To discover a service, the caller can query the registry: @@ -67,18 +66,9 @@ It's also possible to subscribe to changes in the registry, so that the caller can be notified when a service comes or goes, with `registry.registerMonitor()`. However the `GrpcChannelPoolFactory` is a more convenient way to access the services, -it will let the caller create a pool of channels to the services, and manage their +it will let the caller create a pool of channels to the services, and manage their lifecycle, listen to lifecycle notifications and so on. -The ChannelPools exist in two flavors, one for partitioned services, and one for non-partitioned services. - - -### Central Classes - -* [ServiceRegistryIf](src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java) -* [ZkServiceRegistry](src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java) -* [FixedServiceRegistry](src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java) - ## gRPC Channel Pool From the [GrpcChannelPoolFactory](src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java), two types of channel pools can be created @@ -86,8 +76,74 @@ that are aware of the service registry: * [GrpcMultiNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java) - This pool permits 1-n style communication with partitioned services * [GrpcSingleNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java) - This pool permits 1-1 style communication with non-partitioned services. - if multiple instances are running, it will use one of them and fall back - to another if the first is not available. + if multiple instances are running, it will use one of them and fall back + to another if the first is not available. -The pools manage the lifecycle of the gRPC channels, and will permit the caller -to access Stub interfaces for the services. +The pools can generate calls to the gRPC services, and will manage the lifecycle of the channels. + +The API is designed to be simple to use, and will permit the caller to access the Stub interfaces +for the services through a fluent API. + +### Example Usage of the GrpcSingleNodeChannelPool + +```java +// create a pool for a non-partitioned service +channelPool = factory.createSingle( + ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()), + MathApiGrpc::newBlockingStub); + +// blocking call +Response response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .run(request); + +// sequential blocking calls +List response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .runFor(request1, request2); + + +// async call +Future response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .async(myExecutor) + .run(request); + +// multiple async calls +Future> response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .async(myExecutor) + .runFor(request1, request2); +``` + +### Example Usage of the GrpcSingleNodeChannelPool + +```java +// create a pool for a partitioned service +channelPool = factory.createMulti( + ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.multi()), + MathApiGrpc::newBlockingStub); + +// blocking call +List response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .run(request); + +// async call +Future> response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .async(myExecutor) + .runEach(request); + +// async call, will fail or succeed as a group +Future> response = channelPool + .call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .async(myExecutor) + .runAll(request1, request2); +``` + + +### Central Classes + +* [ServiceRegistryIf](src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java) +* [ZkServiceRegistry](src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java) \ No newline at end of file diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index de76ecc9..149c9943 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -100,6 +100,7 @@ public class GrpcMultiNodeChannelPool { this.method = method; } + /** Run the given method on each node, returning a future of a list of results */ public CompletableFuture> runAll(I arg) { var futures = getEligibleNodes().stream() .map(GrpcMultiNodeChannelPool.this::getPoolForNode) @@ -113,6 +114,7 @@ public class GrpcMultiNodeChannelPool { .thenApply(v -> futures.stream().map(CompletableFuture::join).toList()); } + /** Run the given method on each node, returning a list of futures. */ public List> runEach(I arg) { return getEligibleNodes().stream() .map(GrpcMultiNodeChannelPool.this::getPoolForNode) From f4ff7185f0631b24fd64f3a1d71c2cb77b52003b Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 23 Feb 2024 11:18:29 +0100 Subject: [PATCH 26/56] (refac) Move process-mqapi out of api directory --- code/{api => }/process-mqapi/build.gradle | 0 .../src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java | 0 .../main/java/nu/marginalia/mqapi/converting/ConvertAction.java | 0 .../java/nu/marginalia/mqapi/converting/ConvertRequest.java | 0 .../main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java | 0 .../main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java | 0 .../src/main/java/nu/marginalia/mqapi/index/IndexName.java | 0 .../src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java | 0 code/processes/converting-process/build.gradle | 2 +- code/processes/crawling-process/build.gradle | 2 +- code/processes/index-constructor-process/build.gradle | 2 +- code/processes/loading-process/build.gradle | 2 +- code/services-core/control-service/build.gradle | 2 +- code/services-core/executor-service/build.gradle | 2 +- settings.gradle | 2 +- 15 files changed, 7 insertions(+), 7 deletions(-) rename code/{api => }/process-mqapi/build.gradle (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java (100%) rename code/{api => }/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java (100%) diff --git a/code/api/process-mqapi/build.gradle b/code/process-mqapi/build.gradle similarity index 100% rename from code/api/process-mqapi/build.gradle rename to code/process-mqapi/build.gradle diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java b/code/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java similarity index 100% rename from code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java rename to code/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java diff --git a/code/processes/converting-process/build.gradle b/code/processes/converting-process/build.gradle index caa7e4a8..f36b30d0 100644 --- a/code/processes/converting-process/build.gradle +++ b/code/processes/converting-process/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation project(':third-party:count-min-sketch') implementation project(':code:index:api') - implementation project(':code:api:process-mqapi') + implementation project(':code:process-mqapi') implementation project(':code:common:model') implementation project(':code:common:db') diff --git a/code/processes/crawling-process/build.gradle b/code/processes/crawling-process/build.gradle index 6a1a3823..ed53c6a0 100644 --- a/code/processes/crawling-process/build.gradle +++ b/code/processes/crawling-process/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation project(':code:libraries:big-string') implementation project(':code:libraries:blocking-thread-pool') implementation project(':code:index:api') - implementation project(':code:api:process-mqapi') + implementation project(':code:process-mqapi') implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') implementation project(':code:libraries:language-processing') diff --git a/code/processes/index-constructor-process/build.gradle b/code/processes/index-constructor-process/build.gradle index 6b6ae660..f7361358 100644 --- a/code/processes/index-constructor-process/build.gradle +++ b/code/processes/index-constructor-process/build.gradle @@ -19,7 +19,7 @@ application { tasks.distZip.enabled = false dependencies { - implementation project(':code:api:process-mqapi') + implementation project(':code:process-mqapi') implementation project(':code:common:process') implementation project(':code:common:service') implementation project(':code:common:db') diff --git a/code/processes/loading-process/build.gradle b/code/processes/loading-process/build.gradle index db139f36..66db9cb9 100644 --- a/code/processes/loading-process/build.gradle +++ b/code/processes/loading-process/build.gradle @@ -19,7 +19,7 @@ tasks.distZip.enabled = false dependencies { implementation project(':code:common:process') - implementation project(':code:api:process-mqapi') + implementation project(':code:process-mqapi') implementation project(':code:index:api') implementation project(':code:common:model') implementation project(':code:common:db') diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 99390c27..b5e69e98 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -48,7 +48,7 @@ dependencies { implementation project(':code:functions:search-query:api') implementation project(':code:api:executor-api') implementation project(':code:index:api') - implementation project(':code:api:process-mqapi') + implementation project(':code:process-mqapi') implementation project(':code:features-search:screenshots') implementation project(':code:index:index-journal') implementation project(':code:index:query') diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 5139c4eb..f079a675 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -71,7 +71,7 @@ dependencies { implementation project(':code:features-convert:reddit-json') implementation project(':code:index:index-journal') implementation project(':code:index:api') - implementation project(':code:api:process-mqapi') + implementation project(':code:process-mqapi') implementation project(':code:api:executor-api') implementation project(':third-party:encyclopedia-marginalia-nu') diff --git a/settings.gradle b/settings.gradle index b6f2f9c0..15561f1a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -65,7 +65,7 @@ include 'code:features-crawl:crawl-blocklist' include 'code:features-crawl:link-parser' include 'code:features-crawl:content-type' -include 'code:api:process-mqapi' +include 'code:process-mqapi' include 'code:api:executor-api' include 'code:common:service-discovery' From 6154e16951774f2969c85c311da0a25a4ac3fd0a Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 23 Feb 2024 11:22:02 +0100 Subject: [PATCH 27/56] (refac) Remove "distPath" --- .../main/java/nu/marginalia/executor/ExecutorModule.java | 7 ------- .../main/java/nu/marginalia/process/ProcessService.java | 9 +-------- run/env/service.env | 2 +- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java index 0c9f08eb..d49e2a56 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java @@ -1,15 +1,8 @@ package nu.marginalia.executor; import com.google.inject.AbstractModule; -import com.google.inject.name.Names; -import nu.marginalia.WmsaHome; - -import java.nio.file.Path; public class ExecutorModule extends AbstractModule { public void configure() { - - String dist = System.getProperty("distPath", WmsaHome.getHomePath().resolve("/dist").toString()); - bind(Path.class).annotatedWith(Names.named("distPath")).toInstance(Path.of(dist)); } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java index 93f89063..89e87e52 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java @@ -2,14 +2,12 @@ package nu.marginalia.process; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.google.inject.name.Named; import nu.marginalia.WmsaHome; import nu.marginalia.adjacencies.WebsiteAdjacenciesCalculator; import nu.marginalia.converting.ConverterMain; import nu.marginalia.crawl.CrawlerMain; import nu.marginalia.index.IndexConstructorMain; import nu.marginalia.loading.LoaderMain; -import nu.marginalia.service.MainClass; import nu.marginalia.service.ProcessMainClass; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.BaseServiceParams; @@ -21,8 +19,6 @@ import org.slf4j.MarkerFactory; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -34,7 +30,6 @@ public class ProcessService { private final Marker processMarker = MarkerFactory.getMarker("PROCESS"); private final ServiceEventLog eventLog; - private final Path distPath; private final ConcurrentHashMap processes = new ConcurrentHashMap<>(); @@ -81,10 +76,8 @@ public class ProcessService { } @Inject - public ProcessService(BaseServiceParams params, - @Named("distPath") Path distPath) { + public ProcessService(BaseServiceParams params) { this.eventLog = params.eventLog; - this.distPath = distPath; } diff --git a/run/env/service.env b/run/env/service.env index d94f62a9..5ccabc86 100644 --- a/run/env/service.env +++ b/run/env/service.env @@ -1,7 +1,7 @@ WMSA_HOME=run/ JAVA_OPTS="--enable-preview -da -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" JAVA_TOOL_OPTIONS="--enable-preview -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5000" -EXECUTOR_SERVICE_OPTS="-DdistPath=/dist" +EXECUTOR_SERVICE_OPTS="" CONVERTER_PROCESS_OPTS="-Dservice-name=converter" CRAWLER_PROCESS_OPTS="-Dservice-name=crawler" LOADER_PROCESS_OPTS="-Dservice-name=loader" From 5cdb07023b584e1652e6184762085ce6e72fc4c1 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 23 Feb 2024 11:27:20 +0100 Subject: [PATCH 28/56] (refac) Clean up unused imports --- .../nu/marginalia/service/client/GrpcMultiNodeChannelPool.java | 3 --- .../main/java/nu/marginalia/service/module/DatabaseModule.java | 2 -- .../nu/marginalia/service/module/ServiceConfiguration.java | 1 - .../main/java/nu/marginalia/api/domains/DomainInfoClient.java | 1 - .../nu/marginalia/functions/domains/DomainInfoGrpcService.java | 1 - .../functions/domainlinks/PartitionDomainLinksService.java | 1 - .../api/src/main/java/nu/marginalia/api/math/MathClient.java | 2 -- .../ranking/domains/RankingAlgorithmsContainerTest.java | 1 - .../nu/marginalia/search/command/commands/SearchCommand.java | 1 - .../control/sys/svc/ControlDomainRankingSetsService.java | 1 - .../src/main/java/nu/marginalia/query/QueryService.java | 1 - 11 files changed, 15 deletions(-) diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index 149c9943..c725fa0b 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -11,13 +11,10 @@ import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; import java.util.function.BiFunction; -import java.util.function.Consumer; import java.util.function.Function; -import java.util.stream.Stream; /** A pool of gRPC channels for a service, with a separate channel for each node. *

diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java index a196872f..15a70e57 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java @@ -7,7 +7,6 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.WmsaHome; -import nu.marginalia.service.ServiceHomeNotConfiguredException; import org.flywaydb.core.Flyway; import org.mariadb.jdbc.Driver; import org.slf4j.Logger; @@ -18,7 +17,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Optional; import java.util.Properties; public class DatabaseModule extends AbstractModule { diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java index cf152bf0..e64c88a8 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java @@ -1,6 +1,5 @@ package nu.marginalia.service.module; -import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import java.util.UUID; diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java index ee305994..80ca97fb 100644 --- a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java @@ -2,7 +2,6 @@ package nu.marginalia.api.domains; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.api.domains.model.SimilarDomain; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.client.GrpcSingleNodeChannelPool; import nu.marginalia.service.discovery.property.ServiceKey; diff --git a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java index 9bf8a1c2..8af30e56 100644 --- a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java @@ -2,7 +2,6 @@ package nu.marginalia.functions.domains; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.api.domains.DomainInfoAPIGrpc; import nu.marginalia.api.domains.*; public class DomainInfoGrpcService extends DomainInfoAPIGrpc.DomainInfoAPIImplBase { diff --git a/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java b/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java index 323ec51d..bb89fc6b 100644 --- a/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java +++ b/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java @@ -2,7 +2,6 @@ package nu.marginalia.functions.domainlinks; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.api.domainlink.Empty; import nu.marginalia.api.domainlink.*; import nu.marginalia.linkdb.dlinks.DomainLinkDb; diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java index c851aee9..27867b6c 100644 --- a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java @@ -6,12 +6,10 @@ import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.client.GrpcSingleNodeChannelPool; import nu.marginalia.service.discovery.property.ServiceKey; import nu.marginalia.service.discovery.property.ServicePartition; -import nu.marginalia.service.id.ServiceId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Duration; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java b/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java index e27d881c..c25b818a 100644 --- a/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java +++ b/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java @@ -20,7 +20,6 @@ import org.testcontainers.junit.jupiter.Testcontainers; import java.sql.SQLException; import java.util.List; -import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; import static org.mockito.Mockito.when; diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java index 323d744e..ea054a7b 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java @@ -1,7 +1,6 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; -import nu.marginalia.db.DomainBlacklist; import nu.marginalia.search.SearchOperator; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java index ca2c3bc7..e2adf14b 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java @@ -1,7 +1,6 @@ package nu.marginalia.control.sys.svc; import com.google.inject.Inject; -import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.control.ControlRendererFactory; import nu.marginalia.control.ControlValidationError; import nu.marginalia.control.Redirects; diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index 9ccb3f95..b8a4c6c5 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -1,7 +1,6 @@ package nu.marginalia.query; import com.google.inject.Inject; -import io.prometheus.client.Histogram; import lombok.SneakyThrows; import nu.marginalia.functions.domainlinks.AggregateDomainLinksService; import nu.marginalia.functions.searchquery.QueryGRPCService; From 2201b1a506bb08faaf217911ad642ff81fbfe3b4 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 23 Feb 2024 11:39:19 +0100 Subject: [PATCH 29/56] (refac) Clean up code issues --- .../model/src/main/java/nu/marginalia/model/EdgeDomain.java | 2 +- .../src/main/java/nu/marginalia/link_parser/LinkParser.java | 2 +- .../nu/marginalia/index/results/IndexMetadataService.java | 2 +- .../java/nu/marginalia/index/searchset/SearchSetsService.java | 2 +- .../nu/marginalia/control/sys/model/ProcessHeartbeat.java | 4 +--- .../java/nu/marginalia/control/sys/model/TaskHeartbeat.java | 4 +--- .../java/nu/marginalia/actor/task/ActorProcessWatcher.java | 4 ++++ 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java b/code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java index 0686a9ce..86c17824 100644 --- a/code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java +++ b/code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java @@ -127,7 +127,7 @@ public class EdgeDomain implements Serializable { ret.append(topDomain, 0, cutPoint); } - if (!"".equals(subDomain) && !"www".equals(subDomain)) { + if (!subDomain.isEmpty() && !"www".equals(subDomain)) { ret.append(":"); ret.append(subDomain); } diff --git a/code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/LinkParser.java b/code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/LinkParser.java index 08ccc221..652eab8d 100644 --- a/code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/LinkParser.java +++ b/code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/LinkParser.java @@ -209,7 +209,7 @@ public class LinkParser { } private boolean isUrlRelevant(String href) { - if (null == href || "".equals(href)) { + if (null == href || href.isEmpty()) { return false; } if (href.length() > 128) { diff --git a/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java b/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java index 5b082efe..5b8df400 100644 --- a/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java +++ b/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java @@ -56,7 +56,7 @@ public class IndexMetadataService { continue; } - long id = searchTermsUtil.getWordId(term); + long id = SearchTermsUtil.getWordId(term); termIdsList.add(id); termToId.put(term, id); } diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java index 317bc239..52c5ad89 100644 --- a/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java +++ b/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java @@ -93,7 +93,7 @@ public class SearchSetsService { return anySet; } - if ("NONE".equals(searchSetIdentifier) || "".equals(searchSetIdentifier)) { + if ("NONE".equals(searchSetIdentifier) || searchSetIdentifier.isEmpty()) { return anySet; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java index 7cf5205d..f2911ad2 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java @@ -20,9 +20,7 @@ public record ProcessHeartbeat( } public String progressStyle() { if ("RUNNING".equals(status) && progress != null) { - return """ - background: linear-gradient(90deg, #ccc 0%%, #ccc %d%%, #fff %d%%) - """.formatted(progress, progress, progress); + return STR."background: linear-gradient(90deg, #ccc 0%, #ccc \{progress}%, #fff \{progress}%)"; } return ""; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java index 467ae493..203d36be 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java @@ -21,9 +21,7 @@ public record TaskHeartbeat( public String progressStyle() { if ("RUNNING".equals(status) && progress != null) { - return """ - background: linear-gradient(90deg, #ccc 0%%, #ccc %d%%, #fff %d%%) - """.formatted(progress, progress, progress); + return STR."background: linear-gradient(90deg, #ccc 0%, #ccc \{progress}%, #fff \{progress}%)"; } return ""; } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java index 4bae8674..65d59f7e 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java @@ -39,6 +39,10 @@ public class ActorProcessWatcher { public MqMessage waitResponse(MqOutbox outbox, ProcessService.ProcessId processId, long msgId) throws ActorControlFlowException, InterruptedException, SQLException { + // enums values only have a single instance, + // so it's safe to synchronize on them + // even though it looks a bit weird to + // synchronize on a parameter like this: synchronized (processId) { // Wake up the process spawning actor processId.notifyAll(); From 56d35aa5962b4a5330366d3a2b1cb1af4e556926 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 23 Feb 2024 13:26:11 +0100 Subject: [PATCH 30/56] (refac) Move execution API out of executor service --- .../executor/model/transfer/TransferItem.java | 9 - .../executor/model/transfer/TransferSpec.java | 13 - code/api/readme.md | 23 -- .../api}/build.gradle | 6 +- .../executor/client/ExecutorClient.java | 139 +------- .../executor/client/ExecutorCrawlClient.java | 80 +++++ .../executor/client/ExecutorExportClient.java | 74 ++++ .../client/ExecutorRemoteActorFactory.java | 0 .../client/ExecutorSideloadClient.java | 71 ++++ .../executor/model/ActorRunState.java | 0 .../executor/model/ActorRunStates.java | 0 .../executor/storage/FileStorageContent.java | 0 .../executor/storage/FileStorageFile.java | 0 .../executor/upload/UploadDirContents.java | 0 .../executor/upload/UploadDirItem.java | 0 .../api}/src/main/protobuf/executor-api.proto | 28 +- code/execution/build.gradle | 75 ++++ .../java/nu/marginalia/actor/ActorApi.java | 47 +-- .../nu/marginalia/actor/ExecutorActor.java | 0 .../actor/ExecutorActorControlService.java | 0 .../actor/ExecutorActorStateMachines.java | 0 .../monitor/AbstractProcessSpawnerActor.java | 0 .../monitor/FileStorageMonitorActor.java | 0 .../actor/proc/ConverterMonitorActor.java | 0 .../actor/proc/CrawlerMonitorActor.java | 0 .../proc/IndexConstructorMonitorActor.java | 0 .../actor/proc/LoaderMonitorActor.java | 0 .../proc/ProcessLivenessMonitorActor.java | 0 .../actor/task/ActorProcessWatcher.java | 0 .../marginalia/actor/task/ConvertActor.java | 0 .../actor/task/ConvertAndLoadActor.java | 0 .../nu/marginalia/actor/task/CrawlActor.java | 0 .../actor/task/CrawlJobExtractorActor.java | 0 .../actor/task/DownloadSampleActor.java | 0 .../actor/task/ExportAtagsActor.java | 0 .../actor/task/ExportDataActor.java | 0 .../actor/task/ExportFeedsActor.java | 0 .../actor/task/ExportSampleDataActor.java | 0 .../actor/task/ExportTermFreqActor.java | 0 .../marginalia/actor/task/RecrawlActor.java | 0 .../actor/task/RestoreBackupActor.java | 0 .../TriggerAdjacencyCalculationActor.java | 0 .../marginalia/execution/ExecutionInit.java | 26 ++ .../execution/ExecutorCrawlGrpcService.java | 113 ++++++ .../execution/ExecutorExportGrpcService.java | 95 +++++ .../execution/ExecutorGrpcService.java | 237 +++++++++++++ .../ExecutorSideloadGrpcService.java | 96 ++++++ .../marginalia/process/ProcessOutboxes.java | 0 .../nu/marginalia/process/ProcessService.java | 0 .../sideload/RedditSideloadHelper.java | 0 .../marginalia/sideload/SideloadHelper.java | 0 .../sideload/StackExchangeSideloadHelper.java | 0 .../java/nu/marginalia/svc/BackupService.java | 12 +- .../svc/DomainListRefreshService.java | 0 .../svc/DomainListRefreshServiceTest.java | 0 .../control-service/build.gradle | 2 +- .../node/svc/ControlFileStorageService.java | 3 +- .../node/svc/ControlNodeActionsService.java | 43 ++- .../executor-service/build.gradle | 9 +- .../executor/ExecutorGrpcService.java | 325 ------------------ .../nu/marginalia/executor/ExecutorSvc.java | 57 ++- .../executor/svc/BackupService.java | 22 -- .../executor/svc/ExportService.java | 51 --- .../executor/svc/ProcessingService.java | 62 ---- .../executor/svc/SideloadService.java | 96 ------ .../executor/svc/TransferService.java | 97 ------ settings.gradle | 4 +- 67 files changed, 979 insertions(+), 936 deletions(-) delete mode 100644 code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferItem.java delete mode 100644 code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferSpec.java delete mode 100644 code/api/readme.md rename code/{api/executor-api => execution/api}/build.gradle (96%) rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/client/ExecutorClient.java (54%) create mode 100644 code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorCrawlClient.java create mode 100644 code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorExportClient.java rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java (100%) create mode 100644 code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorSideloadClient.java rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/model/ActorRunState.java (100%) rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/model/ActorRunStates.java (100%) rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java (100%) rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java (100%) rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java (100%) rename code/{api/executor-api => execution/api}/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java (100%) rename code/{api/executor-api => execution/api}/src/main/protobuf/executor-api.proto (93%) create mode 100644 code/execution/build.gradle rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/ActorApi.java (60%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/ExecutorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ConvertActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/CrawlActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ExportDataActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/RecrawlActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java (100%) create mode 100644 code/execution/src/main/java/nu/marginalia/execution/ExecutionInit.java create mode 100644 code/execution/src/main/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java create mode 100644 code/execution/src/main/java/nu/marginalia/execution/ExecutorExportGrpcService.java create mode 100644 code/execution/src/main/java/nu/marginalia/execution/ExecutorGrpcService.java create mode 100644 code/execution/src/main/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/process/ProcessOutboxes.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/process/ProcessService.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/sideload/SideloadHelper.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java (100%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/svc/BackupService.java (90%) rename code/{services-core/executor-service => execution}/src/main/java/nu/marginalia/svc/DomainListRefreshService.java (100%) rename code/{services-core/executor-service => execution}/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java (100%) delete mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorGrpcService.java delete mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/BackupService.java delete mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java delete mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ProcessingService.java delete mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java delete mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferItem.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferItem.java deleted file mode 100644 index 17be3dd8..00000000 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferItem.java +++ /dev/null @@ -1,9 +0,0 @@ -package nu.marginalia.executor.model.transfer; - -import nu.marginalia.storage.model.FileStorageId; - -public record TransferItem(String domainName, - int domainId, - FileStorageId fileStorageId, - String path) { -} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferSpec.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferSpec.java deleted file mode 100644 index 8048a411..00000000 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/transfer/TransferSpec.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.executor.model.transfer; - -import java.util.List; - -public record TransferSpec(List items) { - public TransferSpec() { - this(List.of()); - } - - public int size() { - return items.size(); - } -} diff --git a/code/api/readme.md b/code/api/readme.md deleted file mode 100644 index 31000815..00000000 --- a/code/api/readme.md +++ /dev/null @@ -1,23 +0,0 @@ -# Clients - -## Core Services - -* [assistant-api](assistant-api/) -* [query-api](query-api/) -* [index-api](index-api/) - -These are clients for the [core services](../services-core/), along with what models -are necessary for speaking to them. They each implement the abstract client classes from -[service-client](../common/service-client). - -All that is necessary is to `@Inject` them into the constructor and then -requests can be sent. - -**Note:** If you are looking for the public API, it's handled by the api service in [services-application/api-service](../services-application/api-service). - -## MQ-API Process API - -[process-mqapi](process-mqapi/) defines requests and inboxes for the message queue based API used -for interacting with processes. - -See [libraries/message-queue](../libraries/message-queue) and [services-application/control-service](../services-core/control-service). \ No newline at end of file diff --git a/code/api/executor-api/build.gradle b/code/execution/api/build.gradle similarity index 96% rename from code/api/executor-api/build.gradle rename to code/execution/api/build.gradle index 3ea8f63c..58343ecf 100644 --- a/code/api/executor-api/build.gradle +++ b/code/execution/api/build.gradle @@ -4,14 +4,18 @@ plugins { id "com.google.protobuf" version "0.9.4" } +jar.archiveBaseName = 'execution-api' + java { toolchain { languageVersion.set(JavaLanguageVersion.of(21)) } } + apply from: "$rootProject.projectDir/protobuf.gradle" + sourceSets { main { proto { @@ -20,8 +24,6 @@ sourceSets { } } - - dependencies { implementation project(':code:common:model') implementation project(':code:index:api') diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java similarity index 54% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java rename to code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 225e0518..91f890fa 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -2,16 +2,15 @@ package nu.marginalia.executor.client; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; -import nu.marginalia.executor.api.*; -import nu.marginalia.executor.api.ExecutorApiGrpc.ExecutorApiBlockingStub; import nu.marginalia.executor.model.ActorRunState; import nu.marginalia.executor.model.ActorRunStates; import nu.marginalia.executor.storage.FileStorageContent; import nu.marginalia.executor.storage.FileStorageFile; import nu.marginalia.executor.upload.UploadDirContents; import nu.marginalia.executor.upload.UploadDirItem; +import nu.marginalia.functions.execution.api.*; import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; import nu.marginalia.service.discovery.ServiceRegistryIf; import nu.marginalia.service.discovery.property.ServiceKey; import nu.marginalia.service.discovery.property.ServicePartition; @@ -26,9 +25,10 @@ import java.io.OutputStream; import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.util.List; +import static nu.marginalia.functions.execution.api.ExecutorApiGrpc.*; + @Singleton public class ExecutorClient { private final GrpcMultiNodeChannelPool channelPool; @@ -72,131 +72,13 @@ public class ExecutorClient { } - public void triggerCrawl(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::triggerCrawl) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } - - public void triggerRecrawl(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::triggerRecrawl) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } - - public void triggerConvert(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::triggerConvert) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } - - public void triggerConvertAndLoad(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::triggerConvertAndLoad) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } - - public void loadProcessedData(int node, List ids) { - channelPool.call(ExecutorApiBlockingStub::loadProcessedData) - .forNode(node) - .run(RpcFileStorageIds.newBuilder() - .addAllFileStorageIds(ids.stream().map(FileStorageId::id).toList()) - .build()); - } - public void calculateAdjacencies(int node) { channelPool.call(ExecutorApiBlockingStub::calculateAdjacencies) .forNode(node) .run(Empty.getDefaultInstance()); } - public void sideloadEncyclopedia(int node, Path sourcePath, String baseUrl) { - channelPool.call(ExecutorApiBlockingStub::sideloadEncyclopedia) - .forNode(node) - .run(RpcSideloadEncyclopedia.newBuilder() - .setBaseUrl(baseUrl) - .setSourcePath(sourcePath.toString()) - .build()); - } - public void sideloadDirtree(int node, Path sourcePath) { - channelPool.call(ExecutorApiBlockingStub::sideloadDirtree) - .forNode(node) - .run(RpcSideloadDirtree.newBuilder() - .setSourcePath(sourcePath.toString()) - .build()); - } - public void sideloadReddit(int node, Path sourcePath) { - channelPool.call(ExecutorApiBlockingStub::sideloadReddit) - .forNode(node) - .run(RpcSideloadReddit.newBuilder() - .setSourcePath(sourcePath.toString()) - .build()); - } - public void sideloadWarc(int node, Path sourcePath) { - channelPool.call(ExecutorApiBlockingStub::sideloadWarc) - .forNode(node) - .run(RpcSideloadWarc.newBuilder() - .setSourcePath(sourcePath.toString()) - .build()); - } - - public void sideloadStackexchange(int node, Path sourcePath) { - channelPool.call(ExecutorApiBlockingStub::sideloadStackexchange) - .forNode(node) - .run(RpcSideloadStackexchange.newBuilder() - .setSourcePath(sourcePath.toString()) - .build()); - } - - public void createCrawlSpecFromDownload(int node, String description, String url) { - channelPool.call(ExecutorApiBlockingStub::createCrawlSpecFromDownload) - .forNode(node) - .run(RpcCrawlSpecFromDownload.newBuilder() - .setDescription(description) - .setUrl(url) - .build()); - } - - public void exportAtags(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::exportAtags) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } - public void exportSampleData(int node, FileStorageId fid, int size, String name) { - channelPool.call(ExecutorApiBlockingStub::exportSampleData) - .forNode(node) - .run(RpcExportSampleData.newBuilder() - .setFileStorageId(fid.id()) - .setSize(size) - .setName(name) - .build()); - } - - public void exportRssFeeds(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::exportRssFeeds) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } - public void exportTermFrequencies(int node, FileStorageId fid) { - channelPool.call(ExecutorApiBlockingStub::exportTermFrequencies) - .forNode(node) - .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) - .build()); - } public void downloadSampleData(int node, String sampleSet) { channelPool.call(ExecutorApiBlockingStub::downloadSampleData) @@ -206,17 +88,11 @@ public class ExecutorClient { .build()); } - public void exportData(int node) { - channelPool.call(ExecutorApiBlockingStub::exportData) - .forNode(node) - .run(Empty.getDefaultInstance()); - } - - public void restoreBackup(int node, FileStorageId fid) { + public void restoreBackup(int nodeId, FileStorageId toLoad) { channelPool.call(ExecutorApiBlockingStub::restoreBackup) - .forNode(node) + .forNode(nodeId) .run(RpcFileStorageId.newBuilder() - .setFileStorageId(fid.id()) + .setFileStorageId(toLoad.id()) .build()); } @@ -300,5 +176,4 @@ public class ExecutorClient { } } - } diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorCrawlClient.java b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorCrawlClient.java new file mode 100644 index 00000000..b037702d --- /dev/null +++ b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorCrawlClient.java @@ -0,0 +1,80 @@ +package nu.marginalia.executor.client; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.functions.execution.api.*; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import nu.marginalia.storage.model.FileStorageId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.List; + +import static nu.marginalia.functions.execution.api.ExecutorCrawlApiGrpc.*; + +@Singleton +public class ExecutorCrawlClient { + private final GrpcMultiNodeChannelPool channelPool; + private static final Logger logger = LoggerFactory.getLogger(ExecutorCrawlClient.class); + + @Inject + public ExecutorCrawlClient(GrpcChannelPoolFactory grpcChannelPoolFactory) + { + this.channelPool = grpcChannelPoolFactory + .createMulti( + ServiceKey.forGrpcApi(ExecutorCrawlApiGrpc.class, ServicePartition.multi()), + ExecutorCrawlApiGrpc::newBlockingStub); + } + + public void triggerCrawl(int node, FileStorageId fid) { + channelPool.call(ExecutorCrawlApiBlockingStub::triggerCrawl) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + + public void triggerRecrawl(int node, FileStorageId fid) { + channelPool.call(ExecutorCrawlApiBlockingStub::triggerRecrawl) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + + public void triggerConvert(int node, FileStorageId fid) { + channelPool.call(ExecutorCrawlApiBlockingStub::triggerConvert) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + + public void triggerConvertAndLoad(int node, FileStorageId fid) { + channelPool.call(ExecutorCrawlApiBlockingStub::triggerConvertAndLoad) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + + public void loadProcessedData(int node, List ids) { + channelPool.call(ExecutorCrawlApiBlockingStub::loadProcessedData) + .forNode(node) + .run(RpcFileStorageIds.newBuilder() + .addAllFileStorageIds(ids.stream().map(FileStorageId::id).toList()) + .build()); + } + + public void createCrawlSpecFromDownload(int node, String description, String url) { + channelPool.call(ExecutorCrawlApiBlockingStub::createCrawlSpecFromDownload) + .forNode(node) + .run(RpcCrawlSpecFromDownload.newBuilder() + .setDescription(description) + .setUrl(url) + .build()); + } + +} diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorExportClient.java b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorExportClient.java new file mode 100644 index 00000000..a3286a1b --- /dev/null +++ b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorExportClient.java @@ -0,0 +1,74 @@ +package nu.marginalia.executor.client; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.functions.execution.api.Empty; +import nu.marginalia.functions.execution.api.ExecutorExportApiGrpc; +import nu.marginalia.functions.execution.api.RpcExportSampleData; +import nu.marginalia.functions.execution.api.RpcFileStorageId; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import nu.marginalia.storage.model.FileStorageId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static nu.marginalia.functions.execution.api.ExecutorExportApiGrpc.ExecutorExportApiBlockingStub; + +@Singleton +public class ExecutorExportClient { + private final GrpcMultiNodeChannelPool channelPool; + private static final Logger logger = LoggerFactory.getLogger(ExecutorExportClient.class); + + @Inject + public ExecutorExportClient(GrpcChannelPoolFactory grpcChannelPoolFactory) + { + this.channelPool = grpcChannelPoolFactory + .createMulti( + ServiceKey.forGrpcApi(ExecutorExportApiGrpc.class, ServicePartition.multi()), + ExecutorExportApiGrpc::newBlockingStub); + } + + + public void exportAtags(int node, FileStorageId fid) { + channelPool.call(ExecutorExportApiBlockingStub::exportAtags) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + public void exportSampleData(int node, FileStorageId fid, int size, String name) { + channelPool.call(ExecutorExportApiBlockingStub::exportSampleData) + .forNode(node) + .run(RpcExportSampleData.newBuilder() + .setFileStorageId(fid.id()) + .setSize(size) + .setName(name) + .build()); + } + + public void exportRssFeeds(int node, FileStorageId fid) { + channelPool.call(ExecutorExportApiBlockingStub::exportRssFeeds) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + public void exportTermFrequencies(int node, FileStorageId fid) { + channelPool.call(ExecutorExportApiBlockingStub::exportTermFrequencies) + .forNode(node) + .run(RpcFileStorageId.newBuilder() + .setFileStorageId(fid.id()) + .build()); + } + + public void exportData(int node) { + channelPool.call(ExecutorExportApiBlockingStub::exportData) + .forNode(node) + .run(Empty.getDefaultInstance()); + } + + + +} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java rename to code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorSideloadClient.java b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorSideloadClient.java new file mode 100644 index 00000000..0848c009 --- /dev/null +++ b/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorSideloadClient.java @@ -0,0 +1,71 @@ +package nu.marginalia.executor.client; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.functions.execution.api.*; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; + +import static nu.marginalia.functions.execution.api.ExecutorSideloadApiGrpc.ExecutorSideloadApiBlockingStub; + +@Singleton +public class ExecutorSideloadClient { + private final GrpcMultiNodeChannelPool channelPool; + private static final Logger logger = LoggerFactory.getLogger(ExecutorSideloadClient.class); + + @Inject + public ExecutorSideloadClient(GrpcChannelPoolFactory grpcChannelPoolFactory) + { + this.channelPool = grpcChannelPoolFactory + .createMulti( + ServiceKey.forGrpcApi(ExecutorSideloadApiGrpc.class, ServicePartition.multi()), + ExecutorSideloadApiGrpc::newBlockingStub); + } + + + public void sideloadEncyclopedia(int node, Path sourcePath, String baseUrl) { + channelPool.call(ExecutorSideloadApiBlockingStub::sideloadEncyclopedia) + .forNode(node) + .run(RpcSideloadEncyclopedia.newBuilder() + .setBaseUrl(baseUrl) + .setSourcePath(sourcePath.toString()) + .build()); + } + + public void sideloadDirtree(int node, Path sourcePath) { + channelPool.call(ExecutorSideloadApiBlockingStub::sideloadDirtree) + .forNode(node) + .run(RpcSideloadDirtree.newBuilder() + .setSourcePath(sourcePath.toString()) + .build()); + } + public void sideloadReddit(int node, Path sourcePath) { + channelPool.call(ExecutorSideloadApiBlockingStub::sideloadReddit) + .forNode(node) + .run(RpcSideloadReddit.newBuilder() + .setSourcePath(sourcePath.toString()) + .build()); + } + public void sideloadWarc(int node, Path sourcePath) { + channelPool.call(ExecutorSideloadApiBlockingStub::sideloadWarc) + .forNode(node) + .run(RpcSideloadWarc.newBuilder() + .setSourcePath(sourcePath.toString()) + .build()); + } + + public void sideloadStackexchange(int node, Path sourcePath) { + channelPool.call(ExecutorSideloadApiBlockingStub::sideloadStackexchange) + .forNode(node) + .run(RpcSideloadStackexchange.newBuilder() + .setSourcePath(sourcePath.toString()) + .build()); + } + +} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunState.java b/code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunState.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunState.java rename to code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunState.java diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java b/code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java rename to code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java b/code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java rename to code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java b/code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java rename to code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java b/code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java rename to code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java b/code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java similarity index 100% rename from code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java rename to code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java diff --git a/code/api/executor-api/src/main/protobuf/executor-api.proto b/code/execution/api/src/main/protobuf/executor-api.proto similarity index 93% rename from code/api/executor-api/src/main/protobuf/executor-api.proto rename to code/execution/api/src/main/protobuf/executor-api.proto index 2c2756f2..31cffe9b 100644 --- a/code/api/executor-api/src/main/protobuf/executor-api.proto +++ b/code/execution/api/src/main/protobuf/executor-api.proto @@ -1,39 +1,47 @@ syntax="proto3"; -package actorapi; -option java_package="nu.marginalia.executor.api"; +package nu.marginalia.functions.execution.api; + +option java_package="nu.marginalia.functions.execution.api"; option java_multiple_files=true; service ExecutorApi { rpc startFsm(RpcFsmName) returns (Empty) {} rpc stopFsm(RpcFsmName) returns (Empty) {} - rpc stopProcess(RpcProcessId) returns (Empty) {} + rpc getActorStates(Empty) returns (RpcActorRunStates) {} + rpc listSideloadDir(Empty) returns (RpcUploadDirContents) {} + rpc listFileStorage(RpcFileStorageId) returns (RpcFileStorageContent) {} + + rpc downloadSampleData(RpcDownloadSampleData) returns (Empty) {} + rpc calculateAdjacencies(Empty) returns (Empty) {} + rpc restoreBackup(RpcFileStorageId) returns (Empty) {} +} + +service ExecutorCrawlApi { rpc triggerCrawl(RpcFileStorageId) returns (Empty) {} rpc triggerRecrawl(RpcFileStorageId) returns (Empty) {} rpc triggerConvert(RpcFileStorageId) returns (Empty) {} rpc triggerConvertAndLoad(RpcFileStorageId) returns (Empty) {} rpc loadProcessedData(RpcFileStorageIds) returns (Empty) {} - rpc calculateAdjacencies(Empty) returns (Empty) {} + rpc createCrawlSpecFromDownload(RpcCrawlSpecFromDownload) returns (Empty) {} +} +service ExecutorSideloadApi { rpc sideloadEncyclopedia(RpcSideloadEncyclopedia) returns (Empty) {} rpc sideloadDirtree(RpcSideloadDirtree) returns (Empty) {} rpc sideloadWarc(RpcSideloadWarc) returns (Empty) {} rpc sideloadReddit(RpcSideloadReddit) returns (Empty) {} rpc sideloadStackexchange(RpcSideloadStackexchange) returns (Empty) {} +} - rpc createCrawlSpecFromDownload(RpcCrawlSpecFromDownload) returns (Empty) {} +service ExecutorExportApi { rpc exportAtags(RpcFileStorageId) returns (Empty) {} rpc exportSampleData(RpcExportSampleData) returns (Empty) {} rpc exportRssFeeds(RpcFileStorageId) returns (Empty) {} rpc exportTermFrequencies(RpcFileStorageId) returns (Empty) {} - rpc downloadSampleData(RpcDownloadSampleData) returns (Empty) {} rpc exportData(Empty) returns (Empty) {} - rpc restoreBackup(RpcFileStorageId) returns (Empty) {} - rpc getActorStates(Empty) returns (RpcActorRunStates) {} - rpc listSideloadDir(Empty) returns (RpcUploadDirContents) {} - rpc listFileStorage(RpcFileStorageId) returns (RpcFileStorageContent) {} } message Empty {} diff --git a/code/execution/build.gradle b/code/execution/build.gradle new file mode 100644 index 00000000..79eb9c68 --- /dev/null +++ b/code/execution/build.gradle @@ -0,0 +1,75 @@ +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + // These look weird but they're needed to be able to spawn the processes + // from the executor service + + implementation project(':code:processes:website-adjacencies-calculator') + implementation project(':code:processes:crawling-process') + implementation project(':code:processes:loading-process') + implementation project(':code:processes:converting-process') + implementation project(':code:processes:index-constructor-process') + + implementation project(':code:common:config') + implementation project(':code:common:model') + implementation project(':code:common:process') + implementation project(':code:common:db') + implementation project(':code:common:linkdb') + + implementation project(':code:common:service') + implementation project(':code:common:service-discovery') + + implementation project(':third-party:commons-codec') + + implementation project(':code:libraries:message-queue') + + implementation project(':code:functions:domain-links:api') + implementation project(':code:execution:api') + + implementation project(':code:process-models:crawl-spec') + implementation project(':code:process-models:crawling-model') + implementation project(':code:features-crawl:link-parser') + implementation project(':code:features-convert:data-extractors') + implementation project(':code:features-convert:stackexchange-xml') + implementation project(':code:features-convert:reddit-json') + implementation project(':code:index:index-journal') + implementation project(':code:index:api') + implementation project(':code:process-mqapi') + implementation project(':third-party:encyclopedia-marginalia-nu') + + implementation libs.bundles.slf4j + + implementation libs.spark + implementation libs.bundles.grpc + implementation libs.gson + implementation libs.prometheus + implementation libs.notnull + implementation libs.guice + implementation libs.trove + implementation libs.protobuf + implementation libs.zstd + implementation libs.jsoup + implementation libs.commons.io + implementation libs.commons.compress + implementation libs.commons.lang3 + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') + testImplementation 'org.testcontainers:mariadb:1.17.4' + testImplementation 'org.testcontainers:junit-jupiter:1.17.4' + testImplementation project(':code:libraries:test-helpers') + +} \ No newline at end of file diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/ActorApi.java b/code/execution/src/main/java/nu/marginalia/actor/ActorApi.java similarity index 60% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/ActorApi.java rename to code/execution/src/main/java/nu/marginalia/actor/ActorApi.java index 13923302..608e6ff3 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/ActorApi.java +++ b/code/execution/src/main/java/nu/marginalia/actor/ActorApi.java @@ -2,11 +2,7 @@ package nu.marginalia.actor; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.actor.state.ActorStateInstance; -import nu.marginalia.executor.api.RpcActorRunState; -import nu.marginalia.executor.api.RpcActorRunStates; -import nu.marginalia.executor.api.RpcFsmName; -import nu.marginalia.executor.api.RpcProcessId; +import nu.marginalia.functions.execution.api.*; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.persistence.MqPersistence; import nu.marginalia.process.ProcessService; @@ -15,8 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Spark; -import java.util.Comparator; - @Singleton public class ActorApi { private final ExecutorActorControlService actors; @@ -75,45 +69,6 @@ public class ActorApi { return "OK"; } - - public RpcActorRunStates getActorStates() { - var items = actors.getActorStates().entrySet().stream().map(e -> { - final var stateGraph = actors.getActorDefinition(e.getKey()); - - final ActorStateInstance state = e.getValue(); - final String actorDescription = stateGraph.describe(); - - final String machineName = e.getKey().name(); - final String stateName = state.name(); - - final String stateDescription = ""; - - final boolean terminal = state.isFinal(); - final boolean canStart = actors.isDirectlyInitializable(e.getKey()) && terminal; - - return RpcActorRunState - .newBuilder() - .setActorName(machineName) - .setState(stateName) - .setActorDescription(actorDescription) - .setStateDescription(stateDescription) - .setTerminal(terminal) - .setCanStart(canStart) - .build(); - - }) - .filter(s -> !s.getTerminal() || s.getCanStart()) - .sorted(Comparator.comparing(RpcActorRunState::getActorName)) - .toList(); - - return RpcActorRunStates.newBuilder() - .setNode(serviceConfiguration.node()) - .addAllActorRunStates(items) - .build(); - - } - - public ExecutorActor translateActor(String name) { try { return ExecutorActor.valueOf(name.toUpperCase()); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/ExecutorActor.java b/code/execution/src/main/java/nu/marginalia/actor/ExecutorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/ExecutorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/ExecutorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java b/code/execution/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java rename to code/execution/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java b/code/execution/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java rename to code/execution/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java b/code/execution/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java rename to code/execution/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java b/code/execution/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java b/code/execution/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java b/code/execution/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java b/code/execution/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java b/code/execution/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java b/code/execution/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java b/code/execution/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ConvertActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ConvertActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/CrawlActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/CrawlActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ExportDataActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ExportDataActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RecrawlActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/RecrawlActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RecrawlActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/RecrawlActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java b/code/execution/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java rename to code/execution/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutionInit.java b/code/execution/src/main/java/nu/marginalia/execution/ExecutionInit.java new file mode 100644 index 00000000..faaa16f8 --- /dev/null +++ b/code/execution/src/main/java/nu/marginalia/execution/ExecutionInit.java @@ -0,0 +1,26 @@ +package nu.marginalia.execution; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import nu.marginalia.actor.ExecutorActorControlService; +import nu.marginalia.actor.ExecutorActor; + +@Singleton +public class ExecutionInit { + private final ExecutorActorControlService actorControlService; + + @Inject + public ExecutionInit(ExecutorActorControlService actorControlService) { + this.actorControlService = actorControlService; + } + + public void initDefaultActors() throws Exception { + actorControlService.start(ExecutorActor.MONITOR_PROCESS_LIVENESS); + actorControlService.start(ExecutorActor.MONITOR_FILE_STORAGE); + actorControlService.start(ExecutorActor.PROC_CONVERTER_SPAWNER); + actorControlService.start(ExecutorActor.PROC_CRAWLER_SPAWNER); + actorControlService.start(ExecutorActor.PROC_INDEX_CONSTRUCTOR_SPAWNER); + actorControlService.start(ExecutorActor.PROC_LOADER_SPAWNER); + } +} diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java b/code/execution/src/main/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java new file mode 100644 index 00000000..b95f64d0 --- /dev/null +++ b/code/execution/src/main/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java @@ -0,0 +1,113 @@ +package nu.marginalia.execution; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.actor.ExecutorActor; +import nu.marginalia.actor.ExecutorActorControlService; +import nu.marginalia.actor.task.*; +import nu.marginalia.functions.execution.api.*; +import nu.marginalia.storage.model.FileStorageId; + +import java.util.stream.Collectors; + +public class ExecutorCrawlGrpcService extends ExecutorCrawlApiGrpc.ExecutorCrawlApiImplBase { + private final ExecutorActorControlService actorControlService; + + @Inject + public ExecutorCrawlGrpcService(ExecutorActorControlService actorControlService) + { + this.actorControlService = actorControlService; + } + + @Override + public void triggerCrawl(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CRAWL, + new CrawlActor.Initial(FileStorageId.of(request.getFileStorageId()))); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void triggerRecrawl(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.RECRAWL, + new RecrawlActor.Initial(FileStorageId.of(request.getFileStorageId()), false)); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void triggerConvert(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.Convert(FileStorageId.of(request.getFileStorageId()))); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void triggerConvertAndLoad(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT_AND_LOAD, + new ConvertAndLoadActor.Initial(FileStorageId.of(request.getFileStorageId()))); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void loadProcessedData(RpcFileStorageIds request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT_AND_LOAD, + new ConvertAndLoadActor.Load(request.getFileStorageIdsList() + .stream() + .map(FileStorageId::of) + .collect(Collectors.toList())) + ); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void createCrawlSpecFromDownload(RpcCrawlSpecFromDownload request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CRAWL_JOB_EXTRACTOR, + new CrawlJobExtractorActor.CreateFromUrl( + request.getDescription(), + request.getUrl()) + ); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + +} diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorExportGrpcService.java b/code/execution/src/main/java/nu/marginalia/execution/ExecutorExportGrpcService.java new file mode 100644 index 00000000..41c8bb8b --- /dev/null +++ b/code/execution/src/main/java/nu/marginalia/execution/ExecutorExportGrpcService.java @@ -0,0 +1,95 @@ +package nu.marginalia.execution; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.grpc.stub.StreamObserver; +import nu.marginalia.actor.ExecutorActor; +import nu.marginalia.actor.ExecutorActorControlService; +import nu.marginalia.actor.task.*; +import nu.marginalia.functions.execution.api.Empty; +import nu.marginalia.functions.execution.api.ExecutorExportApiGrpc; +import nu.marginalia.functions.execution.api.RpcExportSampleData; +import nu.marginalia.functions.execution.api.RpcFileStorageId; +import nu.marginalia.storage.model.FileStorageId; + +@Singleton +public class ExecutorExportGrpcService extends ExecutorExportApiGrpc.ExecutorExportApiImplBase { + private final ExecutorActorControlService actorControlService; + + @Inject + public ExecutorExportGrpcService(ExecutorActorControlService actorControlService) { + this.actorControlService = actorControlService; + } + + @Override + public void exportAtags(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.EXPORT_ATAGS, + new ExportAtagsActor.Export(FileStorageId.of(request.getFileStorageId())) + ); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void exportSampleData(RpcExportSampleData request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.EXPORT_SAMPLE_DATA, + new ExportSampleDataActor.Export( + FileStorageId.of(request.getFileStorageId()), + request.getSize(), + request.getName() + ) + ); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void exportRssFeeds(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.EXPORT_FEEDS, + new ExportFeedsActor.Export(FileStorageId.of(request.getFileStorageId())) + ); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void exportTermFrequencies(RpcFileStorageId request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.EXPORT_TERM_FREQUENCIES, + new ExportTermFreqActor.Export(FileStorageId.of(request.getFileStorageId())) + ); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void exportData(Empty request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.EXPORT_DATA, new ExportDataActor.Export()); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } +} diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorGrpcService.java b/code/execution/src/main/java/nu/marginalia/execution/ExecutorGrpcService.java new file mode 100644 index 00000000..b4182bb0 --- /dev/null +++ b/code/execution/src/main/java/nu/marginalia/execution/ExecutorGrpcService.java @@ -0,0 +1,237 @@ +package nu.marginalia.execution; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import lombok.SneakyThrows; +import nu.marginalia.WmsaHome; +import nu.marginalia.actor.ActorApi; +import nu.marginalia.actor.ExecutorActor; +import nu.marginalia.actor.ExecutorActorControlService; +import nu.marginalia.actor.state.ActorStateInstance; +import nu.marginalia.actor.task.DownloadSampleActor; +import nu.marginalia.actor.task.RestoreBackupActor; +import nu.marginalia.actor.task.TriggerAdjacencyCalculationActor; +import nu.marginalia.functions.execution.api.*; +import nu.marginalia.service.module.ServiceConfiguration; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageId; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Comparator; + +public class ExecutorGrpcService extends ExecutorApiGrpc.ExecutorApiImplBase { + private final ActorApi actorApi; + private final FileStorageService fileStorageService; + private final ServiceConfiguration serviceConfiguration; + private final ExecutorActorControlService actorControlService; + + @Inject + public ExecutorGrpcService(ActorApi actorApi, + FileStorageService fileStorageService, + ServiceConfiguration serviceConfiguration, + ExecutorActorControlService actorControlService) + { + this.actorApi = actorApi; + this.fileStorageService = fileStorageService; + this.serviceConfiguration = serviceConfiguration; + this.actorControlService = actorControlService; + } + + @Override + public void startFsm(RpcFsmName request, StreamObserver responseObserver) { + try { + actorApi.startActor(request); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void stopFsm(RpcFsmName request, StreamObserver responseObserver) { + try { + actorApi.stopActor(request); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void stopProcess(RpcProcessId request, StreamObserver responseObserver) { + try { + actorApi.stopProcess(request); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void calculateAdjacencies(Empty request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.ADJACENCY_CALCULATION, + new TriggerAdjacencyCalculationActor.Run()); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void downloadSampleData(RpcDownloadSampleData request, StreamObserver responseObserver) { + try { + String sampleSet = request.getSampleSet(); + + actorControlService.startFrom(ExecutorActor.DOWNLOAD_SAMPLE, + new DownloadSampleActor.Run(sampleSet)); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void restoreBackup(RpcFileStorageId request, StreamObserver responseObserver) { + try { + var fid = FileStorageId.of(request.getFileStorageId()); + + actorControlService.startFrom(ExecutorActor.RESTORE_BACKUP, + new RestoreBackupActor.Restore(fid)); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void getActorStates(Empty request, StreamObserver responseObserver) { + var items = actorControlService.getActorStates().entrySet().stream().map(e -> { + final var stateGraph = actorControlService.getActorDefinition(e.getKey()); + + final ActorStateInstance state = e.getValue(); + final String actorDescription = stateGraph.describe(); + + final String machineName = e.getKey().name(); + final String stateName = state.name(); + + final String stateDescription = ""; + + final boolean terminal = state.isFinal(); + final boolean canStart = actorControlService.isDirectlyInitializable(e.getKey()) && terminal; + + return RpcActorRunState + .newBuilder() + .setActorName(machineName) + .setState(stateName) + .setActorDescription(actorDescription) + .setStateDescription(stateDescription) + .setTerminal(terminal) + .setCanStart(canStart) + .build(); + + }) + .filter(s -> !s.getTerminal() || s.getCanStart()) + .sorted(Comparator.comparing(RpcActorRunState::getActorName)) + .toList(); + + responseObserver.onNext(RpcActorRunStates.newBuilder() + .setNode(serviceConfiguration.node()) + .addAllActorRunStates(items) + .build()); + responseObserver.onCompleted(); + } + + @Override + public void listSideloadDir(Empty request, StreamObserver responseObserver) { + try { + Path uploadDir = WmsaHome.getUploadDir(); + + try (var items = Files.list(uploadDir).sorted( + Comparator.comparing((Path d) -> Files.isDirectory(d)).reversed() + .thenComparing(path -> path.getFileName().toString()) + )) { + var builder = RpcUploadDirContents.newBuilder().setPath(uploadDir.toString()); + + var iter = items.iterator(); + while (iter.hasNext()) { + var path = iter.next(); + + boolean isDir = Files.isDirectory(path); + long size = isDir ? 0 : Files.size(path); + var mtime = Files.getLastModifiedTime(path); + + builder.addEntriesBuilder() + .setName(path.toString()) + .setIsDirectory(isDir) + .setLastModifiedTime( + LocalDateTime.ofInstant(mtime.toInstant(), ZoneId.systemDefault()).format(DateTimeFormatter.ISO_DATE_TIME)) + .setSize(size); + } + + responseObserver.onNext(builder.build()); + } + + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void listFileStorage(RpcFileStorageId request, StreamObserver responseObserver) { + try { + FileStorageId fileStorageId = FileStorageId.of(request.getFileStorageId()); + + var storage = fileStorageService.getStorage(fileStorageId); + + var builder = RpcFileStorageContent.newBuilder(); + + + try (var fs = Files.list(storage.asPath())) { + fs.filter(Files::isRegularFile) + .map(this::createFileModel) + .sorted(Comparator.comparing(RpcFileStorageEntry::getName)) + .forEach(builder::addEntries); + } + + responseObserver.onNext(builder.build()); + + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @SneakyThrows + private RpcFileStorageEntry createFileModel(Path path) { + return RpcFileStorageEntry.newBuilder() + .setName(path.toFile().getName()) + .setSize(Files.size(path)) + .setLastModifiedTime(Files.getLastModifiedTime(path).toInstant().toString()) + .build(); + } + + +} diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java b/code/execution/src/main/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java new file mode 100644 index 00000000..945824c0 --- /dev/null +++ b/code/execution/src/main/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java @@ -0,0 +1,96 @@ +package nu.marginalia.execution; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.actor.ExecutorActor; +import nu.marginalia.actor.ExecutorActorControlService; +import nu.marginalia.actor.task.ConvertActor; +import nu.marginalia.functions.execution.api.*; + +public class ExecutorSideloadGrpcService extends ExecutorSideloadApiGrpc.ExecutorSideloadApiImplBase { + private final ExecutorActorControlService actorControlService; + + @Inject + public ExecutorSideloadGrpcService(ExecutorActorControlService actorControlService) + { + this.actorControlService = actorControlService; + } + + @Override + public void sideloadEncyclopedia(RpcSideloadEncyclopedia request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.ConvertEncyclopedia( + request.getSourcePath(), + request.getBaseUrl() + )); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void sideloadDirtree(RpcSideloadDirtree request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.ConvertDirtree(request.getSourcePath()) + ); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void sideloadReddit(RpcSideloadReddit request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.ConvertReddit(request.getSourcePath()) + ); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void sideloadWarc(RpcSideloadWarc request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.ConvertWarc(request.getSourcePath()) + ); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void sideloadStackexchange(RpcSideloadStackexchange request, StreamObserver responseObserver) { + try { + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.ConvertStackexchange(request.getSourcePath()) + ); + + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } + catch (Exception e) { + responseObserver.onError(e); + } + } + +} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessOutboxes.java b/code/execution/src/main/java/nu/marginalia/process/ProcessOutboxes.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessOutboxes.java rename to code/execution/src/main/java/nu/marginalia/process/ProcessOutboxes.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java b/code/execution/src/main/java/nu/marginalia/process/ProcessService.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java rename to code/execution/src/main/java/nu/marginalia/process/ProcessService.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java b/code/execution/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java rename to code/execution/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/SideloadHelper.java b/code/execution/src/main/java/nu/marginalia/sideload/SideloadHelper.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/sideload/SideloadHelper.java rename to code/execution/src/main/java/nu/marginalia/sideload/SideloadHelper.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java b/code/execution/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java rename to code/execution/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/svc/BackupService.java b/code/execution/src/main/java/nu/marginalia/svc/BackupService.java similarity index 90% rename from code/services-core/executor-service/src/main/java/nu/marginalia/svc/BackupService.java rename to code/execution/src/main/java/nu/marginalia/svc/BackupService.java index 46578348..23b95f6c 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/svc/BackupService.java +++ b/code/execution/src/main/java/nu/marginalia/svc/BackupService.java @@ -3,6 +3,7 @@ package nu.marginalia.svc; import com.github.luben.zstd.ZstdInputStream; import com.github.luben.zstd.ZstdOutputStream; import nu.marginalia.IndexLocations; +import nu.marginalia.linkdb.LinkdbFileNames; import nu.marginalia.service.control.ServiceHeartbeat; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; @@ -18,9 +19,6 @@ import java.sql.SQLException; import java.time.LocalDateTime; import java.util.List; -import static nu.marginalia.linkdb.LinkdbFileNames.DOCDB_FILE_NAME; -import static nu.marginalia.linkdb.LinkdbFileNames.DOMAIN_LINKS_FILE_NAME; - public class BackupService { private final FileStorageService storageService; @@ -59,10 +57,10 @@ public class BackupService { try (var heartbeat = serviceHeartbeat.createServiceTaskHeartbeat(BackupHeartbeatSteps.class, "Backup")) { heartbeat.progress(BackupHeartbeatSteps.DOCS); - backupFileCompressed(DOCDB_FILE_NAME, linkdbStagingStorage, backupStorage.asPath()); + backupFileCompressed(LinkdbFileNames.DOCDB_FILE_NAME, linkdbStagingStorage, backupStorage.asPath()); heartbeat.progress(BackupHeartbeatSteps.LINKS); - backupFileCompressed(DOMAIN_LINKS_FILE_NAME, linkdbStagingStorage, backupStorage.asPath()); + backupFileCompressed(LinkdbFileNames.DOMAIN_LINKS_FILE_NAME, linkdbStagingStorage, backupStorage.asPath()); heartbeat.progress(BackupHeartbeatSteps.JOURNAL); // This file format is already compressed @@ -84,10 +82,10 @@ public class BackupService { try (var heartbeat = serviceHeartbeat.createServiceTaskHeartbeat(BackupHeartbeatSteps.class, "Restore Backup")) { heartbeat.progress(BackupHeartbeatSteps.DOCS); - restoreBackupCompressed(DOCDB_FILE_NAME, linkdbStagingStorage, backupStorage); + restoreBackupCompressed(LinkdbFileNames.DOCDB_FILE_NAME, linkdbStagingStorage, backupStorage); heartbeat.progress(BackupHeartbeatSteps.LINKS); - restoreBackupCompressed(DOMAIN_LINKS_FILE_NAME, linkdbStagingStorage, backupStorage); + restoreBackupCompressed(LinkdbFileNames.DOMAIN_LINKS_FILE_NAME, linkdbStagingStorage, backupStorage); heartbeat.progress(BackupHeartbeatSteps.JOURNAL); restoreJournal(indexStagingStorage, backupStorage); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/svc/DomainListRefreshService.java b/code/execution/src/main/java/nu/marginalia/svc/DomainListRefreshService.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/svc/DomainListRefreshService.java rename to code/execution/src/main/java/nu/marginalia/svc/DomainListRefreshService.java diff --git a/code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java b/code/execution/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java similarity index 100% rename from code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java rename to code/execution/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index b5e69e98..634eb1c5 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -46,7 +46,7 @@ dependencies { implementation project(':code:libraries:message-queue') implementation project(':code:common:service-discovery') implementation project(':code:functions:search-query:api') - implementation project(':code:api:executor-api') + implementation project(':code:execution:api') implementation project(':code:index:api') implementation project(':code:process-mqapi') implementation project(':code:features-search:screenshots') diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java index 03a7baab..b385e32e 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java @@ -24,7 +24,8 @@ public class ControlFileStorageService { private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject - public ControlFileStorageService(FileStorageService fileStorageService, ExecutorClient executorClient) + public ControlFileStorageService(FileStorageService fileStorageService, + ExecutorClient executorClient) { this.fileStorageService = fileStorageService; this.executorClient = executorClient; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java index fbcf07f5..2ae09234 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java @@ -5,6 +5,9 @@ import com.google.inject.Singleton; import nu.marginalia.control.ControlValidationError; import nu.marginalia.control.RedirectControl; import nu.marginalia.executor.client.ExecutorClient; +import nu.marginalia.executor.client.ExecutorCrawlClient; +import nu.marginalia.executor.client.ExecutorExportClient; +import nu.marginalia.executor.client.ExecutorSideloadClient; import nu.marginalia.index.api.IndexMqClient; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.storage.FileStorageService; @@ -31,13 +34,16 @@ public class ControlNodeActionsService { private final FileStorageService fileStorageService; private final ServiceEventLog eventLog; private final ExecutorClient executorClient; + private final ExecutorCrawlClient crawlClient; + private final ExecutorSideloadClient sideloadClient; + private final ExecutorExportClient exportClient; @Inject public ControlNodeActionsService(ExecutorClient executorClient, IndexMqClient indexMqClient, RedirectControl redirectControl, FileStorageService fileStorageService, - ServiceEventLog eventLog) + ServiceEventLog eventLog, ExecutorCrawlClient crawlClient, ExecutorSideloadClient sideloadClient, ExecutorExportClient exportClient) { this.executorClient = executorClient; @@ -46,6 +52,9 @@ public class ControlNodeActionsService { this.fileStorageService = fileStorageService; this.eventLog = eventLog; + this.crawlClient = crawlClient; + this.sideloadClient = sideloadClient; + this.exportClient = exportClient; } public void register() { @@ -127,7 +136,7 @@ public class ControlNodeActionsService { eventLog.logEvent("USER-ACTION", "SIDELOAD ENCYCLOPEDIA " + nodeId); - executorClient.sideloadEncyclopedia(nodeId, sourcePath, baseUrl); + sideloadClient.sideloadEncyclopedia(nodeId, sourcePath, baseUrl); return ""; } @@ -140,7 +149,7 @@ public class ControlNodeActionsService { eventLog.logEvent("USER-ACTION", "SIDELOAD DIRTREE " + nodeId); - executorClient.sideloadDirtree(nodeId, sourcePath); + sideloadClient.sideloadDirtree(nodeId, sourcePath); return ""; } @@ -152,7 +161,7 @@ public class ControlNodeActionsService { eventLog.logEvent("USER-ACTION", "SIDELOAD REDDIT " + nodeId); - executorClient.sideloadReddit(nodeId, sourcePath); + sideloadClient.sideloadReddit(nodeId, sourcePath); return ""; } @@ -163,7 +172,7 @@ public class ControlNodeActionsService { eventLog.logEvent("USER-ACTION", "SIDELOAD WARC " + nodeId); - executorClient.sideloadWarc(nodeId, sourcePath); + sideloadClient.sideloadWarc(nodeId, sourcePath); return ""; } @@ -178,7 +187,7 @@ public class ControlNodeActionsService { eventLog.logEvent("USER-ACTION", "SIDELOAD STACKEXCHANGE " + nodeId); - executorClient.sideloadStackexchange(nodeId, sourcePath); + sideloadClient.sideloadStackexchange(nodeId, sourcePath); return ""; } @@ -196,7 +205,7 @@ public class ControlNodeActionsService { changeActiveStorage(nodeId, FileStorageType.CRAWL_DATA, toCrawl); - executorClient.triggerRecrawl( + crawlClient.triggerRecrawl( nodeId, toCrawl ); @@ -211,7 +220,7 @@ public class ControlNodeActionsService { changeActiveStorage(nodeId, FileStorageType.CRAWL_SPEC, toCrawl); - executorClient.triggerCrawl(nodeId, toCrawl); + crawlClient.triggerCrawl(nodeId, toCrawl); return ""; } @@ -224,10 +233,10 @@ public class ControlNodeActionsService { changeActiveStorage(nodeId, FileStorageType.PROCESSED_DATA, toProcess); if (isAutoload) { - executorClient.triggerConvertAndLoad(nodeId, toProcess); + crawlClient.triggerConvertAndLoad(nodeId, toProcess); } else { - executorClient.triggerConvert(nodeId, toProcess); + crawlClient.triggerConvert(nodeId, toProcess); } return ""; @@ -245,7 +254,7 @@ public class ControlNodeActionsService { changeActiveStorage(nodeId, FileStorageType.PROCESSED_DATA, ids.toArray(new FileStorageId[0])); - executorClient.loadProcessedData(nodeId, ids); + crawlClient.loadProcessedData(nodeId, ids); return ""; } @@ -287,13 +296,13 @@ public class ControlNodeActionsService { throw new ControlValidationError("No url specified", "A url must be specified", ".."); } - executorClient.createCrawlSpecFromDownload(nodeId, description, url); + crawlClient.createCrawlSpecFromDownload(nodeId, description, url); return ""; } private Object exportDbData(Request req, Response rsp) { - executorClient.exportData(Integer.parseInt(req.params("id"))); + exportClient.exportData(Integer.parseInt(req.params("id"))); return ""; } @@ -303,9 +312,9 @@ public class ControlNodeActionsService { FileStorageId source = parseSourceFileStorageId(req.queryParams("source")); switch (exportType) { - case "atags" -> executorClient.exportAtags(Integer.parseInt(req.params("id")), source); - case "rss" -> executorClient.exportRssFeeds(Integer.parseInt(req.params("id")), source); - case "termFreq" -> executorClient.exportTermFrequencies(Integer.parseInt(req.params("id")), source); + case "atags" -> exportClient.exportAtags(Integer.parseInt(req.params("id")), source); + case "rss" -> exportClient.exportRssFeeds(Integer.parseInt(req.params("id")), source); + case "termFreq" -> exportClient.exportTermFrequencies(Integer.parseInt(req.params("id")), source); default -> throw new ControlValidationError("No export type specified", "An export type must be specified", ".."); } @@ -317,7 +326,7 @@ public class ControlNodeActionsService { int size = Integer.parseInt(req.queryParams("size")); String name = req.queryParams("name"); - executorClient.exportSampleData(Integer.parseInt(req.params("id")), source, size, name); + exportClient.exportSampleData(Integer.parseInt(req.params("id")), source, size, name); return ""; } diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index f079a675..68a28eff 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -13,10 +13,6 @@ application { tasks.distZip.enabled = false -clean { - delete fileTree('build/dist-extra') -} - jib { from { image = image = rootProject.ext.dockerImageBase @@ -72,7 +68,10 @@ dependencies { implementation project(':code:index:index-journal') implementation project(':code:index:api') implementation project(':code:process-mqapi') - implementation project(':code:api:executor-api') + + implementation project(':code:execution') + implementation project(':code:execution:api') + implementation project(':third-party:encyclopedia-marginalia-nu') implementation libs.bundles.slf4j diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorGrpcService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorGrpcService.java deleted file mode 100644 index e0969196..00000000 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorGrpcService.java +++ /dev/null @@ -1,325 +0,0 @@ -package nu.marginalia.executor; - -import com.google.inject.Inject; -import io.grpc.stub.StreamObserver; -import nu.marginalia.actor.ActorApi; -import nu.marginalia.executor.api.*; -import nu.marginalia.executor.svc.*; - -public class ExecutorGrpcService extends ExecutorApiGrpc.ExecutorApiImplBase { - private final ActorApi actorApi; - private final ExportService exportService; - private final SideloadService sideloadService; - private final BackupService backupService; - private final TransferService transferService; - private final ProcessingService processingService; - - @Inject - public ExecutorGrpcService(ActorApi actorApi, - ExportService exportService, - SideloadService sideloadService, - BackupService backupService, - TransferService transferService, - ProcessingService processingService) - { - this.actorApi = actorApi; - this.exportService = exportService; - this.sideloadService = sideloadService; - this.backupService = backupService; - this.transferService = transferService; - this.processingService = processingService; - } - - @Override - public void startFsm(RpcFsmName request, StreamObserver responseObserver) { - try { - actorApi.startActor(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void stopFsm(RpcFsmName request, StreamObserver responseObserver) { - try { - actorApi.stopActor(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void stopProcess(RpcProcessId request, StreamObserver responseObserver) { - try { - actorApi.stopProcess(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void triggerCrawl(RpcFileStorageId request, StreamObserver responseObserver) { - try { - processingService.startCrawl(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void triggerRecrawl(RpcFileStorageId request, StreamObserver responseObserver) { - try { - processingService.startRecrawl(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void triggerConvert(RpcFileStorageId request, StreamObserver responseObserver) { - try { - processingService.startConversion(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void triggerConvertAndLoad(RpcFileStorageId request, StreamObserver responseObserver) { - try { - processingService.startConvertLoad(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void loadProcessedData(RpcFileStorageIds request, StreamObserver responseObserver) { - try { - processingService.startLoad(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void calculateAdjacencies(Empty request, StreamObserver responseObserver) { - try { - processingService.startAdjacencyCalculation(); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void sideloadEncyclopedia(RpcSideloadEncyclopedia request, StreamObserver responseObserver) { - try { - sideloadService.sideloadEncyclopedia(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void sideloadDirtree(RpcSideloadDirtree request, StreamObserver responseObserver) { - try { - sideloadService.sideloadDirtree(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void sideloadReddit(RpcSideloadReddit request, StreamObserver responseObserver) { - try { - sideloadService.sideloadReddit(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void sideloadWarc(RpcSideloadWarc request, StreamObserver responseObserver) { - try { - sideloadService.sideloadWarc(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void sideloadStackexchange(RpcSideloadStackexchange request, StreamObserver responseObserver) { - try { - sideloadService.sideloadStackexchange(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void createCrawlSpecFromDownload(RpcCrawlSpecFromDownload request, StreamObserver responseObserver) { - try { - processingService.createCrawlSpecFromDownload(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void exportAtags(RpcFileStorageId request, StreamObserver responseObserver) { - try { - exportService.exportAtags(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void exportSampleData(RpcExportSampleData request, StreamObserver responseObserver) { - try { - exportService.exportSampleData(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void exportRssFeeds(RpcFileStorageId request, StreamObserver responseObserver) { - try { - exportService.exportFeeds(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void exportTermFrequencies(RpcFileStorageId request, StreamObserver responseObserver) { - try { - exportService.exportTermFrequencies(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void downloadSampleData(RpcDownloadSampleData request, StreamObserver responseObserver) { - try { - sideloadService.downloadSampleData(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void exportData(Empty request, StreamObserver responseObserver) { - try { - exportService.exportData(); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - - } - - @Override - public void restoreBackup(RpcFileStorageId request, StreamObserver responseObserver) { - try { - backupService.restore(request); - responseObserver.onNext(Empty.getDefaultInstance()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void getActorStates(Empty request, StreamObserver responseObserver) { - responseObserver.onNext(actorApi.getActorStates()); - responseObserver.onCompleted(); - } - - @Override - public void listSideloadDir(Empty request, StreamObserver responseObserver) { - try { - responseObserver.onNext(sideloadService.listUploadDir()); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } - - @Override - public void listFileStorage(RpcFileStorageId request, StreamObserver responseObserver) { - try { - responseObserver.onNext(transferService.listFiles(request)); - responseObserver.onCompleted(); - } - catch (Exception e) { - responseObserver.onError(e); - } - } -} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java index 61769f5b..a84eebd3 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java @@ -1,50 +1,75 @@ package nu.marginalia.executor; import com.google.inject.Inject; -import nu.marginalia.actor.ExecutorActor; -import nu.marginalia.actor.ExecutorActorControlService; -import nu.marginalia.executor.svc.TransferService; +import nu.marginalia.execution.*; import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import nu.marginalia.service.server.mq.MqRequest; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageId; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; import spark.Spark; +import java.io.IOException; +import java.nio.file.Path; +import java.sql.SQLException; import java.util.List; // Weird name for this one to not have clashes with java.util.concurrent.ExecutorService public class ExecutorSvc extends Service { - private final ExecutorActorControlService actorControlService; private static final Logger logger = LoggerFactory.getLogger(ExecutorSvc.class); + private final ExecutionInit executionInit; + private final FileStorageService fileStorageService; @Inject - public ExecutorSvc(BaseServiceParams params, - ExecutorActorControlService actorControlService, ExecutorGrpcService executorGrpcService, - TransferService transferService) + ExecutorCrawlGrpcService executorCrawlGrpcService, + ExecutorSideloadGrpcService executorSideloadGrpcService, + ExecutorExportGrpcService executorExportGrpcService, + ExecutionInit executionInit, + FileStorageService fileStorageService) { super(params, ServicePartition.partition(params.configuration.node()), - List.of(executorGrpcService)); - this.actorControlService = actorControlService; + List.of(executorGrpcService, + executorCrawlGrpcService, + executorSideloadGrpcService, + executorExportGrpcService) + ); - Spark.get("/transfer/file/:fid", transferService::transferFile); + this.executionInit = executionInit; + this.fileStorageService = fileStorageService; + + Spark.get("/transfer/file/:fid", this::transferFile); } @MqRequest(endpoint="FIRST-BOOT") public void setUpDefaultActors(String message) throws Exception { logger.info("Initializing default actors"); - actorControlService.start(ExecutorActor.MONITOR_PROCESS_LIVENESS); - actorControlService.start(ExecutorActor.MONITOR_FILE_STORAGE); - actorControlService.start(ExecutorActor.PROC_CONVERTER_SPAWNER); - actorControlService.start(ExecutorActor.PROC_CRAWLER_SPAWNER); - actorControlService.start(ExecutorActor.PROC_INDEX_CONSTRUCTOR_SPAWNER); - actorControlService.start(ExecutorActor.PROC_LOADER_SPAWNER); + executionInit.initDefaultActors(); + } + + /** Allows transfer of files from each partition */ + private Object transferFile(Request request, Response response) throws SQLException, IOException { + FileStorageId fileStorageId = FileStorageId.parse(request.params("fid")); + + var fileStorage = fileStorageService.getStorage(fileStorageId); + + Path basePath = fileStorage.asPath(); + // This is not a public API so injection isn't a concern + Path filePath = basePath.resolve(request.queryParams("path")); + + response.type("application/octet-stream"); + FileUtils.copyFile(filePath.toFile(), response.raw().getOutputStream()); + return ""; } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/BackupService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/BackupService.java deleted file mode 100644 index 45f8c622..00000000 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/BackupService.java +++ /dev/null @@ -1,22 +0,0 @@ -package nu.marginalia.executor.svc; - -import com.google.inject.Inject; -import nu.marginalia.actor.ExecutorActor; -import nu.marginalia.actor.ExecutorActorControlService; -import nu.marginalia.actor.task.RestoreBackupActor; -import nu.marginalia.executor.api.RpcFileStorageId; -import nu.marginalia.storage.model.FileStorageId; - -public class BackupService { - private final ExecutorActorControlService actorControlService; - - @Inject - public BackupService(ExecutorActorControlService actorControlService) { - this.actorControlService = actorControlService; - } - - public void restore(RpcFileStorageId request) throws Exception { - var fid = FileStorageId.of(request.getFileStorageId()); - actorControlService.startFrom(ExecutorActor.RESTORE_BACKUP, new RestoreBackupActor.Restore(fid)); - } -} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java deleted file mode 100644 index 9f941ab9..00000000 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java +++ /dev/null @@ -1,51 +0,0 @@ -package nu.marginalia.executor.svc; - -import com.google.inject.Inject; -import nu.marginalia.actor.ExecutorActor; -import nu.marginalia.actor.ExecutorActorControlService; -import nu.marginalia.actor.task.*; -import nu.marginalia.executor.api.RpcExportSampleData; -import nu.marginalia.executor.api.RpcFileStorageId; -import nu.marginalia.storage.model.FileStorageId; - -public class ExportService { - private final ExecutorActorControlService actorControlService; - - @Inject - public ExportService(ExecutorActorControlService actorControlService) { - this.actorControlService = actorControlService; - } - - public void exportData() throws Exception { - actorControlService.startFrom(ExecutorActor.EXPORT_DATA, new ExportDataActor.Export()); - } - - public void exportSampleData(RpcExportSampleData request) throws Exception { - actorControlService.startFrom(ExecutorActor.EXPORT_SAMPLE_DATA, - new ExportSampleDataActor.Export( - FileStorageId.of(request.getFileStorageId()), - request.getSize(), - request.getName() - ) - ); - } - - public void exportAtags(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.EXPORT_ATAGS, - new ExportAtagsActor.Export(FileStorageId.of(request.getFileStorageId())) - ); - } - - public void exportFeeds(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.EXPORT_FEEDS, - new ExportFeedsActor.Export(FileStorageId.of(request.getFileStorageId())) - ); - } - - public void exportTermFrequencies(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.EXPORT_TERM_FREQUENCIES, - new ExportTermFreqActor.Export(FileStorageId.of(request.getFileStorageId())) - ); - } - -} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ProcessingService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ProcessingService.java deleted file mode 100644 index bee5c2c1..00000000 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ProcessingService.java +++ /dev/null @@ -1,62 +0,0 @@ -package nu.marginalia.executor.svc; - -import com.google.inject.Inject; -import nu.marginalia.actor.ExecutorActor; -import nu.marginalia.actor.ExecutorActorControlService; -import nu.marginalia.actor.task.*; -import nu.marginalia.executor.api.RpcCrawlSpecFromDownload; -import nu.marginalia.executor.api.RpcFileStorageId; -import nu.marginalia.executor.api.RpcFileStorageIds; -import nu.marginalia.storage.model.FileStorageId; - -import java.util.stream.Collectors; - -public class ProcessingService { - private final ExecutorActorControlService actorControlService; - - @Inject - public ProcessingService(ExecutorActorControlService actorControlService) { - this.actorControlService = actorControlService; - } - - public void startRecrawl(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.RECRAWL, - new RecrawlActor.Initial(FileStorageId.of(request.getFileStorageId()), false)); - } - - public void startCrawl(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.CRAWL, - new CrawlActor.Initial(FileStorageId.of(request.getFileStorageId()))); - } - - public void startConversion(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, - new ConvertActor.Convert(FileStorageId.of(request.getFileStorageId()))); - } - - public void startConvertLoad(RpcFileStorageId request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT_AND_LOAD, - new ConvertAndLoadActor.Initial(FileStorageId.of(request.getFileStorageId()))); - } - - public void startLoad(RpcFileStorageIds request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT_AND_LOAD, - new ConvertAndLoadActor.Load(request.getFileStorageIdsList() - .stream() - .map(FileStorageId::of) - .collect(Collectors.toList())) - ); - } - - public void startAdjacencyCalculation() throws Exception { - actorControlService.startFrom(ExecutorActor.ADJACENCY_CALCULATION, new TriggerAdjacencyCalculationActor.Run()); - } - - public void createCrawlSpecFromDownload(RpcCrawlSpecFromDownload request) throws Exception { - actorControlService.startFrom(ExecutorActor.CRAWL_JOB_EXTRACTOR, - new CrawlJobExtractorActor.CreateFromUrl( - request.getDescription(), - request.getUrl()) - ); - } -} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java deleted file mode 100644 index 136b6f1d..00000000 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java +++ /dev/null @@ -1,96 +0,0 @@ -package nu.marginalia.executor.svc; - -import com.google.inject.Inject; -import nu.marginalia.WmsaHome; -import nu.marginalia.actor.ExecutorActor; -import nu.marginalia.actor.ExecutorActorControlService; -import nu.marginalia.actor.task.ConvertActor; -import nu.marginalia.actor.task.DownloadSampleActor; -import nu.marginalia.executor.api.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Comparator; - -public class SideloadService { - private final ExecutorActorControlService actorControlService; - private static final Logger logger = LoggerFactory.getLogger(SideloadService.class); - - @Inject - public SideloadService(ExecutorActorControlService actorControlService) { - this.actorControlService = actorControlService; - } - - public void sideloadDirtree(RpcSideloadDirtree request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, - new ConvertActor.ConvertDirtree(request.getSourcePath()) - ); - } - - public void sideloadReddit(RpcSideloadReddit request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, - new ConvertActor.ConvertReddit(request.getSourcePath()) - ); - } - - public void sideloadWarc(RpcSideloadWarc request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, - new ConvertActor.ConvertWarc(request.getSourcePath()) - ); - } - - public void sideloadEncyclopedia(RpcSideloadEncyclopedia request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, - new ConvertActor.ConvertEncyclopedia( - request.getSourcePath(), - request.getBaseUrl() - )); - } - - public void sideloadStackexchange(RpcSideloadStackexchange request) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, - new ConvertActor.ConvertStackexchange(request.getSourcePath()) - ); - } - - public RpcUploadDirContents listUploadDir() throws IOException { - Path uploadDir = WmsaHome.getUploadDir(); - - try (var items = Files.list(uploadDir).sorted( - Comparator.comparing((Path d) -> Files.isDirectory(d)).reversed() - .thenComparing(path -> path.getFileName().toString()) - )) { - var builder = RpcUploadDirContents.newBuilder().setPath(uploadDir.toString()); - - var iter = items.iterator(); - while (iter.hasNext()) { - var path = iter.next(); - - boolean isDir = Files.isDirectory(path); - long size = isDir ? 0 : Files.size(path); - var mtime = Files.getLastModifiedTime(path); - - builder.addEntriesBuilder() - .setName(path.toString()) - .setIsDirectory(isDir) - .setLastModifiedTime( - LocalDateTime.ofInstant(mtime.toInstant(), ZoneId.systemDefault()).format(DateTimeFormatter.ISO_DATE_TIME)) - .setSize(size); - } - - return builder.build(); - } - } - - public void downloadSampleData(RpcDownloadSampleData request) throws Exception { - String sampleSet = request.getSampleSet(); - - actorControlService.startFrom(ExecutorActor.DOWNLOAD_SAMPLE, new DownloadSampleActor.Run(sampleSet)); - } -} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java deleted file mode 100644 index 0fdd6c65..00000000 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/TransferService.java +++ /dev/null @@ -1,97 +0,0 @@ -package nu.marginalia.executor.svc; - -import com.google.gson.Gson; -import com.google.inject.Inject; -import com.zaxxer.hikari.HikariDataSource; -import lombok.SneakyThrows; -import nu.marginalia.executor.api.RpcFileStorageContent; -import nu.marginalia.executor.api.RpcFileStorageEntry; -import nu.marginalia.executor.api.RpcFileStorageId; -import nu.marginalia.executor.client.ExecutorClient; -import nu.marginalia.mq.persistence.MqPersistence; -import nu.marginalia.service.module.ServiceConfiguration; -import nu.marginalia.storage.FileStorageService; -import nu.marginalia.storage.model.FileStorageId; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.Comparator; - -public class TransferService { - private final Gson gson; - private final FileStorageService fileStorageService; - private final HikariDataSource dataSource; - private final ExecutorClient executorClient; - private final MqPersistence persistence; - private final String executorServiceName; - private final int nodeId; - - private static final Logger logger = LoggerFactory.getLogger(TransferService.class); - @Inject - public TransferService( - Gson gson, - FileStorageService fileStorageService, - HikariDataSource dataSource, - ExecutorClient executorClient, MqPersistence persistence, ServiceConfiguration config) - { - this.gson = gson; - this.fileStorageService = fileStorageService; - this.dataSource = dataSource; - this.executorClient = executorClient; - this.persistence = persistence; - this.nodeId = config.node(); - this.executorServiceName = config.serviceName(); - } - - public Object transferFile(Request request, Response response) throws SQLException, IOException { - FileStorageId fileStorageId = FileStorageId.parse(request.params("fid")); - - var fileStorage = fileStorageService.getStorage(fileStorageId); - - Path basePath = fileStorage.asPath(); - // This is not a public API so injection isn't a concern - Path filePath = basePath.resolve(request.queryParams("path")); - - response.type("application/octet-stream"); - FileUtils.copyFile(filePath.toFile(), response.raw().getOutputStream()); - return ""; - } - - - public RpcFileStorageContent listFiles(RpcFileStorageId request) throws SQLException, IOException { - FileStorageId fileStorageId = FileStorageId.of(request.getFileStorageId()); - - var storage = fileStorageService.getStorage(fileStorageId); - - var builder = RpcFileStorageContent.newBuilder(); - - - try (var fs = Files.list(storage.asPath())) { - fs.filter(Files::isRegularFile) - .map(this::createFileModel) - .sorted(Comparator.comparing(RpcFileStorageEntry::getName)) - .forEach(builder::addEntries); - } - - return builder.build(); - } - - @SneakyThrows - private RpcFileStorageEntry createFileModel(Path path) { - return RpcFileStorageEntry.newBuilder() - .setName(path.toFile().getName()) - .setSize(Files.size(path)) - .setLastModifiedTime(Files.getLastModifiedTime(path).toInstant().toString()) - .build(); - } - - - public record TransferReq(int sourceNode, int count) { } -} diff --git a/settings.gradle b/settings.gradle index 15561f1a..8fe58024 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,9 @@ include 'code:functions:domain-links:api' include 'code:functions:search-query' include 'code:functions:search-query:api' +include 'code:execution' +include 'code:execution:api' + include 'code:index' include 'code:index:api' include 'code:index:index-journal' @@ -66,7 +69,6 @@ include 'code:features-crawl:link-parser' include 'code:features-crawl:content-type' include 'code:process-mqapi' -include 'code:api:executor-api' include 'code:common:service-discovery' include 'code:common:db' From 1d342244169fedac70e2d6344e4aadaf6c3f65ac Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 23 Feb 2024 16:13:40 +0100 Subject: [PATCH 31/56] (refac) Remove src/main from all source code paths. Look, this will make the git history look funny, but trimming unnecessary depth from the source tree is a very necessary sanity-preserving measure when dealing with a super-modularized codebase like this one. While it makes the project configuration a bit less conventional, it will save you several clicks every time you jump between modules. Which you'll do a lot, because it's *modul*ar. The src/main/java convention makes a lot of sense for a non-modular project though. This ain't that. --- build.gradle | 33 ++++-------- code/common/config/build.gradle | 2 + .../java/nu/marginalia/IndexLocations.java | 0 .../java/nu/marginalia/LanguageModels.java | 0 .../java/nu/marginalia/UserAgent.java | 0 .../java/nu/marginalia/WebsiteUrl.java | 0 .../java/nu/marginalia/WmsaHome.java | 0 .../nodecfg/NodeConfigurationService.java | 0 .../nodecfg/model/NodeConfiguration.java | 0 .../storage/FileStorageManifest.java | 0 .../storage/FileStorageService.java | 0 .../marginalia/storage/model/FileStorage.java | 0 .../storage/model/FileStorageBase.java | 0 .../storage/model/FileStorageBaseId.java | 0 .../storage/model/FileStorageBaseType.java | 0 .../storage/model/FileStorageId.java | 0 .../storage/model/FileStorageState.java | 0 .../storage/model/FileStorageType.java | 0 .../nodecfg/NodeConfigurationServiceTest.java | 0 .../storage/FileStorageServiceTest.java | 0 code/common/db/build.gradle | 2 + .../nu/marginalia/db/DbDomainQueries.java | 0 .../db/DbDomainStatsExportMultitool.java | 0 .../nu/marginalia/db/DomainBlacklist.java | 0 .../nu/marginalia/db/DomainBlacklistImpl.java | 0 .../db/DomainRankingSetsService.java | 0 .../java/nu/marginalia/db/DomainTypes.java | 0 .../db/migration/V23_06_0_000__base.sql | 0 .../db/migration/V23_06_0_001__blacklist.sql | 0 .../db/migration/V23_06_0_002__dictionary.sql | 0 .../migration/V23_06_0_003__crawl-queue.sql | 0 .../db/migration/V23_06_0_004__screenshot.sql | 0 .../V23_06_0_005__domain_complaint.sql | 0 .../db/migration/V23_06_0_006__api_key.sql | 0 .../db/migration/V23_06_0_007__neighbors.sql | 0 .../V23_06_0_008__random_domains.sql | 0 .../db/migration/V23_06_0_009__news_feed.sql | 0 .../migration/V23_07_0_001__domain_type.sql | 0 .../V23_07_0_002__service_status.sql | 0 .../migration/V23_07_0_003__message_queue.sql | 0 .../migration/V23_07_0_004__file_storage.sql | 0 ..._07_0_005__file_storage_default_values.sql | 0 ...3_07_0_006__message_queue_default_jobs.sql | 0 .../migration/V23_07_0_007__task_status.sql | 0 .../migration/V23_07_0_008__events_index.sql | 0 .../V23_09_0_000__filestorage_livedb.sql | 0 .../db/migration/V23_09_1_000__drop_ecurl.sql | 0 .../V23_09_2_000__filestorage_backup.sql | 0 .../V23_09_2_001__filestorage_no_lexicon.sql | 0 .../V23_11_0_000__file_storage_node.sql | 0 .../V23_11_0_001__heartbeat_node.sql | 0 .../V23_11_0_002__file_storage_state.sql | 0 .../V23_11_0_003__node_configuration.sql | 0 .../V23_11_0_004__file_storage_base_type.sql | 0 .../V23_11_0_005__clean_message_queue.sql | 0 .../migration/V23_11_0_006__clean_stores.sql | 0 .../V23_11_0_007__domain_node_affinity.sql | 0 .../V23_11_0_008__purge_procedure.sql | 0 .../V24_01_0_001__node_config__keep_warc.sql | 0 .../db/migration/V24_01_0_002__domain_set.sql | 0 .../db/migration/V24_01_0_003__mqaudit.sql | 0 .../V24_02_0_000__drop_domain_links.sql | 0 .../V24_02_0_001__drop_ranking_set_algo.sql | 0 .../db/DomainRankingSetsServiceTest.java | 0 .../nu/marginalia/db/DomainTypesTest.java | 0 code/common/linkdb/build.gradle | 2 + .../nu/marginalia/linkdb/LinkdbFileNames.java | 0 .../linkdb/dlinks/DelayingDomainLinkDb.java | 0 .../linkdb/dlinks/DomainLinkDb.java | 0 .../linkdb/dlinks/DomainLinkDbLoader.java | 0 .../linkdb/dlinks/DomainLinkDbWriter.java | 0 .../linkdb/dlinks/FileDomainLinkDb.java | 0 .../linkdb/docs/DocumentDbReader.java | 0 .../linkdb/docs/DocumentDbWriter.java | 0 .../linkdb/model/DocdbUrlDetail.java | 0 .../main => }/resources/db/docdb-document.sql | 0 .../linkdb/DocumentDbWriterTest.java | 0 .../marginalia/linkdb/DomainLinkDbTest.java | 0 code/common/model/build.gradle | 2 + .../java/nu/marginalia/model/EdgeDomain.java | 0 .../java/nu/marginalia/model/EdgeUrl.java | 0 .../model/crawl/DomainIndexingState.java | 0 .../marginalia/model/crawl/HtmlFeature.java | 0 .../nu/marginalia/model/crawl/PubDate.java | 0 .../model/crawl/UrlIndexingState.java | 0 .../nu/marginalia/model/gson/GsonFactory.java | 0 .../marginalia/model/html/HtmlStandard.java | 0 .../nu/marginalia/model/id/UrlIdCodec.java | 0 .../marginalia/model/idx/DocumentFlags.java | 0 .../model/idx/DocumentMetadata.java | 0 .../nu/marginalia/model/idx/WordFlags.java | 0 .../nu/marginalia/model/idx/WordMetadata.java | 0 .../java/nu/marginalia/util/QueryParams.java | 0 .../java/nu/marginalia/util/StringPool.java | 0 .../model/DocumentMetadataTest.java | 0 .../nu/marginalia/model/EdgeDomainTest.java | 0 .../nu/marginalia/model/EdgeUrlTest.java | 0 .../nu/marginalia/model/WordMetadataTest.java | 0 .../marginalia/model/id/UrlIdCodecTest.java | 0 code/common/process/build.gradle | 2 + .../nu/marginalia/ProcessConfiguration.java | 0 .../ProcessConfigurationModule.java | 0 .../process/control/FakeProcessHeartbeat.java | 0 .../control/ProcessAdHocTaskHeartbeat.java | 0 .../ProcessAdHocTaskHeartbeatImpl.java | 0 .../process/control/ProcessHeartbeat.java | 0 .../process/control/ProcessHeartbeatImpl.java | 0 .../process/control/ProcessTaskHeartbeat.java | 0 .../control/ProcessTaskHeartbeatImpl.java | 0 .../process/log/WorkLoadIterable.java | 0 .../nu/marginalia/process/log/WorkLog.java | 0 .../marginalia/process/log/WorkLogEntry.java | 0 .../main => }/resources/log4j2.properties | 0 .../marginalia/process/log/WorkLogTest.java | 0 code/common/renderer/build.gradle | 2 + .../marginalia/renderer/MustacheRenderer.java | 0 .../marginalia/renderer/RendererFactory.java | 0 .../renderer/RenderingException.java | 0 .../config/DefaultHandlebarsConfigurator.java | 0 .../config/HandlebarsConfigurator.java | 0 code/common/service-discovery/build.gradle | 2 + .../service/NodeConfigurationWatcher.java | 0 .../service/ServiceDiscoveryModule.java | 0 .../ServiceHomeNotConfiguredException.java | 0 .../marginalia/service/ServiceMonitors.java | 0 .../client/GrpcChannelPoolFactory.java | 0 .../client/GrpcMultiNodeChannelPool.java | 0 .../client/GrpcSingleNodeChannelPool.java | 0 .../client/ServiceNotAvailableException.java | 0 .../service/discovery/ServiceRegistryIf.java | 0 .../service/discovery/ZkServiceRegistry.java | 0 .../monitor/ServiceChangeMonitor.java | 0 .../discovery/monitor/ServiceMonitorIf.java | 0 .../discovery/property/PartitionTraits.java | 0 .../discovery/property/ServiceEndpoint.java | 0 .../discovery/property/ServiceKey.java | 0 .../discovery/property/ServicePartition.java | 0 .../nu/marginalia/service/id/ServiceId.java | 0 .../discovery/ZkServiceRegistryTest.java | 0 code/common/service/build.gradle | 2 + .../nu/marginalia/service/ConfigLoader.java | 0 .../java/nu/marginalia/service/MainClass.java | 0 .../marginalia/service/ProcessMainClass.java | 0 .../service/control/FakeServiceHeartbeat.java | 0 .../service/control/ServiceEventLog.java | 0 .../service/control/ServiceHeartbeat.java | 0 .../service/control/ServiceHeartbeatImpl.java | 0 .../service/control/ServiceTaskHeartbeat.java | 0 .../control/ServiceTaskHeartbeatImpl.java | 0 .../service/module/DatabaseModule.java | 0 .../service/module/ServiceConfiguration.java | 0 .../module/ServiceConfigurationModule.java | 0 .../service/server/BaseServiceParams.java | 0 .../service/server/Initialization.java | 0 .../service/server/MetricsServer.java | 0 .../service/server/NodeStatusWatcher.java | 0 .../service/server/RateLimiter.java | 0 .../nu/marginalia/service/server/Service.java | 0 .../service/server/StaticResources.java | 0 .../service/server/mq/MqRequest.java | 0 .../server/mq/ServiceMqSubscription.java | 0 .../{src/main => }/resources/log4j2-prod.xml | 0 .../{src/main => }/resources/log4j2-test.xml | 0 code/execution/api/build.gradle | 4 +- .../executor/client/ExecutorClient.java | 0 .../executor/client/ExecutorCrawlClient.java | 0 .../executor/client/ExecutorExportClient.java | 0 .../client/ExecutorRemoteActorFactory.java | 0 .../client/ExecutorSideloadClient.java | 0 .../executor/model/ActorRunState.java | 0 .../executor/model/ActorRunStates.java | 0 .../executor/storage/FileStorageContent.java | 0 .../executor/storage/FileStorageFile.java | 0 .../executor/upload/UploadDirContents.java | 0 .../executor/upload/UploadDirItem.java | 0 code/execution/build.gradle | 2 + .../java/nu/marginalia/actor/ActorApi.java | 0 .../nu/marginalia/actor/ExecutorActor.java | 0 .../actor/ExecutorActorControlService.java | 0 .../actor/ExecutorActorStateMachines.java | 0 .../monitor/AbstractProcessSpawnerActor.java | 0 .../monitor/FileStorageMonitorActor.java | 0 .../actor/proc/ConverterMonitorActor.java | 0 .../actor/proc/CrawlerMonitorActor.java | 0 .../proc/IndexConstructorMonitorActor.java | 0 .../actor/proc/LoaderMonitorActor.java | 0 .../proc/ProcessLivenessMonitorActor.java | 0 .../actor/task/ActorProcessWatcher.java | 0 .../marginalia/actor/task/ConvertActor.java | 0 .../actor/task/ConvertAndLoadActor.java | 0 .../nu/marginalia/actor/task/CrawlActor.java | 0 .../actor/task/CrawlJobExtractorActor.java | 0 .../actor/task/DownloadSampleActor.java | 0 .../actor/task/ExportAtagsActor.java | 0 .../actor/task/ExportDataActor.java | 0 .../actor/task/ExportFeedsActor.java | 0 .../actor/task/ExportSampleDataActor.java | 0 .../actor/task/ExportTermFreqActor.java | 0 .../marginalia/actor/task/RecrawlActor.java | 0 .../actor/task/RestoreBackupActor.java | 0 .../TriggerAdjacencyCalculationActor.java | 0 .../marginalia/execution/ExecutionInit.java | 0 .../execution/ExecutorCrawlGrpcService.java | 0 .../execution/ExecutorExportGrpcService.java | 0 .../execution/ExecutorGrpcService.java | 0 .../ExecutorSideloadGrpcService.java | 0 .../marginalia/process/ProcessOutboxes.java | 0 .../nu/marginalia/process/ProcessService.java | 0 .../sideload/RedditSideloadHelper.java | 0 .../marginalia/sideload/SideloadHelper.java | 0 .../sideload/StackExchangeSideloadHelper.java | 0 .../java/nu/marginalia/svc/BackupService.java | 0 .../svc/DomainListRefreshService.java | 0 .../svc/DomainListRefreshServiceTest.java | 0 code/features-convert/adblock/build.gradle | 2 + .../marginalia/adblock/AdblockSimulator.java | 0 .../adblock/GoogleAnwersSpamDetector.java | 0 .../anchor-keywords/build.gradle | 2 + .../marginalia/atags/AnchorTextKeywords.java | 0 .../marginalia/atags/model/DomainLinks.java | 0 .../java/nu/marginalia/atags/model/Link.java | 0 .../marginalia/atags/model/LinkWithText.java | 0 .../atags/source/AnchorTagsImpl.java | 0 .../atags/source/AnchorTagsSource.java | 0 .../atags/source/AnchorTagsSourceFactory.java | 0 .../{src/main => }/resources/atags-stop-list | 0 .../atags/DomainAnchorTagsImplTest.java | 0 .../marginalia/util/TestLanguageModels.java | 0 .../data-extractors/build.gradle | 2 + .../nu/marginalia/extractor/AtagExporter.java | 0 .../nu/marginalia/extractor/ExporterIf.java | 0 .../nu/marginalia/extractor/FeedExporter.java | 0 .../extractor/SampleDataExporter.java | 0 .../extractor/TermFrequencyExporter.java | 0 .../keyword-extraction/build.gradle | 2 + .../keyword/DocumentKeywordExtractor.java | 0 .../marginalia/keyword/KeywordExtractor.java | 0 .../marginalia/keyword/KeywordMetadata.java | 0 .../java/nu/marginalia/keyword/WordReps.java | 0 .../keyword/extractors/ArtifactKeywords.java | 0 .../extractors/KeywordPositionBitmask.java | 0 .../keyword/extractors/NameLikeKeywords.java | 0 .../extractors/SubjectLikeKeywords.java | 0 .../keyword/extractors/TitleKeywords.java | 0 .../keyword/extractors/UrlKeywords.java | 0 .../keyword/extractors/WordsTfIdfCounts.java | 0 .../keyword/model/DocumentKeywords.java | 0 .../model/DocumentKeywordsBuilder.java | 0 .../model/DocumentKeywordsPointer.java | 0 .../test-data/java.html | 0 .../test-data/keyboards.html | 0 .../test-data/madonna.html | 0 .../test-data/spam.html | 0 .../keyword/DocumentKeywordExtractorTest.java | 0 .../keyword/SentenceExtractorTest.java | 0 .../extractors/ArtifactKeywordsTest.java | 0 .../extractors/NameLikeKeywordsTest.java | 0 .../extractors/SubjectLikeKeywordsTest.java | 0 .../keyword/extractors/TitleKeywordsTest.java | 0 .../keyword/extractors/UrlKeywordsTest.java | 0 .../test/util/TestLanguageModels.java | 0 code/features-convert/pubdate/build.gradle | 2 + .../pubdate/PubDateEffortLevel.java | 0 .../pubdate/PubDateFromHtmlStandard.java | 0 .../marginalia/pubdate/PubDateHeuristic.java | 0 .../nu/marginalia/pubdate/PubDateParser.java | 0 .../nu/marginalia/pubdate/PubDateSniffer.java | 0 .../PubDateHeuristicDOMParsingPass1.java | 0 .../PubDateHeuristicDOMParsingPass2.java | 0 ...PubDateHeuristicGuessFromHtmlStandard.java | 0 .../PubDateHeuristicHtml5AnyTimeTag.java | 0 .../PubDateHeuristicHtml5ArticleDateTag.java | 0 .../PubDateHeuristicHtml5ItempropDateTag.java | 0 .../heuristic/PubDateHeuristicJSONLD.java | 0 .../PubDateHeuristicLastModified.java | 0 .../heuristic/PubDateHeuristicMicrodata.java | 0 .../heuristic/PubDateHeuristicOpenGraph.java | 0 .../heuristic/PubDateHeuristicRDFaTag.java | 0 .../PubDateHeuristicUrlPatternPass1.java | 0 .../PubDateHeuristicUrlPatternPass2.java | 0 .../pubdate/PubDateSnifferTest.java | 0 .../nu/marginalia/pubdate/PubDateTest.java | 0 .../features-convert/reddit-json/build.gradle | 2 + .../integration/reddit/RedditEntryReader.java | 0 .../integration/reddit/db/RedditDb.java | 0 .../model/ProcessableRedditComment.java | 0 .../model/ProcessableRedditSubmission.java | 0 .../reddit/model/RawRedditComment.java | 0 .../reddit/model/RawRedditSubmission.java | 0 .../{src/main => }/resources/db/reddit.sql | 0 .../reddit/RedditEntryReaderTest.java | 0 .../integration/reddit/db/RedditDbTest.java | 0 .../stackexchange-xml/build.gradle | 2 + .../model/StackExchangeComment.java | 0 .../model/StackExchangePost.java | 0 .../sqlite/StackExchangePostsDb.java | 0 .../StackExchange7zXmlEventReaderSource.java | 0 .../xml/StackExchangeXmlCommentReader.java | 0 .../xml/StackExchangeXmlIterator.java | 0 .../xml/StackExchangeXmlPostReader.java | 0 .../xml/XmlEventReaderSource.java | 0 .../main => }/resources/db/stackexchange.sql | 0 .../StackExchangeXmlCommentReaderTest.java | 0 .../xml/StackExchangeXmlPostReaderTest.java | 0 .../xml/StringXmlTestEventReader.java | 0 .../summary-extraction/build.gradle | 2 + .../marginalia/summary/SummaryExtractor.java | 0 .../summary/heuristic/DomFilterHeuristic.java | 0 .../summary/heuristic/FallbackHeuristic.java | 0 .../summary/heuristic/HeuristicTextUtil.java | 0 .../heuristic/MetaDescriptionHeuristic.java | 0 .../OpenGraphDescriptionHeuristic.java | 0 .../heuristic/SummarizingDOMFilter.java | 0 .../summary/heuristic/SummaryHeuristic.java | 0 .../heuristic/TagDensityHeuristic.java | 0 .../html/monadnock.html | 0 .../html/readme.md | 0 .../html/summarization/187.shtml | 0 .../html/summarization/surrey.html | 0 .../html/summarization/surrey.html.1 | 0 .../html/theregister.html | 0 .../html/work-set/index | 0 .../html/work-set/url--1021546012 | 0 .../html/work-set/url--1028592943 | 0 .../html/work-set/url--1081293162 | 0 .../html/work-set/url--1105046394 | 0 .../html/work-set/url--1146923296 | 0 .../html/work-set/url--1194694074 | 0 .../html/work-set/url--1207898281 | 0 .../html/work-set/url--1268145073 | 0 .../html/work-set/url--1294876331 | 0 .../html/work-set/url--1314767420 | 0 .../html/work-set/url--1316269786 | 0 .../html/work-set/url--1316766580 | 0 .../html/work-set/url--1319968043 | 0 .../html/work-set/url--1338576987 | 0 .../html/work-set/url--1341909571 | 0 .../html/work-set/url--1369578579 | 0 .../html/work-set/url--1437315645 | 0 .../html/work-set/url--1458954960 | 0 .../html/work-set/url--1475681345 | 0 .../html/work-set/url--1498328446 | 0 .../html/work-set/url--1507779664 | 0 .../html/work-set/url--1540303379 | 0 .../html/work-set/url--154898476 | 0 .../html/work-set/url--1552059399 | 0 .../html/work-set/url--1557688340 | 0 .../html/work-set/url--1584145751 | 0 .../html/work-set/url--1605151204 | 0 .../html/work-set/url--162269247 | 0 .../html/work-set/url--1624294488 | 0 .../html/work-set/url--164108285 | 0 .../html/work-set/url--1645688243 | 0 .../html/work-set/url--1658004609 | 0 .../html/work-set/url--1658558834 | 0 .../html/work-set/url--1698664879 | 0 .../html/work-set/url--169975195 | 0 .../html/work-set/url--1701203332 | 0 .../html/work-set/url--17281998 | 0 .../html/work-set/url--1742070028 | Bin .../html/work-set/url--1745376814 | 0 .../html/work-set/url--1749889035 | 0 .../html/work-set/url--176177364 | 0 .../html/work-set/url--177014197 | 0 .../html/work-set/url--1794527707 | 0 .../html/work-set/url--1797740201 | 0 .../html/work-set/url--1799098579 | 0 .../html/work-set/url--1959637826 | 0 .../html/work-set/url--1971916964 | 0 .../html/work-set/url--1985840368 | 0 .../html/work-set/url--2012610859 | 0 .../html/work-set/url--202178680 | 0 .../html/work-set/url--2043528727 | 0 .../html/work-set/url--2081757477 | 0 .../html/work-set/url--2103982576 | 0 .../html/work-set/url--2111558769 | 0 .../html/work-set/url--213168798 | 0 .../html/work-set/url--232544032 | 0 .../html/work-set/url--253010011 | 0 .../html/work-set/url--274250994 | 0 .../html/work-set/url--332442790 | 0 .../html/work-set/url--353437903 | 0 .../html/work-set/url--364546777 | 0 .../html/work-set/url--379129416 | 0 .../html/work-set/url--399428149 | 0 .../html/work-set/url--425233170 | 0 .../html/work-set/url--434612307 | 0 .../html/work-set/url--439772328 | 0 .../html/work-set/url--458002611 | 0 .../html/work-set/url--506010305 | 0 .../html/work-set/url--546773534 | 0 .../html/work-set/url--551288516 | 0 .../html/work-set/url--602577763 | 0 .../html/work-set/url--611668054 | 0 .../html/work-set/url--634771245 | 0 .../html/work-set/url--639320493 | 0 .../html/work-set/url--643179018 | 0 .../html/work-set/url--663772351 | 0 .../html/work-set/url--670789152 | 0 .../html/work-set/url--6797317 | 0 .../html/work-set/url--700978490 | 0 .../html/work-set/url--708035332 | 0 .../html/work-set/url--804917062 | 0 .../html/work-set/url--819771302 | 0 .../html/work-set/url--840796372 | 0 .../html/work-set/url--841445362 | 0 .../html/work-set/url--862385354 | 0 .../html/work-set/url--879796466 | 0 .../html/work-set/url--89134993 | 0 .../html/work-set/url--905197876 | 0 .../html/work-set/url--920328354 | 0 .../html/work-set/url--952827759 | 0 .../html/work-set/url--964018507 | 0 .../html/work-set/url--972614909 | 0 .../html/work-set/url-10088520 | 0 .../html/work-set/url-1013281103 | 0 .../html/work-set/url-1019241851 | 0 .../html/work-set/url-1059944953 | 0 .../html/work-set/url-1118681302 | 0 .../html/work-set/url-1179298706 | 0 .../html/work-set/url-1191749784 | 0 .../html/work-set/url-1207094790 | 0 .../html/work-set/url-1213989666 | 0 .../html/work-set/url-1222442301 | 0 .../html/work-set/url-130332455 | 0 .../html/work-set/url-1311055461 | 0 .../html/work-set/url-1391842722 | 0 .../html/work-set/url-1457388763 | 0 .../html/work-set/url-1506356272 | 0 .../html/work-set/url-1511762169 | 0 .../html/work-set/url-1534640058 | 0 .../html/work-set/url-1551513871 | 0 .../html/work-set/url-1567632447 | 0 .../html/work-set/url-1623049502 | 0 .../html/work-set/url-163919330 | 0 .../html/work-set/url-1661398327 | 0 .../html/work-set/url-1724309925 | 0 .../html/work-set/url-1736807128 | 0 .../html/work-set/url-1739031345 | 0 .../html/work-set/url-1755745765 | 0 .../html/work-set/url-1802811100 | 0 .../html/work-set/url-1805364707 | 0 .../html/work-set/url-1832702370 | 0 .../html/work-set/url-1853114311 | 0 .../html/work-set/url-1924872844 | 0 .../html/work-set/url-197772804 | 0 .../html/work-set/url-1984259912 | 0 .../html/work-set/url-1990903988 | 0 .../html/work-set/url-2039310951 | 0 .../html/work-set/url-2040857056 | 0 .../html/work-set/url-2052613093 | 0 .../html/work-set/url-2063899866 | 0 .../html/work-set/url-2115548255 | 0 .../html/work-set/url-2127148436 | 0 .../html/work-set/url-2133781904 | 0 .../html/work-set/url-225690385 | 0 .../html/work-set/url-226401955 | 0 .../html/work-set/url-262970770 | 0 .../html/work-set/url-30106798 | 0 .../html/work-set/url-302167335 | 0 .../html/work-set/url-327999153 | 0 .../html/work-set/url-332568225 | 0 .../html/work-set/url-343223418 | 0 .../html/work-set/url-383103932 | 0 .../html/work-set/url-412929678 | 0 .../html/work-set/url-475213997 | 0 .../html/work-set/url-483403121 | 0 .../html/work-set/url-488667993 | 0 .../html/work-set/url-50815201 | 0 .../html/work-set/url-522685905 | 0 .../html/work-set/url-570714305 | 0 .../html/work-set/url-58733529 | 0 .../html/work-set/url-616518304 | 0 .../html/work-set/url-662169426 | 0 .../html/work-set/url-677278788 | 0 .../html/work-set/url-690486170 | 0 .../html/work-set/url-709693331 | 0 .../html/work-set/url-734531556 | 0 .../html/work-set/url-767530276 | 0 .../html/work-set/url-783154014 | 0 .../html/work-set/url-796905237 | 0 .../html/work-set/url-800099955 | 0 .../html/work-set/url-804101946 | 0 .../html/work-set/url-830664902 | 0 .../html/work-set/url-876060686 | 0 .../html/work-set/url-892584998 | 0 .../html/work-set/url-942458463 | 0 .../html/work-set/url-952036171 | 0 .../html/work-set/url-968207276 | 0 .../summary/SummaryExtractorTest.java | 0 .../heuristic/HeuristicTextUtilTest.java | 0 .../topic-detection/build.gradle | 2 + .../nu/marginalia/topic/RecipeDetector.java | 0 .../topic/TextileCraftDetector.java | 0 .../marginalia/topic/WoodworkingDetector.java | 0 code/features-crawl/content-type/build.gradle | 2 + .../marginalia/contenttype/ContentType.java | 0 .../contenttype/ContentTypeParser.java | 0 .../contenttype/DocumentBodyToString.java | 0 .../contenttype/ContentTypeParserTest.java | 0 .../contenttype/DocumentBodyToStringTest.java | 0 .../crawl-blocklist/build.gradle | 2 + .../ip_blocklist/GeoIpBlocklist.java | 0 .../ip_blocklist/InetAddressCache.java | 0 .../marginalia/ip_blocklist/IpBlockList.java | 0 .../marginalia/ip_blocklist/UrlBlocklist.java | 0 .../ip_blocklist/UrlBlocklistTest.java | 0 code/features-crawl/link-parser/build.gradle | 2 + .../marginalia/link_parser/FeedExtractor.java | 0 .../nu/marginalia/link_parser/LinkParser.java | 0 .../feedlot-client/build.gradle | 2 + .../nu/marginalia/feedlot/FeedlotClient.java | 0 .../nu/marginalia/feedlot/model/FeedItem.java | 0 .../marginalia/feedlot/model/FeedItems.java | 0 .../random-websites/build.gradle | 2 + .../browse/DbBrowseDomainsRandom.java | 0 .../browse/DbBrowseDomainsSimilarCosine.java | 0 .../marginalia/browse/model/BrowseResult.java | 0 .../browse/model/BrowseResultSet.java | 0 code/features-search/screenshots/build.gradle | 2 + .../screenshot/ScreenshotService.java | 0 code/functions/domain-info/api/build.gradle | 9 +--- .../api/domains/DomainInfoClient.java | 0 .../api/domains/DomainsProtobufCodec.java | 0 .../api/domains/model/DomainInformation.java | 0 .../api/domains/model/SimilarDomain.java | 0 code/functions/domain-info/build.gradle | 2 + .../domains/DomainInfoGrpcService.java | 0 .../domains/DomainInformationService.java | 0 .../domains/SimilarDomainsService.java | 0 .../domain-links/aggregate/build.gradle | 2 + .../AggregateDomainLinksService.java | 0 code/functions/domain-links/api/build.gradle | 9 +--- .../AggregateDomainLinksClient.java | 0 .../PartitionDomainLinksClient.java | 0 .../domain-links/partition/build.gradle | 2 + .../PartitionDomainLinksService.java | 0 code/functions/math/api/build.gradle | 9 +--- .../nu/marginalia/api/math/MathClient.java | 0 .../api/math/MathProtobufCodec.java | 0 .../api/math/model/DictionaryEntry.java | 0 .../api/math/model/DictionaryResponse.java | 0 code/functions/math/build.gradle | 2 + .../functions/math/MathGrpcService.java | 0 .../math/dict/DictionaryService.java | 0 .../functions/math/dict/SpellChecker.java | 0 .../functions/math/eval/MathParser.java | 0 .../marginalia/functions/math/eval/Unit.java | 0 .../marginalia/functions/math/eval/Units.java | 0 .../math/{src/main => }/resources/units.csv | 0 .../functions/math/eval/MathParserTest.java | 0 .../functions/math/eval/UnitsTest.java | 0 code/functions/search-query/api/build.gradle | 9 +--- .../api/searchquery/IndexProtobufCodec.java | 0 .../api/searchquery/QueryClient.java | 0 .../api/searchquery/QueryProtobufCodec.java | 0 .../model/query/ProcessedQuery.java | 0 .../searchquery/model/query/QueryParams.java | 0 .../model/query/QueryResponse.java | 0 .../model/query/SearchSetIdentifier.java | 0 .../model/query/SearchSpecification.java | 0 .../model/query/SearchSubquery.java | 0 .../model/results/Bm25Parameters.java | 0 .../results/DecoratedSearchResultItem.java | 0 .../model/results/ResultRankingContext.java | 0 .../results/ResultRankingParameters.java | 0 .../model/results/SearchResultItem.java | 0 .../results/SearchResultKeywordScore.java | 0 .../results/SearchResultPreliminaryScore.java | 0 .../model/results/SearchResultSet.java | 0 .../index/client/IndexProtobufCodecTest.java | 0 code/functions/search-query/build.gradle | 2 + .../searchquery/QueryGRPCService.java | 0 .../searchquery/query_parser/QueryParser.java | 0 .../query_parser/QueryPermutation.java | 0 .../query_parser/QueryTokenizer.java | 0 .../query_parser/QueryVariants.java | 0 .../searchquery/query_parser/token/Token.java | 0 .../query_parser/token/TokenType.java | 0 .../query_parser/token/TokenVisitor.java | 0 .../searchquery/svc/QueryFactory.java | 0 .../svc/QueryLimitsAccumulator.java | 0 .../svc/QuerySearchTermsAccumulator.java | 0 .../util/language/EnglishDictionary.java | 0 .../marginalia/util/ngrams/DenseBitMap.java | 0 .../util/ngrams/NGramBloomFilter.java | 0 .../util/transform_list/TransformList.java | 0 .../query/svc/QueryFactoryTest.java | 0 code/index/api/build.gradle | 2 + .../nu/marginalia/index/api/IndexClient.java | 0 .../marginalia/index/api/IndexMqClient.java | 0 .../index/api/IndexMqEndpoints.java | 0 code/index/build.gradle | 6 ++- .../build.gradle | 2 + .../index/forward/ForwardIndexConverter.java | 0 .../index/forward/ForwardIndexFileNames.java | 0 .../index/forward/ForwardIndexParameters.java | 0 .../index/forward/ForwardIndexReader.java | 0 .../readme.md | 0 .../forward/ForwardIndexConverterTest.java | 0 .../test}/nu/marginalia/test/TestUtil.java | 0 code/index/index-journal/build.gradle | 2 + .../index/journal/IndexJournalFileNames.java | 0 .../journal/model/IndexJournalEntry.java | 0 .../model/IndexJournalEntryBuilder.java | 0 .../journal/model/IndexJournalEntryData.java | 0 .../model/IndexJournalEntryHeader.java | 0 .../journal/model/IndexJournalFileHeader.java | 0 .../journal/reader/IndexJournalReadEntry.java | 0 .../journal/reader/IndexJournalReader.java | 0 .../reader/IndexJournalReaderPagingImpl.java | 0 .../reader/IndexJournalReaderSingleFile.java | 0 .../reader/pointer/IndexJournalPointer.java | 0 .../journal/writer/IndexJournalWriter.java | 0 .../writer/IndexJournalWriterPagingImpl.java | 0 .../IndexJournalWriterSingleFileImpl.java | 0 .../index/journal/IndexJournalTest.java | 0 .../pointer/IndexJournalPointerTest.java | 0 .../build.gradle | 2 + .../index.svg | 0 .../index/ReverseIndexEntrySource.java | 0 .../index/ReverseIndexFullFileNames.java | 0 .../index/ReverseIndexParameters.java | 0 .../index/ReverseIndexPrioFileNames.java | 0 .../marginalia/index/ReverseIndexReader.java | 0 .../index/ReverseIndexSelfTest.java | 0 .../CountToOffsetTransformer.java | 0 .../index/construction/DocIdRewriter.java | 0 .../construction/IndexSizeEstimator.java | 0 .../construction/JournalReaderSource.java | 0 .../ReverseIndexBTreeTransformer.java | 0 .../construction/ReverseIndexConstructor.java | 0 .../index/construction/ReversePreindex.java | 0 .../ReversePreindexDocuments.java | 0 .../ReversePreindexReference.java | 0 .../ReversePreindexWordSegments.java | 0 .../index/query/ReverseIndexRejectFilter.java | 0 .../index/query/ReverseIndexRetainFilter.java | 0 .../merging.svg | 0 .../preindex.svg | 0 .../readme.md | 0 .../index/ReverseIndexDebugTest.java | 0 .../index/ReverseIndexReaderTest.java | 0 .../construction/ReversePreindexDocsTest.java | 0 .../ReversePreindexFinalizeTest.java | 0 .../ReversePreindexMergeTest.java | 0 .../ReversePreindexWordSegmentsTest.java | 0 .../construction/TestJournalFactory.java | 0 .../index/construction/TestSegmentData.java | 0 .../test}/nu/marginalia/test/TestUtil.java | 0 .../index/IndexQueryGrpcService.java | 0 .../index/IndexServicesFactory.java | 0 .../nu/marginalia/index/SearchTermsUtil.java | 0 .../index/index/CombinedIndexReader.java | 0 .../index/index/IndexQueryBuilderImpl.java | 0 .../index/index/IndexQueryService.java | 0 .../marginalia/index/index/StatefulIndex.java | 0 .../index/model/IndexQueryParams.java | 0 .../index/model/IndexSearchParameters.java | 0 .../index/model/IndexSearchTerms.java | 0 .../index/results/IndexMetadataService.java | 0 .../IndexResultDomainDeduplicator.java | 0 .../results/IndexResultValuationContext.java | 0 .../results/IndexResultValuatorService.java | 0 .../index/searchset/DbUpdateRanks.java | 0 .../index/searchset/RankingSearchSet.java | 0 .../marginalia/index/searchset/SearchSet.java | 0 .../index/searchset/SearchSetAny.java | 0 .../index/searchset/SearchSetsService.java | 0 .../index/searchset/SmallSearchSet.java | 0 .../ranking/domains/PageRankDomainRanker.java | 0 .../ranking/domains/RankingAlgorithm.java | 0 .../accumulator/RankingResultAccumulator.java | 0 .../RankingResultBitSetAccumulator.java | 0 .../RankingResultHashMapAccumulator.java | 0 .../RankingResultHashSetAccumulator.java | 0 .../RankingResultListAccumulator.java | 0 .../domains/data/AbstractGraphSource.java | 0 .../ranking/domains/data/GraphSource.java | 0 .../domains/data/InvertedLinkGraphSource.java | 0 .../ranking/domains/data/LinkGraphSource.java | 0 .../domains/data/SimilarityGraphSource.java | 0 .../domains/jgrapht/PersonalizedPageRank.java | 0 .../ranking/results/ResultKeywordSet.java | 0 .../ranking/results/ResultValuator.java | 0 .../ranking/results/factors/Bm25Factor.java | 0 .../results/factors/PriorityTermBonus.java | 0 .../results/factors/TermCoherenceFactor.java | 0 code/index/query/build.gradle | 2 + .../index/domainrankings/DomainRankings.java | 0 .../index/query/EmptyEntrySource.java | 0 .../marginalia/index/query/EntrySource.java | 0 .../nu/marginalia/index/query/IndexQuery.java | 0 .../index/query/IndexQueryBuilder.java | 0 .../index/query/IndexQueryPriority.java | 0 .../index/query/IndexSearchBudget.java | 0 .../index/query/filter/QueryFilterAnyOf.java | 0 .../query/filter/QueryFilterLetThrough.java | 0 .../index/query/filter/QueryFilterNoPass.java | 0 .../QueryFilterStepExcludeFromPredicate.java | 0 .../filter/QueryFilterStepFromPredicate.java | 0 .../index/query/filter/QueryFilterStepIf.java | 0 .../index/query/limit/QueryLimits.java | 0 .../index/query/limit/QueryStrategy.java | 0 .../index/query/limit/SpecificationLimit.java | 0 .../query/limit/SpecificationLimitType.java | 0 .../query/filter/QueryFilterStepIfTest.java | 0 ...IndexQueryServiceIntegrationSmokeTest.java | 0 .../IndexQueryServiceIntegrationTest.java | 0 ...ndexQueryServiceIntegrationTestModule.java | 0 .../IndexResultDomainDeduplicatorTest.java | 0 .../index/searchset/RankingSearchSetTest.java | 0 .../nu/marginalia/index/util/TestUtil.java | 0 .../RankingAlgorithmWithRealDataTest.java | 0 .../RankingAlgorithmsContainerTest.java | 0 .../TestGraphSourceForInvertedLinkData.java | 0 .../domains/TestGraphSourceForLinkData.java | 0 .../TestGraphSourceForSimilarityData.java | 0 .../ranking/results/ResultValuatorTest.java | 0 .../factors/TermCoherenceFactorTest.java | 0 code/libraries/array/build.gradle | 2 + .../marginalia/array/ArrayRangeReference.java | 0 .../java/nu/marginalia/array/IntArray.java | 0 .../java/nu/marginalia/array/LongArray.java | 0 .../nu/marginalia/array/LongArrayFactory.java | 0 .../array/algo/BulkTransferArray.java | 0 .../marginalia/array/algo/IntArrayBase.java | 0 .../marginalia/array/algo/IntArraySearch.java | 0 .../marginalia/array/algo/IntArraySort.java | 0 .../array/algo/IntArrayTransformations.java | 0 .../marginalia/array/algo/LongArrayBase.java | 0 .../array/algo/LongArraySearch.java | 0 .../marginalia/array/algo/LongArraySort.java | 0 .../array/algo/LongArrayTransformations.java | 0 .../array/algo/SortAlgoInsertionSort.java | 0 .../array/algo/SortAlgoMergeSort.java | 0 .../array/algo/SortAlgoQuickSort.java | 0 .../marginalia/array/algo/SortingContext.java | 0 .../array/algo/TwoArrayOperations.java | 0 .../array/buffer/IntQueryBuffer.java | 0 .../array/buffer/LongQueryBuffer.java | 0 .../ReferenceImplIntArrayDelegate.java | 0 .../ReferenceImplLongArrayDelegate.java | 0 .../array/delegate/ShiftedIntArray.java | 0 .../array/delegate/ShiftedLongArray.java | 0 .../array/functional/AddressRangeCall.java | 0 .../array/functional/AddressRangeCallIO.java | 0 .../functional/AddressRangeIntFunction.java | 0 .../functional/AddressRangeLongFunction.java | 0 .../functional/IntBinaryIOOperation.java | 0 .../array/functional/IntBinaryOperation.java | 0 .../array/functional/IntIOTransformer.java | 0 .../array/functional/IntTransformer.java | 0 .../functional/LongBinaryIOOperation.java | 0 .../array/functional/LongBinaryOperation.java | 0 .../array/functional/LongIOTransformer.java | 0 .../array/functional/LongIntConsumer.java | 0 .../array/functional/LongLongConsumer.java | 0 .../array/functional/LongTransformer.java | 0 .../array/page/AbstractPagingArray.java | 0 .../marginalia/array/page/PartitionPage.java | 0 .../array/page/SegmentIntArray.java | 0 .../array/page/SegmentLongArray.java | 0 .../array/page/UnsafeLongArray.java | 0 .../marginalia/array/page/UnsafeProvider.java | 0 .../array/scheme/ArrayPartitioningScheme.java | 0 .../scheme/PowerOf2PartitioningScheme.java | 0 .../scheme/SequentialPartitioningScheme.java | 0 .../IntLowBitPartitioningSchemeTest.java | 0 .../array/algo/IntArraySearchTest.java | 0 .../array/algo/IntArraySortTest.java | 0 .../algo/IntArrayTransformations2Test.java | 0 .../algo/IntArrayTransformationsTest.java | 0 .../array/algo/LongArraySearchTest.java | 0 .../array/algo/LongArraySortNTest.java | 0 .../array/algo/LongArraySortTest.java | 0 .../algo/LongArrayTransformations2Test.java | 0 .../algo/LongArrayTransformationsTest.java | 0 .../array/algo/TwoArrayOperationsTest.java | 0 .../scheme/ArrayPartitioningSchemeTest.java | 0 .../nu/marginalia/util/test/TestUtil.java | 0 code/libraries/big-string/build.gradle | 2 + .../nu/marginalia/bigstring/BigString.java | 0 .../bigstring/CompressedBigString.java | 0 .../bigstring/CompressionBuffer.java | 0 .../bigstring/CompressionBufferPool.java | 0 .../marginalia/bigstring/PlainBigString.java | 0 .../bigstring/CompressedBigStringTest.java | 0 .../blocking-thread-pool/build.gradle | 2 + .../marginalia/util/ProcessingIterator.java | 0 .../util/SimpleBlockingThreadPool.java | 0 .../util/ProcessingIteratorTest.java | 0 .../braille-block-punch-cards/build.gradle | 2 + .../bbpc/BrailleBlockPunchCards.java | 0 .../bbpc/BrailleBlockPunchCardsTest.java | 0 code/libraries/btree/build.gradle | 2 + .../java/nu/marginalia/btree/BTreeDogEar.java | 0 .../java/nu/marginalia/btree/BTreeReader.java | 0 .../marginalia/btree/BTreeWriteCallback.java | 0 .../java/nu/marginalia/btree/BTreeWriter.java | 0 .../btree/model/BTreeBlockSize.java | 0 .../marginalia/btree/model/BTreeContext.java | 0 .../marginalia/btree/model/BTreeHeader.java | 0 .../BTreeReaderQueryDataWithIndexTest.java | 0 .../BTreeReaderQueryDataWithoutIndexTest.java | 0 .../BTreeReaderRejectRetainWithIndexTest.java | 0 ...reeReaderRejectRetainWithoutIndexTest.java | 0 .../nu/marginalia/btree/BTreeWriterTest.java | 0 code/libraries/easy-lsh/build.gradle | 2 + .../java/nu/marginalia/lsh/EasyLSH.java | 0 .../nu/marginalia/lsh/EasyLSHTest.java | 0 code/libraries/geo-ip/build.gradle | 2 + .../nu/marginalia/geoip/GeoIpDictionary.java | 0 .../marginalia/geoip/sources/AsnMapping.java | 0 .../nu/marginalia/geoip/sources/AsnTable.java | 0 .../geoip/sources/IP2LocationMapping.java | 0 .../geoip/sources/IpRangeMapping.java | 0 .../marginalia/geoip/GeoIpDictionaryTest.java | 0 code/libraries/guarded-regex/build.gradle | 2 + .../nu/marginalia/gregex/GuardedRegex.java | 0 .../gregex/GuardedRegexFactory.java | 0 .../language-processing/build.gradle | 2 + .../marginalia/language/WordDictionary.java | 0 .../nu/marginalia/language/WordPatterns.java | 0 .../language/encoding/AsciiFlattener.java | 0 .../language/encoding/UnicodeRanges.java | 0 .../FasttextLanguagePredictionModel.java | 0 .../language/filter/LanguageFilter.java | 0 .../filter/LanguagePredictionModel.java | 0 .../UngaBungaLanguagePredictionModel.java | 0 .../language/model/DocumentLanguageData.java | 0 .../language/model/DocumentSentence.java | 0 .../language/model/WordFrequencyData.java | 0 .../nu/marginalia/language/model/WordRep.java | 0 .../language/model/WordSeparator.java | 0 .../marginalia/language/model/WordSpan.java | 0 .../language/sentence/SentenceExtractor.java | 0 .../SentenceExtractorHtmlTagCleaner.java | 0 .../SentenceExtractorStringUtils.java | 0 .../language/sentence/SentencePreCleaner.java | 0 .../sentence/SentenceSegmentSplitter.java | 0 .../ThreadLocalSentenceExtractorProvider.java | 0 .../main => }/resources/dictionary/en-1000 | 0 .../resources/dictionary/en-stopwords | 0 .../main => }/resources/dictionary/en-words | 0 .../main => }/resources/dictionary/latin-1000 | 0 .../main => }/resources/dictionary/swe-1000 | 0 .../resources/dictionary/word-frequency | 0 .../html/jep.html | 0 .../html/spam.html | 0 .../language/encoding/AsciiFlattenerTest.java | 0 .../SentenceExtractorHtmlTagCleanerTest.java | 0 .../language/filter/LanguageFilterTest.java | 0 .../language/filter/TestLanguageModels.java | 0 .../sentence/SentenceExtractorTest.java | 0 code/libraries/message-queue/build.gradle | 2 + .../marginalia/actor/ActorStateMachine.java | 0 .../nu/marginalia/actor/ExpectedMessage.java | 0 .../actor/prototype/ActorPrototype.java | 0 .../actor/prototype/RecordActorPrototype.java | 0 .../state/ActorControlFlowException.java | 0 .../actor/state/ActorResumeBehavior.java | 0 .../actor/state/ActorStateInstance.java | 0 .../actor/state/ActorStateTransition.java | 0 .../nu/marginalia/actor/state/ActorStep.java | 0 .../nu/marginalia/actor/state/Resume.java | 0 .../nu/marginalia/actor/state/Terminal.java | 0 .../nu/marginalia/mq/MessageQueueFactory.java | 0 .../java/nu/marginalia/mq/MqException.java | 0 .../java/nu/marginalia/mq/MqMessage.java | 0 .../java/nu/marginalia/mq/MqMessageState.java | 0 .../mq/inbox/MqAsynchronousInbox.java | 0 .../nu/marginalia/mq/inbox/MqInboxIf.java | 0 .../marginalia/mq/inbox/MqInboxResponse.java | 0 .../marginalia/mq/inbox/MqInboxShredder.java | 0 .../mq/inbox/MqSingleShotInbox.java | 0 .../marginalia/mq/inbox/MqSubscription.java | 0 .../mq/inbox/MqSynchronousInbox.java | 0 .../nu/marginalia/mq/outbox/MqOutbox.java | 0 .../persistence/MqMessageHandlerRegistry.java | 0 .../mq/persistence/MqPersistence.java | 0 .../actor/ActorRecordMachineTest.java | 0 .../nu/marginalia/mq/MqMessageRow.java | 0 .../nu/marginalia/mq/MqTestUtil.java | 0 .../nu/marginalia/mq/outbox/MqOutboxTest.java | 0 .../mq/persistence/MqPersistenceTest.java | 0 code/libraries/next-prime/build.gradle | 2 + .../nu/marginalia/util/NextPrimeUtil.java | 0 .../nu/marginalia/util/NextPrimeUtilTest.java | 0 .../random-write-funnel/build.gradle | 2 + .../marginalia/rwf/RandomFileAssembler.java | 0 .../nu/marginalia/rwf/RandomWriteFunnel.java | 0 .../rwf/RandomFileAssemblerTest.java | 0 .../marginalia/rwf/RandomWriteFunnelTest.java | 0 .../term-frequency-dict/build.gradle | 2 + .../TermFrequencyDict.java | 0 code/libraries/test-helpers/build.gradle | 2 + .../marginalia/test/TestMigrationLoader.java | 0 code/process-models/crawl-spec/build.gradle | 2 + .../crawlspec/CrawlSpecFileNames.java | 0 .../crawlspec/CrawlSpecGenerator.java | 0 .../CrawlSpecRecordParquetFileReader.java | 0 .../CrawlSpecRecordParquetFileWriter.java | 0 .../model/crawlspec/CrawlSpecRecord.java | 0 .../crawling-model/build.gradle | 2 + .../crawling/body/ContentTypeLogic.java | 0 .../crawling/body/DocumentBodyExtractor.java | 0 .../crawling/body/DocumentBodyResult.java | 0 .../crawling/body/HttpFetchResult.java | 0 .../crawling/io/CrawledDomainReader.java | 0 .../crawling/io/CrawledDomainWriter.java | 0 .../crawling/io/CrawlerOutputFile.java | 0 .../io/SerializableCrawlDataStream.java | 0 ...ibleLegacySerializableCrawlDataStream.java | 0 ...FastLegacySerializableCrawlDataStream.java | 0 .../ParquetSerializableCrawlDataStream.java | 0 .../crawling/model/CrawledDocument.java | 0 .../crawling/model/CrawledDomain.java | 0 .../crawling/model/CrawlerDocumentStatus.java | 0 .../crawling/model/CrawlerDomainStatus.java | 0 .../crawling/model/SerializableCrawlData.java | 0 .../parquet/CrawledDocumentParquetRecord.java | 0 ...rawledDocumentParquetRecordFileReader.java | 0 ...rawledDocumentParquetRecordFileWriter.java | 0 .../jwarc/WarcXCookieInformationHeader.java | 0 .../netpreserve/jwarc/WarcXEntityRefused.java | 0 .../jwarc/WarcXResponseReference.java | 0 .../{src/main => }/java/plan/CrawlPlan.java | 0 .../main => }/java/plan/CrawlPlanLoader.java | 0 .../crawling/model/CrawledDocumentTest.java | 0 ...edDocumentParquetRecordFileWriterTest.java | 0 .../processed-data/build.gradle | 2 + .../DocumentRecordParquetFileReader.java | 0 .../DocumentRecordParquetFileWriter.java | 0 .../DomainLinkRecordParquetFileReader.java | 0 .../DomainLinkRecordParquetFileWriter.java | 0 .../DomainRecordParquetFileReader.java | 0 .../DomainRecordParquetFileWriter.java | 0 .../io/processed/ProcessedDataFileNames.java | 0 .../model/processed/DocumentRecord.java | 0 .../DocumentRecordKeywordsProjection.java | 0 .../DocumentRecordMetadataProjection.java | 0 .../model/processed/DomainLinkRecord.java | 0 .../model/processed/DomainRecord.java | 0 .../model/processed/DomainWithIp.java | 0 .../DocumentRecordParquetFileReaderTest.java | 0 ...DomainLinkRecordParquetFileReaderTest.java | 0 .../DomainRecordParquetFileReaderTest.java | 0 code/process-models/work-log/build.gradle | 2 + .../marginalia/worklog/BatchingWorkLog.java | 0 .../worklog/BatchingWorkLogImpl.java | 0 .../worklog/BatchingWorkLogInspector.java | 0 .../worklog/BatchingWorkLogImplTest.java | 0 code/process-mqapi/build.gradle | 2 + .../marginalia/mqapi/ProcessInboxNames.java | 0 .../mqapi/converting/ConvertAction.java | 0 .../mqapi/converting/ConvertRequest.java | 0 .../mqapi/crawling/CrawlRequest.java | 0 .../mqapi/index/CreateIndexRequest.java | 0 .../nu/marginalia/mqapi/index/IndexName.java | 0 .../marginalia/mqapi/loading/LoadRequest.java | 0 .../processes/converting-process/build.gradle | 2 + .../marginalia/converting/ConverterMain.java | 0 .../converting/ConverterModule.java | 0 .../model/DisqualifiedException.java | 0 .../converting/model/GeneratorType.java | 0 .../converting/model/ProcessedDocument.java | 0 .../model/ProcessedDocumentDetails.java | 0 .../converting/model/ProcessedDomain.java | 0 .../converting/processor/AcceptableAds.java | 0 .../processor/ConverterDomainTypes.java | 0 .../converting/processor/DocumentClass.java | 0 .../processor/DocumentDecorator.java | 0 .../processor/DocumentProcessor.java | 0 .../converting/processor/DomainProcessor.java | 0 .../converting/processor/MetaRobotsTag.java | 0 .../converting/processor/SiteWords.java | 0 .../logic/DocumentGeneratorExtractor.java | 0 .../processor/logic/DocumentLengthLogic.java | 0 .../processor/logic/DocumentValuator.java | 0 .../processor/logic/FeatureExtractor.java | 0 .../logic/HtmlStandardExtractor.java | 0 .../logic/LshDocumentDeduplicator.java | 0 .../processor/logic/PlainTextLogic.java | 0 .../processor/logic/TitleExtractor.java | 0 .../processor/logic/dom/DomPruningFilter.java | 0 .../logic/dom/MeasureLengthVisitor.java | 0 .../logic/links/CommonKeywordExtractor.java | 0 .../processor/logic/links/FileLinks.java | 0 .../processor/logic/links/LinkGraph.java | 0 .../processor/logic/links/LinkProcessor.java | 0 .../processor/logic/links/TopKeywords.java | 0 .../AbstractDocumentProcessorPlugin.java | 0 .../plugin/HtmlDocumentProcessorPlugin.java | 0 .../PlainTextDocumentProcessorPlugin.java | 0 .../specialization/BlogSpecialization.java | 0 .../specialization/DefaultSpecialization.java | 0 .../HtmlProcessorSpecializations.java | 0 .../specialization/JavadocSpecialization.java | 0 .../specialization/LemmySpecialization.java | 0 .../MariadbKbSpecialization.java | 0 .../specialization/PhpBBSpecialization.java | 0 .../specialization/WikiSpecialization.java | 0 .../specialization/XenForoSpecialization.java | 0 .../converting/sideload/SideloadSource.java | 0 .../sideload/SideloadSourceFactory.java | 0 .../sideload/SideloaderProcessing.java | 0 .../sideload/dirtree/DirtreeSideloadSpec.java | 0 .../dirtree/DirtreeSideloadSpecList.java | 0 .../sideload/dirtree/DirtreeSideloader.java | 0 .../dirtree/DirtreeSideloaderFactory.java | 0 .../EncyclopediaMarginaliaNuSideloader.java | 0 .../sideload/reddit/RedditSideloader.java | 0 .../StackexchangeSideloader.java | 0 .../sideload/warc/WarcSideloader.java | 0 .../marginalia/converting/util/LineUtils.java | 0 .../writer/ConverterBatchWritableIf.java | 0 .../writer/ConverterBatchWriter.java | 0 .../writer/ConverterBatchWriterIf.java | 0 .../converting/writer/ConverterWriter.java | 0 .../memex-marginalia/05-test.gmi | 0 .../memex-marginalia/code/index.gmi | 0 .../memex-marginalia/commons/dialogue.gmi | 0 .../commons/search-failure-modes.gmi | 0 .../commons/self-interest.gmi | 0 .../memex-marginalia/dead.gmi | 0 .../debian-laptop-install-log.gmi | 0 .../memex-marginalia/debian-laptop.gmi | 0 .../memex-marginalia/feed.gmi | 0 .../memex-marginalia/fragments-old-web.gmi | 0 .../memex-marginalia/good-video-games.gmi | 0 .../memex-marginalia/guestbook/footer.gmi | 0 .../memex-marginalia/guestbook/header.gmi | 0 .../memex-marginalia/guestbook/index.gmi | 0 .../memex-marginalia/index | 0 .../memex-marginalia/index.gmi | 0 .../memex-marginalia/junk/DedupTest.gmi | 0 .../memex-marginalia/junk/bssl.gmi | 0 .../junk/eldritch-oneliner.gmi | 0 .../memex-marginalia/junk/server.gmi | 0 .../junk/very-old-code.cc.gmi | 0 .../memex-marginalia/links/aggregators.gmi | 0 .../memex-marginalia/links/articles.gmi | 0 .../memex-marginalia/links/bookmarks.gmi | 0 .../links/fragments-old-web.gmi | 0 .../memex-marginalia/links/index.gmi | 0 .../memex-marginalia/links/linkpocalypse.gmi | 0 .../memex-marginalia/log/00-linkpocalypse.gmi | 0 .../memex-marginalia/log/01-astrolabe.gmi | 0 .../memex-marginalia/log/02-re-tests.gmi | 0 .../log/03-writing-for-reading.gmi | 0 .../memex-marginalia/log/04-link-farms.gmi | 0 .../memex-marginalia/log/05-minds-field.gmi | 0 .../memex-marginalia/log/06-optimization.gmi | 0 .../log/07-local-backlinks.gmi | 0 .../log/08-whatever-happened-to-the-memex.gmi | 0 .../log/09-system-upgrade.gmi | 0 .../log/10-astrolabe-2-sampling-bias.gmi | 0 .../log/11-dying-every-day.gmi | 0 .../memex-marginalia/log/12-bye-bye-gmail.gmi | 0 .../memex-marginalia/log/13-static-html.gmi | 0 .../memex-marginalia/log/13-test.gmi | 0 .../log/14-enter-the-circle-of-blame.gmi | 0 .../log/15-stages-of-being.gmi | 0 .../log/16-cursed-motivation.gmi | 0 .../log/17-git-isnt-a-web-service.gmi | 0 .../memex-marginalia/log/18-soaring-high.gmi | 0 .../log/19-website-discoverability-crisis.gmi | 0 .../log/20-dot-com-link-farms.gmi | 0 .../log/21-new-solutions-old-problems.gmi | 0 .../log/22-against-the-flood.gmi | 0 .../log/23-re-software-and-branding.gmi | 0 .../memex-marginalia/log/24-silly-hats.gmi | 0 .../log/25-october-update.gmi | 0 .../log/26-personalized-pagerank.gmi | 0 .../log/27-getting-with-the-times.gmi | 0 .../memex-marginalia/log/28-web-browsing.gmi | 0 .../memex-marginalia/log/29-botnet-ddos.gmi | 0 .../log/30-unintuitive-optimization.gmi | 0 .../memex-marginalia/log/31-ngram-needles.gmi | 0 .../log/32-bot-apologetics.gmi | 0 .../memex-marginalia/log/33-rude-guests.gmi | 0 .../log/34-internet-arguments.gmi | 0 .../log/35-keeping-gemini-difficult.gmi | 0 .../36-localized-programming-languages.gmi | 0 .../log/37-keyword-extraction.gmi | 0 .../memex-marginalia/log/38-old-and-new.gmi | 0 .../log/39-normie-hypothesis.gmi | 0 .../log/40-wasted-resources.gmi | 0 .../log/41-search-result-relevance.gmi | 0 .../memex-marginalia/log/42-dark.gmi | 0 .../memex-marginalia/log/43-pseodonymous.gmi | 0 .../log/44-discovery-and-design.gmi | 0 .../45-unfuck-internet-discoverability.gmi | 0 .../log/46-anatomy-of-search-engine-spam.gmi | 0 .../memex-marginalia/log/47-drive-failure.gmi | 0 .../log/48-i-have-no-capslock.gmi | 0 .../log/49-marginalia-1-year.gmi | 0 .../50-meditation-on-software-correctness.gmi | 0 .../log/51-the-file-startup.gmi | 0 .../memex-marginalia/log/52-growing-pains.gmi | 0 .../log/53-better-hard-drive-metaphor.gmi | 0 .../log/54-bargain-bin-btree.gmi | 0 .../log/55-lexicon-rubberduck.gmi | 0 .../log/56-uncertain-future.gmi | 0 .../57-dont-know-how-to-build-software.gmi | 0 .../log/58-marginalia-open-source.gmi | 0 .../memex-marginalia/log/59-anchor-text.gmi | 0 .../log/60-prescriptive-descriptions.gmi | 0 .../log/61-botspam-apocalypse.gmi | 0 .../log/62-marginaliacoin.gmi | 0 .../log/63-marginalia-crawler.gmi | 0 .../log/64-hundred-million.gmi | 0 .../log/65-scaling-doesnt-scale.gmi | 0 .../memex-marginalia/log/66-carbon-dating.gmi | 0 .../log/67-best-ideas-afk.gmi | 0 .../log/68-wizards-vs-sorcerers.gmi | 0 .../log/69-creepy-website-similarity.gmi | 0 .../log/70-faster-index-joins.gmi | 0 .../memex-marginalia/log/71-memex-design.gmi | 0 .../memex-marginalia/log/72-are-you-ok.gmi | 0 .../log/72-new-approach-to-ranking.gmi | 0 .../log/73-new-approach-to-ranking.gmi | 0 .../log/74-marginalia-2-years.gmi | 0 .../log/bargain-bin-btree.gmi | 0 .../memex-marginalia/log/index.gmi | 0 .../memex-marginalia/log/soaring-high.gmi | 0 .../memex-marginalia/log/todo.gmi | 0 .../memex-marginalia/one-weird-trick.gmi | 0 .../memex-marginalia/pics/index.gmi | 0 .../memex-marginalia/pics/links/index.gmi | 0 .../pics/raster-test/index.gmi | 0 .../memex-marginalia/projects/edge.gmi | 0 .../memex-marginalia/projects/edge/about.gmi | 0 .../memex-marginalia/projects/edge/api.gmi | 0 .../projects/edge/changelog.gmi | 0 .../projects/edge/design-notes.gmi | 0 .../memex-marginalia/projects/edge/faq.gmi | 0 .../projects/edge/for-webmasters.gmi | 0 .../memex-marginalia/projects/edge/index.gmi | 0 .../projects/edge/privacy.gmi | 0 .../projects/edge/search-tips.gmi | 0 .../projects/edge/supporting.gmi | 0 .../memex-marginalia/projects/edge/top-20.gmi | 0 .../projects/encyclopedia/index.gmi | 0 .../projects/gemini-server.gmi | 0 .../memex-marginalia/projects/index.gmi | 0 .../memex-marginalia/projects/memex.gmi | 0 .../memex-marginalia/projects/wmsa.gmi | 0 .../memex-marginalia/readme.md | 0 .../memex-marginalia/recipes/chicken-soup.gmi | 0 .../recipes/french-borscht.gmi | 0 .../memex-marginalia/recipes/index.gmi | 0 .../recipes/omelette-bacon.gmi | 0 .../memex-marginalia/search-about.gmi | 0 .../memex-marginalia/server.gmi | 0 .../memex-marginalia/special/index.gmi | 0 .../memex-marginalia/special/redirect.gmi | 0 .../memex-marginalia/special/tombstone.gmi | 0 .../memex-marginalia/test.gmi | 0 .../memex-marginalia/todo.gmi | 0 .../memex-marginalia/todo/done.gmi | 0 .../memex-marginalia/todo/index.gmi | 0 .../memex-marginalia/todo/todo.gmi | 0 .../memex-marginalia/topic/astrolabe.gmi | 0 .../memex-marginalia/topic/cooking.gmi | 0 .../memex-marginalia/topic/index.gmi | 0 .../topic/moral-philosophy.gmi | 0 .../memex-marginalia/topic/nlnet.gmi | 0 .../memex-marginalia/topic/platforms.gmi | 0 .../memex-marginalia/topic/programming.gmi | 0 .../memex-marginalia/topic/satire.gmi | 0 .../memex-marginalia/topic/server.gmi | 0 .../memex-marginalia/topic/web-design.gmi | 0 .../memex-marginalia/topics.gmi | 0 .../memex-marginalia/worklog.gmi | 0 .../memex-marginalia/writing-topics.gmi | 0 .../converting/ConvertingIntegrationTest.java | 0 .../ConvertingIntegrationTestModule.java | 0 ...CrawlingThenConvertingIntegrationTest.java | 0 .../converting/logic/PlainTextLogicTest.java | 0 .../processor/logic/MetaRobotsTagTest.java | 0 .../logic/dom/MeasureLengthVisitorTest.java | 0 .../BlogSpecializationTest.java | 0 .../JavadocSpecializationTest.java | 0 .../LemmySpecializationTest.java | 0 .../WikiSpecializationTest.java | 0 .../XenForoSpecializationTest.java | 0 ...ncyclopediaMarginaliaNuSideloaderTest.java | 0 .../sideload/reddit/RedditSideloaderTest.java | 0 .../sideload/warc/WarcSideloaderTest.java | 0 .../converting/util/LineUtilsTest.java | 0 .../converting/util/TestLanguageModels.java | 0 code/processes/crawling-process/build.gradle | 2 + .../nu/marginalia/crawl/AbortMonitor.java | 0 .../java/nu/marginalia/crawl/CrawlerMain.java | 0 .../nu/marginalia/crawl/CrawlerModule.java | 0 .../marginalia/crawl/retreival/Cookies.java | 0 .../crawl/retreival/CrawlDataReference.java | 0 .../crawl/retreival/CrawlDelayTimer.java | 0 .../retreival/CrawledDocumentFactory.java | 0 .../crawl/retreival/CrawlerRetreiver.java | 0 .../retreival/CrawlerWarcResynchronizer.java | 0 .../crawl/retreival/DomainCrawlFrontier.java | 0 .../crawl/retreival/DomainProber.java | 0 .../crawl/retreival/LinkFilterSelector.java | 0 .../crawl/retreival/RateLimitException.java | 0 .../crawl/retreival/fetcher/ContentTags.java | 0 .../retreival/fetcher/ContentTypeProber.java | 0 .../crawl/retreival/fetcher/FetchResult.java | 0 .../retreival/fetcher/FetchResultState.java | 0 .../crawl/retreival/fetcher/HttpFetcher.java | 0 .../retreival/fetcher/HttpFetcherImpl.java | 0 .../retreival/fetcher/SitemapRetriever.java | 0 .../socket/FastTerminatingSocketFactory.java | 0 .../IpInterceptingNetworkInterceptor.java | 0 .../fetcher/socket/NoSecuritySSL.java | 0 .../fetcher/warc/WarcDigestBuilder.java | 0 .../fetcher/warc/WarcInputBuffer.java | 0 .../warc/WarcProtocolReconstructor.java | 0 .../retreival/fetcher/warc/WarcRecorder.java | 0 .../retreival/revisit/CrawlerRevisitor.java | 0 .../revisit/DocumentWithReference.java | 0 .../retreival/sitemap/SitemapFetcher.java | 0 .../crawl/spec/CrawlSpecProvider.java | 0 .../crawl/spec/DbCrawlSpecProvider.java | 0 .../crawl/spec/ParquetCrawlSpecProvider.java | 0 .../crawl/warc/WarcArchiverFactory.java | 0 .../marginalia/crawl/warc/WarcArchiverIf.java | 0 .../main => }/resources/ip-banned-cidr.txt | 0 .../CrawlerWarcResynchronizerTest.java | 0 .../retreival/DomainCrawlFrontierTest.java | 0 .../fetcher/ContentTypeProberTest.java | 0 ...edDocumentParquetRecordFileWriterTest.java | 0 .../retreival/fetcher/WarcRecorderTest.java | 0 .../revisit/DocumentWithReferenceTest.java | 0 .../crawling/CrawlPlanLoaderTest.java | 0 .../crawling/DomainCrawlerRobotsTxtTest.java | 0 .../marginalia/crawling/HttpFetcherTest.java | 0 .../marginalia/crawling/LinkParserTest.java | 0 .../marginalia/crawling/RssCrawlerTest.java | 0 .../retreival/CrawlerMockFetcherTest.java | 0 .../retreival/CrawlerRetreiverTest.java | 0 .../index-constructor-process/build.gradle | 6 ++- .../index/IndexConstructorMain.java | 0 .../index/IndexConstructorModule.java | 0 code/processes/loading-process/build.gradle | 2 + .../loading/LoaderIndexJournalWriter.java | 0 .../marginalia/loading/LoaderInputData.java | 0 .../nu/marginalia/loading/LoaderMain.java | 0 .../nu/marginalia/loading/LoaderModule.java | 0 .../documents/DocumentLoaderService.java | 0 .../documents/KeywordLoaderService.java | 0 .../loading/domains/DomainIdRegistry.java | 0 .../loading/domains/DomainLoaderService.java | 0 .../links/DomainLinksLoaderService.java | 0 .../nu/marginalia/loader/DbTestUtil.java | 0 .../domains/DomainLoaderServiceTest.java | 0 .../loader/LoaderIndexJournalWriterTest.java | 0 code/processes/test-data/build.gradle | 2 + .../nu/marginalia/test/CommonTestData.java | 0 .../mock-crawl-data/discourse/grid.html | 0 .../mock-crawl-data/discourse/index.html | 0 .../mock-crawl-data/discourse/telegram.html | 0 .../mock-crawl-data/javadoc/stream.html | 0 .../mock-crawl-data/lemmy/108995.html | 0 .../mock-crawl-data/lemmy/c_startrek.html | 0 .../mock-crawl-data/lemmy/index.html | 0 .../mock-crawl-data/mediawiki/doom1.html | 0 .../mock-crawl-data/mediawiki/doom2.html | 0 .../mock-crawl-data/mediawiki/index.html | 0 .../mock-crawl-data/xenforo/thread.html | 0 .../build.gradle | 2 + .../adjacencies/AdjacenciesData.java | 0 .../adjacencies/AdjacenciesLoader.java | 0 .../marginalia/adjacencies/DomainAliases.java | 0 .../adjacencies/SparseBitVector.java | 0 .../WebsiteAdjacenciesCalculator.java | 0 .../adjacencies/AdjacenciesLoaderTest.java | 0 .../adjacencies/SparseBitVectorTest.java | 0 .../api-service/build.gradle | 2 + .../java/nu/marginalia/api/ApiMain.java | 0 .../nu/marginalia/api/ApiSearchOperator.java | 0 .../java/nu/marginalia/api/ApiService.java | 0 .../nu/marginalia/api/model/ApiLicense.java | 0 .../marginalia/api/model/ApiSearchResult.java | 0 .../model/ApiSearchResultQueryDetails.java | 0 .../api/model/ApiSearchResults.java | 0 .../nu/marginalia/api/svc/LicenseService.java | 0 .../api/svc/RateLimiterService.java | 0 .../nu/marginalia/api/svc/ResponseCache.java | 0 .../api/svc/LicenseServiceTest.java | 0 .../api/svc/RateLimiterServiceTest.java | 0 .../marginalia/api/svc/ResponseCacheTest.java | 0 .../dating-service/build.gradle | 2 + .../java/nu/marginalia/dating/DatingMain.java | 0 .../nu/marginalia/dating/DatingModule.java | 0 .../nu/marginalia/dating/DatingService.java | 0 .../dating/DatingSessionObject.java | 0 .../resources/static/dating/robots.txt | 0 .../templates/dating/dating-view.hdb | 0 .../explorer-service/build.gradle | 2 + .../nu/marginalia/explorer/ExplorerMain.java | 0 .../marginalia/explorer/ExplorerModule.java | 0 .../marginalia/explorer/ExplorerService.java | 0 .../resources/static/explore/robots.txt | 0 .../resources/static/explore/style.css | 0 .../templates/explorer/explorer-about.hdb | 0 .../templates/explorer/explorer-messages.hdb | 0 .../templates/explorer/explorer-results.hdb | 0 .../templates/explorer/explorer-search.hdb | 0 .../resources/templates/explorer/explorer.hdb | 0 .../search-service/build.gradle | 2 + .../search/SearchHandlebarsConfigurator.java | 0 .../java/nu/marginalia/search/SearchMain.java | 0 .../nu/marginalia/search/SearchModule.java | 0 .../nu/marginalia/search/SearchOperator.java | 0 .../search/SearchQueryParamFactory.java | 0 .../search/SearchResultClusterer.java | 0 .../nu/marginalia/search/SearchService.java | 0 .../search/command/CommandEvaluator.java | 0 .../search/command/SearchAdtechParameter.java | 0 .../command/SearchCommandInterface.java | 0 .../search/command/SearchJsParameter.java | 0 .../search/command/SearchParameters.java | 0 .../search/command/SearchRecentParameter.java | 0 .../search/command/SearchTitleParameter.java | 0 .../search/command/commands/BangCommand.java | 0 .../command/commands/BrowseCommand.java | 0 .../command/commands/ConvertCommand.java | 0 .../command/commands/DefinitionCommand.java | 0 .../command/commands/SearchCommand.java | 0 .../command/commands/SiteRedirectCommand.java | 0 .../search/db/DbNearDomainsQuery.java | 0 .../search/exceptions/RedirectException.java | 0 .../search/model/ClusteredUrlDetails.java | 0 .../search/model/DecoratedSearchResults.java | 0 .../search/model/SearchFilters.java | 0 .../search/model/SearchProfile.java | 0 .../marginalia/search/model/UrlDetails.java | 0 .../search/results/BrowseResultCleaner.java | 0 .../search/results/UrlDeduplicator.java | 0 .../svc/SearchAddToCrawlQueueService.java | 0 .../search/svc/SearchBrowseService.java | 0 .../search/svc/SearchCrosstalkService.java | 0 .../search/svc/SearchErrorPageService.java | 0 .../search/svc/SearchFlagSiteService.java | 0 .../search/svc/SearchFrontPageService.java | 0 .../search/svc/SearchQueryCountService.java | 0 .../search/svc/SearchQueryIndexService.java | 0 .../search/svc/SearchQueryService.java | 0 .../search/svc/SearchSiteInfoService.java | 0 .../svc/SearchUnitConversionService.java | 0 .../resources/static/search/crawler-ips.txt | 0 .../resources/static/search/favicon.ico | Bin .../main => }/resources/static/search/main.js | 0 .../main => }/resources/static/search/menu.js | 0 .../resources/static/search/opensearch.xml | 0 .../resources/static/search/robots.txt | 0 .../main => }/resources/static/search/rss.svg | 0 .../resources/static/search/serp.scss | 0 .../resources/static/search/style-new.css | 0 .../main => }/resources/static/search/tts.js | 0 .../templates/search/browse-result.hdb | 0 .../templates/search/browse-results.hdb | 0 .../templates/search/conversion-results.hdb | 0 .../templates/search/dictionary-results.hdb | 0 .../templates/search/error-page-search.hdb | 0 .../resources/templates/search/error-page.hdb | 0 .../templates/search/index/index-about.hdb | 0 .../templates/search/index/index-news.hdb | 0 .../templates/search/index/index-tips.hdb | 0 .../templates/search/index/index.hdb | 0 .../templates/search/parts/search-filters.hdb | 0 .../templates/search/parts/search-footer.hdb | 0 .../templates/search/parts/search-form.hdb | 0 .../templates/search/parts/search-header.hdb | 0 .../search/parts/search-result-rest.hdb | 0 .../templates/search/parts/search-result.hdb | 0 .../templates/search/search-results.hdb | 0 .../search/site-info/site-crosstalk.hdb | 0 .../search/site-info/site-info-feed.hdb | 0 .../site-info/site-info-index-blacklisted.hdb | 0 .../site-info/site-info-index-indexed.hdb | 0 .../site-info/site-info-index-suggest.hdb | 0 .../site-info/site-info-index-unknown.hdb | 0 .../search/site-info/site-info-index.hdb | 0 .../search/site-info/site-info-links.hdb | 0 .../search/site-info/site-info-report.hdb | 0 .../search/site-info/site-info-summary.hdb | 0 .../templates/search/site-info/site-info.hdb | 0 .../command/commands/BangCommandTest.java | 0 .../marginalia/util/TestLanguageModels.java | 0 .../assistant-service/build.gradle | 2 + .../marginalia/assistant/AssistantMain.java | 0 .../marginalia/assistant/AssistantModule.java | 0 .../assistant/AssistantService.java | 0 .../assistant/suggest/Suggestions.java | 0 .../control-service/build.gradle | 2 + .../ControlHandlebarsConfigurator.java | 0 .../nu/marginalia/control/ControlMain.java | 0 .../control/ControlProcessModule.java | 0 .../control/ControlRendererFactory.java | 0 .../nu/marginalia/control/ControlService.java | 0 .../control/ControlValidationError.java | 0 .../marginalia/control/RedirectControl.java | 0 .../java/nu/marginalia/control/Redirects.java | 0 .../control/actor/ControlActor.java | 0 .../control/actor/ControlActorService.java | 0 .../control/actor/PrecessionNodes.java | 0 .../monitor/MessageQueueMonitorActor.java | 0 .../monitor/ServiceHeartbeatMonitorActor.java | 0 .../actor/precession/RecrawlAllActor.java | 0 .../actor/precession/ReindexAllActor.java | 0 .../actor/precession/ReprocessAllActor.java | 0 .../actor/rebalance/RebalanceActor.java | 0 .../control/app/model/ApiKeyModel.java | 0 .../app/model/BlacklistedDomainModel.java | 0 .../app/model/DomainComplaintCategory.java | 0 .../app/model/DomainComplaintModel.java | 0 .../control/app/svc/ApiKeyService.java | 0 .../app/svc/ControlBlacklistService.java | 0 .../app/svc/DomainComplaintService.java | 0 .../app/svc/RandomExplorationService.java | 0 .../control/app/svc/SearchToBanService.java | 0 .../model/FileStorageBaseWithStorage.java | 0 .../node/model/FileStorageFileModel.java | 0 .../node/model/FileStorageWithActions.java | 0 .../model/FileStorageWithRelatedEntries.java | 0 .../control/node/model/IndexNodeStatus.java | 0 .../node/svc/ControlFileStorageService.java | 0 .../node/svc/ControlNodeActionsService.java | 0 .../control/node/svc/ControlNodeService.java | 0 .../control/sys/model/AbortedProcess.java | 0 .../control/sys/model/EventLogEntry.java | 0 .../sys/model/EventLogServiceFilter.java | 0 .../control/sys/model/EventLogTypeFilter.java | 0 .../control/sys/model/MessageQueueEntry.java | 0 .../control/sys/model/ProcessHeartbeat.java | 0 .../control/sys/model/ServiceHeartbeat.java | 0 .../control/sys/model/TaskHeartbeat.java | 0 .../sys/svc/AbortedProcessService.java | 0 .../svc/ControlDomainRankingSetsService.java | 0 .../control/sys/svc/ControlErrorHandler.java | 0 .../sys/svc/ControlSysActionsService.java | 0 .../control/sys/svc/DataSetsService.java | 0 .../control/sys/svc/EventLogService.java | 0 .../control/sys/svc/HeartbeatService.java | 0 .../control/sys/svc/MessageQueueService.java | 0 .../resources/static/control/noanimation.css | 0 .../resources/static/control/refresh.js | 0 .../resources/static/control/tables.css | 0 .../resources/templates/control/actions.hdb | 0 .../templates/control/actor-details.hdb | 0 .../templates/control/app/api-keys.hdb | 0 .../templates/control/app/blacklist.hdb | 0 .../control/app/domain-complaints.hdb | 0 .../control/app/review-random-domains.hdb | 0 .../templates/control/app/search-to-ban.hdb | 0 .../resources/templates/control/error.hdb | 0 .../resources/templates/control/index.hdb | 0 .../actions/partial-download-sample-data.hdb | 0 .../node/actions/partial-export-db-data.hdb | 0 .../partial-export-from-crawl-data.hdb | 0 .../actions/partial-export-sample-data.hdb | 0 .../control/node/actions/partial-load.hdb | 0 .../node/actions/partial-new-crawl-specs.hdb | 0 .../node/actions/partial-new-crawl.hdb | 0 .../control/node/actions/partial-process.hdb | 0 .../control/node/actions/partial-recrawl.hdb | 0 .../node/actions/partial-repartition.hdb | 0 .../node/actions/partial-restore-backup.hdb | 0 .../node/actions/partial-sideload-dirtree.hdb | 0 .../actions/partial-sideload-encyclopedia.hdb | 0 .../node/actions/partial-sideload-reddit.hdb | 0 .../partial-sideload-stackexchange.hdb | 0 .../node/actions/partial-sideload-warc.hdb | 0 .../templates/control/node/node-actions.hdb | 0 .../templates/control/node/node-actors.hdb | 0 .../templates/control/node/node-config.hdb | 0 .../templates/control/node/node-overview.hdb | 0 .../control/node/node-storage-conf.hdb | 0 .../control/node/node-storage-details.hdb | 0 .../control/node/node-storage-list.hdb | 0 .../templates/control/node/nodes-list.hdb | 0 .../control/node/partial-node-nav.hdb | 0 .../control/partials/actor-summary-table.hdb | 0 .../control/partials/actors-table.hdb | 0 .../control/partials/events-table-summary.hdb | 0 .../control/partials/events-table.hdb | 0 .../control/partials/foot-includes.hdb | 0 .../control/partials/head-includes.hdb | 0 .../control/partials/message-queue-table.hdb | 0 .../templates/control/partials/nav.hdb | 0 .../control/partials/nodes-table.hdb | 0 .../control/partials/processes-table.hdb | 0 .../control/partials/services-table.hdb | 0 .../partials/storage-details/files.hdb | 0 .../partials/storage-details/related.hdb | 0 .../control/partials/storage-types.hdb | 0 .../templates/control/redirect-ok.hdb | 0 .../control/sys/aborted-processes.hdb | 0 .../templates/control/sys/data-sets.hdb | 0 .../control/sys/domain-ranking-sets.hdb | 0 .../templates/control/sys/events.hdb | 0 .../templates/control/sys/message-queue.hdb | 0 .../control/sys/new-domain-ranking-set.hdb | 0 .../templates/control/sys/new-message.hdb | 0 .../templates/control/sys/service-by-id.hdb | 0 .../templates/control/sys/sys-actions.hdb | 0 .../control/sys/update-domain-ranking-set.hdb | 0 .../control/sys/update-message-state.hdb | 0 .../templates/control/sys/view-message.hdb | 0 .../control/svc/ApiKeyServiceTest.java | 0 .../control/svc/HeartbeatServiceTest.java | 0 .../executor-service/build.gradle | 2 + .../nu/marginalia/executor/ExecutorMain.java | 0 .../marginalia/executor/ExecutorModule.java | 0 .../nu/marginalia/executor/ExecutorSvc.java | 0 code/services-core/index-service/build.gradle | 2 + .../java/nu/marginalia/index/IndexMain.java | 0 .../java/nu/marginalia/index/IndexModule.java | 0 .../nu/marginalia/index/IndexOpsService.java | 0 .../nu/marginalia/index/IndexService.java | 0 code/services-core/query-service/build.gradle | 2 + .../marginalia/query/QueryBasicInterface.java | 0 .../java/nu/marginalia/query/QueryMain.java | 0 .../java/nu/marginalia/query/QueryModule.java | 0 .../nu/marginalia/query/QueryService.java | 0 .../resources/static/public/index.html | 0 .../main => }/resources/templates/search.hdb | 0 code/tools/crawl-data-unfcker/build.gradle | 2 + .../nu/marginalia/tools/CrawlDataUnfcker.java | 0 code/tools/experiment-runner/build.gradle | 2 + .../java/nu/marginalia/tools/Experiment.java | 0 .../tools/ExperimentRunnerMain.java | 0 .../nu/marginalia/tools/LegacyExperiment.java | 0 .../tools/experiments/AdblockExperiment.java | 0 .../tools/experiments/AtagsExperiment.java | 0 .../experiments/DebugConverterExperiment.java | 0 .../ExportExternalLinksExperiment.java | 0 .../SentenceStatisticsExperiment.java | 0 .../experiments/SiteStatisticsExperiment.java | 0 .../tools/experiments/TestExperiment.java | 0 .../tools/experiments/TopicExperiment.java | 0 code/tools/load-test/build.gradle | 2 + .../nu/marginalia/load_test/LoadTestMain.java | 0 .../screenshot-capture-tool/build.gradle | 2 + .../screenshot/ScreenshotCaptureToolMain.java | 0 .../screenshot/ScreenshotLoaderMain.java | 0 protobuf.gradle | 12 ++++- settings.gradle | 6 +-- srcsets.gradle | 51 ++++++++++++++++++ .../encyclopedia-marginalia-nu/build.gradle | 7 +++ .../encyclopedia/EncyclopediaConverter.java | 51 ++++++++---------- 1555 files changed, 261 insertions(+), 93 deletions(-) rename code/common/config/{src/main => }/java/nu/marginalia/IndexLocations.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/LanguageModels.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/UserAgent.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/WebsiteUrl.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/WmsaHome.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/nodecfg/NodeConfigurationService.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/nodecfg/model/NodeConfiguration.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/FileStorageManifest.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/FileStorageService.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorage.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorageBase.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorageBaseId.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorageBaseType.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorageId.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorageState.java (100%) rename code/common/config/{src/main => }/java/nu/marginalia/storage/model/FileStorageType.java (100%) rename code/common/config/{src/test/java => test}/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java (100%) rename code/common/config/{src/test/java => test}/nu/marginalia/storage/FileStorageServiceTest.java (100%) rename code/common/db/{src/main => }/java/nu/marginalia/db/DbDomainQueries.java (100%) rename code/common/db/{src/main => }/java/nu/marginalia/db/DbDomainStatsExportMultitool.java (100%) rename code/common/db/{src/main => }/java/nu/marginalia/db/DomainBlacklist.java (100%) rename code/common/db/{src/main => }/java/nu/marginalia/db/DomainBlacklistImpl.java (100%) rename code/common/db/{src/main => }/java/nu/marginalia/db/DomainRankingSetsService.java (100%) rename code/common/db/{src/main => }/java/nu/marginalia/db/DomainTypes.java (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_000__base.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_001__blacklist.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_002__dictionary.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_003__crawl-queue.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_004__screenshot.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_005__domain_complaint.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_006__api_key.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_007__neighbors.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_008__random_domains.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_06_0_009__news_feed.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_001__domain_type.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_002__service_status.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_003__message_queue.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_004__file_storage.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_005__file_storage_default_values.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_006__message_queue_default_jobs.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_007__task_status.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_07_0_008__events_index.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_09_0_000__filestorage_livedb.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_09_1_000__drop_ecurl.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_09_2_000__filestorage_backup.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_09_2_001__filestorage_no_lexicon.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_000__file_storage_node.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_001__heartbeat_node.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_002__file_storage_state.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_003__node_configuration.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_004__file_storage_base_type.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_005__clean_message_queue.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_006__clean_stores.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_007__domain_node_affinity.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V23_11_0_008__purge_procedure.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V24_01_0_001__node_config__keep_warc.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V24_01_0_002__domain_set.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V24_01_0_003__mqaudit.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V24_02_0_000__drop_domain_links.sql (100%) rename code/common/db/{src/main => }/resources/db/migration/V24_02_0_001__drop_ranking_set_algo.sql (100%) rename code/common/db/{src/test/java => test}/nu/marginalia/db/DomainRankingSetsServiceTest.java (100%) rename code/common/db/{src/test/java => test}/nu/marginalia/db/DomainTypesTest.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/LinkdbFileNames.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/docs/DocumentDbReader.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/docs/DocumentDbWriter.java (100%) rename code/common/linkdb/{src/main => }/java/nu/marginalia/linkdb/model/DocdbUrlDetail.java (100%) rename code/common/linkdb/{src/main => }/resources/db/docdb-document.sql (100%) rename code/common/linkdb/{src/test/java => test}/nu/marginalia/linkdb/DocumentDbWriterTest.java (100%) rename code/common/linkdb/{src/test/java => test}/nu/marginalia/linkdb/DomainLinkDbTest.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/EdgeDomain.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/EdgeUrl.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/crawl/DomainIndexingState.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/crawl/HtmlFeature.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/crawl/PubDate.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/crawl/UrlIndexingState.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/gson/GsonFactory.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/html/HtmlStandard.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/id/UrlIdCodec.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/idx/DocumentFlags.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/idx/DocumentMetadata.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/idx/WordFlags.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/model/idx/WordMetadata.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/util/QueryParams.java (100%) rename code/common/model/{src/main => }/java/nu/marginalia/util/StringPool.java (100%) rename code/common/model/{src/test/java => test}/nu/marginalia/model/DocumentMetadataTest.java (100%) rename code/common/model/{src/test/java => test}/nu/marginalia/model/EdgeDomainTest.java (100%) rename code/common/model/{src/test/java => test}/nu/marginalia/model/EdgeUrlTest.java (100%) rename code/common/model/{src/test/java => test}/nu/marginalia/model/WordMetadataTest.java (100%) rename code/common/model/{src/test/java => test}/nu/marginalia/model/id/UrlIdCodecTest.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/ProcessConfiguration.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/ProcessConfigurationModule.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/FakeProcessHeartbeat.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeat.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/ProcessHeartbeat.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/ProcessTaskHeartbeat.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/log/WorkLoadIterable.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/log/WorkLog.java (100%) rename code/common/process/{src/main => }/java/nu/marginalia/process/log/WorkLogEntry.java (100%) rename code/common/process/{src/main => }/resources/log4j2.properties (100%) rename code/common/process/{src/test/java => test}/nu/marginalia/process/log/WorkLogTest.java (100%) rename code/common/renderer/{src/main => }/java/nu/marginalia/renderer/MustacheRenderer.java (100%) rename code/common/renderer/{src/main => }/java/nu/marginalia/renderer/RendererFactory.java (100%) rename code/common/renderer/{src/main => }/java/nu/marginalia/renderer/RenderingException.java (100%) rename code/common/renderer/{src/main => }/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java (100%) rename code/common/renderer/{src/main => }/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/NodeConfigurationWatcher.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/ServiceDiscoveryModule.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/ServiceMonitors.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/client/ServiceNotAvailableException.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/ServiceRegistryIf.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/ZkServiceRegistry.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/property/PartitionTraits.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/property/ServiceKey.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/discovery/property/ServicePartition.java (100%) rename code/common/service-discovery/{src/main => }/java/nu/marginalia/service/id/ServiceId.java (100%) rename code/common/service-discovery/{src/test/java => test}/nu/marginalia/service/discovery/ZkServiceRegistryTest.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/ConfigLoader.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/MainClass.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/ProcessMainClass.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/control/FakeServiceHeartbeat.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/control/ServiceEventLog.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/control/ServiceHeartbeat.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/control/ServiceTaskHeartbeat.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/module/DatabaseModule.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/module/ServiceConfiguration.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/module/ServiceConfigurationModule.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/BaseServiceParams.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/Initialization.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/MetricsServer.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/NodeStatusWatcher.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/RateLimiter.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/Service.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/StaticResources.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/mq/MqRequest.java (100%) rename code/common/service/{src/main => }/java/nu/marginalia/service/server/mq/ServiceMqSubscription.java (100%) rename code/common/service/{src/main => }/resources/log4j2-prod.xml (100%) rename code/common/service/{src/main => }/resources/log4j2-test.xml (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/client/ExecutorClient.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/client/ExecutorCrawlClient.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/client/ExecutorExportClient.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/client/ExecutorSideloadClient.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/model/ActorRunState.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/model/ActorRunStates.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/storage/FileStorageContent.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/storage/FileStorageFile.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/upload/UploadDirContents.java (100%) rename code/execution/api/{src/main => }/java/nu/marginalia/executor/upload/UploadDirItem.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/ActorApi.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/ExecutorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/ExecutorActorControlService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/ExecutorActorStateMachines.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/proc/ConverterMonitorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/proc/LoaderMonitorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ActorProcessWatcher.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ConvertActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ConvertAndLoadActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/CrawlActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/DownloadSampleActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ExportAtagsActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ExportDataActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ExportFeedsActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ExportSampleDataActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/ExportTermFreqActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/RecrawlActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/RestoreBackupActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/execution/ExecutionInit.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/execution/ExecutorExportGrpcService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/execution/ExecutorGrpcService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/process/ProcessOutboxes.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/process/ProcessService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/sideload/RedditSideloadHelper.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/sideload/SideloadHelper.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/svc/BackupService.java (100%) rename code/execution/{src/main => }/java/nu/marginalia/svc/DomainListRefreshService.java (100%) rename code/execution/{src/test/java => test}/nu/marginalia/svc/DomainListRefreshServiceTest.java (100%) rename code/features-convert/adblock/{src/main => }/java/nu/marginalia/adblock/AdblockSimulator.java (100%) rename code/features-convert/adblock/{src/main => }/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/AnchorTextKeywords.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/model/DomainLinks.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/model/Link.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/model/LinkWithText.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/source/AnchorTagsImpl.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/source/AnchorTagsSource.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/java/nu/marginalia/atags/source/AnchorTagsSourceFactory.java (100%) rename code/features-convert/anchor-keywords/{src/main => }/resources/atags-stop-list (100%) rename code/features-convert/anchor-keywords/{src/test/java => test}/nu/marginalia/atags/DomainAnchorTagsImplTest.java (100%) rename code/features-convert/anchor-keywords/{src/test/java => test}/nu/marginalia/util/TestLanguageModels.java (100%) rename code/features-convert/data-extractors/{src/main => }/java/nu/marginalia/extractor/AtagExporter.java (100%) rename code/features-convert/data-extractors/{src/main => }/java/nu/marginalia/extractor/ExporterIf.java (100%) rename code/features-convert/data-extractors/{src/main => }/java/nu/marginalia/extractor/FeedExporter.java (100%) rename code/features-convert/data-extractors/{src/main => }/java/nu/marginalia/extractor/SampleDataExporter.java (100%) rename code/features-convert/data-extractors/{src/main => }/java/nu/marginalia/extractor/TermFrequencyExporter.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/DocumentKeywordExtractor.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/KeywordExtractor.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/KeywordMetadata.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/WordReps.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/ArtifactKeywords.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/KeywordPositionBitmask.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/NameLikeKeywords.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/SubjectLikeKeywords.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/TitleKeywords.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/UrlKeywords.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/model/DocumentKeywords.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/model/DocumentKeywordsBuilder.java (100%) rename code/features-convert/keyword-extraction/{src/main => }/java/nu/marginalia/keyword/model/DocumentKeywordsPointer.java (100%) rename code/features-convert/keyword-extraction/{src/test/resources => test-resources}/test-data/java.html (100%) rename code/features-convert/keyword-extraction/{src/test/resources => test-resources}/test-data/keyboards.html (100%) rename code/features-convert/keyword-extraction/{src/test/resources => test-resources}/test-data/madonna.html (100%) rename code/features-convert/keyword-extraction/{src/test/resources => test-resources}/test-data/spam.html (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/DocumentKeywordExtractorTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/SentenceExtractorTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/extractors/ArtifactKeywordsTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/extractors/NameLikeKeywordsTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/extractors/SubjectLikeKeywordsTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/extractors/TitleKeywordsTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/keyword/extractors/UrlKeywordsTest.java (100%) rename code/features-convert/keyword-extraction/{src/test/java => test}/nu/marginalia/test/util/TestLanguageModels.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/PubDateEffortLevel.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/PubDateFromHtmlStandard.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/PubDateHeuristic.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/PubDateParser.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/PubDateSniffer.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java (100%) rename code/features-convert/pubdate/{src/main => }/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java (100%) rename code/features-convert/pubdate/{src/test/java => test}/nu/marginalia/pubdate/PubDateSnifferTest.java (100%) rename code/features-convert/pubdate/{src/test/java => test}/nu/marginalia/pubdate/PubDateTest.java (100%) rename code/features-convert/reddit-json/{src/main => }/java/nu/marginalia/integration/reddit/RedditEntryReader.java (100%) rename code/features-convert/reddit-json/{src/main => }/java/nu/marginalia/integration/reddit/db/RedditDb.java (100%) rename code/features-convert/reddit-json/{src/main => }/java/nu/marginalia/integration/reddit/model/ProcessableRedditComment.java (100%) rename code/features-convert/reddit-json/{src/main => }/java/nu/marginalia/integration/reddit/model/ProcessableRedditSubmission.java (100%) rename code/features-convert/reddit-json/{src/main => }/java/nu/marginalia/integration/reddit/model/RawRedditComment.java (100%) rename code/features-convert/reddit-json/{src/main => }/java/nu/marginalia/integration/reddit/model/RawRedditSubmission.java (100%) rename code/features-convert/reddit-json/{src/main => }/resources/db/reddit.sql (100%) rename code/features-convert/reddit-json/{src/test/java => test}/nu/marginalia/integration/reddit/RedditEntryReaderTest.java (100%) rename code/features-convert/reddit-json/{src/test/java => test}/nu/marginalia/integration/reddit/db/RedditDbTest.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/model/StackExchangeComment.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/model/StackExchangePost.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/sqlite/StackExchangePostsDb.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/xml/StackExchange7zXmlEventReaderSource.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReader.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlIterator.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReader.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/java/nu/marginalia/integration/stackexchange/xml/XmlEventReaderSource.java (100%) rename code/features-convert/stackexchange-xml/{src/main => }/resources/db/stackexchange.sql (100%) rename code/features-convert/stackexchange-xml/{src/test/java => test}/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReaderTest.java (100%) rename code/features-convert/stackexchange-xml/{src/test/java => test}/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReaderTest.java (100%) rename code/features-convert/stackexchange-xml/{src/test/java => test}/nu/marginalia/integration/stackexchange/xml/StringXmlTestEventReader.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/SummaryExtractor.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/DomFilterHeuristic.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/FallbackHeuristic.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/HeuristicTextUtil.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/MetaDescriptionHeuristic.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/OpenGraphDescriptionHeuristic.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/SummarizingDOMFilter.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/SummaryHeuristic.java (100%) rename code/features-convert/summary-extraction/{src/main => }/java/nu/marginalia/summary/heuristic/TagDensityHeuristic.java (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/monadnock.html (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/readme.md (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/summarization/187.shtml (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/summarization/surrey.html (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/summarization/surrey.html.1 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/theregister.html (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/index (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1021546012 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1028592943 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1081293162 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1105046394 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1146923296 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1194694074 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1207898281 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1268145073 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1294876331 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1314767420 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1316269786 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1316766580 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1319968043 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1338576987 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1341909571 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1369578579 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1437315645 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1458954960 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1475681345 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1498328446 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1507779664 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1540303379 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--154898476 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1552059399 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1557688340 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1584145751 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1605151204 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--162269247 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1624294488 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--164108285 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1645688243 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1658004609 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1658558834 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1698664879 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--169975195 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1701203332 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--17281998 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1742070028 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1745376814 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1749889035 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--176177364 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--177014197 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1794527707 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1797740201 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1799098579 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1959637826 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1971916964 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--1985840368 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--2012610859 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--202178680 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--2043528727 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--2081757477 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--2103982576 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--2111558769 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--213168798 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--232544032 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--253010011 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--274250994 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--332442790 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--353437903 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--364546777 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--379129416 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--399428149 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--425233170 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--434612307 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--439772328 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--458002611 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--506010305 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--546773534 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--551288516 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--602577763 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--611668054 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--634771245 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--639320493 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--643179018 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--663772351 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--670789152 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--6797317 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--700978490 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--708035332 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--804917062 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--819771302 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--840796372 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--841445362 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--862385354 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--879796466 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--89134993 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--905197876 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--920328354 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--952827759 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--964018507 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url--972614909 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-10088520 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1013281103 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1019241851 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1059944953 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1118681302 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1179298706 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1191749784 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1207094790 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1213989666 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1222442301 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-130332455 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1311055461 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1391842722 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1457388763 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1506356272 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1511762169 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1534640058 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1551513871 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1567632447 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1623049502 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-163919330 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1661398327 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1724309925 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1736807128 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1739031345 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1755745765 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1802811100 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1805364707 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1832702370 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1853114311 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1924872844 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-197772804 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1984259912 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-1990903988 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2039310951 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2040857056 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2052613093 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2063899866 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2115548255 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2127148436 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-2133781904 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-225690385 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-226401955 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-262970770 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-30106798 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-302167335 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-327999153 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-332568225 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-343223418 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-383103932 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-412929678 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-475213997 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-483403121 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-488667993 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-50815201 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-522685905 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-570714305 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-58733529 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-616518304 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-662169426 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-677278788 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-690486170 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-709693331 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-734531556 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-767530276 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-783154014 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-796905237 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-800099955 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-804101946 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-830664902 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-876060686 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-892584998 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-942458463 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-952036171 (100%) rename code/features-convert/summary-extraction/{src/test/resources => test-resources}/html/work-set/url-968207276 (100%) rename code/features-convert/summary-extraction/{src/test/java => test}/nu/marginalia/summary/SummaryExtractorTest.java (100%) rename code/features-convert/summary-extraction/{src/test/java => test}/nu/marginalia/summary/heuristic/HeuristicTextUtilTest.java (100%) rename code/features-convert/topic-detection/{src/main => }/java/nu/marginalia/topic/RecipeDetector.java (100%) rename code/features-convert/topic-detection/{src/main => }/java/nu/marginalia/topic/TextileCraftDetector.java (100%) rename code/features-convert/topic-detection/{src/main => }/java/nu/marginalia/topic/WoodworkingDetector.java (100%) rename code/features-crawl/content-type/{src/main => }/java/nu/marginalia/contenttype/ContentType.java (100%) rename code/features-crawl/content-type/{src/main => }/java/nu/marginalia/contenttype/ContentTypeParser.java (100%) rename code/features-crawl/content-type/{src/main => }/java/nu/marginalia/contenttype/DocumentBodyToString.java (100%) rename code/features-crawl/content-type/{src/test/java => test}/nu/marginalia/contenttype/ContentTypeParserTest.java (100%) rename code/features-crawl/content-type/{src/test/java => test}/nu/marginalia/contenttype/DocumentBodyToStringTest.java (100%) rename code/features-crawl/crawl-blocklist/{src/main => }/java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java (100%) rename code/features-crawl/crawl-blocklist/{src/main => }/java/nu/marginalia/ip_blocklist/InetAddressCache.java (100%) rename code/features-crawl/crawl-blocklist/{src/main => }/java/nu/marginalia/ip_blocklist/IpBlockList.java (100%) rename code/features-crawl/crawl-blocklist/{src/main => }/java/nu/marginalia/ip_blocklist/UrlBlocklist.java (100%) rename code/features-crawl/crawl-blocklist/{src/test/java => test}/nu/marginalia/ip_blocklist/UrlBlocklistTest.java (100%) rename code/features-crawl/link-parser/{src/main => }/java/nu/marginalia/link_parser/FeedExtractor.java (100%) rename code/features-crawl/link-parser/{src/main => }/java/nu/marginalia/link_parser/LinkParser.java (100%) rename code/features-search/feedlot-client/{src/main => }/java/nu/marginalia/feedlot/FeedlotClient.java (100%) rename code/features-search/feedlot-client/{src/main => }/java/nu/marginalia/feedlot/model/FeedItem.java (100%) rename code/features-search/feedlot-client/{src/main => }/java/nu/marginalia/feedlot/model/FeedItems.java (100%) rename code/features-search/random-websites/{src/main => }/java/nu/marginalia/browse/DbBrowseDomainsRandom.java (100%) rename code/features-search/random-websites/{src/main => }/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java (100%) rename code/features-search/random-websites/{src/main => }/java/nu/marginalia/browse/model/BrowseResult.java (100%) rename code/features-search/random-websites/{src/main => }/java/nu/marginalia/browse/model/BrowseResultSet.java (100%) rename code/features-search/screenshots/{src/main => }/java/nu/marginalia/screenshot/ScreenshotService.java (100%) rename code/functions/domain-info/api/{src/main => }/java/nu/marginalia/api/domains/DomainInfoClient.java (100%) rename code/functions/domain-info/api/{src/main => }/java/nu/marginalia/api/domains/DomainsProtobufCodec.java (100%) rename code/functions/domain-info/api/{src/main => }/java/nu/marginalia/api/domains/model/DomainInformation.java (100%) rename code/functions/domain-info/api/{src/main => }/java/nu/marginalia/api/domains/model/SimilarDomain.java (100%) rename code/functions/domain-info/{src/main => }/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java (100%) rename code/functions/domain-info/{src/main => }/java/nu/marginalia/functions/domains/DomainInformationService.java (100%) rename code/functions/domain-info/{src/main => }/java/nu/marginalia/functions/domains/SimilarDomainsService.java (100%) rename code/functions/domain-links/aggregate/{src/main => }/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java (100%) rename code/functions/domain-links/api/{src/main => }/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java (100%) rename code/functions/domain-links/api/{src/main => }/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java (100%) rename code/functions/domain-links/partition/{src/main => }/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java (100%) rename code/functions/math/api/{src/main => }/java/nu/marginalia/api/math/MathClient.java (100%) rename code/functions/math/api/{src/main => }/java/nu/marginalia/api/math/MathProtobufCodec.java (100%) rename code/functions/math/api/{src/main => }/java/nu/marginalia/api/math/model/DictionaryEntry.java (100%) rename code/functions/math/api/{src/main => }/java/nu/marginalia/api/math/model/DictionaryResponse.java (100%) rename code/functions/math/{src/main => }/java/nu/marginalia/functions/math/MathGrpcService.java (100%) rename code/functions/math/{src/main => }/java/nu/marginalia/functions/math/dict/DictionaryService.java (100%) rename code/functions/math/{src/main => }/java/nu/marginalia/functions/math/dict/SpellChecker.java (100%) rename code/functions/math/{src/main => }/java/nu/marginalia/functions/math/eval/MathParser.java (100%) rename code/functions/math/{src/main => }/java/nu/marginalia/functions/math/eval/Unit.java (100%) rename code/functions/math/{src/main => }/java/nu/marginalia/functions/math/eval/Units.java (100%) rename code/functions/math/{src/main => }/resources/units.csv (100%) rename code/functions/math/{src/test/java => test}/nu/marginalia/functions/math/eval/MathParserTest.java (100%) rename code/functions/math/{src/test/java => test}/nu/marginalia/functions/math/eval/UnitsTest.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/QueryClient.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/query/QueryParams.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java (100%) rename code/functions/search-query/api/{src/main => }/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java (100%) rename code/functions/search-query/api/{src/test/java => test}/nu/marginalia/index/client/IndexProtobufCodecTest.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/QueryGRPCService.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/util/language/EnglishDictionary.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/util/ngrams/DenseBitMap.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/util/ngrams/NGramBloomFilter.java (100%) rename code/functions/search-query/{src/main => }/java/nu/marginalia/util/transform_list/TransformList.java (100%) rename code/functions/search-query/{src/test/java => test}/nu/marginalia/query/svc/QueryFactoryTest.java (100%) rename code/index/api/{src/main => }/java/nu/marginalia/index/api/IndexClient.java (100%) rename code/index/api/{src/main => }/java/nu/marginalia/index/api/IndexMqClient.java (100%) rename code/index/api/{src/main => }/java/nu/marginalia/index/api/IndexMqEndpoints.java (100%) rename code/index/{forward-index => index-forward}/build.gradle (93%) rename code/index/{forward-index/src/main => index-forward}/java/nu/marginalia/index/forward/ForwardIndexConverter.java (100%) rename code/index/{forward-index/src/main => index-forward}/java/nu/marginalia/index/forward/ForwardIndexFileNames.java (100%) rename code/index/{forward-index/src/main => index-forward}/java/nu/marginalia/index/forward/ForwardIndexParameters.java (100%) rename code/index/{forward-index/src/main => index-forward}/java/nu/marginalia/index/forward/ForwardIndexReader.java (100%) rename code/index/{forward-index => index-forward}/readme.md (100%) rename code/index/{forward-index/src/test/java => index-forward/test}/nu/marginalia/index/forward/ForwardIndexConverterTest.java (100%) rename code/index/{forward-index/src/test/java => index-forward/test}/nu/marginalia/test/TestUtil.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/IndexJournalFileNames.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/model/IndexJournalEntry.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/reader/IndexJournalReader.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java (100%) rename code/index/index-journal/{src/main => }/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java (100%) rename code/index/index-journal/{src/test/java => test}/nu/marginalia/index/journal/IndexJournalTest.java (100%) rename code/index/index-journal/{src/test/java => test}/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java (100%) rename code/index/{reverse-index => index-reverse}/build.gradle (93%) rename code/index/{reverse-index => index-reverse}/index.svg (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/ReverseIndexEntrySource.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/ReverseIndexFullFileNames.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/ReverseIndexParameters.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/ReverseIndexPrioFileNames.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/ReverseIndexReader.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/ReverseIndexSelfTest.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/CountToOffsetTransformer.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/DocIdRewriter.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/IndexSizeEstimator.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/JournalReaderSource.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/ReverseIndexConstructor.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/ReversePreindex.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/ReversePreindexDocuments.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/ReversePreindexReference.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java (100%) rename code/index/{reverse-index/src/main => index-reverse}/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java (100%) rename code/index/{reverse-index => index-reverse}/merging.svg (100%) rename code/index/{reverse-index => index-reverse}/preindex.svg (100%) rename code/index/{reverse-index => index-reverse}/readme.md (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/ReverseIndexDebugTest.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/ReverseIndexReaderTest.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/construction/ReversePreindexDocsTest.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/construction/ReversePreindexMergeTest.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/construction/TestJournalFactory.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/index/construction/TestSegmentData.java (100%) rename code/index/{reverse-index/src/test/java => index-reverse/test}/nu/marginalia/test/TestUtil.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/IndexQueryGrpcService.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/IndexServicesFactory.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/SearchTermsUtil.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/index/CombinedIndexReader.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/index/IndexQueryService.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/index/StatefulIndex.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/model/IndexQueryParams.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/model/IndexSearchParameters.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/model/IndexSearchTerms.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/results/IndexMetadataService.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/results/IndexResultValuationContext.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/results/IndexResultValuatorService.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/searchset/DbUpdateRanks.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/searchset/RankingSearchSet.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/searchset/SearchSet.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/searchset/SearchSetAny.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/searchset/SearchSetsService.java (100%) rename code/index/{src/main => }/java/nu/marginalia/index/searchset/SmallSearchSet.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/RankingAlgorithm.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/data/GraphSource.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/results/ResultKeywordSet.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/results/ResultValuator.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/results/factors/Bm25Factor.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java (100%) rename code/index/{src/main => }/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/domainrankings/DomainRankings.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/EmptyEntrySource.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/EntrySource.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/IndexQuery.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/IndexQueryBuilder.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/IndexQueryPriority.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/IndexSearchBudget.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/limit/QueryLimits.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/limit/QueryStrategy.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/limit/SpecificationLimit.java (100%) rename code/index/query/{src/main => }/java/nu/marginalia/index/query/limit/SpecificationLimitType.java (100%) rename code/index/query/{src/test/java => test}/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/index/IndexQueryServiceIntegrationTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/index/searchset/RankingSearchSetTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/index/util/TestUtil.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/results/ResultValuatorTest.java (100%) rename code/index/{src/test/java => test}/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/ArrayRangeReference.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/IntArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/LongArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/LongArrayFactory.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/BulkTransferArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/IntArrayBase.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/IntArraySearch.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/IntArraySort.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/IntArrayTransformations.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/LongArrayBase.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/LongArraySearch.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/LongArraySort.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/LongArrayTransformations.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/SortAlgoMergeSort.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/SortAlgoQuickSort.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/SortingContext.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/algo/TwoArrayOperations.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/buffer/IntQueryBuffer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/buffer/LongQueryBuffer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/delegate/ReferenceImplIntArrayDelegate.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/delegate/ReferenceImplLongArrayDelegate.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/delegate/ShiftedIntArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/delegate/ShiftedLongArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/AddressRangeCall.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/AddressRangeCallIO.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/AddressRangeIntFunction.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/AddressRangeLongFunction.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/IntBinaryIOOperation.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/IntBinaryOperation.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/IntIOTransformer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/IntTransformer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/LongBinaryIOOperation.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/LongBinaryOperation.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/LongIOTransformer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/LongIntConsumer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/LongLongConsumer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/functional/LongTransformer.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/page/AbstractPagingArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/page/PartitionPage.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/page/SegmentIntArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/page/SegmentLongArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/page/UnsafeLongArray.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/page/UnsafeProvider.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java (100%) rename code/libraries/array/{src/main => }/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/IntArraySearchTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/IntArraySortTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/IntArrayTransformations2Test.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/IntArrayTransformationsTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/LongArraySearchTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/LongArraySortNTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/LongArraySortTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/LongArrayTransformations2Test.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/LongArrayTransformationsTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/algo/TwoArrayOperationsTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java (100%) rename code/libraries/array/{src/test/java => test}/nu/marginalia/util/test/TestUtil.java (100%) rename code/libraries/big-string/{src/main => }/java/nu/marginalia/bigstring/BigString.java (100%) rename code/libraries/big-string/{src/main => }/java/nu/marginalia/bigstring/CompressedBigString.java (100%) rename code/libraries/big-string/{src/main => }/java/nu/marginalia/bigstring/CompressionBuffer.java (100%) rename code/libraries/big-string/{src/main => }/java/nu/marginalia/bigstring/CompressionBufferPool.java (100%) rename code/libraries/big-string/{src/main => }/java/nu/marginalia/bigstring/PlainBigString.java (100%) rename code/libraries/big-string/{src/test/java => test}/nu/marginalia/bigstring/CompressedBigStringTest.java (100%) rename code/libraries/blocking-thread-pool/{src/main => }/java/nu/marginalia/util/ProcessingIterator.java (100%) rename code/libraries/blocking-thread-pool/{src/main => }/java/nu/marginalia/util/SimpleBlockingThreadPool.java (100%) rename code/libraries/blocking-thread-pool/{src/test/java => test}/nu/marginalia/util/ProcessingIteratorTest.java (100%) rename code/libraries/braille-block-punch-cards/{src/main => }/java/nu/marginalia/bbpc/BrailleBlockPunchCards.java (100%) rename code/libraries/braille-block-punch-cards/{src/test/java => test}/nu/marginalia/bbpc/BrailleBlockPunchCardsTest.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/BTreeDogEar.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/BTreeReader.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/BTreeWriteCallback.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/BTreeWriter.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/model/BTreeBlockSize.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/model/BTreeContext.java (100%) rename code/libraries/btree/{src/main => }/java/nu/marginalia/btree/model/BTreeHeader.java (100%) rename code/libraries/btree/{src/test/java => test}/nu/marginalia/btree/BTreeReaderQueryDataWithIndexTest.java (100%) rename code/libraries/btree/{src/test/java => test}/nu/marginalia/btree/BTreeReaderQueryDataWithoutIndexTest.java (100%) rename code/libraries/btree/{src/test/java => test}/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java (100%) rename code/libraries/btree/{src/test/java => test}/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java (100%) rename code/libraries/btree/{src/test/java => test}/nu/marginalia/btree/BTreeWriterTest.java (100%) rename code/libraries/easy-lsh/{src/main => }/java/nu/marginalia/lsh/EasyLSH.java (100%) rename code/libraries/easy-lsh/{src/test/java => test}/nu/marginalia/lsh/EasyLSHTest.java (100%) rename code/libraries/geo-ip/{src/main => }/java/nu/marginalia/geoip/GeoIpDictionary.java (100%) rename code/libraries/geo-ip/{src/main => }/java/nu/marginalia/geoip/sources/AsnMapping.java (100%) rename code/libraries/geo-ip/{src/main => }/java/nu/marginalia/geoip/sources/AsnTable.java (100%) rename code/libraries/geo-ip/{src/main => }/java/nu/marginalia/geoip/sources/IP2LocationMapping.java (100%) rename code/libraries/geo-ip/{src/main => }/java/nu/marginalia/geoip/sources/IpRangeMapping.java (100%) rename code/libraries/geo-ip/{src/test/java => test}/nu/marginalia/geoip/GeoIpDictionaryTest.java (100%) rename code/libraries/guarded-regex/{src/main => }/java/nu/marginalia/gregex/GuardedRegex.java (100%) rename code/libraries/guarded-regex/{src/main => }/java/nu/marginalia/gregex/GuardedRegexFactory.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/WordDictionary.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/WordPatterns.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/encoding/AsciiFlattener.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/encoding/UnicodeRanges.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/filter/FasttextLanguagePredictionModel.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/filter/LanguageFilter.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/filter/LanguagePredictionModel.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/filter/UngaBungaLanguagePredictionModel.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/model/DocumentLanguageData.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/model/DocumentSentence.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/model/WordFrequencyData.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/model/WordRep.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/model/WordSeparator.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/model/WordSpan.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/sentence/SentenceExtractor.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/sentence/SentenceExtractorHtmlTagCleaner.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/sentence/SentenceExtractorStringUtils.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/sentence/SentencePreCleaner.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java (100%) rename code/libraries/language-processing/{src/main => }/java/nu/marginalia/language/sentence/ThreadLocalSentenceExtractorProvider.java (100%) rename code/libraries/language-processing/{src/main => }/resources/dictionary/en-1000 (100%) rename code/libraries/language-processing/{src/main => }/resources/dictionary/en-stopwords (100%) rename code/libraries/language-processing/{src/main => }/resources/dictionary/en-words (100%) rename code/libraries/language-processing/{src/main => }/resources/dictionary/latin-1000 (100%) rename code/libraries/language-processing/{src/main => }/resources/dictionary/swe-1000 (100%) rename code/libraries/language-processing/{src/main => }/resources/dictionary/word-frequency (100%) rename code/libraries/language-processing/{src/test/resources => test-resources}/html/jep.html (100%) rename code/libraries/language-processing/{src/test/resources => test-resources}/html/spam.html (100%) rename code/libraries/language-processing/{src/test/java => test}/nu/marginalia/language/encoding/AsciiFlattenerTest.java (100%) rename code/libraries/language-processing/{src/test/java => test}/nu/marginalia/language/encoding/SentenceExtractorHtmlTagCleanerTest.java (100%) rename code/libraries/language-processing/{src/test/java => test}/nu/marginalia/language/filter/LanguageFilterTest.java (100%) rename code/libraries/language-processing/{src/test/java => test}/nu/marginalia/language/filter/TestLanguageModels.java (100%) rename code/libraries/language-processing/{src/test/java => test}/nu/marginalia/language/sentence/SentenceExtractorTest.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/ActorStateMachine.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/ExpectedMessage.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/prototype/ActorPrototype.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/prototype/RecordActorPrototype.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/ActorControlFlowException.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/ActorResumeBehavior.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/ActorStateInstance.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/ActorStateTransition.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/ActorStep.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/Resume.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/actor/state/Terminal.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/MessageQueueFactory.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/MqException.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/MqMessage.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/MqMessageState.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqAsynchronousInbox.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqInboxIf.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqInboxResponse.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqInboxShredder.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqSingleShotInbox.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqSubscription.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/inbox/MqSynchronousInbox.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/outbox/MqOutbox.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/persistence/MqMessageHandlerRegistry.java (100%) rename code/libraries/message-queue/{src/main => }/java/nu/marginalia/mq/persistence/MqPersistence.java (100%) rename code/libraries/message-queue/{src/test/java => test}/nu/marginalia/actor/ActorRecordMachineTest.java (100%) rename code/libraries/message-queue/{src/test/java => test}/nu/marginalia/mq/MqMessageRow.java (100%) rename code/libraries/message-queue/{src/test/java => test}/nu/marginalia/mq/MqTestUtil.java (100%) rename code/libraries/message-queue/{src/test/java => test}/nu/marginalia/mq/outbox/MqOutboxTest.java (100%) rename code/libraries/message-queue/{src/test/java => test}/nu/marginalia/mq/persistence/MqPersistenceTest.java (100%) rename code/libraries/next-prime/{src/main => }/java/nu/marginalia/util/NextPrimeUtil.java (100%) rename code/libraries/next-prime/{src/test/java => test}/nu/marginalia/util/NextPrimeUtilTest.java (100%) rename code/libraries/random-write-funnel/{src/main => }/java/nu/marginalia/rwf/RandomFileAssembler.java (100%) rename code/libraries/random-write-funnel/{src/main => }/java/nu/marginalia/rwf/RandomWriteFunnel.java (100%) rename code/libraries/random-write-funnel/{src/test/java => test}/nu/marginalia/rwf/RandomFileAssemblerTest.java (100%) rename code/libraries/random-write-funnel/{src/test/java => test}/nu/marginalia/rwf/RandomWriteFunnelTest.java (100%) rename code/libraries/term-frequency-dict/{src/main => }/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java (100%) rename code/libraries/test-helpers/{src/main => }/java/nu/marginalia/test/TestMigrationLoader.java (100%) rename code/process-models/crawl-spec/{src/main => }/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java (100%) rename code/process-models/crawl-spec/{src/main => }/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java (100%) rename code/process-models/crawl-spec/{src/main => }/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java (100%) rename code/process-models/crawl-spec/{src/main => }/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java (100%) rename code/process-models/crawl-spec/{src/main => }/java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/body/ContentTypeLogic.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/body/DocumentBodyExtractor.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/body/DocumentBodyResult.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/body/HttpFetchResult.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/CrawledDomainReader.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/CrawledDomainWriter.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/CrawlerOutputFile.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/format/FastLegacySerializableCrawlDataStream.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/io/format/ParquetSerializableCrawlDataStream.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/model/CrawledDocument.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/model/CrawledDomain.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/model/SerializableCrawlData.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java (100%) rename code/process-models/crawling-model/{src/main => }/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java (100%) rename code/process-models/crawling-model/{src/main => }/java/org/netpreserve/jwarc/WarcXCookieInformationHeader.java (100%) rename code/process-models/crawling-model/{src/main => }/java/org/netpreserve/jwarc/WarcXEntityRefused.java (100%) rename code/process-models/crawling-model/{src/main => }/java/org/netpreserve/jwarc/WarcXResponseReference.java (100%) rename code/process-models/crawling-model/{src/main => }/java/plan/CrawlPlan.java (100%) rename code/process-models/crawling-model/{src/main => }/java/plan/CrawlPlanLoader.java (100%) rename code/process-models/crawling-model/{src/test/java => test}/nu/marginalia/crawling/model/CrawledDocumentTest.java (100%) rename code/process-models/crawling-model/{src/test/java => test}/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriterTest.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/DocumentRecordParquetFileReader.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/DocumentRecordParquetFileWriter.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileWriter.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/DomainRecordParquetFileReader.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/DomainRecordParquetFileWriter.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/io/processed/ProcessedDataFileNames.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/model/processed/DocumentRecord.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/model/processed/DomainLinkRecord.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/model/processed/DomainRecord.java (100%) rename code/process-models/processed-data/{src/main => }/java/nu/marginalia/model/processed/DomainWithIp.java (100%) rename code/process-models/processed-data/{src/test/java => test}/nu/marginalia/io/processed/DocumentRecordParquetFileReaderTest.java (100%) rename code/process-models/processed-data/{src/test/java => test}/nu/marginalia/io/processed/DomainLinkRecordParquetFileReaderTest.java (100%) rename code/process-models/processed-data/{src/test/java => test}/nu/marginalia/io/processed/DomainRecordParquetFileReaderTest.java (100%) rename code/process-models/work-log/{src/main => }/java/nu/marginalia/worklog/BatchingWorkLog.java (100%) rename code/process-models/work-log/{src/main => }/java/nu/marginalia/worklog/BatchingWorkLogImpl.java (100%) rename code/process-models/work-log/{src/main => }/java/nu/marginalia/worklog/BatchingWorkLogInspector.java (100%) rename code/process-models/work-log/{src/test/java => test}/nu/marginalia/worklog/BatchingWorkLogImplTest.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/ProcessInboxNames.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/converting/ConvertAction.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/converting/ConvertRequest.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/crawling/CrawlRequest.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/index/CreateIndexRequest.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/index/IndexName.java (100%) rename code/process-mqapi/{src/main => }/java/nu/marginalia/mqapi/loading/LoadRequest.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/ConverterMain.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/ConverterModule.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/model/DisqualifiedException.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/model/GeneratorType.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/model/ProcessedDocument.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/model/ProcessedDomain.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/AcceptableAds.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/ConverterDomainTypes.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/DocumentClass.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/DocumentDecorator.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/DocumentProcessor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/DomainProcessor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/MetaRobotsTag.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/SiteWords.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/DocumentGeneratorExtractor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/DocumentLengthLogic.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/DocumentValuator.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/FeatureExtractor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/LshDocumentDeduplicator.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/TitleExtractor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/dom/DomPruningFilter.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/links/CommonKeywordExtractor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/links/FileLinks.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/links/LinkGraph.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/links/LinkProcessor.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/logic/links/TopKeywords.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/DefaultSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/SideloadSource.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/SideloaderProcessing.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpec.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpecList.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloader.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloaderFactory.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/sideload/warc/WarcSideloader.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/util/LineUtils.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/writer/ConverterBatchWritableIf.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/writer/ConverterBatchWriter.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/writer/ConverterBatchWriterIf.java (100%) rename code/processes/converting-process/{src/main => }/java/nu/marginalia/converting/writer/ConverterWriter.java (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/05-test.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/code/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/commons/dialogue.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/commons/search-failure-modes.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/commons/self-interest.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/dead.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/debian-laptop-install-log.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/debian-laptop.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/feed.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/fragments-old-web.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/good-video-games.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/guestbook/footer.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/guestbook/header.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/guestbook/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/index (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/junk/DedupTest.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/junk/bssl.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/junk/eldritch-oneliner.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/junk/server.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/junk/very-old-code.cc.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/links/aggregators.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/links/articles.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/links/bookmarks.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/links/fragments-old-web.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/links/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/links/linkpocalypse.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/00-linkpocalypse.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/01-astrolabe.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/02-re-tests.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/03-writing-for-reading.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/04-link-farms.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/05-minds-field.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/06-optimization.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/07-local-backlinks.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/08-whatever-happened-to-the-memex.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/09-system-upgrade.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/10-astrolabe-2-sampling-bias.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/11-dying-every-day.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/12-bye-bye-gmail.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/13-static-html.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/13-test.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/14-enter-the-circle-of-blame.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/15-stages-of-being.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/16-cursed-motivation.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/17-git-isnt-a-web-service.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/18-soaring-high.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/19-website-discoverability-crisis.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/20-dot-com-link-farms.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/21-new-solutions-old-problems.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/22-against-the-flood.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/23-re-software-and-branding.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/24-silly-hats.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/25-october-update.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/26-personalized-pagerank.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/27-getting-with-the-times.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/28-web-browsing.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/29-botnet-ddos.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/30-unintuitive-optimization.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/31-ngram-needles.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/32-bot-apologetics.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/33-rude-guests.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/34-internet-arguments.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/35-keeping-gemini-difficult.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/36-localized-programming-languages.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/37-keyword-extraction.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/38-old-and-new.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/39-normie-hypothesis.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/40-wasted-resources.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/41-search-result-relevance.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/42-dark.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/43-pseodonymous.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/44-discovery-and-design.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/45-unfuck-internet-discoverability.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/46-anatomy-of-search-engine-spam.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/47-drive-failure.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/48-i-have-no-capslock.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/49-marginalia-1-year.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/50-meditation-on-software-correctness.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/51-the-file-startup.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/52-growing-pains.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/53-better-hard-drive-metaphor.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/54-bargain-bin-btree.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/55-lexicon-rubberduck.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/56-uncertain-future.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/57-dont-know-how-to-build-software.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/58-marginalia-open-source.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/59-anchor-text.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/60-prescriptive-descriptions.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/61-botspam-apocalypse.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/62-marginaliacoin.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/63-marginalia-crawler.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/64-hundred-million.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/65-scaling-doesnt-scale.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/66-carbon-dating.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/67-best-ideas-afk.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/68-wizards-vs-sorcerers.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/69-creepy-website-similarity.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/70-faster-index-joins.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/71-memex-design.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/72-are-you-ok.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/72-new-approach-to-ranking.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/73-new-approach-to-ranking.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/74-marginalia-2-years.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/bargain-bin-btree.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/soaring-high.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/log/todo.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/one-weird-trick.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/pics/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/pics/links/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/pics/raster-test/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/about.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/api.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/changelog.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/design-notes.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/faq.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/for-webmasters.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/privacy.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/search-tips.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/supporting.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/edge/top-20.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/encyclopedia/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/gemini-server.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/memex.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/projects/wmsa.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/readme.md (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/recipes/chicken-soup.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/recipes/french-borscht.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/recipes/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/recipes/omelette-bacon.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/search-about.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/server.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/special/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/special/redirect.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/special/tombstone.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/test.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/todo.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/todo/done.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/todo/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/todo/todo.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/astrolabe.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/cooking.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/index.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/moral-philosophy.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/nlnet.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/platforms.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/programming.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/satire.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/server.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topic/web-design.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/topics.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/worklog.gmi (100%) rename code/processes/converting-process/{src/test/resources => test-resources}/memex-marginalia/writing-topics.gmi (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/ConvertingIntegrationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/ConvertingIntegrationTestModule.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/CrawlingThenConvertingIntegrationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/logic/PlainTextLogicTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/logic/MetaRobotsTagTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitorTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/plugin/specialization/BlogSpecializationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecializationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/plugin/specialization/LemmySpecializationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecializationTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloaderTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/sideload/reddit/RedditSideloaderTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/sideload/warc/WarcSideloaderTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/util/LineUtilsTest.java (100%) rename code/processes/converting-process/{src/test/java => test}/nu/marginalia/converting/util/TestLanguageModels.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/AbortMonitor.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/CrawlerMain.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/CrawlerModule.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/Cookies.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/CrawlDataReference.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/CrawledDocumentFactory.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/DomainCrawlFrontier.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/DomainProber.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/LinkFilterSelector.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/RateLimitException.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/ContentTags.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProber.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/FetchResult.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/FetchResultState.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcher.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/SitemapRetriever.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/socket/FastTerminatingSocketFactory.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/socket/IpInterceptingNetworkInterceptor.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/socket/NoSecuritySSL.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcDigestBuilder.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcInputBuffer.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcProtocolReconstructor.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcRecorder.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/retreival/sitemap/SitemapFetcher.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/spec/CrawlSpecProvider.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/spec/DbCrawlSpecProvider.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/spec/ParquetCrawlSpecProvider.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/warc/WarcArchiverFactory.java (100%) rename code/processes/crawling-process/{src/main => }/java/nu/marginalia/crawl/warc/WarcArchiverIf.java (100%) rename code/processes/crawling-process/{src/main => }/resources/ip-banned-cidr.txt (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawl/retreival/DomainCrawlFrontierTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawl/retreival/fetcher/CrawledDocumentParquetRecordFileWriterTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawl/retreival/revisit/DocumentWithReferenceTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/CrawlPlanLoaderTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/DomainCrawlerRobotsTxtTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/HttpFetcherTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/LinkParserTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/RssCrawlerTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/retreival/CrawlerMockFetcherTest.java (100%) rename code/processes/crawling-process/{src/test/java => test}/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java (100%) rename code/processes/index-constructor-process/{src/main => }/java/nu/marginalia/index/IndexConstructorMain.java (100%) rename code/processes/index-constructor-process/{src/main => }/java/nu/marginalia/index/IndexConstructorModule.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/LoaderIndexJournalWriter.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/LoaderInputData.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/LoaderMain.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/LoaderModule.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/documents/DocumentLoaderService.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/documents/KeywordLoaderService.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/domains/DomainIdRegistry.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/domains/DomainLoaderService.java (100%) rename code/processes/loading-process/{src/main => }/java/nu/marginalia/loading/links/DomainLinksLoaderService.java (100%) rename code/processes/loading-process/{src/test/java => test}/nu/marginalia/loader/DbTestUtil.java (100%) rename code/processes/loading-process/{src/test/java => test}/nu/marginalia/loading/domains/DomainLoaderServiceTest.java (100%) rename code/processes/loading-process/{src/test/java => test}/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java (100%) rename code/processes/test-data/{src/main => }/java/nu/marginalia/test/CommonTestData.java (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/discourse/grid.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/discourse/index.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/discourse/telegram.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/javadoc/stream.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/lemmy/108995.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/lemmy/c_startrek.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/lemmy/index.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/mediawiki/doom1.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/mediawiki/doom2.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/mediawiki/index.html (100%) rename code/processes/test-data/{src/main => }/resources/mock-crawl-data/xenforo/thread.html (100%) rename code/processes/website-adjacencies-calculator/{src/main => }/java/nu/marginalia/adjacencies/AdjacenciesData.java (100%) rename code/processes/website-adjacencies-calculator/{src/main => }/java/nu/marginalia/adjacencies/AdjacenciesLoader.java (100%) rename code/processes/website-adjacencies-calculator/{src/main => }/java/nu/marginalia/adjacencies/DomainAliases.java (100%) rename code/processes/website-adjacencies-calculator/{src/main => }/java/nu/marginalia/adjacencies/SparseBitVector.java (100%) rename code/processes/website-adjacencies-calculator/{src/main => }/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java (100%) rename code/processes/website-adjacencies-calculator/{src/test/java => test}/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java (100%) rename code/processes/website-adjacencies-calculator/{src/test/java => test}/nu/marginalia/adjacencies/SparseBitVectorTest.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/ApiMain.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/ApiSearchOperator.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/ApiService.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/model/ApiLicense.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/model/ApiSearchResult.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/model/ApiSearchResultQueryDetails.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/model/ApiSearchResults.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/svc/LicenseService.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/svc/RateLimiterService.java (100%) rename code/services-application/api-service/{src/main => }/java/nu/marginalia/api/svc/ResponseCache.java (100%) rename code/services-application/api-service/{src/test/java => test}/nu/marginalia/api/svc/LicenseServiceTest.java (100%) rename code/services-application/api-service/{src/test/java => test}/nu/marginalia/api/svc/RateLimiterServiceTest.java (100%) rename code/services-application/api-service/{src/test/java => test}/nu/marginalia/api/svc/ResponseCacheTest.java (100%) rename code/services-application/dating-service/{src/main => }/java/nu/marginalia/dating/DatingMain.java (100%) rename code/services-application/dating-service/{src/main => }/java/nu/marginalia/dating/DatingModule.java (100%) rename code/services-application/dating-service/{src/main => }/java/nu/marginalia/dating/DatingService.java (100%) rename code/services-application/dating-service/{src/main => }/java/nu/marginalia/dating/DatingSessionObject.java (100%) rename code/services-application/dating-service/{src/main => }/resources/static/dating/robots.txt (100%) rename code/services-application/dating-service/{src/main => }/resources/templates/dating/dating-view.hdb (100%) rename code/services-application/explorer-service/{src/main => }/java/nu/marginalia/explorer/ExplorerMain.java (100%) rename code/services-application/explorer-service/{src/main => }/java/nu/marginalia/explorer/ExplorerModule.java (100%) rename code/services-application/explorer-service/{src/main => }/java/nu/marginalia/explorer/ExplorerService.java (100%) rename code/services-application/explorer-service/{src/main => }/resources/static/explore/robots.txt (100%) rename code/services-application/explorer-service/{src/main => }/resources/static/explore/style.css (100%) rename code/services-application/explorer-service/{src/main => }/resources/templates/explorer/explorer-about.hdb (100%) rename code/services-application/explorer-service/{src/main => }/resources/templates/explorer/explorer-messages.hdb (100%) rename code/services-application/explorer-service/{src/main => }/resources/templates/explorer/explorer-results.hdb (100%) rename code/services-application/explorer-service/{src/main => }/resources/templates/explorer/explorer-search.hdb (100%) rename code/services-application/explorer-service/{src/main => }/resources/templates/explorer/explorer.hdb (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchHandlebarsConfigurator.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchMain.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchModule.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchOperator.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchQueryParamFactory.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchResultClusterer.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/SearchService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/CommandEvaluator.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/SearchAdtechParameter.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/SearchCommandInterface.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/SearchJsParameter.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/SearchParameters.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/SearchRecentParameter.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/SearchTitleParameter.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/commands/BangCommand.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/commands/BrowseCommand.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/commands/ConvertCommand.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/commands/DefinitionCommand.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/commands/SearchCommand.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/db/DbNearDomainsQuery.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/exceptions/RedirectException.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/model/ClusteredUrlDetails.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/model/DecoratedSearchResults.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/model/SearchFilters.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/model/SearchProfile.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/model/UrlDetails.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/results/BrowseResultCleaner.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/results/UrlDeduplicator.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchAddToCrawlQueueService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchBrowseService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchCrosstalkService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchErrorPageService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchFlagSiteService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchFrontPageService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchQueryCountService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchQueryIndexService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchQueryService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchSiteInfoService.java (100%) rename code/services-application/search-service/{src/main => }/java/nu/marginalia/search/svc/SearchUnitConversionService.java (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/crawler-ips.txt (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/favicon.ico (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/main.js (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/menu.js (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/opensearch.xml (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/robots.txt (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/rss.svg (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/serp.scss (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/style-new.css (100%) rename code/services-application/search-service/{src/main => }/resources/static/search/tts.js (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/browse-result.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/browse-results.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/conversion-results.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/dictionary-results.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/error-page-search.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/error-page.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/index/index-about.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/index/index-news.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/index/index-tips.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/index/index.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/parts/search-filters.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/parts/search-footer.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/parts/search-form.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/parts/search-header.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/parts/search-result-rest.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/parts/search-result.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/search-results.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-crosstalk.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-feed.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-index-blacklisted.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-index-indexed.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-index-suggest.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-index-unknown.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-index.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-links.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-report.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info-summary.hdb (100%) rename code/services-application/search-service/{src/main => }/resources/templates/search/site-info/site-info.hdb (100%) rename code/services-application/search-service/{src/test/java => test}/nu/marginalia/search/command/commands/BangCommandTest.java (100%) rename code/services-application/search-service/{src/test/java => test}/nu/marginalia/util/TestLanguageModels.java (100%) rename code/services-core/assistant-service/{src/main => }/java/nu/marginalia/assistant/AssistantMain.java (100%) rename code/services-core/assistant-service/{src/main => }/java/nu/marginalia/assistant/AssistantModule.java (100%) rename code/services-core/assistant-service/{src/main => }/java/nu/marginalia/assistant/AssistantService.java (100%) rename code/services-core/assistant-service/{src/main => }/java/nu/marginalia/assistant/suggest/Suggestions.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/ControlHandlebarsConfigurator.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/ControlMain.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/ControlProcessModule.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/ControlRendererFactory.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/ControlService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/ControlValidationError.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/RedirectControl.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/Redirects.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/ControlActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/ControlActorService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/PrecessionNodes.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/monitor/ServiceHeartbeatMonitorActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/precession/ReindexAllActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/precession/ReprocessAllActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/actor/rebalance/RebalanceActor.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/model/ApiKeyModel.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/model/DomainComplaintCategory.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/model/DomainComplaintModel.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/svc/ApiKeyService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/svc/ControlBlacklistService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/svc/DomainComplaintService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/svc/RandomExplorationService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/app/svc/SearchToBanService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/model/FileStorageBaseWithStorage.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/model/FileStorageFileModel.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/model/FileStorageWithActions.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/model/FileStorageWithRelatedEntries.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/model/IndexNodeStatus.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/svc/ControlFileStorageService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/node/svc/ControlNodeService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/AbortedProcess.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/EventLogEntry.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/MessageQueueEntry.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/model/TaskHeartbeat.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/AbortedProcessService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/ControlErrorHandler.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/DataSetsService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/EventLogService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/HeartbeatService.java (100%) rename code/services-core/control-service/{src/main => }/java/nu/marginalia/control/sys/svc/MessageQueueService.java (100%) rename code/services-core/control-service/{src/main => }/resources/static/control/noanimation.css (100%) rename code/services-core/control-service/{src/main => }/resources/static/control/refresh.js (100%) rename code/services-core/control-service/{src/main => }/resources/static/control/tables.css (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/actions.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/actor-details.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/app/api-keys.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/app/blacklist.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/app/domain-complaints.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/app/review-random-domains.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/app/search-to-ban.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/error.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/index.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-download-sample-data.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-export-db-data.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-export-from-crawl-data.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-export-sample-data.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-load.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-new-crawl-specs.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-new-crawl.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-process.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-recrawl.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-repartition.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-restore-backup.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-sideload-dirtree.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-sideload-encyclopedia.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-sideload-reddit.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-sideload-stackexchange.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/actions/partial-sideload-warc.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-actions.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-actors.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-config.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-overview.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-storage-conf.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-storage-details.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/node-storage-list.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/nodes-list.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/node/partial-node-nav.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/actor-summary-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/actors-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/events-table-summary.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/events-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/foot-includes.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/head-includes.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/message-queue-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/nav.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/nodes-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/processes-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/services-table.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/storage-details/files.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/storage-details/related.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/partials/storage-types.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/redirect-ok.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/aborted-processes.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/data-sets.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/domain-ranking-sets.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/events.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/message-queue.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/new-domain-ranking-set.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/new-message.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/service-by-id.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/sys-actions.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/update-domain-ranking-set.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/update-message-state.hdb (100%) rename code/services-core/control-service/{src/main => }/resources/templates/control/sys/view-message.hdb (100%) rename code/services-core/control-service/{src/test/java => test}/nu/marginalia/control/svc/ApiKeyServiceTest.java (100%) rename code/services-core/control-service/{src/test/java => test}/nu/marginalia/control/svc/HeartbeatServiceTest.java (100%) rename code/services-core/executor-service/{src/main => }/java/nu/marginalia/executor/ExecutorMain.java (100%) rename code/services-core/executor-service/{src/main => }/java/nu/marginalia/executor/ExecutorModule.java (100%) rename code/services-core/executor-service/{src/main => }/java/nu/marginalia/executor/ExecutorSvc.java (100%) rename code/services-core/index-service/{src/main => }/java/nu/marginalia/index/IndexMain.java (100%) rename code/services-core/index-service/{src/main => }/java/nu/marginalia/index/IndexModule.java (100%) rename code/services-core/index-service/{src/main => }/java/nu/marginalia/index/IndexOpsService.java (100%) rename code/services-core/index-service/{src/main => }/java/nu/marginalia/index/IndexService.java (100%) rename code/services-core/query-service/{src/main => }/java/nu/marginalia/query/QueryBasicInterface.java (100%) rename code/services-core/query-service/{src/main => }/java/nu/marginalia/query/QueryMain.java (100%) rename code/services-core/query-service/{src/main => }/java/nu/marginalia/query/QueryModule.java (100%) rename code/services-core/query-service/{src/main => }/java/nu/marginalia/query/QueryService.java (100%) rename code/services-core/query-service/{src/main => }/resources/static/public/index.html (100%) rename code/services-core/query-service/{src/main => }/resources/templates/search.hdb (100%) rename code/tools/crawl-data-unfcker/{src/main => }/java/nu/marginalia/tools/CrawlDataUnfcker.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/Experiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/ExperimentRunnerMain.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/LegacyExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/AdblockExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/AtagsExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/ExportExternalLinksExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/SiteStatisticsExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/TestExperiment.java (100%) rename code/tools/experiment-runner/{src/main => }/java/nu/marginalia/tools/experiments/TopicExperiment.java (100%) rename code/tools/load-test/{src/main => }/java/nu/marginalia/load_test/LoadTestMain.java (100%) rename code/tools/screenshot-capture-tool/{src/main => }/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java (100%) rename code/tools/screenshot-capture-tool/{src/main => }/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java (100%) create mode 100644 srcsets.gradle diff --git a/build.gradle b/build.gradle index 12ab5b31..3e4e8c99 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,14 @@ compileTestJava.options.encoding = "UTF-8" subprojects.forEach {it -> // Enable preview features for the entire project + + if (it.path.contains(':code:')) { + sourceSets.main.java.srcDirs += file('java') + sourceSets.main.resources.srcDirs += file('resources') + sourceSets.test.java.srcDirs += file('test') + sourceSets.test.resources.srcDirs += file('test-resources') + } + it.tasks.withType(JavaCompile).configureEach { options.compilerArgs += ['--enable-preview'] } @@ -32,34 +40,11 @@ subprojects.forEach {it -> preserveFileTimestamps = false reproducibleFileOrder = true } + } ext { dockerImageBase='container-registry.oracle.com/graalvm/jdk:21@sha256:1fd33d4d4eba3a9e1a41a728e39ea217178d257694eea1214fec68d2ed4d3d9b' } -allprojects { - apply plugin: 'java' - apply plugin: 'io.freefair.lombok' - dependencies { - implementation libs.lombok - testImplementation libs.lombok - annotationProcessor libs.lombok - - lombok libs.lombok // prevent plugin from downgrading the version to something incompatible with '19 - } - - test { - maxHeapSize = "8G" - useJUnitPlatform() - } - - tasks.register('fastTests', Test) { - maxHeapSize = "8G" - useJUnitPlatform { - excludeTags "slow" - } - } - -} idea { module { diff --git a/code/common/config/build.gradle b/code/common/config/build.gradle index 5d4b5ecc..66a984d4 100644 --- a/code/common/config/build.gradle +++ b/code/common/config/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:db') diff --git a/code/common/config/src/main/java/nu/marginalia/IndexLocations.java b/code/common/config/java/nu/marginalia/IndexLocations.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/IndexLocations.java rename to code/common/config/java/nu/marginalia/IndexLocations.java diff --git a/code/common/config/src/main/java/nu/marginalia/LanguageModels.java b/code/common/config/java/nu/marginalia/LanguageModels.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/LanguageModels.java rename to code/common/config/java/nu/marginalia/LanguageModels.java diff --git a/code/common/config/src/main/java/nu/marginalia/UserAgent.java b/code/common/config/java/nu/marginalia/UserAgent.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/UserAgent.java rename to code/common/config/java/nu/marginalia/UserAgent.java diff --git a/code/common/config/src/main/java/nu/marginalia/WebsiteUrl.java b/code/common/config/java/nu/marginalia/WebsiteUrl.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/WebsiteUrl.java rename to code/common/config/java/nu/marginalia/WebsiteUrl.java diff --git a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java b/code/common/config/java/nu/marginalia/WmsaHome.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/WmsaHome.java rename to code/common/config/java/nu/marginalia/WmsaHome.java diff --git a/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java b/code/common/config/java/nu/marginalia/nodecfg/NodeConfigurationService.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java rename to code/common/config/java/nu/marginalia/nodecfg/NodeConfigurationService.java diff --git a/code/common/config/src/main/java/nu/marginalia/nodecfg/model/NodeConfiguration.java b/code/common/config/java/nu/marginalia/nodecfg/model/NodeConfiguration.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/nodecfg/model/NodeConfiguration.java rename to code/common/config/java/nu/marginalia/nodecfg/model/NodeConfiguration.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageManifest.java b/code/common/config/java/nu/marginalia/storage/FileStorageManifest.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/FileStorageManifest.java rename to code/common/config/java/nu/marginalia/storage/FileStorageManifest.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java b/code/common/config/java/nu/marginalia/storage/FileStorageService.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java rename to code/common/config/java/nu/marginalia/storage/FileStorageService.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java b/code/common/config/java/nu/marginalia/storage/model/FileStorage.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorage.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java b/code/common/config/java/nu/marginalia/storage/model/FileStorageBase.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorageBase.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseId.java b/code/common/config/java/nu/marginalia/storage/model/FileStorageBaseId.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseId.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorageBaseId.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java b/code/common/config/java/nu/marginalia/storage/model/FileStorageBaseType.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorageBaseType.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageId.java b/code/common/config/java/nu/marginalia/storage/model/FileStorageId.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageId.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorageId.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageState.java b/code/common/config/java/nu/marginalia/storage/model/FileStorageState.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageState.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorageState.java diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageType.java b/code/common/config/java/nu/marginalia/storage/model/FileStorageType.java similarity index 100% rename from code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageType.java rename to code/common/config/java/nu/marginalia/storage/model/FileStorageType.java diff --git a/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java b/code/common/config/test/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java similarity index 100% rename from code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java rename to code/common/config/test/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java diff --git a/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java b/code/common/config/test/nu/marginalia/storage/FileStorageServiceTest.java similarity index 100% rename from code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java rename to code/common/config/test/nu/marginalia/storage/FileStorageServiceTest.java diff --git a/code/common/db/build.gradle b/code/common/db/build.gradle index a31e21fb..85ab7dba 100644 --- a/code/common/db/build.gradle +++ b/code/common/db/build.gradle @@ -26,6 +26,8 @@ configurations { flywayMigration.extendsFrom(implementation) } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') diff --git a/code/common/db/src/main/java/nu/marginalia/db/DbDomainQueries.java b/code/common/db/java/nu/marginalia/db/DbDomainQueries.java similarity index 100% rename from code/common/db/src/main/java/nu/marginalia/db/DbDomainQueries.java rename to code/common/db/java/nu/marginalia/db/DbDomainQueries.java diff --git a/code/common/db/src/main/java/nu/marginalia/db/DbDomainStatsExportMultitool.java b/code/common/db/java/nu/marginalia/db/DbDomainStatsExportMultitool.java similarity index 100% rename from code/common/db/src/main/java/nu/marginalia/db/DbDomainStatsExportMultitool.java rename to code/common/db/java/nu/marginalia/db/DbDomainStatsExportMultitool.java diff --git a/code/common/db/src/main/java/nu/marginalia/db/DomainBlacklist.java b/code/common/db/java/nu/marginalia/db/DomainBlacklist.java similarity index 100% rename from code/common/db/src/main/java/nu/marginalia/db/DomainBlacklist.java rename to code/common/db/java/nu/marginalia/db/DomainBlacklist.java diff --git a/code/common/db/src/main/java/nu/marginalia/db/DomainBlacklistImpl.java b/code/common/db/java/nu/marginalia/db/DomainBlacklistImpl.java similarity index 100% rename from code/common/db/src/main/java/nu/marginalia/db/DomainBlacklistImpl.java rename to code/common/db/java/nu/marginalia/db/DomainBlacklistImpl.java diff --git a/code/common/db/src/main/java/nu/marginalia/db/DomainRankingSetsService.java b/code/common/db/java/nu/marginalia/db/DomainRankingSetsService.java similarity index 100% rename from code/common/db/src/main/java/nu/marginalia/db/DomainRankingSetsService.java rename to code/common/db/java/nu/marginalia/db/DomainRankingSetsService.java diff --git a/code/common/db/src/main/java/nu/marginalia/db/DomainTypes.java b/code/common/db/java/nu/marginalia/db/DomainTypes.java similarity index 100% rename from code/common/db/src/main/java/nu/marginalia/db/DomainTypes.java rename to code/common/db/java/nu/marginalia/db/DomainTypes.java diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_000__base.sql b/code/common/db/resources/db/migration/V23_06_0_000__base.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_000__base.sql rename to code/common/db/resources/db/migration/V23_06_0_000__base.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql b/code/common/db/resources/db/migration/V23_06_0_001__blacklist.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql rename to code/common/db/resources/db/migration/V23_06_0_001__blacklist.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_002__dictionary.sql b/code/common/db/resources/db/migration/V23_06_0_002__dictionary.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_002__dictionary.sql rename to code/common/db/resources/db/migration/V23_06_0_002__dictionary.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_003__crawl-queue.sql b/code/common/db/resources/db/migration/V23_06_0_003__crawl-queue.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_003__crawl-queue.sql rename to code/common/db/resources/db/migration/V23_06_0_003__crawl-queue.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_004__screenshot.sql b/code/common/db/resources/db/migration/V23_06_0_004__screenshot.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_004__screenshot.sql rename to code/common/db/resources/db/migration/V23_06_0_004__screenshot.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_005__domain_complaint.sql b/code/common/db/resources/db/migration/V23_06_0_005__domain_complaint.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_005__domain_complaint.sql rename to code/common/db/resources/db/migration/V23_06_0_005__domain_complaint.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_006__api_key.sql b/code/common/db/resources/db/migration/V23_06_0_006__api_key.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_006__api_key.sql rename to code/common/db/resources/db/migration/V23_06_0_006__api_key.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_007__neighbors.sql b/code/common/db/resources/db/migration/V23_06_0_007__neighbors.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_007__neighbors.sql rename to code/common/db/resources/db/migration/V23_06_0_007__neighbors.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_008__random_domains.sql b/code/common/db/resources/db/migration/V23_06_0_008__random_domains.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_008__random_domains.sql rename to code/common/db/resources/db/migration/V23_06_0_008__random_domains.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_009__news_feed.sql b/code/common/db/resources/db/migration/V23_06_0_009__news_feed.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_06_0_009__news_feed.sql rename to code/common/db/resources/db/migration/V23_06_0_009__news_feed.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_001__domain_type.sql b/code/common/db/resources/db/migration/V23_07_0_001__domain_type.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_001__domain_type.sql rename to code/common/db/resources/db/migration/V23_07_0_001__domain_type.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_002__service_status.sql b/code/common/db/resources/db/migration/V23_07_0_002__service_status.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_002__service_status.sql rename to code/common/db/resources/db/migration/V23_07_0_002__service_status.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_003__message_queue.sql b/code/common/db/resources/db/migration/V23_07_0_003__message_queue.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_003__message_queue.sql rename to code/common/db/resources/db/migration/V23_07_0_003__message_queue.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_004__file_storage.sql b/code/common/db/resources/db/migration/V23_07_0_004__file_storage.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_004__file_storage.sql rename to code/common/db/resources/db/migration/V23_07_0_004__file_storage.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql b/code/common/db/resources/db/migration/V23_07_0_005__file_storage_default_values.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql rename to code/common/db/resources/db/migration/V23_07_0_005__file_storage_default_values.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_006__message_queue_default_jobs.sql b/code/common/db/resources/db/migration/V23_07_0_006__message_queue_default_jobs.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_006__message_queue_default_jobs.sql rename to code/common/db/resources/db/migration/V23_07_0_006__message_queue_default_jobs.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_007__task_status.sql b/code/common/db/resources/db/migration/V23_07_0_007__task_status.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_007__task_status.sql rename to code/common/db/resources/db/migration/V23_07_0_007__task_status.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_07_0_008__events_index.sql b/code/common/db/resources/db/migration/V23_07_0_008__events_index.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_07_0_008__events_index.sql rename to code/common/db/resources/db/migration/V23_07_0_008__events_index.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_09_0_000__filestorage_livedb.sql b/code/common/db/resources/db/migration/V23_09_0_000__filestorage_livedb.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_09_0_000__filestorage_livedb.sql rename to code/common/db/resources/db/migration/V23_09_0_000__filestorage_livedb.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_09_1_000__drop_ecurl.sql b/code/common/db/resources/db/migration/V23_09_1_000__drop_ecurl.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_09_1_000__drop_ecurl.sql rename to code/common/db/resources/db/migration/V23_09_1_000__drop_ecurl.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_09_2_000__filestorage_backup.sql b/code/common/db/resources/db/migration/V23_09_2_000__filestorage_backup.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_09_2_000__filestorage_backup.sql rename to code/common/db/resources/db/migration/V23_09_2_000__filestorage_backup.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_09_2_001__filestorage_no_lexicon.sql b/code/common/db/resources/db/migration/V23_09_2_001__filestorage_no_lexicon.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_09_2_001__filestorage_no_lexicon.sql rename to code/common/db/resources/db/migration/V23_09_2_001__filestorage_no_lexicon.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_000__file_storage_node.sql b/code/common/db/resources/db/migration/V23_11_0_000__file_storage_node.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_000__file_storage_node.sql rename to code/common/db/resources/db/migration/V23_11_0_000__file_storage_node.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_001__heartbeat_node.sql b/code/common/db/resources/db/migration/V23_11_0_001__heartbeat_node.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_001__heartbeat_node.sql rename to code/common/db/resources/db/migration/V23_11_0_001__heartbeat_node.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_002__file_storage_state.sql b/code/common/db/resources/db/migration/V23_11_0_002__file_storage_state.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_002__file_storage_state.sql rename to code/common/db/resources/db/migration/V23_11_0_002__file_storage_state.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql b/code/common/db/resources/db/migration/V23_11_0_003__node_configuration.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql rename to code/common/db/resources/db/migration/V23_11_0_003__node_configuration.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_004__file_storage_base_type.sql b/code/common/db/resources/db/migration/V23_11_0_004__file_storage_base_type.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_004__file_storage_base_type.sql rename to code/common/db/resources/db/migration/V23_11_0_004__file_storage_base_type.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_005__clean_message_queue.sql b/code/common/db/resources/db/migration/V23_11_0_005__clean_message_queue.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_005__clean_message_queue.sql rename to code/common/db/resources/db/migration/V23_11_0_005__clean_message_queue.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_006__clean_stores.sql b/code/common/db/resources/db/migration/V23_11_0_006__clean_stores.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_006__clean_stores.sql rename to code/common/db/resources/db/migration/V23_11_0_006__clean_stores.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_007__domain_node_affinity.sql b/code/common/db/resources/db/migration/V23_11_0_007__domain_node_affinity.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_007__domain_node_affinity.sql rename to code/common/db/resources/db/migration/V23_11_0_007__domain_node_affinity.sql diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_008__purge_procedure.sql b/code/common/db/resources/db/migration/V23_11_0_008__purge_procedure.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V23_11_0_008__purge_procedure.sql rename to code/common/db/resources/db/migration/V23_11_0_008__purge_procedure.sql diff --git a/code/common/db/src/main/resources/db/migration/V24_01_0_001__node_config__keep_warc.sql b/code/common/db/resources/db/migration/V24_01_0_001__node_config__keep_warc.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V24_01_0_001__node_config__keep_warc.sql rename to code/common/db/resources/db/migration/V24_01_0_001__node_config__keep_warc.sql diff --git a/code/common/db/src/main/resources/db/migration/V24_01_0_002__domain_set.sql b/code/common/db/resources/db/migration/V24_01_0_002__domain_set.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V24_01_0_002__domain_set.sql rename to code/common/db/resources/db/migration/V24_01_0_002__domain_set.sql diff --git a/code/common/db/src/main/resources/db/migration/V24_01_0_003__mqaudit.sql b/code/common/db/resources/db/migration/V24_01_0_003__mqaudit.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V24_01_0_003__mqaudit.sql rename to code/common/db/resources/db/migration/V24_01_0_003__mqaudit.sql diff --git a/code/common/db/src/main/resources/db/migration/V24_02_0_000__drop_domain_links.sql b/code/common/db/resources/db/migration/V24_02_0_000__drop_domain_links.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V24_02_0_000__drop_domain_links.sql rename to code/common/db/resources/db/migration/V24_02_0_000__drop_domain_links.sql diff --git a/code/common/db/src/main/resources/db/migration/V24_02_0_001__drop_ranking_set_algo.sql b/code/common/db/resources/db/migration/V24_02_0_001__drop_ranking_set_algo.sql similarity index 100% rename from code/common/db/src/main/resources/db/migration/V24_02_0_001__drop_ranking_set_algo.sql rename to code/common/db/resources/db/migration/V24_02_0_001__drop_ranking_set_algo.sql diff --git a/code/common/db/src/test/java/nu/marginalia/db/DomainRankingSetsServiceTest.java b/code/common/db/test/nu/marginalia/db/DomainRankingSetsServiceTest.java similarity index 100% rename from code/common/db/src/test/java/nu/marginalia/db/DomainRankingSetsServiceTest.java rename to code/common/db/test/nu/marginalia/db/DomainRankingSetsServiceTest.java diff --git a/code/common/db/src/test/java/nu/marginalia/db/DomainTypesTest.java b/code/common/db/test/nu/marginalia/db/DomainTypesTest.java similarity index 100% rename from code/common/db/src/test/java/nu/marginalia/db/DomainTypesTest.java rename to code/common/db/test/nu/marginalia/db/DomainTypesTest.java diff --git a/code/common/linkdb/build.gradle b/code/common/linkdb/build.gradle index 41ec618d..811aa577 100644 --- a/code/common/linkdb/build.gradle +++ b/code/common/linkdb/build.gradle @@ -14,6 +14,8 @@ configurations { flywayMigration.extendsFrom(implementation) } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:service') diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbFileNames.java b/code/common/linkdb/java/nu/marginalia/linkdb/LinkdbFileNames.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbFileNames.java rename to code/common/linkdb/java/nu/marginalia/linkdb/LinkdbFileNames.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java b/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java rename to code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java b/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java rename to code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java b/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java rename to code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java b/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java rename to code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java b/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java rename to code/common/linkdb/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/docs/DocumentDbReader.java b/code/common/linkdb/java/nu/marginalia/linkdb/docs/DocumentDbReader.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/docs/DocumentDbReader.java rename to code/common/linkdb/java/nu/marginalia/linkdb/docs/DocumentDbReader.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/docs/DocumentDbWriter.java b/code/common/linkdb/java/nu/marginalia/linkdb/docs/DocumentDbWriter.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/docs/DocumentDbWriter.java rename to code/common/linkdb/java/nu/marginalia/linkdb/docs/DocumentDbWriter.java diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/model/DocdbUrlDetail.java b/code/common/linkdb/java/nu/marginalia/linkdb/model/DocdbUrlDetail.java similarity index 100% rename from code/common/linkdb/src/main/java/nu/marginalia/linkdb/model/DocdbUrlDetail.java rename to code/common/linkdb/java/nu/marginalia/linkdb/model/DocdbUrlDetail.java diff --git a/code/common/linkdb/src/main/resources/db/docdb-document.sql b/code/common/linkdb/resources/db/docdb-document.sql similarity index 100% rename from code/common/linkdb/src/main/resources/db/docdb-document.sql rename to code/common/linkdb/resources/db/docdb-document.sql diff --git a/code/common/linkdb/src/test/java/nu/marginalia/linkdb/DocumentDbWriterTest.java b/code/common/linkdb/test/nu/marginalia/linkdb/DocumentDbWriterTest.java similarity index 100% rename from code/common/linkdb/src/test/java/nu/marginalia/linkdb/DocumentDbWriterTest.java rename to code/common/linkdb/test/nu/marginalia/linkdb/DocumentDbWriterTest.java diff --git a/code/common/linkdb/src/test/java/nu/marginalia/linkdb/DomainLinkDbTest.java b/code/common/linkdb/test/nu/marginalia/linkdb/DomainLinkDbTest.java similarity index 100% rename from code/common/linkdb/src/test/java/nu/marginalia/linkdb/DomainLinkDbTest.java rename to code/common/linkdb/test/nu/marginalia/linkdb/DomainLinkDbTest.java diff --git a/code/common/model/build.gradle b/code/common/model/build.gradle index 9a3b81d8..80d9f247 100644 --- a/code/common/model/build.gradle +++ b/code/common/model/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:service-discovery') implementation project(':code:libraries:big-string') diff --git a/code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java b/code/common/model/java/nu/marginalia/model/EdgeDomain.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java rename to code/common/model/java/nu/marginalia/model/EdgeDomain.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java b/code/common/model/java/nu/marginalia/model/EdgeUrl.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java rename to code/common/model/java/nu/marginalia/model/EdgeUrl.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/crawl/DomainIndexingState.java b/code/common/model/java/nu/marginalia/model/crawl/DomainIndexingState.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/crawl/DomainIndexingState.java rename to code/common/model/java/nu/marginalia/model/crawl/DomainIndexingState.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/crawl/HtmlFeature.java b/code/common/model/java/nu/marginalia/model/crawl/HtmlFeature.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/crawl/HtmlFeature.java rename to code/common/model/java/nu/marginalia/model/crawl/HtmlFeature.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/crawl/PubDate.java b/code/common/model/java/nu/marginalia/model/crawl/PubDate.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/crawl/PubDate.java rename to code/common/model/java/nu/marginalia/model/crawl/PubDate.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/crawl/UrlIndexingState.java b/code/common/model/java/nu/marginalia/model/crawl/UrlIndexingState.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/crawl/UrlIndexingState.java rename to code/common/model/java/nu/marginalia/model/crawl/UrlIndexingState.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/gson/GsonFactory.java b/code/common/model/java/nu/marginalia/model/gson/GsonFactory.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/gson/GsonFactory.java rename to code/common/model/java/nu/marginalia/model/gson/GsonFactory.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/html/HtmlStandard.java b/code/common/model/java/nu/marginalia/model/html/HtmlStandard.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/html/HtmlStandard.java rename to code/common/model/java/nu/marginalia/model/html/HtmlStandard.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/id/UrlIdCodec.java b/code/common/model/java/nu/marginalia/model/id/UrlIdCodec.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/id/UrlIdCodec.java rename to code/common/model/java/nu/marginalia/model/id/UrlIdCodec.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/idx/DocumentFlags.java b/code/common/model/java/nu/marginalia/model/idx/DocumentFlags.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/idx/DocumentFlags.java rename to code/common/model/java/nu/marginalia/model/idx/DocumentFlags.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/idx/DocumentMetadata.java b/code/common/model/java/nu/marginalia/model/idx/DocumentMetadata.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/idx/DocumentMetadata.java rename to code/common/model/java/nu/marginalia/model/idx/DocumentMetadata.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/idx/WordFlags.java b/code/common/model/java/nu/marginalia/model/idx/WordFlags.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/idx/WordFlags.java rename to code/common/model/java/nu/marginalia/model/idx/WordFlags.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/idx/WordMetadata.java b/code/common/model/java/nu/marginalia/model/idx/WordMetadata.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/model/idx/WordMetadata.java rename to code/common/model/java/nu/marginalia/model/idx/WordMetadata.java diff --git a/code/common/model/src/main/java/nu/marginalia/util/QueryParams.java b/code/common/model/java/nu/marginalia/util/QueryParams.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/util/QueryParams.java rename to code/common/model/java/nu/marginalia/util/QueryParams.java diff --git a/code/common/model/src/main/java/nu/marginalia/util/StringPool.java b/code/common/model/java/nu/marginalia/util/StringPool.java similarity index 100% rename from code/common/model/src/main/java/nu/marginalia/util/StringPool.java rename to code/common/model/java/nu/marginalia/util/StringPool.java diff --git a/code/common/model/src/test/java/nu/marginalia/model/DocumentMetadataTest.java b/code/common/model/test/nu/marginalia/model/DocumentMetadataTest.java similarity index 100% rename from code/common/model/src/test/java/nu/marginalia/model/DocumentMetadataTest.java rename to code/common/model/test/nu/marginalia/model/DocumentMetadataTest.java diff --git a/code/common/model/src/test/java/nu/marginalia/model/EdgeDomainTest.java b/code/common/model/test/nu/marginalia/model/EdgeDomainTest.java similarity index 100% rename from code/common/model/src/test/java/nu/marginalia/model/EdgeDomainTest.java rename to code/common/model/test/nu/marginalia/model/EdgeDomainTest.java diff --git a/code/common/model/src/test/java/nu/marginalia/model/EdgeUrlTest.java b/code/common/model/test/nu/marginalia/model/EdgeUrlTest.java similarity index 100% rename from code/common/model/src/test/java/nu/marginalia/model/EdgeUrlTest.java rename to code/common/model/test/nu/marginalia/model/EdgeUrlTest.java diff --git a/code/common/model/src/test/java/nu/marginalia/model/WordMetadataTest.java b/code/common/model/test/nu/marginalia/model/WordMetadataTest.java similarity index 100% rename from code/common/model/src/test/java/nu/marginalia/model/WordMetadataTest.java rename to code/common/model/test/nu/marginalia/model/WordMetadataTest.java diff --git a/code/common/model/src/test/java/nu/marginalia/model/id/UrlIdCodecTest.java b/code/common/model/test/nu/marginalia/model/id/UrlIdCodecTest.java similarity index 100% rename from code/common/model/src/test/java/nu/marginalia/model/id/UrlIdCodecTest.java rename to code/common/model/test/nu/marginalia/model/id/UrlIdCodecTest.java diff --git a/code/common/process/build.gradle b/code/common/process/build.gradle index fb68b32f..7fcff1ab 100644 --- a/code/common/process/build.gradle +++ b/code/common/process/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.notnull diff --git a/code/common/process/src/main/java/nu/marginalia/ProcessConfiguration.java b/code/common/process/java/nu/marginalia/ProcessConfiguration.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/ProcessConfiguration.java rename to code/common/process/java/nu/marginalia/ProcessConfiguration.java diff --git a/code/common/process/src/main/java/nu/marginalia/ProcessConfigurationModule.java b/code/common/process/java/nu/marginalia/ProcessConfigurationModule.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/ProcessConfigurationModule.java rename to code/common/process/java/nu/marginalia/ProcessConfigurationModule.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/FakeProcessHeartbeat.java b/code/common/process/java/nu/marginalia/process/control/FakeProcessHeartbeat.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/FakeProcessHeartbeat.java rename to code/common/process/java/nu/marginalia/process/control/FakeProcessHeartbeat.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeat.java b/code/common/process/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeat.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeat.java rename to code/common/process/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeat.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java b/code/common/process/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java rename to code/common/process/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeat.java b/code/common/process/java/nu/marginalia/process/control/ProcessHeartbeat.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeat.java rename to code/common/process/java/nu/marginalia/process/control/ProcessHeartbeat.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java b/code/common/process/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java rename to code/common/process/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeat.java b/code/common/process/java/nu/marginalia/process/control/ProcessTaskHeartbeat.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeat.java rename to code/common/process/java/nu/marginalia/process/control/ProcessTaskHeartbeat.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java b/code/common/process/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java rename to code/common/process/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/log/WorkLoadIterable.java b/code/common/process/java/nu/marginalia/process/log/WorkLoadIterable.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/log/WorkLoadIterable.java rename to code/common/process/java/nu/marginalia/process/log/WorkLoadIterable.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/log/WorkLog.java b/code/common/process/java/nu/marginalia/process/log/WorkLog.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/log/WorkLog.java rename to code/common/process/java/nu/marginalia/process/log/WorkLog.java diff --git a/code/common/process/src/main/java/nu/marginalia/process/log/WorkLogEntry.java b/code/common/process/java/nu/marginalia/process/log/WorkLogEntry.java similarity index 100% rename from code/common/process/src/main/java/nu/marginalia/process/log/WorkLogEntry.java rename to code/common/process/java/nu/marginalia/process/log/WorkLogEntry.java diff --git a/code/common/process/src/main/resources/log4j2.properties b/code/common/process/resources/log4j2.properties similarity index 100% rename from code/common/process/src/main/resources/log4j2.properties rename to code/common/process/resources/log4j2.properties diff --git a/code/common/process/src/test/java/nu/marginalia/process/log/WorkLogTest.java b/code/common/process/test/nu/marginalia/process/log/WorkLogTest.java similarity index 100% rename from code/common/process/src/test/java/nu/marginalia/process/log/WorkLogTest.java rename to code/common/process/test/nu/marginalia/process/log/WorkLogTest.java diff --git a/code/common/renderer/build.gradle b/code/common/renderer/build.gradle index b40aaee2..83957546 100644 --- a/code/common/renderer/build.gradle +++ b/code/common/renderer/build.gradle @@ -11,6 +11,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java b/code/common/renderer/java/nu/marginalia/renderer/MustacheRenderer.java similarity index 100% rename from code/common/renderer/src/main/java/nu/marginalia/renderer/MustacheRenderer.java rename to code/common/renderer/java/nu/marginalia/renderer/MustacheRenderer.java diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/RendererFactory.java b/code/common/renderer/java/nu/marginalia/renderer/RendererFactory.java similarity index 100% rename from code/common/renderer/src/main/java/nu/marginalia/renderer/RendererFactory.java rename to code/common/renderer/java/nu/marginalia/renderer/RendererFactory.java diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/RenderingException.java b/code/common/renderer/java/nu/marginalia/renderer/RenderingException.java similarity index 100% rename from code/common/renderer/src/main/java/nu/marginalia/renderer/RenderingException.java rename to code/common/renderer/java/nu/marginalia/renderer/RenderingException.java diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java b/code/common/renderer/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java similarity index 100% rename from code/common/renderer/src/main/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java rename to code/common/renderer/java/nu/marginalia/renderer/config/DefaultHandlebarsConfigurator.java diff --git a/code/common/renderer/src/main/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java b/code/common/renderer/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java similarity index 100% rename from code/common/renderer/src/main/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java rename to code/common/renderer/java/nu/marginalia/renderer/config/HandlebarsConfigurator.java diff --git a/code/common/service-discovery/build.gradle b/code/common/service-discovery/build.gradle index d86ef9f4..db87eb95 100644 --- a/code/common/service-discovery/build.gradle +++ b/code/common/service-discovery/build.gradle @@ -18,6 +18,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/NodeConfigurationWatcher.java b/code/common/service-discovery/java/nu/marginalia/service/NodeConfigurationWatcher.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/NodeConfigurationWatcher.java rename to code/common/service-discovery/java/nu/marginalia/service/NodeConfigurationWatcher.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java b/code/common/service-discovery/java/nu/marginalia/service/ServiceDiscoveryModule.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java rename to code/common/service-discovery/java/nu/marginalia/service/ServiceDiscoveryModule.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java b/code/common/service-discovery/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java rename to code/common/service-discovery/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceMonitors.java b/code/common/service-discovery/java/nu/marginalia/service/ServiceMonitors.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceMonitors.java rename to code/common/service-discovery/java/nu/marginalia/service/ServiceMonitors.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java rename to code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java rename to code/common/service-discovery/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java rename to code/common/service-discovery/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java b/code/common/service-discovery/java/nu/marginalia/service/client/ServiceNotAvailableException.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java rename to code/common/service-discovery/java/nu/marginalia/service/client/ServiceNotAvailableException.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/ServiceRegistryIf.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/ServiceRegistryIf.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/ZkServiceRegistry.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/ZkServiceRegistry.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/property/PartitionTraits.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/property/PartitionTraits.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/property/ServiceKey.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/property/ServiceKey.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/property/ServicePartition.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java rename to code/common/service-discovery/java/nu/marginalia/service/discovery/property/ServicePartition.java diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java b/code/common/service-discovery/java/nu/marginalia/service/id/ServiceId.java similarity index 100% rename from code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java rename to code/common/service-discovery/java/nu/marginalia/service/id/ServiceId.java diff --git a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java b/code/common/service-discovery/test/nu/marginalia/service/discovery/ZkServiceRegistryTest.java similarity index 100% rename from code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java rename to code/common/service-discovery/test/nu/marginalia/service/discovery/ZkServiceRegistryTest.java diff --git a/code/common/service/build.gradle b/code/common/service/build.gradle index 4e4e8ed1..75fab4ca 100644 --- a/code/common/service/build.gradle +++ b/code/common/service/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') diff --git a/code/common/service/src/main/java/nu/marginalia/service/ConfigLoader.java b/code/common/service/java/nu/marginalia/service/ConfigLoader.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/ConfigLoader.java rename to code/common/service/java/nu/marginalia/service/ConfigLoader.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/MainClass.java b/code/common/service/java/nu/marginalia/service/MainClass.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/MainClass.java rename to code/common/service/java/nu/marginalia/service/MainClass.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/ProcessMainClass.java b/code/common/service/java/nu/marginalia/service/ProcessMainClass.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/ProcessMainClass.java rename to code/common/service/java/nu/marginalia/service/ProcessMainClass.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/FakeServiceHeartbeat.java b/code/common/service/java/nu/marginalia/service/control/FakeServiceHeartbeat.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/control/FakeServiceHeartbeat.java rename to code/common/service/java/nu/marginalia/service/control/FakeServiceHeartbeat.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceEventLog.java b/code/common/service/java/nu/marginalia/service/control/ServiceEventLog.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/control/ServiceEventLog.java rename to code/common/service/java/nu/marginalia/service/control/ServiceEventLog.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeat.java b/code/common/service/java/nu/marginalia/service/control/ServiceHeartbeat.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeat.java rename to code/common/service/java/nu/marginalia/service/control/ServiceHeartbeat.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java b/code/common/service/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java rename to code/common/service/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeat.java b/code/common/service/java/nu/marginalia/service/control/ServiceTaskHeartbeat.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeat.java rename to code/common/service/java/nu/marginalia/service/control/ServiceTaskHeartbeat.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java b/code/common/service/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java rename to code/common/service/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java b/code/common/service/java/nu/marginalia/service/module/DatabaseModule.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java rename to code/common/service/java/nu/marginalia/service/module/DatabaseModule.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java b/code/common/service/java/nu/marginalia/service/module/ServiceConfiguration.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java rename to code/common/service/java/nu/marginalia/service/module/ServiceConfiguration.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java b/code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java rename to code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/BaseServiceParams.java b/code/common/service/java/nu/marginalia/service/server/BaseServiceParams.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/BaseServiceParams.java rename to code/common/service/java/nu/marginalia/service/server/BaseServiceParams.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Initialization.java b/code/common/service/java/nu/marginalia/service/server/Initialization.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/Initialization.java rename to code/common/service/java/nu/marginalia/service/server/Initialization.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/MetricsServer.java b/code/common/service/java/nu/marginalia/service/server/MetricsServer.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/MetricsServer.java rename to code/common/service/java/nu/marginalia/service/server/MetricsServer.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java b/code/common/service/java/nu/marginalia/service/server/NodeStatusWatcher.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java rename to code/common/service/java/nu/marginalia/service/server/NodeStatusWatcher.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/RateLimiter.java b/code/common/service/java/nu/marginalia/service/server/RateLimiter.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/RateLimiter.java rename to code/common/service/java/nu/marginalia/service/server/RateLimiter.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java b/code/common/service/java/nu/marginalia/service/server/Service.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/Service.java rename to code/common/service/java/nu/marginalia/service/server/Service.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/StaticResources.java b/code/common/service/java/nu/marginalia/service/server/StaticResources.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/StaticResources.java rename to code/common/service/java/nu/marginalia/service/server/StaticResources.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/mq/MqRequest.java b/code/common/service/java/nu/marginalia/service/server/mq/MqRequest.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/mq/MqRequest.java rename to code/common/service/java/nu/marginalia/service/server/mq/MqRequest.java diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/mq/ServiceMqSubscription.java b/code/common/service/java/nu/marginalia/service/server/mq/ServiceMqSubscription.java similarity index 100% rename from code/common/service/src/main/java/nu/marginalia/service/server/mq/ServiceMqSubscription.java rename to code/common/service/java/nu/marginalia/service/server/mq/ServiceMqSubscription.java diff --git a/code/common/service/src/main/resources/log4j2-prod.xml b/code/common/service/resources/log4j2-prod.xml similarity index 100% rename from code/common/service/src/main/resources/log4j2-prod.xml rename to code/common/service/resources/log4j2-prod.xml diff --git a/code/common/service/src/main/resources/log4j2-test.xml b/code/common/service/resources/log4j2-test.xml similarity index 100% rename from code/common/service/src/main/resources/log4j2-test.xml rename to code/common/service/resources/log4j2-test.xml diff --git a/code/execution/api/build.gradle b/code/execution/api/build.gradle index 58343ecf..b31d87d3 100644 --- a/code/execution/api/build.gradle +++ b/code/execution/api/build.gradle @@ -19,11 +19,13 @@ apply from: "$rootProject.projectDir/protobuf.gradle" sourceSets { main { proto { - srcDir 'src/main/protobuf' + srcDir 'protobuf' } } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:index:api') diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/execution/api/java/nu/marginalia/executor/client/ExecutorClient.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java rename to code/execution/api/java/nu/marginalia/executor/client/ExecutorClient.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorCrawlClient.java b/code/execution/api/java/nu/marginalia/executor/client/ExecutorCrawlClient.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorCrawlClient.java rename to code/execution/api/java/nu/marginalia/executor/client/ExecutorCrawlClient.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorExportClient.java b/code/execution/api/java/nu/marginalia/executor/client/ExecutorExportClient.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorExportClient.java rename to code/execution/api/java/nu/marginalia/executor/client/ExecutorExportClient.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java b/code/execution/api/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java rename to code/execution/api/java/nu/marginalia/executor/client/ExecutorRemoteActorFactory.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorSideloadClient.java b/code/execution/api/java/nu/marginalia/executor/client/ExecutorSideloadClient.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/client/ExecutorSideloadClient.java rename to code/execution/api/java/nu/marginalia/executor/client/ExecutorSideloadClient.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunState.java b/code/execution/api/java/nu/marginalia/executor/model/ActorRunState.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunState.java rename to code/execution/api/java/nu/marginalia/executor/model/ActorRunState.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java b/code/execution/api/java/nu/marginalia/executor/model/ActorRunStates.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java rename to code/execution/api/java/nu/marginalia/executor/model/ActorRunStates.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java b/code/execution/api/java/nu/marginalia/executor/storage/FileStorageContent.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java rename to code/execution/api/java/nu/marginalia/executor/storage/FileStorageContent.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java b/code/execution/api/java/nu/marginalia/executor/storage/FileStorageFile.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java rename to code/execution/api/java/nu/marginalia/executor/storage/FileStorageFile.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java b/code/execution/api/java/nu/marginalia/executor/upload/UploadDirContents.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java rename to code/execution/api/java/nu/marginalia/executor/upload/UploadDirContents.java diff --git a/code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java b/code/execution/api/java/nu/marginalia/executor/upload/UploadDirItem.java similarity index 100% rename from code/execution/api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java rename to code/execution/api/java/nu/marginalia/executor/upload/UploadDirItem.java diff --git a/code/execution/build.gradle b/code/execution/build.gradle index 79eb9c68..842bcab7 100644 --- a/code/execution/build.gradle +++ b/code/execution/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { // These look weird but they're needed to be able to spawn the processes // from the executor service diff --git a/code/execution/src/main/java/nu/marginalia/actor/ActorApi.java b/code/execution/java/nu/marginalia/actor/ActorApi.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/ActorApi.java rename to code/execution/java/nu/marginalia/actor/ActorApi.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/ExecutorActor.java b/code/execution/java/nu/marginalia/actor/ExecutorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/ExecutorActor.java rename to code/execution/java/nu/marginalia/actor/ExecutorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java b/code/execution/java/nu/marginalia/actor/ExecutorActorControlService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/ExecutorActorControlService.java rename to code/execution/java/nu/marginalia/actor/ExecutorActorControlService.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java b/code/execution/java/nu/marginalia/actor/ExecutorActorStateMachines.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/ExecutorActorStateMachines.java rename to code/execution/java/nu/marginalia/actor/ExecutorActorStateMachines.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java b/code/execution/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java rename to code/execution/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java b/code/execution/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java rename to code/execution/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java b/code/execution/java/nu/marginalia/actor/proc/ConverterMonitorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/proc/ConverterMonitorActor.java rename to code/execution/java/nu/marginalia/actor/proc/ConverterMonitorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java b/code/execution/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java rename to code/execution/java/nu/marginalia/actor/proc/CrawlerMonitorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java b/code/execution/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java rename to code/execution/java/nu/marginalia/actor/proc/IndexConstructorMonitorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java b/code/execution/java/nu/marginalia/actor/proc/LoaderMonitorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/proc/LoaderMonitorActor.java rename to code/execution/java/nu/marginalia/actor/proc/LoaderMonitorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java b/code/execution/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java rename to code/execution/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java b/code/execution/java/nu/marginalia/actor/task/ActorProcessWatcher.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java rename to code/execution/java/nu/marginalia/actor/task/ActorProcessWatcher.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ConvertActor.java b/code/execution/java/nu/marginalia/actor/task/ConvertActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ConvertActor.java rename to code/execution/java/nu/marginalia/actor/task/ConvertActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/execution/java/nu/marginalia/actor/task/ConvertAndLoadActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java rename to code/execution/java/nu/marginalia/actor/task/ConvertAndLoadActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/CrawlActor.java b/code/execution/java/nu/marginalia/actor/task/CrawlActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/CrawlActor.java rename to code/execution/java/nu/marginalia/actor/task/CrawlActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java b/code/execution/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java rename to code/execution/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java b/code/execution/java/nu/marginalia/actor/task/DownloadSampleActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/DownloadSampleActor.java rename to code/execution/java/nu/marginalia/actor/task/DownloadSampleActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java b/code/execution/java/nu/marginalia/actor/task/ExportAtagsActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ExportAtagsActor.java rename to code/execution/java/nu/marginalia/actor/task/ExportAtagsActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ExportDataActor.java b/code/execution/java/nu/marginalia/actor/task/ExportDataActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ExportDataActor.java rename to code/execution/java/nu/marginalia/actor/task/ExportDataActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java b/code/execution/java/nu/marginalia/actor/task/ExportFeedsActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ExportFeedsActor.java rename to code/execution/java/nu/marginalia/actor/task/ExportFeedsActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java b/code/execution/java/nu/marginalia/actor/task/ExportSampleDataActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ExportSampleDataActor.java rename to code/execution/java/nu/marginalia/actor/task/ExportSampleDataActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java b/code/execution/java/nu/marginalia/actor/task/ExportTermFreqActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/ExportTermFreqActor.java rename to code/execution/java/nu/marginalia/actor/task/ExportTermFreqActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/RecrawlActor.java b/code/execution/java/nu/marginalia/actor/task/RecrawlActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/RecrawlActor.java rename to code/execution/java/nu/marginalia/actor/task/RecrawlActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java b/code/execution/java/nu/marginalia/actor/task/RestoreBackupActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java rename to code/execution/java/nu/marginalia/actor/task/RestoreBackupActor.java diff --git a/code/execution/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java b/code/execution/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java rename to code/execution/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutionInit.java b/code/execution/java/nu/marginalia/execution/ExecutionInit.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/execution/ExecutionInit.java rename to code/execution/java/nu/marginalia/execution/ExecutionInit.java diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java b/code/execution/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java rename to code/execution/java/nu/marginalia/execution/ExecutorCrawlGrpcService.java diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorExportGrpcService.java b/code/execution/java/nu/marginalia/execution/ExecutorExportGrpcService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/execution/ExecutorExportGrpcService.java rename to code/execution/java/nu/marginalia/execution/ExecutorExportGrpcService.java diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorGrpcService.java b/code/execution/java/nu/marginalia/execution/ExecutorGrpcService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/execution/ExecutorGrpcService.java rename to code/execution/java/nu/marginalia/execution/ExecutorGrpcService.java diff --git a/code/execution/src/main/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java b/code/execution/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java rename to code/execution/java/nu/marginalia/execution/ExecutorSideloadGrpcService.java diff --git a/code/execution/src/main/java/nu/marginalia/process/ProcessOutboxes.java b/code/execution/java/nu/marginalia/process/ProcessOutboxes.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/process/ProcessOutboxes.java rename to code/execution/java/nu/marginalia/process/ProcessOutboxes.java diff --git a/code/execution/src/main/java/nu/marginalia/process/ProcessService.java b/code/execution/java/nu/marginalia/process/ProcessService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/process/ProcessService.java rename to code/execution/java/nu/marginalia/process/ProcessService.java diff --git a/code/execution/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java b/code/execution/java/nu/marginalia/sideload/RedditSideloadHelper.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/sideload/RedditSideloadHelper.java rename to code/execution/java/nu/marginalia/sideload/RedditSideloadHelper.java diff --git a/code/execution/src/main/java/nu/marginalia/sideload/SideloadHelper.java b/code/execution/java/nu/marginalia/sideload/SideloadHelper.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/sideload/SideloadHelper.java rename to code/execution/java/nu/marginalia/sideload/SideloadHelper.java diff --git a/code/execution/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java b/code/execution/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java rename to code/execution/java/nu/marginalia/sideload/StackExchangeSideloadHelper.java diff --git a/code/execution/src/main/java/nu/marginalia/svc/BackupService.java b/code/execution/java/nu/marginalia/svc/BackupService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/svc/BackupService.java rename to code/execution/java/nu/marginalia/svc/BackupService.java diff --git a/code/execution/src/main/java/nu/marginalia/svc/DomainListRefreshService.java b/code/execution/java/nu/marginalia/svc/DomainListRefreshService.java similarity index 100% rename from code/execution/src/main/java/nu/marginalia/svc/DomainListRefreshService.java rename to code/execution/java/nu/marginalia/svc/DomainListRefreshService.java diff --git a/code/execution/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java b/code/execution/test/nu/marginalia/svc/DomainListRefreshServiceTest.java similarity index 100% rename from code/execution/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java rename to code/execution/test/nu/marginalia/svc/DomainListRefreshServiceTest.java diff --git a/code/features-convert/adblock/build.gradle b/code/features-convert/adblock/build.gradle index 90e3e205..f28fe998 100644 --- a/code/features-convert/adblock/build.gradle +++ b/code/features-convert/adblock/build.gradle @@ -13,6 +13,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') diff --git a/code/features-convert/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java b/code/features-convert/adblock/java/nu/marginalia/adblock/AdblockSimulator.java similarity index 100% rename from code/features-convert/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java rename to code/features-convert/adblock/java/nu/marginalia/adblock/AdblockSimulator.java diff --git a/code/features-convert/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java b/code/features-convert/adblock/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java similarity index 100% rename from code/features-convert/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java rename to code/features-convert/adblock/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java diff --git a/code/features-convert/anchor-keywords/build.gradle b/code/features-convert/anchor-keywords/build.gradle index 122046f8..23e415b9 100644 --- a/code/features-convert/anchor-keywords/build.gradle +++ b/code/features-convert/anchor-keywords/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') implementation project(':code:common:model') diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/AnchorTextKeywords.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/AnchorTextKeywords.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/AnchorTextKeywords.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/AnchorTextKeywords.java diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/model/DomainLinks.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/model/DomainLinks.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/model/DomainLinks.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/model/DomainLinks.java diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/model/Link.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/model/Link.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/model/Link.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/model/Link.java diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/model/LinkWithText.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/model/LinkWithText.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/model/LinkWithText.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/model/LinkWithText.java diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/source/AnchorTagsImpl.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/source/AnchorTagsImpl.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/source/AnchorTagsImpl.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/source/AnchorTagsImpl.java diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/source/AnchorTagsSource.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/source/AnchorTagsSource.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/source/AnchorTagsSource.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/source/AnchorTagsSource.java diff --git a/code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/source/AnchorTagsSourceFactory.java b/code/features-convert/anchor-keywords/java/nu/marginalia/atags/source/AnchorTagsSourceFactory.java similarity index 100% rename from code/features-convert/anchor-keywords/src/main/java/nu/marginalia/atags/source/AnchorTagsSourceFactory.java rename to code/features-convert/anchor-keywords/java/nu/marginalia/atags/source/AnchorTagsSourceFactory.java diff --git a/code/features-convert/anchor-keywords/src/main/resources/atags-stop-list b/code/features-convert/anchor-keywords/resources/atags-stop-list similarity index 100% rename from code/features-convert/anchor-keywords/src/main/resources/atags-stop-list rename to code/features-convert/anchor-keywords/resources/atags-stop-list diff --git a/code/features-convert/anchor-keywords/src/test/java/nu/marginalia/atags/DomainAnchorTagsImplTest.java b/code/features-convert/anchor-keywords/test/nu/marginalia/atags/DomainAnchorTagsImplTest.java similarity index 100% rename from code/features-convert/anchor-keywords/src/test/java/nu/marginalia/atags/DomainAnchorTagsImplTest.java rename to code/features-convert/anchor-keywords/test/nu/marginalia/atags/DomainAnchorTagsImplTest.java diff --git a/code/features-convert/anchor-keywords/src/test/java/nu/marginalia/util/TestLanguageModels.java b/code/features-convert/anchor-keywords/test/nu/marginalia/util/TestLanguageModels.java similarity index 100% rename from code/features-convert/anchor-keywords/src/test/java/nu/marginalia/util/TestLanguageModels.java rename to code/features-convert/anchor-keywords/test/nu/marginalia/util/TestLanguageModels.java diff --git a/code/features-convert/data-extractors/build.gradle b/code/features-convert/data-extractors/build.gradle index 67c42633..6fd5671d 100644 --- a/code/features-convert/data-extractors/build.gradle +++ b/code/features-convert/data-extractors/build.gradle @@ -13,6 +13,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') implementation project(':code:common:process') diff --git a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/AtagExporter.java b/code/features-convert/data-extractors/java/nu/marginalia/extractor/AtagExporter.java similarity index 100% rename from code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/AtagExporter.java rename to code/features-convert/data-extractors/java/nu/marginalia/extractor/AtagExporter.java diff --git a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/ExporterIf.java b/code/features-convert/data-extractors/java/nu/marginalia/extractor/ExporterIf.java similarity index 100% rename from code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/ExporterIf.java rename to code/features-convert/data-extractors/java/nu/marginalia/extractor/ExporterIf.java diff --git a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/FeedExporter.java b/code/features-convert/data-extractors/java/nu/marginalia/extractor/FeedExporter.java similarity index 100% rename from code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/FeedExporter.java rename to code/features-convert/data-extractors/java/nu/marginalia/extractor/FeedExporter.java diff --git a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/SampleDataExporter.java b/code/features-convert/data-extractors/java/nu/marginalia/extractor/SampleDataExporter.java similarity index 100% rename from code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/SampleDataExporter.java rename to code/features-convert/data-extractors/java/nu/marginalia/extractor/SampleDataExporter.java diff --git a/code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/TermFrequencyExporter.java b/code/features-convert/data-extractors/java/nu/marginalia/extractor/TermFrequencyExporter.java similarity index 100% rename from code/features-convert/data-extractors/src/main/java/nu/marginalia/extractor/TermFrequencyExporter.java rename to code/features-convert/data-extractors/java/nu/marginalia/extractor/TermFrequencyExporter.java diff --git a/code/features-convert/keyword-extraction/build.gradle b/code/features-convert/keyword-extraction/build.gradle index 9afcc61c..7912e246 100644 --- a/code/features-convert/keyword-extraction/build.gradle +++ b/code/features-convert/keyword-extraction/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:porterstemmer') implementation project(':code:common:model') diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/DocumentKeywordExtractor.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/DocumentKeywordExtractor.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordExtractor.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/KeywordExtractor.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordExtractor.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/KeywordExtractor.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordMetadata.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/KeywordMetadata.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordMetadata.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/KeywordMetadata.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/WordReps.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/WordReps.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/WordReps.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/WordReps.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/ArtifactKeywords.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/ArtifactKeywords.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/ArtifactKeywords.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/ArtifactKeywords.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/KeywordPositionBitmask.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/KeywordPositionBitmask.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/KeywordPositionBitmask.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/KeywordPositionBitmask.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/NameLikeKeywords.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/NameLikeKeywords.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/NameLikeKeywords.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/NameLikeKeywords.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/SubjectLikeKeywords.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/SubjectLikeKeywords.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/SubjectLikeKeywords.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/SubjectLikeKeywords.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/TitleKeywords.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/TitleKeywords.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/TitleKeywords.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/TitleKeywords.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/UrlKeywords.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/UrlKeywords.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/UrlKeywords.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/UrlKeywords.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/extractors/WordsTfIdfCounts.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/model/DocumentKeywords.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywords.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/model/DocumentKeywords.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywords.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/model/DocumentKeywordsBuilder.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywordsBuilder.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/model/DocumentKeywordsBuilder.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywordsBuilder.java diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/model/DocumentKeywordsPointer.java b/code/features-convert/keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywordsPointer.java similarity index 100% rename from code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/model/DocumentKeywordsPointer.java rename to code/features-convert/keyword-extraction/java/nu/marginalia/keyword/model/DocumentKeywordsPointer.java diff --git a/code/features-convert/keyword-extraction/src/test/resources/test-data/java.html b/code/features-convert/keyword-extraction/test-resources/test-data/java.html similarity index 100% rename from code/features-convert/keyword-extraction/src/test/resources/test-data/java.html rename to code/features-convert/keyword-extraction/test-resources/test-data/java.html diff --git a/code/features-convert/keyword-extraction/src/test/resources/test-data/keyboards.html b/code/features-convert/keyword-extraction/test-resources/test-data/keyboards.html similarity index 100% rename from code/features-convert/keyword-extraction/src/test/resources/test-data/keyboards.html rename to code/features-convert/keyword-extraction/test-resources/test-data/keyboards.html diff --git a/code/features-convert/keyword-extraction/src/test/resources/test-data/madonna.html b/code/features-convert/keyword-extraction/test-resources/test-data/madonna.html similarity index 100% rename from code/features-convert/keyword-extraction/src/test/resources/test-data/madonna.html rename to code/features-convert/keyword-extraction/test-resources/test-data/madonna.html diff --git a/code/features-convert/keyword-extraction/src/test/resources/test-data/spam.html b/code/features-convert/keyword-extraction/test-resources/test-data/spam.html similarity index 100% rename from code/features-convert/keyword-extraction/src/test/resources/test-data/spam.html rename to code/features-convert/keyword-extraction/test-resources/test-data/spam.html diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/DocumentKeywordExtractorTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/DocumentKeywordExtractorTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/DocumentKeywordExtractorTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/DocumentKeywordExtractorTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/SentenceExtractorTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/SentenceExtractorTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/SentenceExtractorTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/SentenceExtractorTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/ArtifactKeywordsTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/ArtifactKeywordsTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/ArtifactKeywordsTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/ArtifactKeywordsTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/NameLikeKeywordsTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/NameLikeKeywordsTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/NameLikeKeywordsTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/NameLikeKeywordsTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/SubjectLikeKeywordsTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/SubjectLikeKeywordsTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/SubjectLikeKeywordsTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/SubjectLikeKeywordsTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/TitleKeywordsTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/TitleKeywordsTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/TitleKeywordsTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/TitleKeywordsTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/UrlKeywordsTest.java b/code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/UrlKeywordsTest.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/keyword/extractors/UrlKeywordsTest.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/keyword/extractors/UrlKeywordsTest.java diff --git a/code/features-convert/keyword-extraction/src/test/java/nu/marginalia/test/util/TestLanguageModels.java b/code/features-convert/keyword-extraction/test/nu/marginalia/test/util/TestLanguageModels.java similarity index 100% rename from code/features-convert/keyword-extraction/src/test/java/nu/marginalia/test/util/TestLanguageModels.java rename to code/features-convert/keyword-extraction/test/nu/marginalia/test/util/TestLanguageModels.java diff --git a/code/features-convert/pubdate/build.gradle b/code/features-convert/pubdate/build.gradle index 0a08ea74..bebd3d8e 100644 --- a/code/features-convert/pubdate/build.gradle +++ b/code/features-convert/pubdate/build.gradle @@ -13,6 +13,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateEffortLevel.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateEffortLevel.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateFromHtmlStandard.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateFromHtmlStandard.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateFromHtmlStandard.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateFromHtmlStandard.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateHeuristic.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateHeuristic.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateParser.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateParser.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateSniffer.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/PubDateSniffer.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java diff --git a/code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java b/code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java similarity index 100% rename from code/features-convert/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java rename to code/features-convert/pubdate/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java diff --git a/code/features-convert/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java b/code/features-convert/pubdate/test/nu/marginalia/pubdate/PubDateSnifferTest.java similarity index 100% rename from code/features-convert/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java rename to code/features-convert/pubdate/test/nu/marginalia/pubdate/PubDateSnifferTest.java diff --git a/code/features-convert/pubdate/src/test/java/nu/marginalia/pubdate/PubDateTest.java b/code/features-convert/pubdate/test/nu/marginalia/pubdate/PubDateTest.java similarity index 100% rename from code/features-convert/pubdate/src/test/java/nu/marginalia/pubdate/PubDateTest.java rename to code/features-convert/pubdate/test/nu/marginalia/pubdate/PubDateTest.java diff --git a/code/features-convert/reddit-json/build.gradle b/code/features-convert/reddit-json/build.gradle index 08420127..a9aa3bd6 100644 --- a/code/features-convert/reddit-json/build.gradle +++ b/code/features-convert/reddit-json/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/RedditEntryReader.java b/code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/RedditEntryReader.java similarity index 100% rename from code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/RedditEntryReader.java rename to code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/RedditEntryReader.java diff --git a/code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/db/RedditDb.java b/code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/db/RedditDb.java similarity index 100% rename from code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/db/RedditDb.java rename to code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/db/RedditDb.java diff --git a/code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/ProcessableRedditComment.java b/code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/ProcessableRedditComment.java similarity index 100% rename from code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/ProcessableRedditComment.java rename to code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/ProcessableRedditComment.java diff --git a/code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/ProcessableRedditSubmission.java b/code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/ProcessableRedditSubmission.java similarity index 100% rename from code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/ProcessableRedditSubmission.java rename to code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/ProcessableRedditSubmission.java diff --git a/code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/RawRedditComment.java b/code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/RawRedditComment.java similarity index 100% rename from code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/RawRedditComment.java rename to code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/RawRedditComment.java diff --git a/code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/RawRedditSubmission.java b/code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/RawRedditSubmission.java similarity index 100% rename from code/features-convert/reddit-json/src/main/java/nu/marginalia/integration/reddit/model/RawRedditSubmission.java rename to code/features-convert/reddit-json/java/nu/marginalia/integration/reddit/model/RawRedditSubmission.java diff --git a/code/features-convert/reddit-json/src/main/resources/db/reddit.sql b/code/features-convert/reddit-json/resources/db/reddit.sql similarity index 100% rename from code/features-convert/reddit-json/src/main/resources/db/reddit.sql rename to code/features-convert/reddit-json/resources/db/reddit.sql diff --git a/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/RedditEntryReaderTest.java b/code/features-convert/reddit-json/test/nu/marginalia/integration/reddit/RedditEntryReaderTest.java similarity index 100% rename from code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/RedditEntryReaderTest.java rename to code/features-convert/reddit-json/test/nu/marginalia/integration/reddit/RedditEntryReaderTest.java diff --git a/code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/db/RedditDbTest.java b/code/features-convert/reddit-json/test/nu/marginalia/integration/reddit/db/RedditDbTest.java similarity index 100% rename from code/features-convert/reddit-json/src/test/java/nu/marginalia/integration/reddit/db/RedditDbTest.java rename to code/features-convert/reddit-json/test/nu/marginalia/integration/reddit/db/RedditDbTest.java diff --git a/code/features-convert/stackexchange-xml/build.gradle b/code/features-convert/stackexchange-xml/build.gradle index 8126187d..d693047b 100644 --- a/code/features-convert/stackexchange-xml/build.gradle +++ b/code/features-convert/stackexchange-xml/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/model/StackExchangeComment.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/model/StackExchangeComment.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/model/StackExchangeComment.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/model/StackExchangeComment.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/model/StackExchangePost.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/model/StackExchangePost.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/model/StackExchangePost.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/model/StackExchangePost.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/sqlite/StackExchangePostsDb.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/sqlite/StackExchangePostsDb.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/sqlite/StackExchangePostsDb.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/sqlite/StackExchangePostsDb.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchange7zXmlEventReaderSource.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchange7zXmlEventReaderSource.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchange7zXmlEventReaderSource.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchange7zXmlEventReaderSource.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReader.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReader.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReader.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReader.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlIterator.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlIterator.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlIterator.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlIterator.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReader.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReader.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReader.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReader.java diff --git a/code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/XmlEventReaderSource.java b/code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/XmlEventReaderSource.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/java/nu/marginalia/integration/stackexchange/xml/XmlEventReaderSource.java rename to code/features-convert/stackexchange-xml/java/nu/marginalia/integration/stackexchange/xml/XmlEventReaderSource.java diff --git a/code/features-convert/stackexchange-xml/src/main/resources/db/stackexchange.sql b/code/features-convert/stackexchange-xml/resources/db/stackexchange.sql similarity index 100% rename from code/features-convert/stackexchange-xml/src/main/resources/db/stackexchange.sql rename to code/features-convert/stackexchange-xml/resources/db/stackexchange.sql diff --git a/code/features-convert/stackexchange-xml/src/test/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReaderTest.java b/code/features-convert/stackexchange-xml/test/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReaderTest.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/test/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReaderTest.java rename to code/features-convert/stackexchange-xml/test/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlCommentReaderTest.java diff --git a/code/features-convert/stackexchange-xml/src/test/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReaderTest.java b/code/features-convert/stackexchange-xml/test/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReaderTest.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/test/java/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReaderTest.java rename to code/features-convert/stackexchange-xml/test/nu/marginalia/integration/stackexchange/xml/StackExchangeXmlPostReaderTest.java diff --git a/code/features-convert/stackexchange-xml/src/test/java/nu/marginalia/integration/stackexchange/xml/StringXmlTestEventReader.java b/code/features-convert/stackexchange-xml/test/nu/marginalia/integration/stackexchange/xml/StringXmlTestEventReader.java similarity index 100% rename from code/features-convert/stackexchange-xml/src/test/java/nu/marginalia/integration/stackexchange/xml/StringXmlTestEventReader.java rename to code/features-convert/stackexchange-xml/test/nu/marginalia/integration/stackexchange/xml/StringXmlTestEventReader.java diff --git a/code/features-convert/summary-extraction/build.gradle b/code/features-convert/summary-extraction/build.gradle index 7ff4f23c..bf6a87c5 100644 --- a/code/features-convert/summary-extraction/build.gradle +++ b/code/features-convert/summary-extraction/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/SummaryExtractor.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/SummaryExtractor.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/SummaryExtractor.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/SummaryExtractor.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/DomFilterHeuristic.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/DomFilterHeuristic.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/DomFilterHeuristic.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/DomFilterHeuristic.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/FallbackHeuristic.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/FallbackHeuristic.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/FallbackHeuristic.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/FallbackHeuristic.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/HeuristicTextUtil.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/HeuristicTextUtil.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/HeuristicTextUtil.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/HeuristicTextUtil.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/MetaDescriptionHeuristic.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/MetaDescriptionHeuristic.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/MetaDescriptionHeuristic.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/MetaDescriptionHeuristic.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/OpenGraphDescriptionHeuristic.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/OpenGraphDescriptionHeuristic.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/OpenGraphDescriptionHeuristic.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/OpenGraphDescriptionHeuristic.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/SummarizingDOMFilter.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/SummarizingDOMFilter.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/SummarizingDOMFilter.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/SummarizingDOMFilter.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/SummaryHeuristic.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/SummaryHeuristic.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/SummaryHeuristic.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/SummaryHeuristic.java diff --git a/code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/TagDensityHeuristic.java b/code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/TagDensityHeuristic.java similarity index 100% rename from code/features-convert/summary-extraction/src/main/java/nu/marginalia/summary/heuristic/TagDensityHeuristic.java rename to code/features-convert/summary-extraction/java/nu/marginalia/summary/heuristic/TagDensityHeuristic.java diff --git a/code/features-convert/summary-extraction/src/test/resources/html/monadnock.html b/code/features-convert/summary-extraction/test-resources/html/monadnock.html similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/monadnock.html rename to code/features-convert/summary-extraction/test-resources/html/monadnock.html diff --git a/code/features-convert/summary-extraction/src/test/resources/html/readme.md b/code/features-convert/summary-extraction/test-resources/html/readme.md similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/readme.md rename to code/features-convert/summary-extraction/test-resources/html/readme.md diff --git a/code/features-convert/summary-extraction/src/test/resources/html/summarization/187.shtml b/code/features-convert/summary-extraction/test-resources/html/summarization/187.shtml similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/summarization/187.shtml rename to code/features-convert/summary-extraction/test-resources/html/summarization/187.shtml diff --git a/code/features-convert/summary-extraction/src/test/resources/html/summarization/surrey.html b/code/features-convert/summary-extraction/test-resources/html/summarization/surrey.html similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/summarization/surrey.html rename to code/features-convert/summary-extraction/test-resources/html/summarization/surrey.html diff --git a/code/features-convert/summary-extraction/src/test/resources/html/summarization/surrey.html.1 b/code/features-convert/summary-extraction/test-resources/html/summarization/surrey.html.1 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/summarization/surrey.html.1 rename to code/features-convert/summary-extraction/test-resources/html/summarization/surrey.html.1 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/theregister.html b/code/features-convert/summary-extraction/test-resources/html/theregister.html similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/theregister.html rename to code/features-convert/summary-extraction/test-resources/html/theregister.html diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/index b/code/features-convert/summary-extraction/test-resources/html/work-set/index similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/index rename to code/features-convert/summary-extraction/test-resources/html/work-set/index diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1021546012 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1021546012 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1021546012 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1021546012 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1028592943 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1028592943 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1028592943 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1028592943 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1081293162 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1081293162 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1081293162 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1081293162 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1105046394 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1105046394 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1105046394 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1105046394 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1146923296 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1146923296 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1146923296 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1146923296 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1194694074 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1194694074 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1194694074 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1194694074 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1207898281 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1207898281 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1207898281 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1207898281 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1268145073 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1268145073 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1268145073 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1268145073 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1294876331 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1294876331 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1294876331 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1294876331 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1314767420 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1314767420 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1314767420 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1314767420 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1316269786 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1316269786 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1316269786 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1316269786 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1316766580 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1316766580 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1316766580 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1316766580 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1319968043 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1319968043 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1319968043 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1319968043 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1338576987 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1338576987 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1338576987 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1338576987 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1341909571 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1341909571 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1341909571 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1341909571 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1369578579 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1369578579 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1369578579 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1369578579 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1437315645 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1437315645 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1437315645 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1437315645 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1458954960 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1458954960 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1458954960 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1458954960 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1475681345 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1475681345 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1475681345 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1475681345 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1498328446 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1498328446 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1498328446 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1498328446 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1507779664 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1507779664 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1507779664 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1507779664 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1540303379 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1540303379 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1540303379 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1540303379 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--154898476 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--154898476 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--154898476 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--154898476 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1552059399 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1552059399 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1552059399 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1552059399 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1557688340 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1557688340 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1557688340 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1557688340 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1584145751 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1584145751 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1584145751 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1584145751 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1605151204 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1605151204 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1605151204 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1605151204 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--162269247 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--162269247 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--162269247 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--162269247 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1624294488 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1624294488 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1624294488 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1624294488 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--164108285 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--164108285 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--164108285 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--164108285 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1645688243 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1645688243 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1645688243 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1645688243 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1658004609 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1658004609 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1658004609 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1658004609 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1658558834 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1658558834 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1658558834 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1658558834 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1698664879 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1698664879 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1698664879 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1698664879 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--169975195 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--169975195 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--169975195 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--169975195 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1701203332 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1701203332 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1701203332 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1701203332 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--17281998 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--17281998 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--17281998 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--17281998 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1742070028 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1742070028 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1742070028 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1742070028 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1745376814 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1745376814 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1745376814 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1745376814 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1749889035 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1749889035 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1749889035 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1749889035 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--176177364 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--176177364 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--176177364 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--176177364 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--177014197 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--177014197 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--177014197 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--177014197 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1794527707 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1794527707 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1794527707 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1794527707 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1797740201 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1797740201 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1797740201 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1797740201 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1799098579 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1799098579 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1799098579 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1799098579 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1959637826 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1959637826 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1959637826 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1959637826 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1971916964 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1971916964 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1971916964 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1971916964 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1985840368 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--1985840368 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--1985840368 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--1985840368 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2012610859 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--2012610859 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2012610859 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--2012610859 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--202178680 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--202178680 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--202178680 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--202178680 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2043528727 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--2043528727 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2043528727 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--2043528727 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2081757477 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--2081757477 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2081757477 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--2081757477 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2103982576 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--2103982576 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2103982576 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--2103982576 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2111558769 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--2111558769 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--2111558769 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--2111558769 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--213168798 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--213168798 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--213168798 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--213168798 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--232544032 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--232544032 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--232544032 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--232544032 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--253010011 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--253010011 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--253010011 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--253010011 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--274250994 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--274250994 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--274250994 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--274250994 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--332442790 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--332442790 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--332442790 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--332442790 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--353437903 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--353437903 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--353437903 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--353437903 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--364546777 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--364546777 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--364546777 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--364546777 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--379129416 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--379129416 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--379129416 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--379129416 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--399428149 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--399428149 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--399428149 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--399428149 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--425233170 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--425233170 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--425233170 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--425233170 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--434612307 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--434612307 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--434612307 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--434612307 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--439772328 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--439772328 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--439772328 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--439772328 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--458002611 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--458002611 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--458002611 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--458002611 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--506010305 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--506010305 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--506010305 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--506010305 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--546773534 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--546773534 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--546773534 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--546773534 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--551288516 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--551288516 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--551288516 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--551288516 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--602577763 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--602577763 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--602577763 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--602577763 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--611668054 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--611668054 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--611668054 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--611668054 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--634771245 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--634771245 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--634771245 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--634771245 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--639320493 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--639320493 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--639320493 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--639320493 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--643179018 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--643179018 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--643179018 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--643179018 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--663772351 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--663772351 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--663772351 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--663772351 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--670789152 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--670789152 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--670789152 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--670789152 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--6797317 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--6797317 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--6797317 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--6797317 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--700978490 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--700978490 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--700978490 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--700978490 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--708035332 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--708035332 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--708035332 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--708035332 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--804917062 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--804917062 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--804917062 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--804917062 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--819771302 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--819771302 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--819771302 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--819771302 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--840796372 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--840796372 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--840796372 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--840796372 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--841445362 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--841445362 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--841445362 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--841445362 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--862385354 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--862385354 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--862385354 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--862385354 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--879796466 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--879796466 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--879796466 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--879796466 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--89134993 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--89134993 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--89134993 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--89134993 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--905197876 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--905197876 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--905197876 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--905197876 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--920328354 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--920328354 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--920328354 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--920328354 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--952827759 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--952827759 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--952827759 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--952827759 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--964018507 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--964018507 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--964018507 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--964018507 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url--972614909 b/code/features-convert/summary-extraction/test-resources/html/work-set/url--972614909 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url--972614909 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url--972614909 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-10088520 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-10088520 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-10088520 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-10088520 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1013281103 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1013281103 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1013281103 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1013281103 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1019241851 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1019241851 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1019241851 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1019241851 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1059944953 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1059944953 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1059944953 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1059944953 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1118681302 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1118681302 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1118681302 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1118681302 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1179298706 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1179298706 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1179298706 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1179298706 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1191749784 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1191749784 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1191749784 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1191749784 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1207094790 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1207094790 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1207094790 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1207094790 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1213989666 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1213989666 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1213989666 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1213989666 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1222442301 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1222442301 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1222442301 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1222442301 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-130332455 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-130332455 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-130332455 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-130332455 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1311055461 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1311055461 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1311055461 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1311055461 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1391842722 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1391842722 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1391842722 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1391842722 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1457388763 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1457388763 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1457388763 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1457388763 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1506356272 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1506356272 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1506356272 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1506356272 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1511762169 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1511762169 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1511762169 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1511762169 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1534640058 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1534640058 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1534640058 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1534640058 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1551513871 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1551513871 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1551513871 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1551513871 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1567632447 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1567632447 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1567632447 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1567632447 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1623049502 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1623049502 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1623049502 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1623049502 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-163919330 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-163919330 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-163919330 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-163919330 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1661398327 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1661398327 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1661398327 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1661398327 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1724309925 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1724309925 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1724309925 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1724309925 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1736807128 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1736807128 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1736807128 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1736807128 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1739031345 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1739031345 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1739031345 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1739031345 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1755745765 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1755745765 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1755745765 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1755745765 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1802811100 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1802811100 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1802811100 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1802811100 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1805364707 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1805364707 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1805364707 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1805364707 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1832702370 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1832702370 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1832702370 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1832702370 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1853114311 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1853114311 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1853114311 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1853114311 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1924872844 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1924872844 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1924872844 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1924872844 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-197772804 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-197772804 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-197772804 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-197772804 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1984259912 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1984259912 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1984259912 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1984259912 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1990903988 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-1990903988 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-1990903988 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-1990903988 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2039310951 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2039310951 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2039310951 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2039310951 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2040857056 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2040857056 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2040857056 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2040857056 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2052613093 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2052613093 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2052613093 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2052613093 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2063899866 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2063899866 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2063899866 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2063899866 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2115548255 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2115548255 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2115548255 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2115548255 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2127148436 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2127148436 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2127148436 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2127148436 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2133781904 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-2133781904 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-2133781904 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-2133781904 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-225690385 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-225690385 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-225690385 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-225690385 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-226401955 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-226401955 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-226401955 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-226401955 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-262970770 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-262970770 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-262970770 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-262970770 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-30106798 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-30106798 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-30106798 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-30106798 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-302167335 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-302167335 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-302167335 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-302167335 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-327999153 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-327999153 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-327999153 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-327999153 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-332568225 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-332568225 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-332568225 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-332568225 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-343223418 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-343223418 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-343223418 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-343223418 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-383103932 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-383103932 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-383103932 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-383103932 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-412929678 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-412929678 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-412929678 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-412929678 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-475213997 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-475213997 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-475213997 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-475213997 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-483403121 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-483403121 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-483403121 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-483403121 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-488667993 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-488667993 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-488667993 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-488667993 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-50815201 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-50815201 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-50815201 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-50815201 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-522685905 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-522685905 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-522685905 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-522685905 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-570714305 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-570714305 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-570714305 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-570714305 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-58733529 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-58733529 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-58733529 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-58733529 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-616518304 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-616518304 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-616518304 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-616518304 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-662169426 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-662169426 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-662169426 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-662169426 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-677278788 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-677278788 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-677278788 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-677278788 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-690486170 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-690486170 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-690486170 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-690486170 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-709693331 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-709693331 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-709693331 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-709693331 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-734531556 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-734531556 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-734531556 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-734531556 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-767530276 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-767530276 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-767530276 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-767530276 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-783154014 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-783154014 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-783154014 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-783154014 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-796905237 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-796905237 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-796905237 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-796905237 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-800099955 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-800099955 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-800099955 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-800099955 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-804101946 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-804101946 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-804101946 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-804101946 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-830664902 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-830664902 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-830664902 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-830664902 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-876060686 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-876060686 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-876060686 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-876060686 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-892584998 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-892584998 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-892584998 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-892584998 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-942458463 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-942458463 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-942458463 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-942458463 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-952036171 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-952036171 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-952036171 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-952036171 diff --git a/code/features-convert/summary-extraction/src/test/resources/html/work-set/url-968207276 b/code/features-convert/summary-extraction/test-resources/html/work-set/url-968207276 similarity index 100% rename from code/features-convert/summary-extraction/src/test/resources/html/work-set/url-968207276 rename to code/features-convert/summary-extraction/test-resources/html/work-set/url-968207276 diff --git a/code/features-convert/summary-extraction/src/test/java/nu/marginalia/summary/SummaryExtractorTest.java b/code/features-convert/summary-extraction/test/nu/marginalia/summary/SummaryExtractorTest.java similarity index 100% rename from code/features-convert/summary-extraction/src/test/java/nu/marginalia/summary/SummaryExtractorTest.java rename to code/features-convert/summary-extraction/test/nu/marginalia/summary/SummaryExtractorTest.java diff --git a/code/features-convert/summary-extraction/src/test/java/nu/marginalia/summary/heuristic/HeuristicTextUtilTest.java b/code/features-convert/summary-extraction/test/nu/marginalia/summary/heuristic/HeuristicTextUtilTest.java similarity index 100% rename from code/features-convert/summary-extraction/src/test/java/nu/marginalia/summary/heuristic/HeuristicTextUtilTest.java rename to code/features-convert/summary-extraction/test/nu/marginalia/summary/heuristic/HeuristicTextUtilTest.java diff --git a/code/features-convert/topic-detection/build.gradle b/code/features-convert/topic-detection/build.gradle index 5836dc2b..49661c25 100644 --- a/code/features-convert/topic-detection/build.gradle +++ b/code/features-convert/topic-detection/build.gradle @@ -13,6 +13,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') implementation project(':code:libraries:language-processing') diff --git a/code/features-convert/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java b/code/features-convert/topic-detection/java/nu/marginalia/topic/RecipeDetector.java similarity index 100% rename from code/features-convert/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java rename to code/features-convert/topic-detection/java/nu/marginalia/topic/RecipeDetector.java diff --git a/code/features-convert/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java b/code/features-convert/topic-detection/java/nu/marginalia/topic/TextileCraftDetector.java similarity index 100% rename from code/features-convert/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java rename to code/features-convert/topic-detection/java/nu/marginalia/topic/TextileCraftDetector.java diff --git a/code/features-convert/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java b/code/features-convert/topic-detection/java/nu/marginalia/topic/WoodworkingDetector.java similarity index 100% rename from code/features-convert/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java rename to code/features-convert/topic-detection/java/nu/marginalia/topic/WoodworkingDetector.java diff --git a/code/features-crawl/content-type/build.gradle b/code/features-crawl/content-type/build.gradle index 73a155cb..c807b86e 100644 --- a/code/features-crawl/content-type/build.gradle +++ b/code/features-crawl/content-type/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation libs.crawlercommons diff --git a/code/features-crawl/content-type/src/main/java/nu/marginalia/contenttype/ContentType.java b/code/features-crawl/content-type/java/nu/marginalia/contenttype/ContentType.java similarity index 100% rename from code/features-crawl/content-type/src/main/java/nu/marginalia/contenttype/ContentType.java rename to code/features-crawl/content-type/java/nu/marginalia/contenttype/ContentType.java diff --git a/code/features-crawl/content-type/src/main/java/nu/marginalia/contenttype/ContentTypeParser.java b/code/features-crawl/content-type/java/nu/marginalia/contenttype/ContentTypeParser.java similarity index 100% rename from code/features-crawl/content-type/src/main/java/nu/marginalia/contenttype/ContentTypeParser.java rename to code/features-crawl/content-type/java/nu/marginalia/contenttype/ContentTypeParser.java diff --git a/code/features-crawl/content-type/src/main/java/nu/marginalia/contenttype/DocumentBodyToString.java b/code/features-crawl/content-type/java/nu/marginalia/contenttype/DocumentBodyToString.java similarity index 100% rename from code/features-crawl/content-type/src/main/java/nu/marginalia/contenttype/DocumentBodyToString.java rename to code/features-crawl/content-type/java/nu/marginalia/contenttype/DocumentBodyToString.java diff --git a/code/features-crawl/content-type/src/test/java/nu/marginalia/contenttype/ContentTypeParserTest.java b/code/features-crawl/content-type/test/nu/marginalia/contenttype/ContentTypeParserTest.java similarity index 100% rename from code/features-crawl/content-type/src/test/java/nu/marginalia/contenttype/ContentTypeParserTest.java rename to code/features-crawl/content-type/test/nu/marginalia/contenttype/ContentTypeParserTest.java diff --git a/code/features-crawl/content-type/src/test/java/nu/marginalia/contenttype/DocumentBodyToStringTest.java b/code/features-crawl/content-type/test/nu/marginalia/contenttype/DocumentBodyToStringTest.java similarity index 100% rename from code/features-crawl/content-type/src/test/java/nu/marginalia/contenttype/DocumentBodyToStringTest.java rename to code/features-crawl/content-type/test/nu/marginalia/contenttype/DocumentBodyToStringTest.java diff --git a/code/features-crawl/crawl-blocklist/build.gradle b/code/features-crawl/crawl-blocklist/build.gradle index fa849eef..a6fc4f91 100644 --- a/code/features-crawl/crawl-blocklist/build.gradle +++ b/code/features-crawl/crawl-blocklist/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:config') diff --git a/code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java b/code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java similarity index 100% rename from code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java rename to code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java diff --git a/code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/InetAddressCache.java b/code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/InetAddressCache.java similarity index 100% rename from code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/InetAddressCache.java rename to code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/InetAddressCache.java diff --git a/code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/IpBlockList.java b/code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/IpBlockList.java similarity index 100% rename from code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/IpBlockList.java rename to code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/IpBlockList.java diff --git a/code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/UrlBlocklist.java b/code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/UrlBlocklist.java similarity index 100% rename from code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/UrlBlocklist.java rename to code/features-crawl/crawl-blocklist/java/nu/marginalia/ip_blocklist/UrlBlocklist.java diff --git a/code/features-crawl/crawl-blocklist/src/test/java/nu/marginalia/ip_blocklist/UrlBlocklistTest.java b/code/features-crawl/crawl-blocklist/test/nu/marginalia/ip_blocklist/UrlBlocklistTest.java similarity index 100% rename from code/features-crawl/crawl-blocklist/src/test/java/nu/marginalia/ip_blocklist/UrlBlocklistTest.java rename to code/features-crawl/crawl-blocklist/test/nu/marginalia/ip_blocklist/UrlBlocklistTest.java diff --git a/code/features-crawl/link-parser/build.gradle b/code/features-crawl/link-parser/build.gradle index dc551943..2dd04a5c 100644 --- a/code/features-crawl/link-parser/build.gradle +++ b/code/features-crawl/link-parser/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') diff --git a/code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/FeedExtractor.java b/code/features-crawl/link-parser/java/nu/marginalia/link_parser/FeedExtractor.java similarity index 100% rename from code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/FeedExtractor.java rename to code/features-crawl/link-parser/java/nu/marginalia/link_parser/FeedExtractor.java diff --git a/code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/LinkParser.java b/code/features-crawl/link-parser/java/nu/marginalia/link_parser/LinkParser.java similarity index 100% rename from code/features-crawl/link-parser/src/main/java/nu/marginalia/link_parser/LinkParser.java rename to code/features-crawl/link-parser/java/nu/marginalia/link_parser/LinkParser.java diff --git a/code/features-search/feedlot-client/build.gradle b/code/features-search/feedlot-client/build.gradle index 808c9ca6..c62182fe 100644 --- a/code/features-search/feedlot-client/build.gradle +++ b/code/features-search/feedlot-client/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/features-search/feedlot-client/src/main/java/nu/marginalia/feedlot/FeedlotClient.java b/code/features-search/feedlot-client/java/nu/marginalia/feedlot/FeedlotClient.java similarity index 100% rename from code/features-search/feedlot-client/src/main/java/nu/marginalia/feedlot/FeedlotClient.java rename to code/features-search/feedlot-client/java/nu/marginalia/feedlot/FeedlotClient.java diff --git a/code/features-search/feedlot-client/src/main/java/nu/marginalia/feedlot/model/FeedItem.java b/code/features-search/feedlot-client/java/nu/marginalia/feedlot/model/FeedItem.java similarity index 100% rename from code/features-search/feedlot-client/src/main/java/nu/marginalia/feedlot/model/FeedItem.java rename to code/features-search/feedlot-client/java/nu/marginalia/feedlot/model/FeedItem.java diff --git a/code/features-search/feedlot-client/src/main/java/nu/marginalia/feedlot/model/FeedItems.java b/code/features-search/feedlot-client/java/nu/marginalia/feedlot/model/FeedItems.java similarity index 100% rename from code/features-search/feedlot-client/src/main/java/nu/marginalia/feedlot/model/FeedItems.java rename to code/features-search/feedlot-client/java/nu/marginalia/feedlot/model/FeedItems.java diff --git a/code/features-search/random-websites/build.gradle b/code/features-search/random-websites/build.gradle index b95e7aa1..ec4201ec 100644 --- a/code/features-search/random-websites/build.gradle +++ b/code/features-search/random-websites/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:db') diff --git a/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java b/code/features-search/random-websites/java/nu/marginalia/browse/DbBrowseDomainsRandom.java similarity index 100% rename from code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java rename to code/features-search/random-websites/java/nu/marginalia/browse/DbBrowseDomainsRandom.java diff --git a/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java b/code/features-search/random-websites/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java similarity index 100% rename from code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java rename to code/features-search/random-websites/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java diff --git a/code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java b/code/features-search/random-websites/java/nu/marginalia/browse/model/BrowseResult.java similarity index 100% rename from code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java rename to code/features-search/random-websites/java/nu/marginalia/browse/model/BrowseResult.java diff --git a/code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java b/code/features-search/random-websites/java/nu/marginalia/browse/model/BrowseResultSet.java similarity index 100% rename from code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java rename to code/features-search/random-websites/java/nu/marginalia/browse/model/BrowseResultSet.java diff --git a/code/features-search/screenshots/build.gradle b/code/features-search/screenshots/build.gradle index ac2779f6..52572e91 100644 --- a/code/features-search/screenshots/build.gradle +++ b/code/features-search/screenshots/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:db') diff --git a/code/features-search/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java b/code/features-search/screenshots/java/nu/marginalia/screenshot/ScreenshotService.java similarity index 100% rename from code/features-search/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java rename to code/features-search/screenshots/java/nu/marginalia/screenshot/ScreenshotService.java diff --git a/code/functions/domain-info/api/build.gradle b/code/functions/domain-info/api/build.gradle index 74c0548e..c99c4fbf 100644 --- a/code/functions/domain-info/api/build.gradle +++ b/code/functions/domain-info/api/build.gradle @@ -13,15 +13,8 @@ java { jar.archiveBaseName = 'domain-info-api' -sourceSets { - main { - proto { - srcDir 'src/main/protobuf' - } - } -} - apply from: "$rootProject.projectDir/protobuf.gradle" +apply from: "$rootProject.projectDir/srcsets.gradle" dependencies { implementation project(':code:common:model') diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java b/code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java similarity index 100% rename from code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java rename to code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java b/code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainsProtobufCodec.java similarity index 100% rename from code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java rename to code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainsProtobufCodec.java diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java b/code/functions/domain-info/api/java/nu/marginalia/api/domains/model/DomainInformation.java similarity index 100% rename from code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java rename to code/functions/domain-info/api/java/nu/marginalia/api/domains/model/DomainInformation.java diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java b/code/functions/domain-info/api/java/nu/marginalia/api/domains/model/SimilarDomain.java similarity index 100% rename from code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java rename to code/functions/domain-info/api/java/nu/marginalia/api/domains/model/SimilarDomain.java diff --git a/code/functions/domain-info/build.gradle b/code/functions/domain-info/build.gradle index c9f641d0..11e3f08c 100644 --- a/code/functions/domain-info/build.gradle +++ b/code/functions/domain-info/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:functions:domain-info:api') implementation project(':code:functions:domain-links:api') diff --git a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java b/code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java similarity index 100% rename from code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java rename to code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java diff --git a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java b/code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInformationService.java similarity index 100% rename from code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java rename to code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInformationService.java diff --git a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java b/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java similarity index 100% rename from code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java rename to code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java diff --git a/code/functions/domain-links/aggregate/build.gradle b/code/functions/domain-links/aggregate/build.gradle index b8702030..11260f1f 100644 --- a/code/functions/domain-links/aggregate/build.gradle +++ b/code/functions/domain-links/aggregate/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:functions:domain-links:api') diff --git a/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java b/code/functions/domain-links/aggregate/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java similarity index 100% rename from code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java rename to code/functions/domain-links/aggregate/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java diff --git a/code/functions/domain-links/api/build.gradle b/code/functions/domain-links/api/build.gradle index 081e3167..3232a623 100644 --- a/code/functions/domain-links/api/build.gradle +++ b/code/functions/domain-links/api/build.gradle @@ -13,15 +13,8 @@ java { jar.archiveBaseName = 'index-domain-links-api' -sourceSets { - main { - proto { - srcDir 'src/main/protobuf' - } - } -} - apply from: "$rootProject.projectDir/protobuf.gradle" +apply from: "$rootProject.projectDir/srcsets.gradle" dependencies { implementation project(':code:common:model') diff --git a/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java b/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java similarity index 100% rename from code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java rename to code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java diff --git a/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java b/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java similarity index 100% rename from code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java rename to code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java diff --git a/code/functions/domain-links/partition/build.gradle b/code/functions/domain-links/partition/build.gradle index 1a5f9dde..940cecfe 100644 --- a/code/functions/domain-links/partition/build.gradle +++ b/code/functions/domain-links/partition/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:functions:domain-links:api') diff --git a/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java b/code/functions/domain-links/partition/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java similarity index 100% rename from code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java rename to code/functions/domain-links/partition/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java diff --git a/code/functions/math/api/build.gradle b/code/functions/math/api/build.gradle index 0ef23547..779c4508 100644 --- a/code/functions/math/api/build.gradle +++ b/code/functions/math/api/build.gradle @@ -13,15 +13,8 @@ java { jar.archiveBaseName = 'math-api' -sourceSets { - main { - proto { - srcDir 'src/main/protobuf' - } - } -} - apply from: "$rootProject.projectDir/protobuf.gradle" +apply from: "$rootProject.projectDir/srcsets.gradle" dependencies { implementation project(':code:common:model') diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java b/code/functions/math/api/java/nu/marginalia/api/math/MathClient.java similarity index 100% rename from code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java rename to code/functions/math/api/java/nu/marginalia/api/math/MathClient.java diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java b/code/functions/math/api/java/nu/marginalia/api/math/MathProtobufCodec.java similarity index 100% rename from code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java rename to code/functions/math/api/java/nu/marginalia/api/math/MathProtobufCodec.java diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java b/code/functions/math/api/java/nu/marginalia/api/math/model/DictionaryEntry.java similarity index 100% rename from code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java rename to code/functions/math/api/java/nu/marginalia/api/math/model/DictionaryEntry.java diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java b/code/functions/math/api/java/nu/marginalia/api/math/model/DictionaryResponse.java similarity index 100% rename from code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java rename to code/functions/math/api/java/nu/marginalia/api/math/model/DictionaryResponse.java diff --git a/code/functions/math/build.gradle b/code/functions/math/build.gradle index 2a843de8..c1fe528d 100644 --- a/code/functions/math/build.gradle +++ b/code/functions/math/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:symspell') implementation project(':code:functions:math:api') diff --git a/code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java b/code/functions/math/java/nu/marginalia/functions/math/MathGrpcService.java similarity index 100% rename from code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java rename to code/functions/math/java/nu/marginalia/functions/math/MathGrpcService.java diff --git a/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java b/code/functions/math/java/nu/marginalia/functions/math/dict/DictionaryService.java similarity index 100% rename from code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java rename to code/functions/math/java/nu/marginalia/functions/math/dict/DictionaryService.java diff --git a/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java b/code/functions/math/java/nu/marginalia/functions/math/dict/SpellChecker.java similarity index 100% rename from code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java rename to code/functions/math/java/nu/marginalia/functions/math/dict/SpellChecker.java diff --git a/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java b/code/functions/math/java/nu/marginalia/functions/math/eval/MathParser.java similarity index 100% rename from code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java rename to code/functions/math/java/nu/marginalia/functions/math/eval/MathParser.java diff --git a/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java b/code/functions/math/java/nu/marginalia/functions/math/eval/Unit.java similarity index 100% rename from code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java rename to code/functions/math/java/nu/marginalia/functions/math/eval/Unit.java diff --git a/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java b/code/functions/math/java/nu/marginalia/functions/math/eval/Units.java similarity index 100% rename from code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java rename to code/functions/math/java/nu/marginalia/functions/math/eval/Units.java diff --git a/code/functions/math/src/main/resources/units.csv b/code/functions/math/resources/units.csv similarity index 100% rename from code/functions/math/src/main/resources/units.csv rename to code/functions/math/resources/units.csv diff --git a/code/functions/math/src/test/java/nu/marginalia/functions/math/eval/MathParserTest.java b/code/functions/math/test/nu/marginalia/functions/math/eval/MathParserTest.java similarity index 100% rename from code/functions/math/src/test/java/nu/marginalia/functions/math/eval/MathParserTest.java rename to code/functions/math/test/nu/marginalia/functions/math/eval/MathParserTest.java diff --git a/code/functions/math/src/test/java/nu/marginalia/functions/math/eval/UnitsTest.java b/code/functions/math/test/nu/marginalia/functions/math/eval/UnitsTest.java similarity index 100% rename from code/functions/math/src/test/java/nu/marginalia/functions/math/eval/UnitsTest.java rename to code/functions/math/test/nu/marginalia/functions/math/eval/UnitsTest.java diff --git a/code/functions/search-query/api/build.gradle b/code/functions/search-query/api/build.gradle index 5605473c..8da48164 100644 --- a/code/functions/search-query/api/build.gradle +++ b/code/functions/search-query/api/build.gradle @@ -13,16 +13,11 @@ java { jar.archiveBaseName = 'search-query-api' -sourceSets { - main { - proto { - srcDir 'src/main/protobuf' - } - } -} apply from: "$rootProject.projectDir/protobuf.gradle" +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:config') diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryClient.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryClient.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryClient.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryClient.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/ProcessedQuery.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryParams.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryParams.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryResponse.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSetIdentifier.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSubquery.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/Bm25Parameters.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java diff --git a/code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java similarity index 100% rename from code/functions/search-query/api/src/main/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java rename to code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultSet.java diff --git a/code/functions/search-query/api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java b/code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java similarity index 100% rename from code/functions/search-query/api/src/test/java/nu/marginalia/index/client/IndexProtobufCodecTest.java rename to code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java diff --git a/code/functions/search-query/build.gradle b/code/functions/search-query/build.gradle index d75504fe..14b662f9 100644 --- a/code/functions/search-query/build.gradle +++ b/code/functions/search-query/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:index:api') diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/QueryGRPCService.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryPermutation.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryTokenizer.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryVariants.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java rename to code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/util/language/EnglishDictionary.java b/code/functions/search-query/java/nu/marginalia/util/language/EnglishDictionary.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/util/language/EnglishDictionary.java rename to code/functions/search-query/java/nu/marginalia/util/language/EnglishDictionary.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/DenseBitMap.java b/code/functions/search-query/java/nu/marginalia/util/ngrams/DenseBitMap.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/DenseBitMap.java rename to code/functions/search-query/java/nu/marginalia/util/ngrams/DenseBitMap.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/NGramBloomFilter.java b/code/functions/search-query/java/nu/marginalia/util/ngrams/NGramBloomFilter.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/util/ngrams/NGramBloomFilter.java rename to code/functions/search-query/java/nu/marginalia/util/ngrams/NGramBloomFilter.java diff --git a/code/functions/search-query/src/main/java/nu/marginalia/util/transform_list/TransformList.java b/code/functions/search-query/java/nu/marginalia/util/transform_list/TransformList.java similarity index 100% rename from code/functions/search-query/src/main/java/nu/marginalia/util/transform_list/TransformList.java rename to code/functions/search-query/java/nu/marginalia/util/transform_list/TransformList.java diff --git a/code/functions/search-query/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java b/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java similarity index 100% rename from code/functions/search-query/src/test/java/nu/marginalia/query/svc/QueryFactoryTest.java rename to code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java diff --git a/code/index/api/build.gradle b/code/index/api/build.gradle index 69c5afff..895f75b0 100644 --- a/code/index/api/build.gradle +++ b/code/index/api/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:config') diff --git a/code/index/api/src/main/java/nu/marginalia/index/api/IndexClient.java b/code/index/api/java/nu/marginalia/index/api/IndexClient.java similarity index 100% rename from code/index/api/src/main/java/nu/marginalia/index/api/IndexClient.java rename to code/index/api/java/nu/marginalia/index/api/IndexClient.java diff --git a/code/index/api/src/main/java/nu/marginalia/index/api/IndexMqClient.java b/code/index/api/java/nu/marginalia/index/api/IndexMqClient.java similarity index 100% rename from code/index/api/src/main/java/nu/marginalia/index/api/IndexMqClient.java rename to code/index/api/java/nu/marginalia/index/api/IndexMqClient.java diff --git a/code/index/api/src/main/java/nu/marginalia/index/api/IndexMqEndpoints.java b/code/index/api/java/nu/marginalia/index/api/IndexMqEndpoints.java similarity index 100% rename from code/index/api/src/main/java/nu/marginalia/index/api/IndexMqEndpoints.java rename to code/index/api/java/nu/marginalia/index/api/IndexMqEndpoints.java diff --git a/code/index/build.gradle b/code/index/build.gradle index cfeb66d8..403da54a 100644 --- a/code/index/build.gradle +++ b/code/index/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation 'org.jgrapht:jgrapht-core:1.5.2' @@ -29,8 +31,8 @@ dependencies { implementation project(':code:functions:search-query:api') - implementation project(':code:index:forward-index') - implementation project(':code:index:reverse-index') + implementation project(':code:index:index-forward') + implementation project(':code:index:index-reverse') implementation project(':code:index:query') implementation project(':code:index:index-journal') diff --git a/code/index/forward-index/build.gradle b/code/index/index-forward/build.gradle similarity index 93% rename from code/index/forward-index/build.gradle rename to code/index/index-forward/build.gradle index 35e0146e..daf9bc0b 100644 --- a/code/index/forward-index/build.gradle +++ b/code/index/index-forward/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') diff --git a/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java b/code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexConverter.java similarity index 100% rename from code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java rename to code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexConverter.java diff --git a/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java b/code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexFileNames.java similarity index 100% rename from code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexFileNames.java rename to code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexFileNames.java diff --git a/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java b/code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexParameters.java similarity index 100% rename from code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java rename to code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexParameters.java diff --git a/code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java b/code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexReader.java similarity index 100% rename from code/index/forward-index/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java rename to code/index/index-forward/java/nu/marginalia/index/forward/ForwardIndexReader.java diff --git a/code/index/forward-index/readme.md b/code/index/index-forward/readme.md similarity index 100% rename from code/index/forward-index/readme.md rename to code/index/index-forward/readme.md diff --git a/code/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java b/code/index/index-forward/test/nu/marginalia/index/forward/ForwardIndexConverterTest.java similarity index 100% rename from code/index/forward-index/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java rename to code/index/index-forward/test/nu/marginalia/index/forward/ForwardIndexConverterTest.java diff --git a/code/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java b/code/index/index-forward/test/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/index/forward-index/src/test/java/nu/marginalia/test/TestUtil.java rename to code/index/index-forward/test/nu/marginalia/test/TestUtil.java diff --git a/code/index/index-journal/build.gradle b/code/index/index-journal/build.gradle index 26e7855a..c6186cf7 100644 --- a/code/index/index-journal/build.gradle +++ b/code/index/index-journal/build.gradle @@ -10,6 +10,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:libraries:array') implementation project(':code:common:model') diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java b/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalFileNames.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/IndexJournalFileNames.java rename to code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalFileNames.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java b/code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntry.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java rename to code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntry.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java b/code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java rename to code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntryBuilder.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java b/code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java rename to code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java b/code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java rename to code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java b/code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java rename to code/index/index-journal/java/nu/marginalia/index/journal/model/IndexJournalFileHeader.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java b/code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java rename to code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReadEntry.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java b/code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReader.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java rename to code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReader.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java b/code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java rename to code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReaderPagingImpl.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java b/code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java rename to code/index/index-journal/java/nu/marginalia/index/journal/reader/IndexJournalReaderSingleFile.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java b/code/index/index-journal/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java rename to code/index/index-journal/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointer.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java b/code/index/index-journal/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java rename to code/index/index-journal/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java b/code/index/index-journal/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java rename to code/index/index-journal/java/nu/marginalia/index/journal/writer/IndexJournalWriterPagingImpl.java diff --git a/code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java b/code/index/index-journal/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java similarity index 100% rename from code/index/index-journal/src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java rename to code/index/index-journal/java/nu/marginalia/index/journal/writer/IndexJournalWriterSingleFileImpl.java diff --git a/code/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java b/code/index/index-journal/test/nu/marginalia/index/journal/IndexJournalTest.java similarity index 100% rename from code/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java rename to code/index/index-journal/test/nu/marginalia/index/journal/IndexJournalTest.java diff --git a/code/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java b/code/index/index-journal/test/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java similarity index 100% rename from code/index/index-journal/src/test/java/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java rename to code/index/index-journal/test/nu/marginalia/index/journal/reader/pointer/IndexJournalPointerTest.java diff --git a/code/index/reverse-index/build.gradle b/code/index/index-reverse/build.gradle similarity index 93% rename from code/index/reverse-index/build.gradle rename to code/index/index-reverse/build.gradle index 216d1142..a9523656 100644 --- a/code/index/reverse-index/build.gradle +++ b/code/index/index-reverse/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') diff --git a/code/index/reverse-index/index.svg b/code/index/index-reverse/index.svg similarity index 100% rename from code/index/reverse-index/index.svg rename to code/index/index-reverse/index.svg diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexEntrySource.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexEntrySource.java rename to code/index/index-reverse/java/nu/marginalia/index/ReverseIndexEntrySource.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexFullFileNames.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexFullFileNames.java rename to code/index/index-reverse/java/nu/marginalia/index/ReverseIndexFullFileNames.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexParameters.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexParameters.java rename to code/index/index-reverse/java/nu/marginalia/index/ReverseIndexParameters.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexPrioFileNames.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexPrioFileNames.java rename to code/index/index-reverse/java/nu/marginalia/index/ReverseIndexPrioFileNames.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexReader.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexReader.java rename to code/index/index-reverse/java/nu/marginalia/index/ReverseIndexReader.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexSelfTest.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/ReverseIndexSelfTest.java rename to code/index/index-reverse/java/nu/marginalia/index/ReverseIndexSelfTest.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java b/code/index/index-reverse/java/nu/marginalia/index/construction/CountToOffsetTransformer.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/CountToOffsetTransformer.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/CountToOffsetTransformer.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java b/code/index/index-reverse/java/nu/marginalia/index/construction/DocIdRewriter.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/DocIdRewriter.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/DocIdRewriter.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java b/code/index/index-reverse/java/nu/marginalia/index/construction/IndexSizeEstimator.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/IndexSizeEstimator.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/IndexSizeEstimator.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java b/code/index/index-reverse/java/nu/marginalia/index/construction/JournalReaderSource.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/JournalReaderSource.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/JournalReaderSource.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexConstructor.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexConstructor.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindex.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindex.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindex.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexDocuments.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexDocuments.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexDocuments.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexReference.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexReference.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexReference.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java rename to code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java b/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java rename to code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java diff --git a/code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java b/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java similarity index 100% rename from code/index/reverse-index/src/main/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java rename to code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java diff --git a/code/index/reverse-index/merging.svg b/code/index/index-reverse/merging.svg similarity index 100% rename from code/index/reverse-index/merging.svg rename to code/index/index-reverse/merging.svg diff --git a/code/index/reverse-index/preindex.svg b/code/index/index-reverse/preindex.svg similarity index 100% rename from code/index/reverse-index/preindex.svg rename to code/index/index-reverse/preindex.svg diff --git a/code/index/reverse-index/readme.md b/code/index/index-reverse/readme.md similarity index 100% rename from code/index/reverse-index/readme.md rename to code/index/index-reverse/readme.md diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java b/code/index/index-reverse/test/nu/marginalia/index/ReverseIndexDebugTest.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexDebugTest.java rename to code/index/index-reverse/test/nu/marginalia/index/ReverseIndexDebugTest.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java b/code/index/index-reverse/test/nu/marginalia/index/ReverseIndexReaderTest.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/ReverseIndexReaderTest.java rename to code/index/index-reverse/test/nu/marginalia/index/ReverseIndexReaderTest.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexDocsTest.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexDocsTest.java rename to code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexDocsTest.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java rename to code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexMergeTest.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexMergeTest.java rename to code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexMergeTest.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java rename to code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexWordSegmentsTest.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java b/code/index/index-reverse/test/nu/marginalia/index/construction/TestJournalFactory.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestJournalFactory.java rename to code/index/index-reverse/test/nu/marginalia/index/construction/TestJournalFactory.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java b/code/index/index-reverse/test/nu/marginalia/index/construction/TestSegmentData.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/index/construction/TestSegmentData.java rename to code/index/index-reverse/test/nu/marginalia/index/construction/TestSegmentData.java diff --git a/code/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java b/code/index/index-reverse/test/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/index/reverse-index/src/test/java/nu/marginalia/test/TestUtil.java rename to code/index/index-reverse/test/nu/marginalia/test/TestUtil.java diff --git a/code/index/src/main/java/nu/marginalia/index/IndexQueryGrpcService.java b/code/index/java/nu/marginalia/index/IndexQueryGrpcService.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/IndexQueryGrpcService.java rename to code/index/java/nu/marginalia/index/IndexQueryGrpcService.java diff --git a/code/index/src/main/java/nu/marginalia/index/IndexServicesFactory.java b/code/index/java/nu/marginalia/index/IndexServicesFactory.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/IndexServicesFactory.java rename to code/index/java/nu/marginalia/index/IndexServicesFactory.java diff --git a/code/index/src/main/java/nu/marginalia/index/SearchTermsUtil.java b/code/index/java/nu/marginalia/index/SearchTermsUtil.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/SearchTermsUtil.java rename to code/index/java/nu/marginalia/index/SearchTermsUtil.java diff --git a/code/index/src/main/java/nu/marginalia/index/index/CombinedIndexReader.java b/code/index/java/nu/marginalia/index/index/CombinedIndexReader.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/index/CombinedIndexReader.java rename to code/index/java/nu/marginalia/index/index/CombinedIndexReader.java diff --git a/code/index/src/main/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java b/code/index/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java rename to code/index/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java diff --git a/code/index/src/main/java/nu/marginalia/index/index/IndexQueryService.java b/code/index/java/nu/marginalia/index/index/IndexQueryService.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/index/IndexQueryService.java rename to code/index/java/nu/marginalia/index/index/IndexQueryService.java diff --git a/code/index/src/main/java/nu/marginalia/index/index/StatefulIndex.java b/code/index/java/nu/marginalia/index/index/StatefulIndex.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/index/StatefulIndex.java rename to code/index/java/nu/marginalia/index/index/StatefulIndex.java diff --git a/code/index/src/main/java/nu/marginalia/index/model/IndexQueryParams.java b/code/index/java/nu/marginalia/index/model/IndexQueryParams.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/model/IndexQueryParams.java rename to code/index/java/nu/marginalia/index/model/IndexQueryParams.java diff --git a/code/index/src/main/java/nu/marginalia/index/model/IndexSearchParameters.java b/code/index/java/nu/marginalia/index/model/IndexSearchParameters.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/model/IndexSearchParameters.java rename to code/index/java/nu/marginalia/index/model/IndexSearchParameters.java diff --git a/code/index/src/main/java/nu/marginalia/index/model/IndexSearchTerms.java b/code/index/java/nu/marginalia/index/model/IndexSearchTerms.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/model/IndexSearchTerms.java rename to code/index/java/nu/marginalia/index/model/IndexSearchTerms.java diff --git a/code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java b/code/index/java/nu/marginalia/index/results/IndexMetadataService.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/results/IndexMetadataService.java rename to code/index/java/nu/marginalia/index/results/IndexMetadataService.java diff --git a/code/index/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java b/code/index/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java rename to code/index/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java diff --git a/code/index/src/main/java/nu/marginalia/index/results/IndexResultValuationContext.java b/code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/results/IndexResultValuationContext.java rename to code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java diff --git a/code/index/src/main/java/nu/marginalia/index/results/IndexResultValuatorService.java b/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/results/IndexResultValuatorService.java rename to code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/DbUpdateRanks.java b/code/index/java/nu/marginalia/index/searchset/DbUpdateRanks.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/searchset/DbUpdateRanks.java rename to code/index/java/nu/marginalia/index/searchset/DbUpdateRanks.java diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/RankingSearchSet.java b/code/index/java/nu/marginalia/index/searchset/RankingSearchSet.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/searchset/RankingSearchSet.java rename to code/index/java/nu/marginalia/index/searchset/RankingSearchSet.java diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/SearchSet.java b/code/index/java/nu/marginalia/index/searchset/SearchSet.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/searchset/SearchSet.java rename to code/index/java/nu/marginalia/index/searchset/SearchSet.java diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetAny.java b/code/index/java/nu/marginalia/index/searchset/SearchSetAny.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/searchset/SearchSetAny.java rename to code/index/java/nu/marginalia/index/searchset/SearchSetAny.java diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java b/code/index/java/nu/marginalia/index/searchset/SearchSetsService.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/searchset/SearchSetsService.java rename to code/index/java/nu/marginalia/index/searchset/SearchSetsService.java diff --git a/code/index/src/main/java/nu/marginalia/index/searchset/SmallSearchSet.java b/code/index/java/nu/marginalia/index/searchset/SmallSearchSet.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/index/searchset/SmallSearchSet.java rename to code/index/java/nu/marginalia/index/searchset/SmallSearchSet.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java b/code/index/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java rename to code/index/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/RankingAlgorithm.java b/code/index/java/nu/marginalia/ranking/domains/RankingAlgorithm.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/RankingAlgorithm.java rename to code/index/java/nu/marginalia/ranking/domains/RankingAlgorithm.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java b/code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java rename to code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultAccumulator.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java b/code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java rename to code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultBitSetAccumulator.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java b/code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java rename to code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashMapAccumulator.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java b/code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java rename to code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultHashSetAccumulator.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java b/code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java rename to code/index/java/nu/marginalia/ranking/domains/accumulator/RankingResultListAccumulator.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java rename to code/index/java/nu/marginalia/ranking/domains/data/AbstractGraphSource.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/data/GraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/GraphSource.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/data/GraphSource.java rename to code/index/java/nu/marginalia/ranking/domains/data/GraphSource.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java rename to code/index/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java rename to code/index/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java rename to code/index/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java b/code/index/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java rename to code/index/java/nu/marginalia/ranking/domains/jgrapht/PersonalizedPageRank.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/results/ResultKeywordSet.java b/code/index/java/nu/marginalia/ranking/results/ResultKeywordSet.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/results/ResultKeywordSet.java rename to code/index/java/nu/marginalia/ranking/results/ResultKeywordSet.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/results/ResultValuator.java b/code/index/java/nu/marginalia/ranking/results/ResultValuator.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/results/ResultValuator.java rename to code/index/java/nu/marginalia/ranking/results/ResultValuator.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/results/factors/Bm25Factor.java b/code/index/java/nu/marginalia/ranking/results/factors/Bm25Factor.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/results/factors/Bm25Factor.java rename to code/index/java/nu/marginalia/ranking/results/factors/Bm25Factor.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java b/code/index/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java rename to code/index/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java diff --git a/code/index/src/main/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java b/code/index/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java similarity index 100% rename from code/index/src/main/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java rename to code/index/java/nu/marginalia/ranking/results/factors/TermCoherenceFactor.java diff --git a/code/index/query/build.gradle b/code/index/query/build.gradle index c54b7fc4..7977ad73 100644 --- a/code/index/query/build.gradle +++ b/code/index/query/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:libraries:array') implementation project(':code:common:model') diff --git a/code/index/query/src/main/java/nu/marginalia/index/domainrankings/DomainRankings.java b/code/index/query/java/nu/marginalia/index/domainrankings/DomainRankings.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/domainrankings/DomainRankings.java rename to code/index/query/java/nu/marginalia/index/domainrankings/DomainRankings.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java b/code/index/query/java/nu/marginalia/index/query/EmptyEntrySource.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java rename to code/index/query/java/nu/marginalia/index/query/EmptyEntrySource.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java b/code/index/query/java/nu/marginalia/index/query/EntrySource.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/EntrySource.java rename to code/index/query/java/nu/marginalia/index/query/EntrySource.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java b/code/index/query/java/nu/marginalia/index/query/IndexQuery.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/IndexQuery.java rename to code/index/query/java/nu/marginalia/index/query/IndexQuery.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java b/code/index/query/java/nu/marginalia/index/query/IndexQueryBuilder.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java rename to code/index/query/java/nu/marginalia/index/query/IndexQueryBuilder.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java b/code/index/query/java/nu/marginalia/index/query/IndexQueryPriority.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java rename to code/index/query/java/nu/marginalia/index/query/IndexQueryPriority.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java b/code/index/query/java/nu/marginalia/index/query/IndexSearchBudget.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java rename to code/index/query/java/nu/marginalia/index/query/IndexSearchBudget.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java rename to code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java rename to code/index/query/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java rename to code/index/query/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java rename to code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java rename to code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java rename to code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java b/code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java rename to code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java b/code/index/query/java/nu/marginalia/index/query/limit/QueryStrategy.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java rename to code/index/query/java/nu/marginalia/index/query/limit/QueryStrategy.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java b/code/index/query/java/nu/marginalia/index/query/limit/SpecificationLimit.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java rename to code/index/query/java/nu/marginalia/index/query/limit/SpecificationLimit.java diff --git a/code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java b/code/index/query/java/nu/marginalia/index/query/limit/SpecificationLimitType.java similarity index 100% rename from code/index/query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java rename to code/index/query/java/nu/marginalia/index/query/limit/SpecificationLimitType.java diff --git a/code/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java b/code/index/query/test/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java similarity index 100% rename from code/index/query/src/test/java/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java rename to code/index/query/test/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java diff --git a/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java rename to code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java diff --git a/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTest.java rename to code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java diff --git a/code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java rename to code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTestModule.java diff --git a/code/index/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java b/code/index/test/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java rename to code/index/test/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java diff --git a/code/index/src/test/java/nu/marginalia/index/searchset/RankingSearchSetTest.java b/code/index/test/nu/marginalia/index/searchset/RankingSearchSetTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/index/searchset/RankingSearchSetTest.java rename to code/index/test/nu/marginalia/index/searchset/RankingSearchSetTest.java diff --git a/code/index/src/test/java/nu/marginalia/index/util/TestUtil.java b/code/index/test/nu/marginalia/index/util/TestUtil.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/index/util/TestUtil.java rename to code/index/test/nu/marginalia/index/util/TestUtil.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java b/code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java rename to code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmWithRealDataTest.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java b/code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java rename to code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java b/code/index/test/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java rename to code/index/test/nu/marginalia/ranking/domains/TestGraphSourceForInvertedLinkData.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java b/code/index/test/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java rename to code/index/test/nu/marginalia/ranking/domains/TestGraphSourceForLinkData.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java b/code/index/test/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java rename to code/index/test/nu/marginalia/ranking/domains/TestGraphSourceForSimilarityData.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/results/ResultValuatorTest.java b/code/index/test/nu/marginalia/ranking/results/ResultValuatorTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/results/ResultValuatorTest.java rename to code/index/test/nu/marginalia/ranking/results/ResultValuatorTest.java diff --git a/code/index/src/test/java/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java b/code/index/test/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java similarity index 100% rename from code/index/src/test/java/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java rename to code/index/test/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java diff --git a/code/libraries/array/build.gradle b/code/libraries/array/build.gradle index 306ab2f9..88e27107 100644 --- a/code/libraries/array/build.gradle +++ b/code/libraries/array/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/ArrayRangeReference.java b/code/libraries/array/java/nu/marginalia/array/ArrayRangeReference.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/ArrayRangeReference.java rename to code/libraries/array/java/nu/marginalia/array/ArrayRangeReference.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java b/code/libraries/array/java/nu/marginalia/array/IntArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java rename to code/libraries/array/java/nu/marginalia/array/IntArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java b/code/libraries/array/java/nu/marginalia/array/LongArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java rename to code/libraries/array/java/nu/marginalia/array/LongArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/LongArrayFactory.java b/code/libraries/array/java/nu/marginalia/array/LongArrayFactory.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/LongArrayFactory.java rename to code/libraries/array/java/nu/marginalia/array/LongArrayFactory.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/BulkTransferArray.java b/code/libraries/array/java/nu/marginalia/array/algo/BulkTransferArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/BulkTransferArray.java rename to code/libraries/array/java/nu/marginalia/array/algo/BulkTransferArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArrayBase.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArrayBase.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArrayBase.java rename to code/libraries/array/java/nu/marginalia/array/algo/IntArrayBase.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySearch.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArraySearch.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySearch.java rename to code/libraries/array/java/nu/marginalia/array/algo/IntArraySearch.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySort.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArraySort.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArraySort.java rename to code/libraries/array/java/nu/marginalia/array/algo/IntArraySort.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArrayTransformations.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArrayTransformations.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/IntArrayTransformations.java rename to code/libraries/array/java/nu/marginalia/array/algo/IntArrayTransformations.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArrayBase.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArrayBase.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArrayBase.java rename to code/libraries/array/java/nu/marginalia/array/algo/LongArrayBase.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArraySearch.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArraySearch.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArraySearch.java rename to code/libraries/array/java/nu/marginalia/array/algo/LongArraySearch.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArraySort.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArraySort.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArraySort.java rename to code/libraries/array/java/nu/marginalia/array/algo/LongArraySort.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArrayTransformations.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArrayTransformations.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/LongArrayTransformations.java rename to code/libraries/array/java/nu/marginalia/array/algo/LongArrayTransformations.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java rename to code/libraries/array/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/SortAlgoMergeSort.java b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoMergeSort.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/SortAlgoMergeSort.java rename to code/libraries/array/java/nu/marginalia/array/algo/SortAlgoMergeSort.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/SortAlgoQuickSort.java b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoQuickSort.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/SortAlgoQuickSort.java rename to code/libraries/array/java/nu/marginalia/array/algo/SortAlgoQuickSort.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/SortingContext.java b/code/libraries/array/java/nu/marginalia/array/algo/SortingContext.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/SortingContext.java rename to code/libraries/array/java/nu/marginalia/array/algo/SortingContext.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/algo/TwoArrayOperations.java b/code/libraries/array/java/nu/marginalia/array/algo/TwoArrayOperations.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/algo/TwoArrayOperations.java rename to code/libraries/array/java/nu/marginalia/array/algo/TwoArrayOperations.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/buffer/IntQueryBuffer.java b/code/libraries/array/java/nu/marginalia/array/buffer/IntQueryBuffer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/buffer/IntQueryBuffer.java rename to code/libraries/array/java/nu/marginalia/array/buffer/IntQueryBuffer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/buffer/LongQueryBuffer.java b/code/libraries/array/java/nu/marginalia/array/buffer/LongQueryBuffer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/buffer/LongQueryBuffer.java rename to code/libraries/array/java/nu/marginalia/array/buffer/LongQueryBuffer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/delegate/ReferenceImplIntArrayDelegate.java b/code/libraries/array/java/nu/marginalia/array/delegate/ReferenceImplIntArrayDelegate.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/delegate/ReferenceImplIntArrayDelegate.java rename to code/libraries/array/java/nu/marginalia/array/delegate/ReferenceImplIntArrayDelegate.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/delegate/ReferenceImplLongArrayDelegate.java b/code/libraries/array/java/nu/marginalia/array/delegate/ReferenceImplLongArrayDelegate.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/delegate/ReferenceImplLongArrayDelegate.java rename to code/libraries/array/java/nu/marginalia/array/delegate/ReferenceImplLongArrayDelegate.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/delegate/ShiftedIntArray.java b/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedIntArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/delegate/ShiftedIntArray.java rename to code/libraries/array/java/nu/marginalia/array/delegate/ShiftedIntArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/delegate/ShiftedLongArray.java b/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedLongArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/delegate/ShiftedLongArray.java rename to code/libraries/array/java/nu/marginalia/array/delegate/ShiftedLongArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeCall.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCall.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeCall.java rename to code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCall.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeCallIO.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCallIO.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeCallIO.java rename to code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCallIO.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeIntFunction.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeIntFunction.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeIntFunction.java rename to code/libraries/array/java/nu/marginalia/array/functional/AddressRangeIntFunction.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeLongFunction.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeLongFunction.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/AddressRangeLongFunction.java rename to code/libraries/array/java/nu/marginalia/array/functional/AddressRangeLongFunction.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/IntBinaryIOOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryIOOperation.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/IntBinaryIOOperation.java rename to code/libraries/array/java/nu/marginalia/array/functional/IntBinaryIOOperation.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/IntBinaryOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryOperation.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/IntBinaryOperation.java rename to code/libraries/array/java/nu/marginalia/array/functional/IntBinaryOperation.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/IntIOTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/IntIOTransformer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/IntIOTransformer.java rename to code/libraries/array/java/nu/marginalia/array/functional/IntIOTransformer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/IntTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/IntTransformer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/IntTransformer.java rename to code/libraries/array/java/nu/marginalia/array/functional/IntTransformer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/LongBinaryIOOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryIOOperation.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/LongBinaryIOOperation.java rename to code/libraries/array/java/nu/marginalia/array/functional/LongBinaryIOOperation.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/LongBinaryOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryOperation.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/LongBinaryOperation.java rename to code/libraries/array/java/nu/marginalia/array/functional/LongBinaryOperation.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/LongIOTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongIOTransformer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/LongIOTransformer.java rename to code/libraries/array/java/nu/marginalia/array/functional/LongIOTransformer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/LongIntConsumer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongIntConsumer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/LongIntConsumer.java rename to code/libraries/array/java/nu/marginalia/array/functional/LongIntConsumer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/LongLongConsumer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongLongConsumer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/LongLongConsumer.java rename to code/libraries/array/java/nu/marginalia/array/functional/LongLongConsumer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/functional/LongTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongTransformer.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/functional/LongTransformer.java rename to code/libraries/array/java/nu/marginalia/array/functional/LongTransformer.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/AbstractPagingArray.java b/code/libraries/array/java/nu/marginalia/array/page/AbstractPagingArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/page/AbstractPagingArray.java rename to code/libraries/array/java/nu/marginalia/array/page/AbstractPagingArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/PartitionPage.java b/code/libraries/array/java/nu/marginalia/array/page/PartitionPage.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/page/PartitionPage.java rename to code/libraries/array/java/nu/marginalia/array/page/PartitionPage.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentIntArray.java b/code/libraries/array/java/nu/marginalia/array/page/SegmentIntArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentIntArray.java rename to code/libraries/array/java/nu/marginalia/array/page/SegmentIntArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentLongArray.java b/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentLongArray.java rename to code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/UnsafeLongArray.java b/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/page/UnsafeLongArray.java rename to code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/UnsafeProvider.java b/code/libraries/array/java/nu/marginalia/array/page/UnsafeProvider.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/page/UnsafeProvider.java rename to code/libraries/array/java/nu/marginalia/array/page/UnsafeProvider.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java b/code/libraries/array/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java rename to code/libraries/array/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java b/code/libraries/array/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java rename to code/libraries/array/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java b/code/libraries/array/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java similarity index 100% rename from code/libraries/array/src/main/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java rename to code/libraries/array/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java b/code/libraries/array/test/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java rename to code/libraries/array/test/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySearchTest.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArraySearchTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySearchTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/IntArraySearchTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySortTest.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArraySortTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArraySortTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/IntArraySortTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformations2Test.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java rename to code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformations2Test.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformationsTest.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformationsTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformationsTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformationsTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySearchTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySearchTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySearchTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/LongArraySearchTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortNTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/LongArraySortNTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/LongArraySortTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformations2Test.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java rename to code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformations2Test.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformationsTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformationsTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformationsTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformationsTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/TwoArrayOperationsTest.java b/code/libraries/array/test/nu/marginalia/array/algo/TwoArrayOperationsTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/algo/TwoArrayOperationsTest.java rename to code/libraries/array/test/nu/marginalia/array/algo/TwoArrayOperationsTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java b/code/libraries/array/test/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java rename to code/libraries/array/test/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java diff --git a/code/libraries/array/src/test/java/nu/marginalia/util/test/TestUtil.java b/code/libraries/array/test/nu/marginalia/util/test/TestUtil.java similarity index 100% rename from code/libraries/array/src/test/java/nu/marginalia/util/test/TestUtil.java rename to code/libraries/array/test/nu/marginalia/util/test/TestUtil.java diff --git a/code/libraries/big-string/build.gradle b/code/libraries/big-string/build.gradle index c769bc12..2f67aa7f 100644 --- a/code/libraries/big-string/build.gradle +++ b/code/libraries/big-string/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/BigString.java b/code/libraries/big-string/java/nu/marginalia/bigstring/BigString.java similarity index 100% rename from code/libraries/big-string/src/main/java/nu/marginalia/bigstring/BigString.java rename to code/libraries/big-string/java/nu/marginalia/bigstring/BigString.java diff --git a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/CompressedBigString.java b/code/libraries/big-string/java/nu/marginalia/bigstring/CompressedBigString.java similarity index 100% rename from code/libraries/big-string/src/main/java/nu/marginalia/bigstring/CompressedBigString.java rename to code/libraries/big-string/java/nu/marginalia/bigstring/CompressedBigString.java diff --git a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/CompressionBuffer.java b/code/libraries/big-string/java/nu/marginalia/bigstring/CompressionBuffer.java similarity index 100% rename from code/libraries/big-string/src/main/java/nu/marginalia/bigstring/CompressionBuffer.java rename to code/libraries/big-string/java/nu/marginalia/bigstring/CompressionBuffer.java diff --git a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/CompressionBufferPool.java b/code/libraries/big-string/java/nu/marginalia/bigstring/CompressionBufferPool.java similarity index 100% rename from code/libraries/big-string/src/main/java/nu/marginalia/bigstring/CompressionBufferPool.java rename to code/libraries/big-string/java/nu/marginalia/bigstring/CompressionBufferPool.java diff --git a/code/libraries/big-string/src/main/java/nu/marginalia/bigstring/PlainBigString.java b/code/libraries/big-string/java/nu/marginalia/bigstring/PlainBigString.java similarity index 100% rename from code/libraries/big-string/src/main/java/nu/marginalia/bigstring/PlainBigString.java rename to code/libraries/big-string/java/nu/marginalia/bigstring/PlainBigString.java diff --git a/code/libraries/big-string/src/test/java/nu/marginalia/bigstring/CompressedBigStringTest.java b/code/libraries/big-string/test/nu/marginalia/bigstring/CompressedBigStringTest.java similarity index 100% rename from code/libraries/big-string/src/test/java/nu/marginalia/bigstring/CompressedBigStringTest.java rename to code/libraries/big-string/test/nu/marginalia/bigstring/CompressedBigStringTest.java diff --git a/code/libraries/blocking-thread-pool/build.gradle b/code/libraries/blocking-thread-pool/build.gradle index e716f1c8..8c5609a5 100644 --- a/code/libraries/blocking-thread-pool/build.gradle +++ b/code/libraries/blocking-thread-pool/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/ProcessingIterator.java b/code/libraries/blocking-thread-pool/java/nu/marginalia/util/ProcessingIterator.java similarity index 100% rename from code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/ProcessingIterator.java rename to code/libraries/blocking-thread-pool/java/nu/marginalia/util/ProcessingIterator.java diff --git a/code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java b/code/libraries/blocking-thread-pool/java/nu/marginalia/util/SimpleBlockingThreadPool.java similarity index 100% rename from code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java rename to code/libraries/blocking-thread-pool/java/nu/marginalia/util/SimpleBlockingThreadPool.java diff --git a/code/libraries/blocking-thread-pool/src/test/java/nu/marginalia/util/ProcessingIteratorTest.java b/code/libraries/blocking-thread-pool/test/nu/marginalia/util/ProcessingIteratorTest.java similarity index 100% rename from code/libraries/blocking-thread-pool/src/test/java/nu/marginalia/util/ProcessingIteratorTest.java rename to code/libraries/blocking-thread-pool/test/nu/marginalia/util/ProcessingIteratorTest.java diff --git a/code/libraries/braille-block-punch-cards/build.gradle b/code/libraries/braille-block-punch-cards/build.gradle index 4d9c5997..673d944f 100644 --- a/code/libraries/braille-block-punch-cards/build.gradle +++ b/code/libraries/braille-block-punch-cards/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j implementation libs.notnull diff --git a/code/libraries/braille-block-punch-cards/src/main/java/nu/marginalia/bbpc/BrailleBlockPunchCards.java b/code/libraries/braille-block-punch-cards/java/nu/marginalia/bbpc/BrailleBlockPunchCards.java similarity index 100% rename from code/libraries/braille-block-punch-cards/src/main/java/nu/marginalia/bbpc/BrailleBlockPunchCards.java rename to code/libraries/braille-block-punch-cards/java/nu/marginalia/bbpc/BrailleBlockPunchCards.java diff --git a/code/libraries/braille-block-punch-cards/src/test/java/nu/marginalia/bbpc/BrailleBlockPunchCardsTest.java b/code/libraries/braille-block-punch-cards/test/nu/marginalia/bbpc/BrailleBlockPunchCardsTest.java similarity index 100% rename from code/libraries/braille-block-punch-cards/src/test/java/nu/marginalia/bbpc/BrailleBlockPunchCardsTest.java rename to code/libraries/braille-block-punch-cards/test/nu/marginalia/bbpc/BrailleBlockPunchCardsTest.java diff --git a/code/libraries/btree/build.gradle b/code/libraries/btree/build.gradle index d39af03c..99917255 100644 --- a/code/libraries/btree/build.gradle +++ b/code/libraries/btree/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:next-prime') diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeDogEar.java b/code/libraries/btree/java/nu/marginalia/btree/BTreeDogEar.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeDogEar.java rename to code/libraries/btree/java/nu/marginalia/btree/BTreeDogEar.java diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java b/code/libraries/btree/java/nu/marginalia/btree/BTreeReader.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java rename to code/libraries/btree/java/nu/marginalia/btree/BTreeReader.java diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriteCallback.java b/code/libraries/btree/java/nu/marginalia/btree/BTreeWriteCallback.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriteCallback.java rename to code/libraries/btree/java/nu/marginalia/btree/BTreeWriteCallback.java diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java b/code/libraries/btree/java/nu/marginalia/btree/BTreeWriter.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java rename to code/libraries/btree/java/nu/marginalia/btree/BTreeWriter.java diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/model/BTreeBlockSize.java b/code/libraries/btree/java/nu/marginalia/btree/model/BTreeBlockSize.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/model/BTreeBlockSize.java rename to code/libraries/btree/java/nu/marginalia/btree/model/BTreeBlockSize.java diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/model/BTreeContext.java b/code/libraries/btree/java/nu/marginalia/btree/model/BTreeContext.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/model/BTreeContext.java rename to code/libraries/btree/java/nu/marginalia/btree/model/BTreeContext.java diff --git a/code/libraries/btree/src/main/java/nu/marginalia/btree/model/BTreeHeader.java b/code/libraries/btree/java/nu/marginalia/btree/model/BTreeHeader.java similarity index 100% rename from code/libraries/btree/src/main/java/nu/marginalia/btree/model/BTreeHeader.java rename to code/libraries/btree/java/nu/marginalia/btree/model/BTreeHeader.java diff --git a/code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderQueryDataWithIndexTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderQueryDataWithIndexTest.java similarity index 100% rename from code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderQueryDataWithIndexTest.java rename to code/libraries/btree/test/nu/marginalia/btree/BTreeReaderQueryDataWithIndexTest.java diff --git a/code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderQueryDataWithoutIndexTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderQueryDataWithoutIndexTest.java similarity index 100% rename from code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderQueryDataWithoutIndexTest.java rename to code/libraries/btree/test/nu/marginalia/btree/BTreeReaderQueryDataWithoutIndexTest.java diff --git a/code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java similarity index 100% rename from code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java rename to code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java diff --git a/code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java similarity index 100% rename from code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java rename to code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java diff --git a/code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeWriterTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeWriterTest.java similarity index 100% rename from code/libraries/btree/src/test/java/nu/marginalia/btree/BTreeWriterTest.java rename to code/libraries/btree/test/nu/marginalia/btree/BTreeWriterTest.java diff --git a/code/libraries/easy-lsh/build.gradle b/code/libraries/easy-lsh/build.gradle index e8b11046..74fd976e 100644 --- a/code/libraries/easy-lsh/build.gradle +++ b/code/libraries/easy-lsh/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/libraries/easy-lsh/src/main/java/nu/marginalia/lsh/EasyLSH.java b/code/libraries/easy-lsh/java/nu/marginalia/lsh/EasyLSH.java similarity index 100% rename from code/libraries/easy-lsh/src/main/java/nu/marginalia/lsh/EasyLSH.java rename to code/libraries/easy-lsh/java/nu/marginalia/lsh/EasyLSH.java diff --git a/code/libraries/easy-lsh/src/test/java/nu/marginalia/lsh/EasyLSHTest.java b/code/libraries/easy-lsh/test/nu/marginalia/lsh/EasyLSHTest.java similarity index 100% rename from code/libraries/easy-lsh/src/test/java/nu/marginalia/lsh/EasyLSHTest.java rename to code/libraries/easy-lsh/test/nu/marginalia/lsh/EasyLSHTest.java diff --git a/code/libraries/geo-ip/build.gradle b/code/libraries/geo-ip/build.gradle index b0180ef8..a47f64a5 100644 --- a/code/libraries/geo-ip/build.gradle +++ b/code/libraries/geo-ip/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') diff --git a/code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/GeoIpDictionary.java b/code/libraries/geo-ip/java/nu/marginalia/geoip/GeoIpDictionary.java similarity index 100% rename from code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/GeoIpDictionary.java rename to code/libraries/geo-ip/java/nu/marginalia/geoip/GeoIpDictionary.java diff --git a/code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/AsnMapping.java b/code/libraries/geo-ip/java/nu/marginalia/geoip/sources/AsnMapping.java similarity index 100% rename from code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/AsnMapping.java rename to code/libraries/geo-ip/java/nu/marginalia/geoip/sources/AsnMapping.java diff --git a/code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/AsnTable.java b/code/libraries/geo-ip/java/nu/marginalia/geoip/sources/AsnTable.java similarity index 100% rename from code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/AsnTable.java rename to code/libraries/geo-ip/java/nu/marginalia/geoip/sources/AsnTable.java diff --git a/code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/IP2LocationMapping.java b/code/libraries/geo-ip/java/nu/marginalia/geoip/sources/IP2LocationMapping.java similarity index 100% rename from code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/IP2LocationMapping.java rename to code/libraries/geo-ip/java/nu/marginalia/geoip/sources/IP2LocationMapping.java diff --git a/code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/IpRangeMapping.java b/code/libraries/geo-ip/java/nu/marginalia/geoip/sources/IpRangeMapping.java similarity index 100% rename from code/libraries/geo-ip/src/main/java/nu/marginalia/geoip/sources/IpRangeMapping.java rename to code/libraries/geo-ip/java/nu/marginalia/geoip/sources/IpRangeMapping.java diff --git a/code/libraries/geo-ip/src/test/java/nu/marginalia/geoip/GeoIpDictionaryTest.java b/code/libraries/geo-ip/test/nu/marginalia/geoip/GeoIpDictionaryTest.java similarity index 100% rename from code/libraries/geo-ip/src/test/java/nu/marginalia/geoip/GeoIpDictionaryTest.java rename to code/libraries/geo-ip/test/nu/marginalia/geoip/GeoIpDictionaryTest.java diff --git a/code/libraries/guarded-regex/build.gradle b/code/libraries/guarded-regex/build.gradle index 8640776d..6faa6f3c 100644 --- a/code/libraries/guarded-regex/build.gradle +++ b/code/libraries/guarded-regex/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/libraries/guarded-regex/src/main/java/nu/marginalia/gregex/GuardedRegex.java b/code/libraries/guarded-regex/java/nu/marginalia/gregex/GuardedRegex.java similarity index 100% rename from code/libraries/guarded-regex/src/main/java/nu/marginalia/gregex/GuardedRegex.java rename to code/libraries/guarded-regex/java/nu/marginalia/gregex/GuardedRegex.java diff --git a/code/libraries/guarded-regex/src/main/java/nu/marginalia/gregex/GuardedRegexFactory.java b/code/libraries/guarded-regex/java/nu/marginalia/gregex/GuardedRegexFactory.java similarity index 100% rename from code/libraries/guarded-regex/src/main/java/nu/marginalia/gregex/GuardedRegexFactory.java rename to code/libraries/guarded-regex/java/nu/marginalia/gregex/GuardedRegexFactory.java diff --git a/code/libraries/language-processing/build.gradle b/code/libraries/language-processing/build.gradle index 266bcfb4..56abe731 100644 --- a/code/libraries/language-processing/build.gradle +++ b/code/libraries/language-processing/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:rdrpostagger') implementation project(':third-party:porterstemmer') diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/WordDictionary.java b/code/libraries/language-processing/java/nu/marginalia/language/WordDictionary.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/WordDictionary.java rename to code/libraries/language-processing/java/nu/marginalia/language/WordDictionary.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/WordPatterns.java b/code/libraries/language-processing/java/nu/marginalia/language/WordPatterns.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/WordPatterns.java rename to code/libraries/language-processing/java/nu/marginalia/language/WordPatterns.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/encoding/AsciiFlattener.java b/code/libraries/language-processing/java/nu/marginalia/language/encoding/AsciiFlattener.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/encoding/AsciiFlattener.java rename to code/libraries/language-processing/java/nu/marginalia/language/encoding/AsciiFlattener.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/encoding/UnicodeRanges.java b/code/libraries/language-processing/java/nu/marginalia/language/encoding/UnicodeRanges.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/encoding/UnicodeRanges.java rename to code/libraries/language-processing/java/nu/marginalia/language/encoding/UnicodeRanges.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/FasttextLanguagePredictionModel.java b/code/libraries/language-processing/java/nu/marginalia/language/filter/FasttextLanguagePredictionModel.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/FasttextLanguagePredictionModel.java rename to code/libraries/language-processing/java/nu/marginalia/language/filter/FasttextLanguagePredictionModel.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/LanguageFilter.java b/code/libraries/language-processing/java/nu/marginalia/language/filter/LanguageFilter.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/LanguageFilter.java rename to code/libraries/language-processing/java/nu/marginalia/language/filter/LanguageFilter.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/LanguagePredictionModel.java b/code/libraries/language-processing/java/nu/marginalia/language/filter/LanguagePredictionModel.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/LanguagePredictionModel.java rename to code/libraries/language-processing/java/nu/marginalia/language/filter/LanguagePredictionModel.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/UngaBungaLanguagePredictionModel.java b/code/libraries/language-processing/java/nu/marginalia/language/filter/UngaBungaLanguagePredictionModel.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/filter/UngaBungaLanguagePredictionModel.java rename to code/libraries/language-processing/java/nu/marginalia/language/filter/UngaBungaLanguagePredictionModel.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentLanguageData.java b/code/libraries/language-processing/java/nu/marginalia/language/model/DocumentLanguageData.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentLanguageData.java rename to code/libraries/language-processing/java/nu/marginalia/language/model/DocumentLanguageData.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentSentence.java b/code/libraries/language-processing/java/nu/marginalia/language/model/DocumentSentence.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/model/DocumentSentence.java rename to code/libraries/language-processing/java/nu/marginalia/language/model/DocumentSentence.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordFrequencyData.java b/code/libraries/language-processing/java/nu/marginalia/language/model/WordFrequencyData.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordFrequencyData.java rename to code/libraries/language-processing/java/nu/marginalia/language/model/WordFrequencyData.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordRep.java b/code/libraries/language-processing/java/nu/marginalia/language/model/WordRep.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordRep.java rename to code/libraries/language-processing/java/nu/marginalia/language/model/WordRep.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordSeparator.java b/code/libraries/language-processing/java/nu/marginalia/language/model/WordSeparator.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordSeparator.java rename to code/libraries/language-processing/java/nu/marginalia/language/model/WordSeparator.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordSpan.java b/code/libraries/language-processing/java/nu/marginalia/language/model/WordSpan.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/model/WordSpan.java rename to code/libraries/language-processing/java/nu/marginalia/language/model/WordSpan.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceExtractor.java b/code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractor.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceExtractor.java rename to code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractor.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceExtractorHtmlTagCleaner.java b/code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractorHtmlTagCleaner.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceExtractorHtmlTagCleaner.java rename to code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractorHtmlTagCleaner.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceExtractorStringUtils.java b/code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractorStringUtils.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceExtractorStringUtils.java rename to code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractorStringUtils.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentencePreCleaner.java b/code/libraries/language-processing/java/nu/marginalia/language/sentence/SentencePreCleaner.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentencePreCleaner.java rename to code/libraries/language-processing/java/nu/marginalia/language/sentence/SentencePreCleaner.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java b/code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java rename to code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/ThreadLocalSentenceExtractorProvider.java b/code/libraries/language-processing/java/nu/marginalia/language/sentence/ThreadLocalSentenceExtractorProvider.java similarity index 100% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/sentence/ThreadLocalSentenceExtractorProvider.java rename to code/libraries/language-processing/java/nu/marginalia/language/sentence/ThreadLocalSentenceExtractorProvider.java diff --git a/code/libraries/language-processing/src/main/resources/dictionary/en-1000 b/code/libraries/language-processing/resources/dictionary/en-1000 similarity index 100% rename from code/libraries/language-processing/src/main/resources/dictionary/en-1000 rename to code/libraries/language-processing/resources/dictionary/en-1000 diff --git a/code/libraries/language-processing/src/main/resources/dictionary/en-stopwords b/code/libraries/language-processing/resources/dictionary/en-stopwords similarity index 100% rename from code/libraries/language-processing/src/main/resources/dictionary/en-stopwords rename to code/libraries/language-processing/resources/dictionary/en-stopwords diff --git a/code/libraries/language-processing/src/main/resources/dictionary/en-words b/code/libraries/language-processing/resources/dictionary/en-words similarity index 100% rename from code/libraries/language-processing/src/main/resources/dictionary/en-words rename to code/libraries/language-processing/resources/dictionary/en-words diff --git a/code/libraries/language-processing/src/main/resources/dictionary/latin-1000 b/code/libraries/language-processing/resources/dictionary/latin-1000 similarity index 100% rename from code/libraries/language-processing/src/main/resources/dictionary/latin-1000 rename to code/libraries/language-processing/resources/dictionary/latin-1000 diff --git a/code/libraries/language-processing/src/main/resources/dictionary/swe-1000 b/code/libraries/language-processing/resources/dictionary/swe-1000 similarity index 100% rename from code/libraries/language-processing/src/main/resources/dictionary/swe-1000 rename to code/libraries/language-processing/resources/dictionary/swe-1000 diff --git a/code/libraries/language-processing/src/main/resources/dictionary/word-frequency b/code/libraries/language-processing/resources/dictionary/word-frequency similarity index 100% rename from code/libraries/language-processing/src/main/resources/dictionary/word-frequency rename to code/libraries/language-processing/resources/dictionary/word-frequency diff --git a/code/libraries/language-processing/src/test/resources/html/jep.html b/code/libraries/language-processing/test-resources/html/jep.html similarity index 100% rename from code/libraries/language-processing/src/test/resources/html/jep.html rename to code/libraries/language-processing/test-resources/html/jep.html diff --git a/code/libraries/language-processing/src/test/resources/html/spam.html b/code/libraries/language-processing/test-resources/html/spam.html similarity index 100% rename from code/libraries/language-processing/src/test/resources/html/spam.html rename to code/libraries/language-processing/test-resources/html/spam.html diff --git a/code/libraries/language-processing/src/test/java/nu/marginalia/language/encoding/AsciiFlattenerTest.java b/code/libraries/language-processing/test/nu/marginalia/language/encoding/AsciiFlattenerTest.java similarity index 100% rename from code/libraries/language-processing/src/test/java/nu/marginalia/language/encoding/AsciiFlattenerTest.java rename to code/libraries/language-processing/test/nu/marginalia/language/encoding/AsciiFlattenerTest.java diff --git a/code/libraries/language-processing/src/test/java/nu/marginalia/language/encoding/SentenceExtractorHtmlTagCleanerTest.java b/code/libraries/language-processing/test/nu/marginalia/language/encoding/SentenceExtractorHtmlTagCleanerTest.java similarity index 100% rename from code/libraries/language-processing/src/test/java/nu/marginalia/language/encoding/SentenceExtractorHtmlTagCleanerTest.java rename to code/libraries/language-processing/test/nu/marginalia/language/encoding/SentenceExtractorHtmlTagCleanerTest.java diff --git a/code/libraries/language-processing/src/test/java/nu/marginalia/language/filter/LanguageFilterTest.java b/code/libraries/language-processing/test/nu/marginalia/language/filter/LanguageFilterTest.java similarity index 100% rename from code/libraries/language-processing/src/test/java/nu/marginalia/language/filter/LanguageFilterTest.java rename to code/libraries/language-processing/test/nu/marginalia/language/filter/LanguageFilterTest.java diff --git a/code/libraries/language-processing/src/test/java/nu/marginalia/language/filter/TestLanguageModels.java b/code/libraries/language-processing/test/nu/marginalia/language/filter/TestLanguageModels.java similarity index 100% rename from code/libraries/language-processing/src/test/java/nu/marginalia/language/filter/TestLanguageModels.java rename to code/libraries/language-processing/test/nu/marginalia/language/filter/TestLanguageModels.java diff --git a/code/libraries/language-processing/src/test/java/nu/marginalia/language/sentence/SentenceExtractorTest.java b/code/libraries/language-processing/test/nu/marginalia/language/sentence/SentenceExtractorTest.java similarity index 100% rename from code/libraries/language-processing/src/test/java/nu/marginalia/language/sentence/SentenceExtractorTest.java rename to code/libraries/language-processing/test/nu/marginalia/language/sentence/SentenceExtractorTest.java diff --git a/code/libraries/message-queue/build.gradle b/code/libraries/message-queue/build.gradle index e3572cf6..5931a76e 100644 --- a/code/libraries/message-queue/build.gradle +++ b/code/libraries/message-queue/build.gradle @@ -9,6 +9,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.spark implementation libs.guice diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java b/code/libraries/message-queue/java/nu/marginalia/actor/ActorStateMachine.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java rename to code/libraries/message-queue/java/nu/marginalia/actor/ActorStateMachine.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ExpectedMessage.java b/code/libraries/message-queue/java/nu/marginalia/actor/ExpectedMessage.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/ExpectedMessage.java rename to code/libraries/message-queue/java/nu/marginalia/actor/ExpectedMessage.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/prototype/ActorPrototype.java b/code/libraries/message-queue/java/nu/marginalia/actor/prototype/ActorPrototype.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/prototype/ActorPrototype.java rename to code/libraries/message-queue/java/nu/marginalia/actor/prototype/ActorPrototype.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/prototype/RecordActorPrototype.java b/code/libraries/message-queue/java/nu/marginalia/actor/prototype/RecordActorPrototype.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/prototype/RecordActorPrototype.java rename to code/libraries/message-queue/java/nu/marginalia/actor/prototype/RecordActorPrototype.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorControlFlowException.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/ActorControlFlowException.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorControlFlowException.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/ActorControlFlowException.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorResumeBehavior.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/ActorResumeBehavior.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorResumeBehavior.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/ActorResumeBehavior.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorStateInstance.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStateInstance.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorStateInstance.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStateInstance.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorStateTransition.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStateTransition.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorStateTransition.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStateTransition.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorStep.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStep.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/ActorStep.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/ActorStep.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/Resume.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/Resume.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/Resume.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/Resume.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/Terminal.java b/code/libraries/message-queue/java/nu/marginalia/actor/state/Terminal.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/actor/state/Terminal.java rename to code/libraries/message-queue/java/nu/marginalia/actor/state/Terminal.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java b/code/libraries/message-queue/java/nu/marginalia/mq/MessageQueueFactory.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java rename to code/libraries/message-queue/java/nu/marginalia/mq/MessageQueueFactory.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MqException.java b/code/libraries/message-queue/java/nu/marginalia/mq/MqException.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/MqException.java rename to code/libraries/message-queue/java/nu/marginalia/mq/MqException.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MqMessage.java b/code/libraries/message-queue/java/nu/marginalia/mq/MqMessage.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/MqMessage.java rename to code/libraries/message-queue/java/nu/marginalia/mq/MqMessage.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MqMessageState.java b/code/libraries/message-queue/java/nu/marginalia/mq/MqMessageState.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/MqMessageState.java rename to code/libraries/message-queue/java/nu/marginalia/mq/MqMessageState.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqAsynchronousInbox.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqAsynchronousInbox.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqAsynchronousInbox.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqAsynchronousInbox.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxIf.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqInboxIf.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxIf.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqInboxIf.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxResponse.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqInboxResponse.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxResponse.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqInboxResponse.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxShredder.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqInboxShredder.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxShredder.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqInboxShredder.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSingleShotInbox.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqSingleShotInbox.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSingleShotInbox.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqSingleShotInbox.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSubscription.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqSubscription.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSubscription.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqSubscription.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSynchronousInbox.java b/code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqSynchronousInbox.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSynchronousInbox.java rename to code/libraries/message-queue/java/nu/marginalia/mq/inbox/MqSynchronousInbox.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java b/code/libraries/message-queue/java/nu/marginalia/mq/outbox/MqOutbox.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java rename to code/libraries/message-queue/java/nu/marginalia/mq/outbox/MqOutbox.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqMessageHandlerRegistry.java b/code/libraries/message-queue/java/nu/marginalia/mq/persistence/MqMessageHandlerRegistry.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqMessageHandlerRegistry.java rename to code/libraries/message-queue/java/nu/marginalia/mq/persistence/MqMessageHandlerRegistry.java diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java b/code/libraries/message-queue/java/nu/marginalia/mq/persistence/MqPersistence.java similarity index 100% rename from code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java rename to code/libraries/message-queue/java/nu/marginalia/mq/persistence/MqPersistence.java diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java b/code/libraries/message-queue/test/nu/marginalia/actor/ActorRecordMachineTest.java similarity index 100% rename from code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java rename to code/libraries/message-queue/test/nu/marginalia/actor/ActorRecordMachineTest.java diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqMessageRow.java b/code/libraries/message-queue/test/nu/marginalia/mq/MqMessageRow.java similarity index 100% rename from code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqMessageRow.java rename to code/libraries/message-queue/test/nu/marginalia/mq/MqMessageRow.java diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java b/code/libraries/message-queue/test/nu/marginalia/mq/MqTestUtil.java similarity index 100% rename from code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java rename to code/libraries/message-queue/test/nu/marginalia/mq/MqTestUtil.java diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java b/code/libraries/message-queue/test/nu/marginalia/mq/outbox/MqOutboxTest.java similarity index 100% rename from code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java rename to code/libraries/message-queue/test/nu/marginalia/mq/outbox/MqOutboxTest.java diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java b/code/libraries/message-queue/test/nu/marginalia/mq/persistence/MqPersistenceTest.java similarity index 100% rename from code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java rename to code/libraries/message-queue/test/nu/marginalia/mq/persistence/MqPersistenceTest.java diff --git a/code/libraries/next-prime/build.gradle b/code/libraries/next-prime/build.gradle index e716f1c8..8c5609a5 100644 --- a/code/libraries/next-prime/build.gradle +++ b/code/libraries/next-prime/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/libraries/next-prime/src/main/java/nu/marginalia/util/NextPrimeUtil.java b/code/libraries/next-prime/java/nu/marginalia/util/NextPrimeUtil.java similarity index 100% rename from code/libraries/next-prime/src/main/java/nu/marginalia/util/NextPrimeUtil.java rename to code/libraries/next-prime/java/nu/marginalia/util/NextPrimeUtil.java diff --git a/code/libraries/next-prime/src/test/java/nu/marginalia/util/NextPrimeUtilTest.java b/code/libraries/next-prime/test/nu/marginalia/util/NextPrimeUtilTest.java similarity index 100% rename from code/libraries/next-prime/src/test/java/nu/marginalia/util/NextPrimeUtilTest.java rename to code/libraries/next-prime/test/nu/marginalia/util/NextPrimeUtilTest.java diff --git a/code/libraries/random-write-funnel/build.gradle b/code/libraries/random-write-funnel/build.gradle index 997ff739..9d23af16 100644 --- a/code/libraries/random-write-funnel/build.gradle +++ b/code/libraries/random-write-funnel/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:libraries:array') diff --git a/code/libraries/random-write-funnel/src/main/java/nu/marginalia/rwf/RandomFileAssembler.java b/code/libraries/random-write-funnel/java/nu/marginalia/rwf/RandomFileAssembler.java similarity index 100% rename from code/libraries/random-write-funnel/src/main/java/nu/marginalia/rwf/RandomFileAssembler.java rename to code/libraries/random-write-funnel/java/nu/marginalia/rwf/RandomFileAssembler.java diff --git a/code/libraries/random-write-funnel/src/main/java/nu/marginalia/rwf/RandomWriteFunnel.java b/code/libraries/random-write-funnel/java/nu/marginalia/rwf/RandomWriteFunnel.java similarity index 100% rename from code/libraries/random-write-funnel/src/main/java/nu/marginalia/rwf/RandomWriteFunnel.java rename to code/libraries/random-write-funnel/java/nu/marginalia/rwf/RandomWriteFunnel.java diff --git a/code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomFileAssemblerTest.java b/code/libraries/random-write-funnel/test/nu/marginalia/rwf/RandomFileAssemblerTest.java similarity index 100% rename from code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomFileAssemblerTest.java rename to code/libraries/random-write-funnel/test/nu/marginalia/rwf/RandomFileAssemblerTest.java diff --git a/code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomWriteFunnelTest.java b/code/libraries/random-write-funnel/test/nu/marginalia/rwf/RandomWriteFunnelTest.java similarity index 100% rename from code/libraries/random-write-funnel/src/test/java/nu/marginalia/rwf/RandomWriteFunnelTest.java rename to code/libraries/random-write-funnel/test/nu/marginalia/rwf/RandomWriteFunnelTest.java diff --git a/code/libraries/term-frequency-dict/build.gradle b/code/libraries/term-frequency-dict/build.gradle index d7cd4029..0fe311b6 100644 --- a/code/libraries/term-frequency-dict/build.gradle +++ b/code/libraries/term-frequency-dict/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:rdrpostagger') implementation project(':third-party:porterstemmer') diff --git a/code/libraries/term-frequency-dict/src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java b/code/libraries/term-frequency-dict/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java similarity index 100% rename from code/libraries/term-frequency-dict/src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java rename to code/libraries/term-frequency-dict/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java diff --git a/code/libraries/test-helpers/build.gradle b/code/libraries/test-helpers/build.gradle index b50556a1..16244d0a 100644 --- a/code/libraries/test-helpers/build.gradle +++ b/code/libraries/test-helpers/build.gradle @@ -8,6 +8,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j implementation libs.bundles.mariadb diff --git a/code/libraries/test-helpers/src/main/java/nu/marginalia/test/TestMigrationLoader.java b/code/libraries/test-helpers/java/nu/marginalia/test/TestMigrationLoader.java similarity index 100% rename from code/libraries/test-helpers/src/main/java/nu/marginalia/test/TestMigrationLoader.java rename to code/libraries/test-helpers/java/nu/marginalia/test/TestMigrationLoader.java diff --git a/code/process-models/crawl-spec/build.gradle b/code/process-models/crawl-spec/build.gradle index a0045a22..551f2c6e 100644 --- a/code/process-models/crawl-spec/build.gradle +++ b/code/process-models/crawl-spec/build.gradle @@ -10,6 +10,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java b/code/process-models/crawl-spec/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java similarity index 100% rename from code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java rename to code/process-models/crawl-spec/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java b/code/process-models/crawl-spec/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java similarity index 100% rename from code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java rename to code/process-models/crawl-spec/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java b/code/process-models/crawl-spec/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java similarity index 100% rename from code/process-models/crawl-spec/src/main/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java rename to code/process-models/crawl-spec/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java b/code/process-models/crawl-spec/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java similarity index 100% rename from code/process-models/crawl-spec/src/main/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java rename to code/process-models/crawl-spec/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java b/code/process-models/crawl-spec/java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java similarity index 100% rename from code/process-models/crawl-spec/src/main/java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java rename to code/process-models/crawl-spec/java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java diff --git a/code/process-models/crawling-model/build.gradle b/code/process-models/crawling-model/build.gradle index f7b5cf49..505db80f 100644 --- a/code/process-models/crawling-model/build.gradle +++ b/code/process-models/crawling-model/build.gradle @@ -12,6 +12,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:db') diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/ContentTypeLogic.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/body/ContentTypeLogic.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/ContentTypeLogic.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/body/ContentTypeLogic.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/DocumentBodyExtractor.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/body/DocumentBodyExtractor.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/DocumentBodyExtractor.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/body/DocumentBodyExtractor.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/DocumentBodyResult.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/body/DocumentBodyResult.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/DocumentBodyResult.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/body/DocumentBodyResult.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/HttpFetchResult.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/body/HttpFetchResult.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/body/HttpFetchResult.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/body/HttpFetchResult.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/CrawledDomainReader.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/CrawledDomainReader.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/CrawledDomainWriter.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/CrawledDomainWriter.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawlerOutputFile.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/CrawlerOutputFile.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawlerOutputFile.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/CrawlerOutputFile.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/format/CompatibleLegacySerializableCrawlDataStream.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/FastLegacySerializableCrawlDataStream.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/format/FastLegacySerializableCrawlDataStream.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/FastLegacySerializableCrawlDataStream.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/format/FastLegacySerializableCrawlDataStream.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/ParquetSerializableCrawlDataStream.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/io/format/ParquetSerializableCrawlDataStream.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/io/format/ParquetSerializableCrawlDataStream.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/io/format/ParquetSerializableCrawlDataStream.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawledDocument.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawledDocument.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawledDomain.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawledDomain.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/model/SerializableCrawlData.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/model/SerializableCrawlData.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java diff --git a/code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java b/code/process-models/crawling-model/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java rename to code/process-models/crawling-model/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java diff --git a/code/process-models/crawling-model/src/main/java/org/netpreserve/jwarc/WarcXCookieInformationHeader.java b/code/process-models/crawling-model/java/org/netpreserve/jwarc/WarcXCookieInformationHeader.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/org/netpreserve/jwarc/WarcXCookieInformationHeader.java rename to code/process-models/crawling-model/java/org/netpreserve/jwarc/WarcXCookieInformationHeader.java diff --git a/code/process-models/crawling-model/src/main/java/org/netpreserve/jwarc/WarcXEntityRefused.java b/code/process-models/crawling-model/java/org/netpreserve/jwarc/WarcXEntityRefused.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/org/netpreserve/jwarc/WarcXEntityRefused.java rename to code/process-models/crawling-model/java/org/netpreserve/jwarc/WarcXEntityRefused.java diff --git a/code/process-models/crawling-model/src/main/java/org/netpreserve/jwarc/WarcXResponseReference.java b/code/process-models/crawling-model/java/org/netpreserve/jwarc/WarcXResponseReference.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/org/netpreserve/jwarc/WarcXResponseReference.java rename to code/process-models/crawling-model/java/org/netpreserve/jwarc/WarcXResponseReference.java diff --git a/code/process-models/crawling-model/src/main/java/plan/CrawlPlan.java b/code/process-models/crawling-model/java/plan/CrawlPlan.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/plan/CrawlPlan.java rename to code/process-models/crawling-model/java/plan/CrawlPlan.java diff --git a/code/process-models/crawling-model/src/main/java/plan/CrawlPlanLoader.java b/code/process-models/crawling-model/java/plan/CrawlPlanLoader.java similarity index 100% rename from code/process-models/crawling-model/src/main/java/plan/CrawlPlanLoader.java rename to code/process-models/crawling-model/java/plan/CrawlPlanLoader.java diff --git a/code/process-models/crawling-model/src/test/java/nu/marginalia/crawling/model/CrawledDocumentTest.java b/code/process-models/crawling-model/test/nu/marginalia/crawling/model/CrawledDocumentTest.java similarity index 100% rename from code/process-models/crawling-model/src/test/java/nu/marginalia/crawling/model/CrawledDocumentTest.java rename to code/process-models/crawling-model/test/nu/marginalia/crawling/model/CrawledDocumentTest.java diff --git a/code/process-models/crawling-model/src/test/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriterTest.java b/code/process-models/crawling-model/test/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriterTest.java similarity index 100% rename from code/process-models/crawling-model/src/test/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriterTest.java rename to code/process-models/crawling-model/test/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriterTest.java diff --git a/code/process-models/processed-data/build.gradle b/code/process-models/processed-data/build.gradle index d77ebab5..910cecc6 100644 --- a/code/process-models/processed-data/build.gradle +++ b/code/process-models/processed-data/build.gradle @@ -10,6 +10,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DocumentRecordParquetFileReader.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/DocumentRecordParquetFileReader.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DocumentRecordParquetFileReader.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/DocumentRecordParquetFileReader.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DocumentRecordParquetFileWriter.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/DocumentRecordParquetFileWriter.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DocumentRecordParquetFileWriter.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/DocumentRecordParquetFileWriter.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReader.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileWriter.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileWriter.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileWriter.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileWriter.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainRecordParquetFileReader.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/DomainRecordParquetFileReader.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainRecordParquetFileReader.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/DomainRecordParquetFileReader.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainRecordParquetFileWriter.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/DomainRecordParquetFileWriter.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/DomainRecordParquetFileWriter.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/DomainRecordParquetFileWriter.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/ProcessedDataFileNames.java b/code/process-models/processed-data/java/nu/marginalia/io/processed/ProcessedDataFileNames.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/io/processed/ProcessedDataFileNames.java rename to code/process-models/processed-data/java/nu/marginalia/io/processed/ProcessedDataFileNames.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DocumentRecord.java b/code/process-models/processed-data/java/nu/marginalia/model/processed/DocumentRecord.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DocumentRecord.java rename to code/process-models/processed-data/java/nu/marginalia/model/processed/DocumentRecord.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java b/code/process-models/processed-data/java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java rename to code/process-models/processed-data/java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java b/code/process-models/processed-data/java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java rename to code/process-models/processed-data/java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DomainLinkRecord.java b/code/process-models/processed-data/java/nu/marginalia/model/processed/DomainLinkRecord.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DomainLinkRecord.java rename to code/process-models/processed-data/java/nu/marginalia/model/processed/DomainLinkRecord.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DomainRecord.java b/code/process-models/processed-data/java/nu/marginalia/model/processed/DomainRecord.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DomainRecord.java rename to code/process-models/processed-data/java/nu/marginalia/model/processed/DomainRecord.java diff --git a/code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DomainWithIp.java b/code/process-models/processed-data/java/nu/marginalia/model/processed/DomainWithIp.java similarity index 100% rename from code/process-models/processed-data/src/main/java/nu/marginalia/model/processed/DomainWithIp.java rename to code/process-models/processed-data/java/nu/marginalia/model/processed/DomainWithIp.java diff --git a/code/process-models/processed-data/src/test/java/nu/marginalia/io/processed/DocumentRecordParquetFileReaderTest.java b/code/process-models/processed-data/test/nu/marginalia/io/processed/DocumentRecordParquetFileReaderTest.java similarity index 100% rename from code/process-models/processed-data/src/test/java/nu/marginalia/io/processed/DocumentRecordParquetFileReaderTest.java rename to code/process-models/processed-data/test/nu/marginalia/io/processed/DocumentRecordParquetFileReaderTest.java diff --git a/code/process-models/processed-data/src/test/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReaderTest.java b/code/process-models/processed-data/test/nu/marginalia/io/processed/DomainLinkRecordParquetFileReaderTest.java similarity index 100% rename from code/process-models/processed-data/src/test/java/nu/marginalia/io/processed/DomainLinkRecordParquetFileReaderTest.java rename to code/process-models/processed-data/test/nu/marginalia/io/processed/DomainLinkRecordParquetFileReaderTest.java diff --git a/code/process-models/processed-data/src/test/java/nu/marginalia/io/processed/DomainRecordParquetFileReaderTest.java b/code/process-models/processed-data/test/nu/marginalia/io/processed/DomainRecordParquetFileReaderTest.java similarity index 100% rename from code/process-models/processed-data/src/test/java/nu/marginalia/io/processed/DomainRecordParquetFileReaderTest.java rename to code/process-models/processed-data/test/nu/marginalia/io/processed/DomainRecordParquetFileReaderTest.java diff --git a/code/process-models/work-log/build.gradle b/code/process-models/work-log/build.gradle index 25fbf22e..fbade272 100644 --- a/code/process-models/work-log/build.gradle +++ b/code/process-models/work-log/build.gradle @@ -10,6 +10,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.slf4j diff --git a/code/process-models/work-log/src/main/java/nu/marginalia/worklog/BatchingWorkLog.java b/code/process-models/work-log/java/nu/marginalia/worklog/BatchingWorkLog.java similarity index 100% rename from code/process-models/work-log/src/main/java/nu/marginalia/worklog/BatchingWorkLog.java rename to code/process-models/work-log/java/nu/marginalia/worklog/BatchingWorkLog.java diff --git a/code/process-models/work-log/src/main/java/nu/marginalia/worklog/BatchingWorkLogImpl.java b/code/process-models/work-log/java/nu/marginalia/worklog/BatchingWorkLogImpl.java similarity index 100% rename from code/process-models/work-log/src/main/java/nu/marginalia/worklog/BatchingWorkLogImpl.java rename to code/process-models/work-log/java/nu/marginalia/worklog/BatchingWorkLogImpl.java diff --git a/code/process-models/work-log/src/main/java/nu/marginalia/worklog/BatchingWorkLogInspector.java b/code/process-models/work-log/java/nu/marginalia/worklog/BatchingWorkLogInspector.java similarity index 100% rename from code/process-models/work-log/src/main/java/nu/marginalia/worklog/BatchingWorkLogInspector.java rename to code/process-models/work-log/java/nu/marginalia/worklog/BatchingWorkLogInspector.java diff --git a/code/process-models/work-log/src/test/java/nu/marginalia/worklog/BatchingWorkLogImplTest.java b/code/process-models/work-log/test/nu/marginalia/worklog/BatchingWorkLogImplTest.java similarity index 100% rename from code/process-models/work-log/src/test/java/nu/marginalia/worklog/BatchingWorkLogImplTest.java rename to code/process-models/work-log/test/nu/marginalia/worklog/BatchingWorkLogImplTest.java diff --git a/code/process-mqapi/build.gradle b/code/process-mqapi/build.gradle index b99fdf75..b1313c62 100644 --- a/code/process-mqapi/build.gradle +++ b/code/process-mqapi/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java b/code/process-mqapi/java/nu/marginalia/mqapi/ProcessInboxNames.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java rename to code/process-mqapi/java/nu/marginalia/mqapi/ProcessInboxNames.java diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java b/code/process-mqapi/java/nu/marginalia/mqapi/converting/ConvertAction.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java rename to code/process-mqapi/java/nu/marginalia/mqapi/converting/ConvertAction.java diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java b/code/process-mqapi/java/nu/marginalia/mqapi/converting/ConvertRequest.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java rename to code/process-mqapi/java/nu/marginalia/mqapi/converting/ConvertRequest.java diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java b/code/process-mqapi/java/nu/marginalia/mqapi/crawling/CrawlRequest.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java rename to code/process-mqapi/java/nu/marginalia/mqapi/crawling/CrawlRequest.java diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java b/code/process-mqapi/java/nu/marginalia/mqapi/index/CreateIndexRequest.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/CreateIndexRequest.java rename to code/process-mqapi/java/nu/marginalia/mqapi/index/CreateIndexRequest.java diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java b/code/process-mqapi/java/nu/marginalia/mqapi/index/IndexName.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/index/IndexName.java rename to code/process-mqapi/java/nu/marginalia/mqapi/index/IndexName.java diff --git a/code/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java b/code/process-mqapi/java/nu/marginalia/mqapi/loading/LoadRequest.java similarity index 100% rename from code/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java rename to code/process-mqapi/java/nu/marginalia/mqapi/loading/LoadRequest.java diff --git a/code/processes/converting-process/build.gradle b/code/processes/converting-process/build.gradle index f36b30d0..8c6b616e 100644 --- a/code/processes/converting-process/build.gradle +++ b/code/processes/converting-process/build.gradle @@ -18,6 +18,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:process') diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java b/code/processes/converting-process/java/nu/marginalia/converting/ConverterMain.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java rename to code/processes/converting-process/java/nu/marginalia/converting/ConverterMain.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterModule.java b/code/processes/converting-process/java/nu/marginalia/converting/ConverterModule.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterModule.java rename to code/processes/converting-process/java/nu/marginalia/converting/ConverterModule.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/DisqualifiedException.java b/code/processes/converting-process/java/nu/marginalia/converting/model/DisqualifiedException.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/model/DisqualifiedException.java rename to code/processes/converting-process/java/nu/marginalia/converting/model/DisqualifiedException.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/GeneratorType.java b/code/processes/converting-process/java/nu/marginalia/converting/model/GeneratorType.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/model/GeneratorType.java rename to code/processes/converting-process/java/nu/marginalia/converting/model/GeneratorType.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocument.java b/code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDocument.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocument.java rename to code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDocument.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java b/code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java rename to code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDocumentDetails.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDomain.java b/code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDomain.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDomain.java rename to code/processes/converting-process/java/nu/marginalia/converting/model/ProcessedDomain.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/AcceptableAds.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/AcceptableAds.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/AcceptableAds.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/AcceptableAds.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/ConverterDomainTypes.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/ConverterDomainTypes.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/ConverterDomainTypes.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/ConverterDomainTypes.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DocumentClass.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentClass.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DocumentClass.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentClass.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DocumentDecorator.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentDecorator.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DocumentDecorator.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentDecorator.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DocumentProcessor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentProcessor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DocumentProcessor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/DocumentProcessor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DomainProcessor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/DomainProcessor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/DomainProcessor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/DomainProcessor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/MetaRobotsTag.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/MetaRobotsTag.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/MetaRobotsTag.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/MetaRobotsTag.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/SiteWords.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/SiteWords.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/SiteWords.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/SiteWords.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentGeneratorExtractor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentGeneratorExtractor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentGeneratorExtractor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentGeneratorExtractor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentLengthLogic.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentLengthLogic.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentLengthLogic.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentLengthLogic.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentValuator.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentValuator.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/DocumentValuator.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/DocumentValuator.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/FeatureExtractor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/FeatureExtractor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/FeatureExtractor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/FeatureExtractor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/HtmlStandardExtractor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/LshDocumentDeduplicator.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/LshDocumentDeduplicator.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/LshDocumentDeduplicator.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/LshDocumentDeduplicator.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/PlainTextLogic.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/TitleExtractor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/TitleExtractor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/TitleExtractor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/TitleExtractor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/dom/DomPruningFilter.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/dom/DomPruningFilter.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/dom/DomPruningFilter.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/dom/DomPruningFilter.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/CommonKeywordExtractor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/CommonKeywordExtractor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/CommonKeywordExtractor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/CommonKeywordExtractor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/FileLinks.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/FileLinks.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/FileLinks.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/FileLinks.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/LinkGraph.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/LinkGraph.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/LinkGraph.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/LinkGraph.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/LinkProcessor.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/LinkProcessor.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/LinkProcessor.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/LinkProcessor.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/TopKeywords.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/TopKeywords.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/logic/links/TopKeywords.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/TopKeywords.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/DefaultSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/DefaultSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/DefaultSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/DefaultSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/MariadbKbSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/PhpBBSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java b/code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java rename to code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSource.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloadSource.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSource.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloadSource.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloaderProcessing.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloaderProcessing.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloaderProcessing.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/SideloaderProcessing.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpec.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpec.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpec.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpec.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpecList.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpecList.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpecList.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloadSpecList.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloader.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloader.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloader.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloader.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloaderFactory.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloaderFactory.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloaderFactory.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/dirtree/DirtreeSideloaderFactory.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/reddit/RedditSideloader.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/stackexchange/StackexchangeSideloader.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/warc/WarcSideloader.java b/code/processes/converting-process/java/nu/marginalia/converting/sideload/warc/WarcSideloader.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/warc/WarcSideloader.java rename to code/processes/converting-process/java/nu/marginalia/converting/sideload/warc/WarcSideloader.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/util/LineUtils.java b/code/processes/converting-process/java/nu/marginalia/converting/util/LineUtils.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/util/LineUtils.java rename to code/processes/converting-process/java/nu/marginalia/converting/util/LineUtils.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterBatchWritableIf.java b/code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWritableIf.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterBatchWritableIf.java rename to code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWritableIf.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterBatchWriter.java b/code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWriter.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterBatchWriter.java rename to code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWriter.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterBatchWriterIf.java b/code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWriterIf.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterBatchWriterIf.java rename to code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterBatchWriterIf.java diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterWriter.java b/code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterWriter.java similarity index 100% rename from code/processes/converting-process/src/main/java/nu/marginalia/converting/writer/ConverterWriter.java rename to code/processes/converting-process/java/nu/marginalia/converting/writer/ConverterWriter.java diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/05-test.gmi b/code/processes/converting-process/test-resources/memex-marginalia/05-test.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/05-test.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/05-test.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/code/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/code/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/code/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/code/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/commons/dialogue.gmi b/code/processes/converting-process/test-resources/memex-marginalia/commons/dialogue.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/commons/dialogue.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/commons/dialogue.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/commons/search-failure-modes.gmi b/code/processes/converting-process/test-resources/memex-marginalia/commons/search-failure-modes.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/commons/search-failure-modes.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/commons/search-failure-modes.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/commons/self-interest.gmi b/code/processes/converting-process/test-resources/memex-marginalia/commons/self-interest.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/commons/self-interest.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/commons/self-interest.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/dead.gmi b/code/processes/converting-process/test-resources/memex-marginalia/dead.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/dead.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/dead.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/debian-laptop-install-log.gmi b/code/processes/converting-process/test-resources/memex-marginalia/debian-laptop-install-log.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/debian-laptop-install-log.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/debian-laptop-install-log.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/debian-laptop.gmi b/code/processes/converting-process/test-resources/memex-marginalia/debian-laptop.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/debian-laptop.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/debian-laptop.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/feed.gmi b/code/processes/converting-process/test-resources/memex-marginalia/feed.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/feed.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/feed.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/fragments-old-web.gmi b/code/processes/converting-process/test-resources/memex-marginalia/fragments-old-web.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/fragments-old-web.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/fragments-old-web.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/good-video-games.gmi b/code/processes/converting-process/test-resources/memex-marginalia/good-video-games.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/good-video-games.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/good-video-games.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/guestbook/footer.gmi b/code/processes/converting-process/test-resources/memex-marginalia/guestbook/footer.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/guestbook/footer.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/guestbook/footer.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/guestbook/header.gmi b/code/processes/converting-process/test-resources/memex-marginalia/guestbook/header.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/guestbook/header.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/guestbook/header.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/guestbook/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/guestbook/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/guestbook/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/guestbook/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/index b/code/processes/converting-process/test-resources/memex-marginalia/index similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/index rename to code/processes/converting-process/test-resources/memex-marginalia/index diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/junk/DedupTest.gmi b/code/processes/converting-process/test-resources/memex-marginalia/junk/DedupTest.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/junk/DedupTest.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/junk/DedupTest.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/junk/bssl.gmi b/code/processes/converting-process/test-resources/memex-marginalia/junk/bssl.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/junk/bssl.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/junk/bssl.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/junk/eldritch-oneliner.gmi b/code/processes/converting-process/test-resources/memex-marginalia/junk/eldritch-oneliner.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/junk/eldritch-oneliner.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/junk/eldritch-oneliner.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/junk/server.gmi b/code/processes/converting-process/test-resources/memex-marginalia/junk/server.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/junk/server.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/junk/server.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/junk/very-old-code.cc.gmi b/code/processes/converting-process/test-resources/memex-marginalia/junk/very-old-code.cc.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/junk/very-old-code.cc.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/junk/very-old-code.cc.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/links/aggregators.gmi b/code/processes/converting-process/test-resources/memex-marginalia/links/aggregators.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/links/aggregators.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/links/aggregators.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/links/articles.gmi b/code/processes/converting-process/test-resources/memex-marginalia/links/articles.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/links/articles.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/links/articles.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/links/bookmarks.gmi b/code/processes/converting-process/test-resources/memex-marginalia/links/bookmarks.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/links/bookmarks.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/links/bookmarks.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/links/fragments-old-web.gmi b/code/processes/converting-process/test-resources/memex-marginalia/links/fragments-old-web.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/links/fragments-old-web.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/links/fragments-old-web.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/links/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/links/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/links/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/links/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/links/linkpocalypse.gmi b/code/processes/converting-process/test-resources/memex-marginalia/links/linkpocalypse.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/links/linkpocalypse.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/links/linkpocalypse.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/00-linkpocalypse.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/00-linkpocalypse.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/00-linkpocalypse.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/00-linkpocalypse.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/01-astrolabe.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/01-astrolabe.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/01-astrolabe.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/01-astrolabe.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/02-re-tests.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/02-re-tests.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/02-re-tests.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/02-re-tests.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/03-writing-for-reading.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/03-writing-for-reading.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/03-writing-for-reading.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/03-writing-for-reading.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/04-link-farms.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/04-link-farms.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/04-link-farms.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/04-link-farms.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/05-minds-field.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/05-minds-field.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/05-minds-field.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/05-minds-field.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/06-optimization.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/06-optimization.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/06-optimization.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/06-optimization.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/07-local-backlinks.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/07-local-backlinks.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/07-local-backlinks.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/07-local-backlinks.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/08-whatever-happened-to-the-memex.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/08-whatever-happened-to-the-memex.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/08-whatever-happened-to-the-memex.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/08-whatever-happened-to-the-memex.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/09-system-upgrade.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/09-system-upgrade.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/09-system-upgrade.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/09-system-upgrade.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/10-astrolabe-2-sampling-bias.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/10-astrolabe-2-sampling-bias.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/10-astrolabe-2-sampling-bias.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/10-astrolabe-2-sampling-bias.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/11-dying-every-day.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/11-dying-every-day.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/11-dying-every-day.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/11-dying-every-day.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/12-bye-bye-gmail.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/12-bye-bye-gmail.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/12-bye-bye-gmail.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/12-bye-bye-gmail.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/13-static-html.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/13-static-html.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/13-static-html.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/13-static-html.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/13-test.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/13-test.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/13-test.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/13-test.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/14-enter-the-circle-of-blame.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/14-enter-the-circle-of-blame.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/14-enter-the-circle-of-blame.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/14-enter-the-circle-of-blame.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/15-stages-of-being.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/15-stages-of-being.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/15-stages-of-being.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/15-stages-of-being.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/16-cursed-motivation.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/16-cursed-motivation.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/16-cursed-motivation.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/16-cursed-motivation.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/17-git-isnt-a-web-service.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/17-git-isnt-a-web-service.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/17-git-isnt-a-web-service.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/17-git-isnt-a-web-service.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/18-soaring-high.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/18-soaring-high.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/18-soaring-high.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/18-soaring-high.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/19-website-discoverability-crisis.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/19-website-discoverability-crisis.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/19-website-discoverability-crisis.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/19-website-discoverability-crisis.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/20-dot-com-link-farms.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/20-dot-com-link-farms.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/20-dot-com-link-farms.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/20-dot-com-link-farms.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/21-new-solutions-old-problems.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/21-new-solutions-old-problems.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/21-new-solutions-old-problems.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/21-new-solutions-old-problems.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/22-against-the-flood.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/22-against-the-flood.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/22-against-the-flood.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/22-against-the-flood.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/23-re-software-and-branding.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/23-re-software-and-branding.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/23-re-software-and-branding.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/23-re-software-and-branding.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/24-silly-hats.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/24-silly-hats.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/24-silly-hats.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/24-silly-hats.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/25-october-update.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/25-october-update.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/25-october-update.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/25-october-update.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/26-personalized-pagerank.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/26-personalized-pagerank.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/26-personalized-pagerank.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/26-personalized-pagerank.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/27-getting-with-the-times.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/27-getting-with-the-times.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/27-getting-with-the-times.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/27-getting-with-the-times.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/28-web-browsing.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/28-web-browsing.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/28-web-browsing.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/28-web-browsing.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/29-botnet-ddos.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/29-botnet-ddos.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/29-botnet-ddos.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/29-botnet-ddos.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/30-unintuitive-optimization.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/30-unintuitive-optimization.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/30-unintuitive-optimization.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/30-unintuitive-optimization.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/31-ngram-needles.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/31-ngram-needles.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/31-ngram-needles.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/31-ngram-needles.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/32-bot-apologetics.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/32-bot-apologetics.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/32-bot-apologetics.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/32-bot-apologetics.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/33-rude-guests.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/33-rude-guests.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/33-rude-guests.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/33-rude-guests.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/34-internet-arguments.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/34-internet-arguments.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/34-internet-arguments.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/34-internet-arguments.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/35-keeping-gemini-difficult.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/35-keeping-gemini-difficult.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/35-keeping-gemini-difficult.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/35-keeping-gemini-difficult.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/36-localized-programming-languages.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/36-localized-programming-languages.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/36-localized-programming-languages.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/36-localized-programming-languages.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/37-keyword-extraction.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/37-keyword-extraction.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/37-keyword-extraction.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/37-keyword-extraction.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/38-old-and-new.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/38-old-and-new.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/38-old-and-new.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/38-old-and-new.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/39-normie-hypothesis.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/39-normie-hypothesis.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/39-normie-hypothesis.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/39-normie-hypothesis.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/40-wasted-resources.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/40-wasted-resources.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/40-wasted-resources.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/40-wasted-resources.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/41-search-result-relevance.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/41-search-result-relevance.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/41-search-result-relevance.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/41-search-result-relevance.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/42-dark.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/42-dark.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/42-dark.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/42-dark.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/43-pseodonymous.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/43-pseodonymous.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/43-pseodonymous.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/43-pseodonymous.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/44-discovery-and-design.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/44-discovery-and-design.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/44-discovery-and-design.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/44-discovery-and-design.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/45-unfuck-internet-discoverability.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/45-unfuck-internet-discoverability.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/45-unfuck-internet-discoverability.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/45-unfuck-internet-discoverability.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/46-anatomy-of-search-engine-spam.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/46-anatomy-of-search-engine-spam.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/46-anatomy-of-search-engine-spam.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/46-anatomy-of-search-engine-spam.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/47-drive-failure.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/47-drive-failure.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/47-drive-failure.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/47-drive-failure.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/48-i-have-no-capslock.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/48-i-have-no-capslock.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/48-i-have-no-capslock.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/48-i-have-no-capslock.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/49-marginalia-1-year.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/49-marginalia-1-year.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/49-marginalia-1-year.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/49-marginalia-1-year.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/50-meditation-on-software-correctness.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/50-meditation-on-software-correctness.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/50-meditation-on-software-correctness.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/50-meditation-on-software-correctness.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/51-the-file-startup.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/51-the-file-startup.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/51-the-file-startup.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/51-the-file-startup.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/52-growing-pains.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/52-growing-pains.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/52-growing-pains.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/52-growing-pains.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/53-better-hard-drive-metaphor.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/53-better-hard-drive-metaphor.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/53-better-hard-drive-metaphor.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/53-better-hard-drive-metaphor.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/54-bargain-bin-btree.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/54-bargain-bin-btree.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/54-bargain-bin-btree.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/54-bargain-bin-btree.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/55-lexicon-rubberduck.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/55-lexicon-rubberduck.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/55-lexicon-rubberduck.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/55-lexicon-rubberduck.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/56-uncertain-future.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/56-uncertain-future.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/56-uncertain-future.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/56-uncertain-future.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/57-dont-know-how-to-build-software.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/57-dont-know-how-to-build-software.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/57-dont-know-how-to-build-software.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/57-dont-know-how-to-build-software.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/58-marginalia-open-source.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/58-marginalia-open-source.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/58-marginalia-open-source.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/58-marginalia-open-source.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/59-anchor-text.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/59-anchor-text.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/59-anchor-text.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/59-anchor-text.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/60-prescriptive-descriptions.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/60-prescriptive-descriptions.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/60-prescriptive-descriptions.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/60-prescriptive-descriptions.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/61-botspam-apocalypse.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/61-botspam-apocalypse.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/61-botspam-apocalypse.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/61-botspam-apocalypse.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/62-marginaliacoin.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/62-marginaliacoin.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/62-marginaliacoin.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/62-marginaliacoin.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/63-marginalia-crawler.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/63-marginalia-crawler.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/63-marginalia-crawler.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/63-marginalia-crawler.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/64-hundred-million.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/64-hundred-million.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/64-hundred-million.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/64-hundred-million.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/65-scaling-doesnt-scale.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/65-scaling-doesnt-scale.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/65-scaling-doesnt-scale.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/65-scaling-doesnt-scale.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/66-carbon-dating.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/66-carbon-dating.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/66-carbon-dating.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/66-carbon-dating.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/67-best-ideas-afk.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/67-best-ideas-afk.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/67-best-ideas-afk.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/67-best-ideas-afk.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/68-wizards-vs-sorcerers.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/68-wizards-vs-sorcerers.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/68-wizards-vs-sorcerers.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/68-wizards-vs-sorcerers.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/69-creepy-website-similarity.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/69-creepy-website-similarity.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/69-creepy-website-similarity.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/69-creepy-website-similarity.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/70-faster-index-joins.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/70-faster-index-joins.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/70-faster-index-joins.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/70-faster-index-joins.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/71-memex-design.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/71-memex-design.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/71-memex-design.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/71-memex-design.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/72-are-you-ok.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/72-are-you-ok.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/72-are-you-ok.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/72-are-you-ok.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/72-new-approach-to-ranking.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/72-new-approach-to-ranking.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/72-new-approach-to-ranking.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/72-new-approach-to-ranking.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/73-new-approach-to-ranking.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/73-new-approach-to-ranking.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/73-new-approach-to-ranking.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/73-new-approach-to-ranking.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/74-marginalia-2-years.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/74-marginalia-2-years.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/74-marginalia-2-years.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/74-marginalia-2-years.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/bargain-bin-btree.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/bargain-bin-btree.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/bargain-bin-btree.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/bargain-bin-btree.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/soaring-high.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/soaring-high.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/soaring-high.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/soaring-high.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/log/todo.gmi b/code/processes/converting-process/test-resources/memex-marginalia/log/todo.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/log/todo.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/log/todo.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/one-weird-trick.gmi b/code/processes/converting-process/test-resources/memex-marginalia/one-weird-trick.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/one-weird-trick.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/one-weird-trick.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/pics/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/pics/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/pics/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/pics/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/pics/links/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/pics/links/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/pics/links/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/pics/links/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/pics/raster-test/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/pics/raster-test/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/pics/raster-test/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/pics/raster-test/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/about.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/about.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/about.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/about.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/api.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/api.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/api.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/api.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/changelog.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/changelog.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/changelog.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/changelog.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/design-notes.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/design-notes.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/design-notes.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/design-notes.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/faq.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/faq.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/faq.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/faq.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/for-webmasters.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/for-webmasters.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/for-webmasters.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/for-webmasters.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/privacy.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/privacy.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/privacy.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/privacy.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/search-tips.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/search-tips.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/search-tips.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/search-tips.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/supporting.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/supporting.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/supporting.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/supporting.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/top-20.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/edge/top-20.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/edge/top-20.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/edge/top-20.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/encyclopedia/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/encyclopedia/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/encyclopedia/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/encyclopedia/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/gemini-server.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/gemini-server.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/gemini-server.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/gemini-server.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/memex.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/memex.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/memex.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/memex.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/projects/wmsa.gmi b/code/processes/converting-process/test-resources/memex-marginalia/projects/wmsa.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/projects/wmsa.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/projects/wmsa.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/readme.md b/code/processes/converting-process/test-resources/memex-marginalia/readme.md similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/readme.md rename to code/processes/converting-process/test-resources/memex-marginalia/readme.md diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/recipes/chicken-soup.gmi b/code/processes/converting-process/test-resources/memex-marginalia/recipes/chicken-soup.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/recipes/chicken-soup.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/recipes/chicken-soup.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/recipes/french-borscht.gmi b/code/processes/converting-process/test-resources/memex-marginalia/recipes/french-borscht.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/recipes/french-borscht.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/recipes/french-borscht.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/recipes/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/recipes/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/recipes/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/recipes/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/recipes/omelette-bacon.gmi b/code/processes/converting-process/test-resources/memex-marginalia/recipes/omelette-bacon.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/recipes/omelette-bacon.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/recipes/omelette-bacon.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/search-about.gmi b/code/processes/converting-process/test-resources/memex-marginalia/search-about.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/search-about.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/search-about.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/server.gmi b/code/processes/converting-process/test-resources/memex-marginalia/server.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/server.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/server.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/special/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/special/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/special/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/special/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/special/redirect.gmi b/code/processes/converting-process/test-resources/memex-marginalia/special/redirect.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/special/redirect.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/special/redirect.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/special/tombstone.gmi b/code/processes/converting-process/test-resources/memex-marginalia/special/tombstone.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/special/tombstone.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/special/tombstone.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/test.gmi b/code/processes/converting-process/test-resources/memex-marginalia/test.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/test.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/test.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/todo.gmi b/code/processes/converting-process/test-resources/memex-marginalia/todo.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/todo.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/todo.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/todo/done.gmi b/code/processes/converting-process/test-resources/memex-marginalia/todo/done.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/todo/done.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/todo/done.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/todo/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/todo/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/todo/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/todo/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/todo/todo.gmi b/code/processes/converting-process/test-resources/memex-marginalia/todo/todo.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/todo/todo.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/todo/todo.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/astrolabe.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/astrolabe.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/astrolabe.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/astrolabe.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/cooking.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/cooking.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/cooking.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/cooking.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/index.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/index.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/index.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/index.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/moral-philosophy.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/moral-philosophy.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/moral-philosophy.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/moral-philosophy.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/nlnet.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/nlnet.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/nlnet.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/nlnet.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/platforms.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/platforms.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/platforms.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/platforms.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/programming.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/programming.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/programming.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/programming.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/satire.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/satire.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/satire.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/satire.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/server.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/server.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/server.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/server.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topic/web-design.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topic/web-design.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topic/web-design.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topic/web-design.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/topics.gmi b/code/processes/converting-process/test-resources/memex-marginalia/topics.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/topics.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/topics.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/worklog.gmi b/code/processes/converting-process/test-resources/memex-marginalia/worklog.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/worklog.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/worklog.gmi diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/writing-topics.gmi b/code/processes/converting-process/test-resources/memex-marginalia/writing-topics.gmi similarity index 100% rename from code/processes/converting-process/src/test/resources/memex-marginalia/writing-topics.gmi rename to code/processes/converting-process/test-resources/memex-marginalia/writing-topics.gmi diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/ConvertingIntegrationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/ConvertingIntegrationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTestModule.java b/code/processes/converting-process/test/nu/marginalia/converting/ConvertingIntegrationTestModule.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTestModule.java rename to code/processes/converting-process/test/nu/marginalia/converting/ConvertingIntegrationTestModule.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/CrawlingThenConvertingIntegrationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/CrawlingThenConvertingIntegrationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/CrawlingThenConvertingIntegrationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/CrawlingThenConvertingIntegrationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/logic/PlainTextLogicTest.java b/code/processes/converting-process/test/nu/marginalia/converting/logic/PlainTextLogicTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/logic/PlainTextLogicTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/logic/PlainTextLogicTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/logic/MetaRobotsTagTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/logic/MetaRobotsTagTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/logic/MetaRobotsTagTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/logic/MetaRobotsTagTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitorTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitorTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitorTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/logic/dom/MeasureLengthVisitorTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecializationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/BlogSpecializationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/BlogSpecializationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/BlogSpecializationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecializationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecializationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecializationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/JavadocSpecializationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecializationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/LemmySpecializationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/LemmySpecializationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/LemmySpecializationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/WikiSpecializationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecializationTest.java b/code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecializationTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecializationTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecializationTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloaderTest.java b/code/processes/converting-process/test/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloaderTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloaderTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloaderTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/sideload/reddit/RedditSideloaderTest.java b/code/processes/converting-process/test/nu/marginalia/converting/sideload/reddit/RedditSideloaderTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/sideload/reddit/RedditSideloaderTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/sideload/reddit/RedditSideloaderTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/sideload/warc/WarcSideloaderTest.java b/code/processes/converting-process/test/nu/marginalia/converting/sideload/warc/WarcSideloaderTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/sideload/warc/WarcSideloaderTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/sideload/warc/WarcSideloaderTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/util/LineUtilsTest.java b/code/processes/converting-process/test/nu/marginalia/converting/util/LineUtilsTest.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/util/LineUtilsTest.java rename to code/processes/converting-process/test/nu/marginalia/converting/util/LineUtilsTest.java diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/util/TestLanguageModels.java b/code/processes/converting-process/test/nu/marginalia/converting/util/TestLanguageModels.java similarity index 100% rename from code/processes/converting-process/src/test/java/nu/marginalia/converting/util/TestLanguageModels.java rename to code/processes/converting-process/test/nu/marginalia/converting/util/TestLanguageModels.java diff --git a/code/processes/crawling-process/build.gradle b/code/processes/crawling-process/build.gradle index ed53c6a0..94a60b11 100644 --- a/code/processes/crawling-process/build.gradle +++ b/code/processes/crawling-process/build.gradle @@ -18,6 +18,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:process') diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/AbortMonitor.java b/code/processes/crawling-process/java/nu/marginalia/crawl/AbortMonitor.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/AbortMonitor.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/AbortMonitor.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java b/code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerModule.java b/code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerModule.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerModule.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerModule.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/Cookies.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/Cookies.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/Cookies.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/Cookies.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlDataReference.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDataReference.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlDataReference.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDataReference.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlDelayTimer.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawledDocumentFactory.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawledDocumentFactory.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawledDocumentFactory.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawledDocumentFactory.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizer.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainCrawlFrontier.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/DomainCrawlFrontier.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainCrawlFrontier.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/DomainCrawlFrontier.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainProber.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/DomainProber.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/DomainProber.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/DomainProber.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/LinkFilterSelector.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/LinkFilterSelector.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/LinkFilterSelector.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/LinkFilterSelector.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/RateLimitException.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/RateLimitException.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/RateLimitException.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/RateLimitException.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/ContentTags.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/ContentTags.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/ContentTags.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/ContentTags.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProber.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProber.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProber.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProber.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/FetchResult.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/FetchResult.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/FetchResult.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/FetchResult.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/FetchResultState.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/FetchResultState.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/FetchResultState.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/FetchResultState.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcher.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcher.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcher.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcher.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/SitemapRetriever.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/SitemapRetriever.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/SitemapRetriever.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/SitemapRetriever.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/socket/FastTerminatingSocketFactory.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/socket/FastTerminatingSocketFactory.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/socket/FastTerminatingSocketFactory.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/socket/FastTerminatingSocketFactory.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/socket/IpInterceptingNetworkInterceptor.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/socket/IpInterceptingNetworkInterceptor.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/socket/IpInterceptingNetworkInterceptor.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/socket/IpInterceptingNetworkInterceptor.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/socket/NoSecuritySSL.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/socket/NoSecuritySSL.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/socket/NoSecuritySSL.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/socket/NoSecuritySSL.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcDigestBuilder.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcDigestBuilder.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcDigestBuilder.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcDigestBuilder.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcInputBuffer.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcInputBuffer.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcInputBuffer.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcInputBuffer.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcProtocolReconstructor.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcProtocolReconstructor.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcProtocolReconstructor.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcProtocolReconstructor.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcRecorder.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcRecorder.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcRecorder.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/warc/WarcRecorder.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/CrawlerRevisitor.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReference.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/sitemap/SitemapFetcher.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/sitemap/SitemapFetcher.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/sitemap/SitemapFetcher.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/retreival/sitemap/SitemapFetcher.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/spec/CrawlSpecProvider.java b/code/processes/crawling-process/java/nu/marginalia/crawl/spec/CrawlSpecProvider.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/spec/CrawlSpecProvider.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/spec/CrawlSpecProvider.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/spec/DbCrawlSpecProvider.java b/code/processes/crawling-process/java/nu/marginalia/crawl/spec/DbCrawlSpecProvider.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/spec/DbCrawlSpecProvider.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/spec/DbCrawlSpecProvider.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/spec/ParquetCrawlSpecProvider.java b/code/processes/crawling-process/java/nu/marginalia/crawl/spec/ParquetCrawlSpecProvider.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/spec/ParquetCrawlSpecProvider.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/spec/ParquetCrawlSpecProvider.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/warc/WarcArchiverFactory.java b/code/processes/crawling-process/java/nu/marginalia/crawl/warc/WarcArchiverFactory.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/warc/WarcArchiverFactory.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/warc/WarcArchiverFactory.java diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/warc/WarcArchiverIf.java b/code/processes/crawling-process/java/nu/marginalia/crawl/warc/WarcArchiverIf.java similarity index 100% rename from code/processes/crawling-process/src/main/java/nu/marginalia/crawl/warc/WarcArchiverIf.java rename to code/processes/crawling-process/java/nu/marginalia/crawl/warc/WarcArchiverIf.java diff --git a/code/processes/crawling-process/src/main/resources/ip-banned-cidr.txt b/code/processes/crawling-process/resources/ip-banned-cidr.txt similarity index 100% rename from code/processes/crawling-process/src/main/resources/ip-banned-cidr.txt rename to code/processes/crawling-process/resources/ip-banned-cidr.txt diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java b/code/processes/crawling-process/test/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawl/retreival/CrawlerWarcResynchronizerTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/DomainCrawlFrontierTest.java b/code/processes/crawling-process/test/nu/marginalia/crawl/retreival/DomainCrawlFrontierTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/DomainCrawlFrontierTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawl/retreival/DomainCrawlFrontierTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java b/code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/ContentTypeProberTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/fetcher/CrawledDocumentParquetRecordFileWriterTest.java b/code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/CrawledDocumentParquetRecordFileWriterTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/fetcher/CrawledDocumentParquetRecordFileWriterTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/CrawledDocumentParquetRecordFileWriterTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java b/code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawl/retreival/fetcher/WarcRecorderTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReferenceTest.java b/code/processes/crawling-process/test/nu/marginalia/crawl/retreival/revisit/DocumentWithReferenceTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawl/retreival/revisit/DocumentWithReferenceTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawl/retreival/revisit/DocumentWithReferenceTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/CrawlPlanLoaderTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/CrawlPlanLoaderTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/CrawlPlanLoaderTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/CrawlPlanLoaderTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/DomainCrawlerRobotsTxtTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/DomainCrawlerRobotsTxtTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/DomainCrawlerRobotsTxtTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/DomainCrawlerRobotsTxtTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/HttpFetcherTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/HttpFetcherTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/HttpFetcherTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/HttpFetcherTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/LinkParserTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/LinkParserTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/LinkParserTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/LinkParserTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/RssCrawlerTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/RssCrawlerTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/RssCrawlerTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/RssCrawlerTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/retreival/CrawlerMockFetcherTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerMockFetcherTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/retreival/CrawlerMockFetcherTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerMockFetcherTest.java diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java b/code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java similarity index 100% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java rename to code/processes/crawling-process/test/nu/marginalia/crawling/retreival/CrawlerRetreiverTest.java diff --git a/code/processes/index-constructor-process/build.gradle b/code/processes/index-constructor-process/build.gradle index f7361358..5e48deea 100644 --- a/code/processes/index-constructor-process/build.gradle +++ b/code/processes/index-constructor-process/build.gradle @@ -18,6 +18,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:process-mqapi') implementation project(':code:common:process') @@ -28,8 +30,8 @@ dependencies { implementation project(':code:libraries:message-queue') implementation project(':code:index:query') - implementation project(':code:index:forward-index') - implementation project(':code:index:reverse-index') + implementation project(':code:index:index-forward') + implementation project(':code:index:index-reverse') implementation project(':code:index:index-journal') implementation libs.bundles.slf4j diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java b/code/processes/index-constructor-process/java/nu/marginalia/index/IndexConstructorMain.java similarity index 100% rename from code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java rename to code/processes/index-constructor-process/java/nu/marginalia/index/IndexConstructorMain.java diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorModule.java b/code/processes/index-constructor-process/java/nu/marginalia/index/IndexConstructorModule.java similarity index 100% rename from code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorModule.java rename to code/processes/index-constructor-process/java/nu/marginalia/index/IndexConstructorModule.java diff --git a/code/processes/loading-process/build.gradle b/code/processes/loading-process/build.gradle index 66db9cb9..cf3ef16b 100644 --- a/code/processes/loading-process/build.gradle +++ b/code/processes/loading-process/build.gradle @@ -17,6 +17,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:process') implementation project(':code:process-mqapi') diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderIndexJournalWriter.java b/code/processes/loading-process/java/nu/marginalia/loading/LoaderIndexJournalWriter.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderIndexJournalWriter.java rename to code/processes/loading-process/java/nu/marginalia/loading/LoaderIndexJournalWriter.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderInputData.java b/code/processes/loading-process/java/nu/marginalia/loading/LoaderInputData.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderInputData.java rename to code/processes/loading-process/java/nu/marginalia/loading/LoaderInputData.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java b/code/processes/loading-process/java/nu/marginalia/loading/LoaderMain.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java rename to code/processes/loading-process/java/nu/marginalia/loading/LoaderMain.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java b/code/processes/loading-process/java/nu/marginalia/loading/LoaderModule.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java rename to code/processes/loading-process/java/nu/marginalia/loading/LoaderModule.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/documents/DocumentLoaderService.java b/code/processes/loading-process/java/nu/marginalia/loading/documents/DocumentLoaderService.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/documents/DocumentLoaderService.java rename to code/processes/loading-process/java/nu/marginalia/loading/documents/DocumentLoaderService.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/documents/KeywordLoaderService.java b/code/processes/loading-process/java/nu/marginalia/loading/documents/KeywordLoaderService.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/documents/KeywordLoaderService.java rename to code/processes/loading-process/java/nu/marginalia/loading/documents/KeywordLoaderService.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainIdRegistry.java b/code/processes/loading-process/java/nu/marginalia/loading/domains/DomainIdRegistry.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainIdRegistry.java rename to code/processes/loading-process/java/nu/marginalia/loading/domains/DomainIdRegistry.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java b/code/processes/loading-process/java/nu/marginalia/loading/domains/DomainLoaderService.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java rename to code/processes/loading-process/java/nu/marginalia/loading/domains/DomainLoaderService.java diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/links/DomainLinksLoaderService.java b/code/processes/loading-process/java/nu/marginalia/loading/links/DomainLinksLoaderService.java similarity index 100% rename from code/processes/loading-process/src/main/java/nu/marginalia/loading/links/DomainLinksLoaderService.java rename to code/processes/loading-process/java/nu/marginalia/loading/links/DomainLinksLoaderService.java diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loader/DbTestUtil.java b/code/processes/loading-process/test/nu/marginalia/loader/DbTestUtil.java similarity index 100% rename from code/processes/loading-process/src/test/java/nu/marginalia/loader/DbTestUtil.java rename to code/processes/loading-process/test/nu/marginalia/loader/DbTestUtil.java diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java b/code/processes/loading-process/test/nu/marginalia/loading/domains/DomainLoaderServiceTest.java similarity index 100% rename from code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java rename to code/processes/loading-process/test/nu/marginalia/loading/domains/DomainLoaderServiceTest.java diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java b/code/processes/loading-process/test/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java similarity index 100% rename from code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java rename to code/processes/loading-process/test/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java diff --git a/code/processes/test-data/build.gradle b/code/processes/test-data/build.gradle index 462be8c3..4f184c80 100644 --- a/code/processes/test-data/build.gradle +++ b/code/processes/test-data/build.gradle @@ -10,5 +10,7 @@ java { } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { } diff --git a/code/processes/test-data/src/main/java/nu/marginalia/test/CommonTestData.java b/code/processes/test-data/java/nu/marginalia/test/CommonTestData.java similarity index 100% rename from code/processes/test-data/src/main/java/nu/marginalia/test/CommonTestData.java rename to code/processes/test-data/java/nu/marginalia/test/CommonTestData.java diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/discourse/grid.html b/code/processes/test-data/resources/mock-crawl-data/discourse/grid.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/discourse/grid.html rename to code/processes/test-data/resources/mock-crawl-data/discourse/grid.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/discourse/index.html b/code/processes/test-data/resources/mock-crawl-data/discourse/index.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/discourse/index.html rename to code/processes/test-data/resources/mock-crawl-data/discourse/index.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/discourse/telegram.html b/code/processes/test-data/resources/mock-crawl-data/discourse/telegram.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/discourse/telegram.html rename to code/processes/test-data/resources/mock-crawl-data/discourse/telegram.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/javadoc/stream.html b/code/processes/test-data/resources/mock-crawl-data/javadoc/stream.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/javadoc/stream.html rename to code/processes/test-data/resources/mock-crawl-data/javadoc/stream.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/lemmy/108995.html b/code/processes/test-data/resources/mock-crawl-data/lemmy/108995.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/lemmy/108995.html rename to code/processes/test-data/resources/mock-crawl-data/lemmy/108995.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/lemmy/c_startrek.html b/code/processes/test-data/resources/mock-crawl-data/lemmy/c_startrek.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/lemmy/c_startrek.html rename to code/processes/test-data/resources/mock-crawl-data/lemmy/c_startrek.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/lemmy/index.html b/code/processes/test-data/resources/mock-crawl-data/lemmy/index.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/lemmy/index.html rename to code/processes/test-data/resources/mock-crawl-data/lemmy/index.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/mediawiki/doom1.html b/code/processes/test-data/resources/mock-crawl-data/mediawiki/doom1.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/mediawiki/doom1.html rename to code/processes/test-data/resources/mock-crawl-data/mediawiki/doom1.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/mediawiki/doom2.html b/code/processes/test-data/resources/mock-crawl-data/mediawiki/doom2.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/mediawiki/doom2.html rename to code/processes/test-data/resources/mock-crawl-data/mediawiki/doom2.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/mediawiki/index.html b/code/processes/test-data/resources/mock-crawl-data/mediawiki/index.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/mediawiki/index.html rename to code/processes/test-data/resources/mock-crawl-data/mediawiki/index.html diff --git a/code/processes/test-data/src/main/resources/mock-crawl-data/xenforo/thread.html b/code/processes/test-data/resources/mock-crawl-data/xenforo/thread.html similarity index 100% rename from code/processes/test-data/src/main/resources/mock-crawl-data/xenforo/thread.html rename to code/processes/test-data/resources/mock-crawl-data/xenforo/thread.html diff --git a/code/processes/website-adjacencies-calculator/build.gradle b/code/processes/website-adjacencies-calculator/build.gradle index d262934e..7c4b9623 100644 --- a/code/processes/website-adjacencies-calculator/build.gradle +++ b/code/processes/website-adjacencies-calculator/build.gradle @@ -17,6 +17,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:db') diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesData.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java rename to code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesData.java diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesLoader.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java rename to code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesLoader.java diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/DomainAliases.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java rename to code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/DomainAliases.java diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/SparseBitVector.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java rename to code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/SparseBitVector.java diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java rename to code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java diff --git a/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java b/code/processes/website-adjacencies-calculator/test/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java rename to code/processes/website-adjacencies-calculator/test/nu/marginalia/adjacencies/AdjacenciesLoaderTest.java diff --git a/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java b/code/processes/website-adjacencies-calculator/test/nu/marginalia/adjacencies/SparseBitVectorTest.java similarity index 100% rename from code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java rename to code/processes/website-adjacencies-calculator/test/nu/marginalia/adjacencies/SparseBitVectorTest.java diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index 24293012..b7b29038 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -35,6 +35,8 @@ jib { } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:db') implementation project(':code:common:model') diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java b/code/services-application/api-service/java/nu/marginalia/api/ApiMain.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java rename to code/services-application/api-service/java/nu/marginalia/api/ApiMain.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java b/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java rename to code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java b/code/services-application/api-service/java/nu/marginalia/api/ApiService.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java rename to code/services-application/api-service/java/nu/marginalia/api/ApiService.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiLicense.java b/code/services-application/api-service/java/nu/marginalia/api/model/ApiLicense.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiLicense.java rename to code/services-application/api-service/java/nu/marginalia/api/model/ApiLicense.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResult.java b/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResult.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResult.java rename to code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResult.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResultQueryDetails.java b/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResultQueryDetails.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResultQueryDetails.java rename to code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResultQueryDetails.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResults.java b/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResults.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResults.java rename to code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResults.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/svc/LicenseService.java b/code/services-application/api-service/java/nu/marginalia/api/svc/LicenseService.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/svc/LicenseService.java rename to code/services-application/api-service/java/nu/marginalia/api/svc/LicenseService.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/svc/RateLimiterService.java b/code/services-application/api-service/java/nu/marginalia/api/svc/RateLimiterService.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/svc/RateLimiterService.java rename to code/services-application/api-service/java/nu/marginalia/api/svc/RateLimiterService.java diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/svc/ResponseCache.java b/code/services-application/api-service/java/nu/marginalia/api/svc/ResponseCache.java similarity index 100% rename from code/services-application/api-service/src/main/java/nu/marginalia/api/svc/ResponseCache.java rename to code/services-application/api-service/java/nu/marginalia/api/svc/ResponseCache.java diff --git a/code/services-application/api-service/src/test/java/nu/marginalia/api/svc/LicenseServiceTest.java b/code/services-application/api-service/test/nu/marginalia/api/svc/LicenseServiceTest.java similarity index 100% rename from code/services-application/api-service/src/test/java/nu/marginalia/api/svc/LicenseServiceTest.java rename to code/services-application/api-service/test/nu/marginalia/api/svc/LicenseServiceTest.java diff --git a/code/services-application/api-service/src/test/java/nu/marginalia/api/svc/RateLimiterServiceTest.java b/code/services-application/api-service/test/nu/marginalia/api/svc/RateLimiterServiceTest.java similarity index 100% rename from code/services-application/api-service/src/test/java/nu/marginalia/api/svc/RateLimiterServiceTest.java rename to code/services-application/api-service/test/nu/marginalia/api/svc/RateLimiterServiceTest.java diff --git a/code/services-application/api-service/src/test/java/nu/marginalia/api/svc/ResponseCacheTest.java b/code/services-application/api-service/test/nu/marginalia/api/svc/ResponseCacheTest.java similarity index 100% rename from code/services-application/api-service/src/test/java/nu/marginalia/api/svc/ResponseCacheTest.java rename to code/services-application/api-service/test/nu/marginalia/api/svc/ResponseCacheTest.java diff --git a/code/services-application/dating-service/build.gradle b/code/services-application/dating-service/build.gradle index b83eff9a..cae0cb59 100644 --- a/code/services-application/dating-service/build.gradle +++ b/code/services-application/dating-service/build.gradle @@ -35,6 +35,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:db') implementation project(':code:common:model') diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java b/code/services-application/dating-service/java/nu/marginalia/dating/DatingMain.java similarity index 100% rename from code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java rename to code/services-application/dating-service/java/nu/marginalia/dating/DatingMain.java diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingModule.java b/code/services-application/dating-service/java/nu/marginalia/dating/DatingModule.java similarity index 100% rename from code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingModule.java rename to code/services-application/dating-service/java/nu/marginalia/dating/DatingModule.java diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingService.java b/code/services-application/dating-service/java/nu/marginalia/dating/DatingService.java similarity index 100% rename from code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingService.java rename to code/services-application/dating-service/java/nu/marginalia/dating/DatingService.java diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingSessionObject.java b/code/services-application/dating-service/java/nu/marginalia/dating/DatingSessionObject.java similarity index 100% rename from code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingSessionObject.java rename to code/services-application/dating-service/java/nu/marginalia/dating/DatingSessionObject.java diff --git a/code/services-application/dating-service/src/main/resources/static/dating/robots.txt b/code/services-application/dating-service/resources/static/dating/robots.txt similarity index 100% rename from code/services-application/dating-service/src/main/resources/static/dating/robots.txt rename to code/services-application/dating-service/resources/static/dating/robots.txt diff --git a/code/services-application/dating-service/src/main/resources/templates/dating/dating-view.hdb b/code/services-application/dating-service/resources/templates/dating/dating-view.hdb similarity index 100% rename from code/services-application/dating-service/src/main/resources/templates/dating/dating-view.hdb rename to code/services-application/dating-service/resources/templates/dating/dating-view.hdb diff --git a/code/services-application/explorer-service/build.gradle b/code/services-application/explorer-service/build.gradle index 75607fae..cf35c610 100644 --- a/code/services-application/explorer-service/build.gradle +++ b/code/services-application/explorer-service/build.gradle @@ -35,6 +35,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:common:service') diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java b/code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerMain.java similarity index 100% rename from code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java rename to code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerMain.java diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerModule.java b/code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerModule.java similarity index 100% rename from code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerModule.java rename to code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerModule.java diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerService.java b/code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerService.java similarity index 100% rename from code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerService.java rename to code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerService.java diff --git a/code/services-application/explorer-service/src/main/resources/static/explore/robots.txt b/code/services-application/explorer-service/resources/static/explore/robots.txt similarity index 100% rename from code/services-application/explorer-service/src/main/resources/static/explore/robots.txt rename to code/services-application/explorer-service/resources/static/explore/robots.txt diff --git a/code/services-application/explorer-service/src/main/resources/static/explore/style.css b/code/services-application/explorer-service/resources/static/explore/style.css similarity index 100% rename from code/services-application/explorer-service/src/main/resources/static/explore/style.css rename to code/services-application/explorer-service/resources/static/explore/style.css diff --git a/code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-about.hdb b/code/services-application/explorer-service/resources/templates/explorer/explorer-about.hdb similarity index 100% rename from code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-about.hdb rename to code/services-application/explorer-service/resources/templates/explorer/explorer-about.hdb diff --git a/code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-messages.hdb b/code/services-application/explorer-service/resources/templates/explorer/explorer-messages.hdb similarity index 100% rename from code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-messages.hdb rename to code/services-application/explorer-service/resources/templates/explorer/explorer-messages.hdb diff --git a/code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-results.hdb b/code/services-application/explorer-service/resources/templates/explorer/explorer-results.hdb similarity index 100% rename from code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-results.hdb rename to code/services-application/explorer-service/resources/templates/explorer/explorer-results.hdb diff --git a/code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-search.hdb b/code/services-application/explorer-service/resources/templates/explorer/explorer-search.hdb similarity index 100% rename from code/services-application/explorer-service/src/main/resources/templates/explorer/explorer-search.hdb rename to code/services-application/explorer-service/resources/templates/explorer/explorer-search.hdb diff --git a/code/services-application/explorer-service/src/main/resources/templates/explorer/explorer.hdb b/code/services-application/explorer-service/resources/templates/explorer/explorer.hdb similarity index 100% rename from code/services-application/explorer-service/src/main/resources/templates/explorer/explorer.hdb rename to code/services-application/explorer-service/resources/templates/explorer/explorer.hdb diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index a32dc71a..cb799ab9 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -42,6 +42,8 @@ sass { outputStyle = EXPANDED } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:db') implementation project(':code:common:model') diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchHandlebarsConfigurator.java b/code/services-application/search-service/java/nu/marginalia/search/SearchHandlebarsConfigurator.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchHandlebarsConfigurator.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchHandlebarsConfigurator.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java b/code/services-application/search-service/java/nu/marginalia/search/SearchMain.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchMain.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchModule.java b/code/services-application/search-service/java/nu/marginalia/search/SearchModule.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchModule.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchModule.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchQueryParamFactory.java b/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchQueryParamFactory.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchResultClusterer.java b/code/services-application/search-service/java/nu/marginalia/search/SearchResultClusterer.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchResultClusterer.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchResultClusterer.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java b/code/services-application/search-service/java/nu/marginalia/search/SearchService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java rename to code/services-application/search-service/java/nu/marginalia/search/SearchService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/CommandEvaluator.java b/code/services-application/search-service/java/nu/marginalia/search/command/CommandEvaluator.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/CommandEvaluator.java rename to code/services-application/search-service/java/nu/marginalia/search/command/CommandEvaluator.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchAdtechParameter.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchAdtechParameter.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchAdtechParameter.java rename to code/services-application/search-service/java/nu/marginalia/search/command/SearchAdtechParameter.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchCommandInterface.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchCommandInterface.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchCommandInterface.java rename to code/services-application/search-service/java/nu/marginalia/search/command/SearchCommandInterface.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchJsParameter.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchJsParameter.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchJsParameter.java rename to code/services-application/search-service/java/nu/marginalia/search/command/SearchJsParameter.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchParameters.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchParameters.java rename to code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchRecentParameter.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchRecentParameter.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchRecentParameter.java rename to code/services-application/search-service/java/nu/marginalia/search/command/SearchRecentParameter.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchTitleParameter.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchTitleParameter.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/SearchTitleParameter.java rename to code/services-application/search-service/java/nu/marginalia/search/command/SearchTitleParameter.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BangCommand.java b/code/services-application/search-service/java/nu/marginalia/search/command/commands/BangCommand.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BangCommand.java rename to code/services-application/search-service/java/nu/marginalia/search/command/commands/BangCommand.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BrowseCommand.java b/code/services-application/search-service/java/nu/marginalia/search/command/commands/BrowseCommand.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/BrowseCommand.java rename to code/services-application/search-service/java/nu/marginalia/search/command/commands/BrowseCommand.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/ConvertCommand.java b/code/services-application/search-service/java/nu/marginalia/search/command/commands/ConvertCommand.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/ConvertCommand.java rename to code/services-application/search-service/java/nu/marginalia/search/command/commands/ConvertCommand.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java b/code/services-application/search-service/java/nu/marginalia/search/command/commands/DefinitionCommand.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java rename to code/services-application/search-service/java/nu/marginalia/search/command/commands/DefinitionCommand.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java b/code/services-application/search-service/java/nu/marginalia/search/command/commands/SearchCommand.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java rename to code/services-application/search-service/java/nu/marginalia/search/command/commands/SearchCommand.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java b/code/services-application/search-service/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java rename to code/services-application/search-service/java/nu/marginalia/search/command/commands/SiteRedirectCommand.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/db/DbNearDomainsQuery.java b/code/services-application/search-service/java/nu/marginalia/search/db/DbNearDomainsQuery.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/db/DbNearDomainsQuery.java rename to code/services-application/search-service/java/nu/marginalia/search/db/DbNearDomainsQuery.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/exceptions/RedirectException.java b/code/services-application/search-service/java/nu/marginalia/search/exceptions/RedirectException.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/exceptions/RedirectException.java rename to code/services-application/search-service/java/nu/marginalia/search/exceptions/RedirectException.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/ClusteredUrlDetails.java b/code/services-application/search-service/java/nu/marginalia/search/model/ClusteredUrlDetails.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/model/ClusteredUrlDetails.java rename to code/services-application/search-service/java/nu/marginalia/search/model/ClusteredUrlDetails.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/DecoratedSearchResults.java b/code/services-application/search-service/java/nu/marginalia/search/model/DecoratedSearchResults.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/model/DecoratedSearchResults.java rename to code/services-application/search-service/java/nu/marginalia/search/model/DecoratedSearchResults.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchFilters.java b/code/services-application/search-service/java/nu/marginalia/search/model/SearchFilters.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchFilters.java rename to code/services-application/search-service/java/nu/marginalia/search/model/SearchFilters.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchProfile.java b/code/services-application/search-service/java/nu/marginalia/search/model/SearchProfile.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/model/SearchProfile.java rename to code/services-application/search-service/java/nu/marginalia/search/model/SearchProfile.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/model/UrlDetails.java b/code/services-application/search-service/java/nu/marginalia/search/model/UrlDetails.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/model/UrlDetails.java rename to code/services-application/search-service/java/nu/marginalia/search/model/UrlDetails.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/BrowseResultCleaner.java b/code/services-application/search-service/java/nu/marginalia/search/results/BrowseResultCleaner.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/results/BrowseResultCleaner.java rename to code/services-application/search-service/java/nu/marginalia/search/results/BrowseResultCleaner.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java b/code/services-application/search-service/java/nu/marginalia/search/results/UrlDeduplicator.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/results/UrlDeduplicator.java rename to code/services-application/search-service/java/nu/marginalia/search/results/UrlDeduplicator.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchAddToCrawlQueueService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchAddToCrawlQueueService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchAddToCrawlQueueService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchAddToCrawlQueueService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchBrowseService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchBrowseService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchCrosstalkService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchCrosstalkService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchCrosstalkService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchCrosstalkService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchErrorPageService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchErrorPageService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchErrorPageService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchFlagSiteService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchFlagSiteService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFrontPageService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchFrontPageService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryCountService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryCountService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryCountService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryCountService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryIndexService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryIndexService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchQueryService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchUnitConversionService.java similarity index 100% rename from code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java rename to code/services-application/search-service/java/nu/marginalia/search/svc/SearchUnitConversionService.java diff --git a/code/services-application/search-service/src/main/resources/static/search/crawler-ips.txt b/code/services-application/search-service/resources/static/search/crawler-ips.txt similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/crawler-ips.txt rename to code/services-application/search-service/resources/static/search/crawler-ips.txt diff --git a/code/services-application/search-service/src/main/resources/static/search/favicon.ico b/code/services-application/search-service/resources/static/search/favicon.ico similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/favicon.ico rename to code/services-application/search-service/resources/static/search/favicon.ico diff --git a/code/services-application/search-service/src/main/resources/static/search/main.js b/code/services-application/search-service/resources/static/search/main.js similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/main.js rename to code/services-application/search-service/resources/static/search/main.js diff --git a/code/services-application/search-service/src/main/resources/static/search/menu.js b/code/services-application/search-service/resources/static/search/menu.js similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/menu.js rename to code/services-application/search-service/resources/static/search/menu.js diff --git a/code/services-application/search-service/src/main/resources/static/search/opensearch.xml b/code/services-application/search-service/resources/static/search/opensearch.xml similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/opensearch.xml rename to code/services-application/search-service/resources/static/search/opensearch.xml diff --git a/code/services-application/search-service/src/main/resources/static/search/robots.txt b/code/services-application/search-service/resources/static/search/robots.txt similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/robots.txt rename to code/services-application/search-service/resources/static/search/robots.txt diff --git a/code/services-application/search-service/src/main/resources/static/search/rss.svg b/code/services-application/search-service/resources/static/search/rss.svg similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/rss.svg rename to code/services-application/search-service/resources/static/search/rss.svg diff --git a/code/services-application/search-service/src/main/resources/static/search/serp.scss b/code/services-application/search-service/resources/static/search/serp.scss similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/serp.scss rename to code/services-application/search-service/resources/static/search/serp.scss diff --git a/code/services-application/search-service/src/main/resources/static/search/style-new.css b/code/services-application/search-service/resources/static/search/style-new.css similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/style-new.css rename to code/services-application/search-service/resources/static/search/style-new.css diff --git a/code/services-application/search-service/src/main/resources/static/search/tts.js b/code/services-application/search-service/resources/static/search/tts.js similarity index 100% rename from code/services-application/search-service/src/main/resources/static/search/tts.js rename to code/services-application/search-service/resources/static/search/tts.js diff --git a/code/services-application/search-service/src/main/resources/templates/search/browse-result.hdb b/code/services-application/search-service/resources/templates/search/browse-result.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/browse-result.hdb rename to code/services-application/search-service/resources/templates/search/browse-result.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/browse-results.hdb b/code/services-application/search-service/resources/templates/search/browse-results.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/browse-results.hdb rename to code/services-application/search-service/resources/templates/search/browse-results.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/conversion-results.hdb b/code/services-application/search-service/resources/templates/search/conversion-results.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/conversion-results.hdb rename to code/services-application/search-service/resources/templates/search/conversion-results.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/dictionary-results.hdb b/code/services-application/search-service/resources/templates/search/dictionary-results.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/dictionary-results.hdb rename to code/services-application/search-service/resources/templates/search/dictionary-results.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/error-page-search.hdb b/code/services-application/search-service/resources/templates/search/error-page-search.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/error-page-search.hdb rename to code/services-application/search-service/resources/templates/search/error-page-search.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/error-page.hdb b/code/services-application/search-service/resources/templates/search/error-page.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/error-page.hdb rename to code/services-application/search-service/resources/templates/search/error-page.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/index/index-about.hdb b/code/services-application/search-service/resources/templates/search/index/index-about.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/index/index-about.hdb rename to code/services-application/search-service/resources/templates/search/index/index-about.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/index/index-news.hdb b/code/services-application/search-service/resources/templates/search/index/index-news.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/index/index-news.hdb rename to code/services-application/search-service/resources/templates/search/index/index-news.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/index/index-tips.hdb b/code/services-application/search-service/resources/templates/search/index/index-tips.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/index/index-tips.hdb rename to code/services-application/search-service/resources/templates/search/index/index-tips.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/index/index.hdb b/code/services-application/search-service/resources/templates/search/index/index.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/index/index.hdb rename to code/services-application/search-service/resources/templates/search/index/index.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/parts/search-filters.hdb b/code/services-application/search-service/resources/templates/search/parts/search-filters.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/parts/search-filters.hdb rename to code/services-application/search-service/resources/templates/search/parts/search-filters.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/parts/search-footer.hdb b/code/services-application/search-service/resources/templates/search/parts/search-footer.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/parts/search-footer.hdb rename to code/services-application/search-service/resources/templates/search/parts/search-footer.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/parts/search-form.hdb b/code/services-application/search-service/resources/templates/search/parts/search-form.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/parts/search-form.hdb rename to code/services-application/search-service/resources/templates/search/parts/search-form.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/parts/search-header.hdb b/code/services-application/search-service/resources/templates/search/parts/search-header.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/parts/search-header.hdb rename to code/services-application/search-service/resources/templates/search/parts/search-header.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/parts/search-result-rest.hdb b/code/services-application/search-service/resources/templates/search/parts/search-result-rest.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/parts/search-result-rest.hdb rename to code/services-application/search-service/resources/templates/search/parts/search-result-rest.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/parts/search-result.hdb b/code/services-application/search-service/resources/templates/search/parts/search-result.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/parts/search-result.hdb rename to code/services-application/search-service/resources/templates/search/parts/search-result.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/search-results.hdb b/code/services-application/search-service/resources/templates/search/search-results.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/search-results.hdb rename to code/services-application/search-service/resources/templates/search/search-results.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-crosstalk.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-crosstalk.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-crosstalk.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-crosstalk.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-feed.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-feed.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-feed.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-feed.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-blacklisted.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-index-blacklisted.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-blacklisted.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-index-blacklisted.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-indexed.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-index-indexed.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-indexed.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-index-indexed.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-suggest.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-index-suggest.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-suggest.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-index-suggest.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-unknown.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-index-unknown.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index-unknown.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-index-unknown.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-index.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-index.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-index.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-links.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-links.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-links.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-links.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-report.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-report.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-report.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-report.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-summary.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info-summary.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info-summary.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info-summary.hdb diff --git a/code/services-application/search-service/src/main/resources/templates/search/site-info/site-info.hdb b/code/services-application/search-service/resources/templates/search/site-info/site-info.hdb similarity index 100% rename from code/services-application/search-service/src/main/resources/templates/search/site-info/site-info.hdb rename to code/services-application/search-service/resources/templates/search/site-info/site-info.hdb diff --git a/code/services-application/search-service/src/test/java/nu/marginalia/search/command/commands/BangCommandTest.java b/code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java similarity index 100% rename from code/services-application/search-service/src/test/java/nu/marginalia/search/command/commands/BangCommandTest.java rename to code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java diff --git a/code/services-application/search-service/src/test/java/nu/marginalia/util/TestLanguageModels.java b/code/services-application/search-service/test/nu/marginalia/util/TestLanguageModels.java similarity index 100% rename from code/services-application/search-service/src/test/java/nu/marginalia/util/TestLanguageModels.java rename to code/services-application/search-service/test/nu/marginalia/util/TestLanguageModels.java diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index ce7e8843..27f89012 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -36,6 +36,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:symspell') diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java b/code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantMain.java similarity index 100% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java rename to code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantMain.java diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantModule.java b/code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantModule.java similarity index 100% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantModule.java rename to code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantModule.java diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java b/code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantService.java similarity index 100% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java rename to code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantService.java diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java b/code/services-core/assistant-service/java/nu/marginalia/assistant/suggest/Suggestions.java similarity index 100% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java rename to code/services-core/assistant-service/java/nu/marginalia/assistant/suggest/Suggestions.java diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 634eb1c5..2ecd1f92 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -35,6 +35,8 @@ jib { } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation libs.bundles.gson diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java b/code/services-core/control-service/java/nu/marginalia/control/ControlHandlebarsConfigurator.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java rename to code/services-core/control-service/java/nu/marginalia/control/ControlHandlebarsConfigurator.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java b/code/services-core/control-service/java/nu/marginalia/control/ControlMain.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java rename to code/services-core/control-service/java/nu/marginalia/control/ControlMain.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java b/code/services-core/control-service/java/nu/marginalia/control/ControlProcessModule.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java rename to code/services-core/control-service/java/nu/marginalia/control/ControlProcessModule.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlRendererFactory.java b/code/services-core/control-service/java/nu/marginalia/control/ControlRendererFactory.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/ControlRendererFactory.java rename to code/services-core/control-service/java/nu/marginalia/control/ControlRendererFactory.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/java/nu/marginalia/control/ControlService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java rename to code/services-core/control-service/java/nu/marginalia/control/ControlService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlValidationError.java b/code/services-core/control-service/java/nu/marginalia/control/ControlValidationError.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/ControlValidationError.java rename to code/services-core/control-service/java/nu/marginalia/control/ControlValidationError.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/RedirectControl.java b/code/services-core/control-service/java/nu/marginalia/control/RedirectControl.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/RedirectControl.java rename to code/services-core/control-service/java/nu/marginalia/control/RedirectControl.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/Redirects.java b/code/services-core/control-service/java/nu/marginalia/control/Redirects.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/Redirects.java rename to code/services-core/control-service/java/nu/marginalia/control/Redirects.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/ControlActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/ControlActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActorService.java b/code/services-core/control-service/java/nu/marginalia/control/actor/ControlActorService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActorService.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/ControlActorService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/PrecessionNodes.java b/code/services-core/control-service/java/nu/marginalia/control/actor/PrecessionNodes.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/PrecessionNodes.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/PrecessionNodes.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ServiceHeartbeatMonitorActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/monitor/ServiceHeartbeatMonitorActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ServiceHeartbeatMonitorActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/monitor/ServiceHeartbeatMonitorActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/precession/RecrawlAllActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/precession/ReindexAllActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReindexAllActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/precession/ReindexAllActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReprocessAllActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/precession/ReprocessAllActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/precession/ReprocessAllActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/precession/ReprocessAllActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/rebalance/RebalanceActor.java b/code/services-core/control-service/java/nu/marginalia/control/actor/rebalance/RebalanceActor.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/rebalance/RebalanceActor.java rename to code/services-core/control-service/java/nu/marginalia/control/actor/rebalance/RebalanceActor.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/ApiKeyModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/ApiKeyModel.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/ApiKeyModel.java rename to code/services-core/control-service/java/nu/marginalia/control/app/model/ApiKeyModel.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java rename to code/services-core/control-service/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintCategory.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainComplaintCategory.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintCategory.java rename to code/services-core/control-service/java/nu/marginalia/control/app/model/DomainComplaintCategory.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainComplaintModel.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintModel.java rename to code/services-core/control-service/java/nu/marginalia/control/app/model/DomainComplaintModel.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/ApiKeyService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java rename to code/services-core/control-service/java/nu/marginalia/control/app/svc/ApiKeyService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/ControlBlacklistService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java rename to code/services-core/control-service/java/nu/marginalia/control/app/svc/ControlBlacklistService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainComplaintService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java rename to code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainComplaintService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/RandomExplorationService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java rename to code/services-core/control-service/java/nu/marginalia/control/app/svc/RandomExplorationService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java rename to code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageBaseWithStorage.java b/code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageBaseWithStorage.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageBaseWithStorage.java rename to code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageBaseWithStorage.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageFileModel.java b/code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageFileModel.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageFileModel.java rename to code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageFileModel.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithActions.java b/code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageWithActions.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithActions.java rename to code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageWithActions.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithRelatedEntries.java b/code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageWithRelatedEntries.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithRelatedEntries.java rename to code/services-core/control-service/java/nu/marginalia/control/node/model/FileStorageWithRelatedEntries.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNodeStatus.java b/code/services-core/control-service/java/nu/marginalia/control/node/model/IndexNodeStatus.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNodeStatus.java rename to code/services-core/control-service/java/nu/marginalia/control/node/model/IndexNodeStatus.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java b/code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlFileStorageService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java rename to code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlFileStorageService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java rename to code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlNodeService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java rename to code/services-core/control-service/java/nu/marginalia/control/node/svc/ControlNodeService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/AbortedProcess.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/AbortedProcess.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/EventLogEntry.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/EventLogEntry.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/MessageQueueEntry.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/MessageQueueEntry.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java b/code/services-core/control-service/java/nu/marginalia/control/sys/model/TaskHeartbeat.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/model/TaskHeartbeat.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/AbortedProcessService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/AbortedProcessService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/ControlDomainRankingSetsService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlErrorHandler.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/ControlErrorHandler.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlErrorHandler.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/ControlErrorHandler.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/DataSetsService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/DataSetsService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/EventLogService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/EventLogService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/HeartbeatService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/HeartbeatService.java diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/MessageQueueService.java b/code/services-core/control-service/java/nu/marginalia/control/sys/svc/MessageQueueService.java similarity index 100% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/MessageQueueService.java rename to code/services-core/control-service/java/nu/marginalia/control/sys/svc/MessageQueueService.java diff --git a/code/services-core/control-service/src/main/resources/static/control/noanimation.css b/code/services-core/control-service/resources/static/control/noanimation.css similarity index 100% rename from code/services-core/control-service/src/main/resources/static/control/noanimation.css rename to code/services-core/control-service/resources/static/control/noanimation.css diff --git a/code/services-core/control-service/src/main/resources/static/control/refresh.js b/code/services-core/control-service/resources/static/control/refresh.js similarity index 100% rename from code/services-core/control-service/src/main/resources/static/control/refresh.js rename to code/services-core/control-service/resources/static/control/refresh.js diff --git a/code/services-core/control-service/src/main/resources/static/control/tables.css b/code/services-core/control-service/resources/static/control/tables.css similarity index 100% rename from code/services-core/control-service/src/main/resources/static/control/tables.css rename to code/services-core/control-service/resources/static/control/tables.css diff --git a/code/services-core/control-service/src/main/resources/templates/control/actions.hdb b/code/services-core/control-service/resources/templates/control/actions.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/actions.hdb rename to code/services-core/control-service/resources/templates/control/actions.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/actor-details.hdb b/code/services-core/control-service/resources/templates/control/actor-details.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/actor-details.hdb rename to code/services-core/control-service/resources/templates/control/actor-details.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/api-keys.hdb b/code/services-core/control-service/resources/templates/control/app/api-keys.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/app/api-keys.hdb rename to code/services-core/control-service/resources/templates/control/app/api-keys.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/blacklist.hdb b/code/services-core/control-service/resources/templates/control/app/blacklist.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/app/blacklist.hdb rename to code/services-core/control-service/resources/templates/control/app/blacklist.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/domain-complaints.hdb b/code/services-core/control-service/resources/templates/control/app/domain-complaints.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/app/domain-complaints.hdb rename to code/services-core/control-service/resources/templates/control/app/domain-complaints.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/review-random-domains.hdb b/code/services-core/control-service/resources/templates/control/app/review-random-domains.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/app/review-random-domains.hdb rename to code/services-core/control-service/resources/templates/control/app/review-random-domains.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/search-to-ban.hdb b/code/services-core/control-service/resources/templates/control/app/search-to-ban.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/app/search-to-ban.hdb rename to code/services-core/control-service/resources/templates/control/app/search-to-ban.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/error.hdb b/code/services-core/control-service/resources/templates/control/error.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/error.hdb rename to code/services-core/control-service/resources/templates/control/error.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/index.hdb b/code/services-core/control-service/resources/templates/control/index.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/index.hdb rename to code/services-core/control-service/resources/templates/control/index.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-download-sample-data.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-download-sample-data.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-download-sample-data.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-download-sample-data.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-db-data.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-export-db-data.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-db-data.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-export-db-data.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-from-crawl-data.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-export-from-crawl-data.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-from-crawl-data.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-export-from-crawl-data.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-sample-data.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-export-sample-data.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-sample-data.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-export-sample-data.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-load.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-load.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-new-crawl-specs.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-new-crawl-specs.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-new-crawl-specs.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-new-crawl-specs.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-new-crawl.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-new-crawl.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-new-crawl.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-new-crawl.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-process.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-process.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-recrawl.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-recrawl.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-recrawl.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-recrawl.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-repartition.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-repartition.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-repartition.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-repartition.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-restore-backup.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-restore-backup.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-restore-backup.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-restore-backup.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-dirtree.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-dirtree.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-dirtree.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-dirtree.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-encyclopedia.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-encyclopedia.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-encyclopedia.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-encyclopedia.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-reddit.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-reddit.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-reddit.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-reddit.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-stackexchange.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-stackexchange.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-stackexchange.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-stackexchange.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-warc.hdb b/code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-warc.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-warc.hdb rename to code/services-core/control-service/resources/templates/control/node/actions/partial-sideload-warc.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb b/code/services-core/control-service/resources/templates/control/node/node-actions.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb rename to code/services-core/control-service/resources/templates/control/node/node-actions.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb b/code/services-core/control-service/resources/templates/control/node/node-actors.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb rename to code/services-core/control-service/resources/templates/control/node/node-actors.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb b/code/services-core/control-service/resources/templates/control/node/node-config.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb rename to code/services-core/control-service/resources/templates/control/node/node-config.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-overview.hdb b/code/services-core/control-service/resources/templates/control/node/node-overview.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-overview.hdb rename to code/services-core/control-service/resources/templates/control/node/node-overview.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-conf.hdb b/code/services-core/control-service/resources/templates/control/node/node-storage-conf.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-storage-conf.hdb rename to code/services-core/control-service/resources/templates/control/node/node-storage-conf.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb b/code/services-core/control-service/resources/templates/control/node/node-storage-details.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb rename to code/services-core/control-service/resources/templates/control/node/node-storage-details.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb b/code/services-core/control-service/resources/templates/control/node/node-storage-list.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb rename to code/services-core/control-service/resources/templates/control/node/node-storage-list.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/nodes-list.hdb b/code/services-core/control-service/resources/templates/control/node/nodes-list.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/nodes-list.hdb rename to code/services-core/control-service/resources/templates/control/node/nodes-list.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/partial-node-nav.hdb b/code/services-core/control-service/resources/templates/control/node/partial-node-nav.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/node/partial-node-nav.hdb rename to code/services-core/control-service/resources/templates/control/node/partial-node-nav.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/actor-summary-table.hdb b/code/services-core/control-service/resources/templates/control/partials/actor-summary-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/actor-summary-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/actor-summary-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb b/code/services-core/control-service/resources/templates/control/partials/actors-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/actors-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb b/code/services-core/control-service/resources/templates/control/partials/events-table-summary.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb rename to code/services-core/control-service/resources/templates/control/partials/events-table-summary.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb b/code/services-core/control-service/resources/templates/control/partials/events-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/events-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/foot-includes.hdb b/code/services-core/control-service/resources/templates/control/partials/foot-includes.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/foot-includes.hdb rename to code/services-core/control-service/resources/templates/control/partials/foot-includes.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/head-includes.hdb b/code/services-core/control-service/resources/templates/control/partials/head-includes.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/head-includes.hdb rename to code/services-core/control-service/resources/templates/control/partials/head-includes.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb b/code/services-core/control-service/resources/templates/control/partials/message-queue-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/message-queue-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb b/code/services-core/control-service/resources/templates/control/partials/nav.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb rename to code/services-core/control-service/resources/templates/control/partials/nav.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/nodes-table.hdb b/code/services-core/control-service/resources/templates/control/partials/nodes-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/nodes-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/nodes-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/processes-table.hdb b/code/services-core/control-service/resources/templates/control/partials/processes-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/processes-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/processes-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/services-table.hdb b/code/services-core/control-service/resources/templates/control/partials/services-table.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/services-table.hdb rename to code/services-core/control-service/resources/templates/control/partials/services-table.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/storage-details/files.hdb b/code/services-core/control-service/resources/templates/control/partials/storage-details/files.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/storage-details/files.hdb rename to code/services-core/control-service/resources/templates/control/partials/storage-details/files.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/storage-details/related.hdb b/code/services-core/control-service/resources/templates/control/partials/storage-details/related.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/storage-details/related.hdb rename to code/services-core/control-service/resources/templates/control/partials/storage-details/related.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/storage-types.hdb b/code/services-core/control-service/resources/templates/control/partials/storage-types.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/partials/storage-types.hdb rename to code/services-core/control-service/resources/templates/control/partials/storage-types.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/redirect-ok.hdb b/code/services-core/control-service/resources/templates/control/redirect-ok.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/redirect-ok.hdb rename to code/services-core/control-service/resources/templates/control/redirect-ok.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb b/code/services-core/control-service/resources/templates/control/sys/aborted-processes.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb rename to code/services-core/control-service/resources/templates/control/sys/aborted-processes.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/data-sets.hdb b/code/services-core/control-service/resources/templates/control/sys/data-sets.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/data-sets.hdb rename to code/services-core/control-service/resources/templates/control/sys/data-sets.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/domain-ranking-sets.hdb b/code/services-core/control-service/resources/templates/control/sys/domain-ranking-sets.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/domain-ranking-sets.hdb rename to code/services-core/control-service/resources/templates/control/sys/domain-ranking-sets.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/events.hdb b/code/services-core/control-service/resources/templates/control/sys/events.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/events.hdb rename to code/services-core/control-service/resources/templates/control/sys/events.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/message-queue.hdb b/code/services-core/control-service/resources/templates/control/sys/message-queue.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/message-queue.hdb rename to code/services-core/control-service/resources/templates/control/sys/message-queue.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/new-domain-ranking-set.hdb b/code/services-core/control-service/resources/templates/control/sys/new-domain-ranking-set.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/new-domain-ranking-set.hdb rename to code/services-core/control-service/resources/templates/control/sys/new-domain-ranking-set.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/new-message.hdb b/code/services-core/control-service/resources/templates/control/sys/new-message.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/new-message.hdb rename to code/services-core/control-service/resources/templates/control/sys/new-message.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/service-by-id.hdb b/code/services-core/control-service/resources/templates/control/sys/service-by-id.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/service-by-id.hdb rename to code/services-core/control-service/resources/templates/control/sys/service-by-id.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/sys-actions.hdb b/code/services-core/control-service/resources/templates/control/sys/sys-actions.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/sys-actions.hdb rename to code/services-core/control-service/resources/templates/control/sys/sys-actions.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/update-domain-ranking-set.hdb b/code/services-core/control-service/resources/templates/control/sys/update-domain-ranking-set.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/update-domain-ranking-set.hdb rename to code/services-core/control-service/resources/templates/control/sys/update-domain-ranking-set.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/update-message-state.hdb b/code/services-core/control-service/resources/templates/control/sys/update-message-state.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/update-message-state.hdb rename to code/services-core/control-service/resources/templates/control/sys/update-message-state.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb b/code/services-core/control-service/resources/templates/control/sys/view-message.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb rename to code/services-core/control-service/resources/templates/control/sys/view-message.hdb diff --git a/code/services-core/control-service/src/test/java/nu/marginalia/control/svc/ApiKeyServiceTest.java b/code/services-core/control-service/test/nu/marginalia/control/svc/ApiKeyServiceTest.java similarity index 100% rename from code/services-core/control-service/src/test/java/nu/marginalia/control/svc/ApiKeyServiceTest.java rename to code/services-core/control-service/test/nu/marginalia/control/svc/ApiKeyServiceTest.java diff --git a/code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java b/code/services-core/control-service/test/nu/marginalia/control/svc/HeartbeatServiceTest.java similarity index 100% rename from code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java rename to code/services-core/control-service/test/nu/marginalia/control/svc/HeartbeatServiceTest.java diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 68a28eff..62324c7f 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -34,6 +34,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { // These look weird but they're needed to be able to spawn the processes // from the executor service diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java b/code/services-core/executor-service/java/nu/marginalia/executor/ExecutorMain.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java rename to code/services-core/executor-service/java/nu/marginalia/executor/ExecutorMain.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java b/code/services-core/executor-service/java/nu/marginalia/executor/ExecutorModule.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java rename to code/services-core/executor-service/java/nu/marginalia/executor/ExecutorModule.java diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/java/nu/marginalia/executor/ExecutorSvc.java similarity index 100% rename from code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java rename to code/services-core/executor-service/java/nu/marginalia/executor/ExecutorSvc.java diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 1c7f81fe..69f1f38d 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -35,6 +35,8 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') implementation project(':code:common:service') diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java b/code/services-core/index-service/java/nu/marginalia/index/IndexMain.java similarity index 100% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java rename to code/services-core/index-service/java/nu/marginalia/index/IndexMain.java diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexModule.java b/code/services-core/index-service/java/nu/marginalia/index/IndexModule.java similarity index 100% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/IndexModule.java rename to code/services-core/index-service/java/nu/marginalia/index/IndexModule.java diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java b/code/services-core/index-service/java/nu/marginalia/index/IndexOpsService.java similarity index 100% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/IndexOpsService.java rename to code/services-core/index-service/java/nu/marginalia/index/IndexOpsService.java diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/java/nu/marginalia/index/IndexService.java similarity index 100% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java rename to code/services-core/index-service/java/nu/marginalia/index/IndexService.java diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index a013e86d..0e283788 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -36,6 +36,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') implementation project(':code:common:model') diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java b/code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java similarity index 100% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/QueryBasicInterface.java rename to code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java b/code/services-core/query-service/java/nu/marginalia/query/QueryMain.java similarity index 100% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java rename to code/services-core/query-service/java/nu/marginalia/query/QueryMain.java diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryModule.java b/code/services-core/query-service/java/nu/marginalia/query/QueryModule.java similarity index 100% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/QueryModule.java rename to code/services-core/query-service/java/nu/marginalia/query/QueryModule.java diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/java/nu/marginalia/query/QueryService.java similarity index 100% rename from code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java rename to code/services-core/query-service/java/nu/marginalia/query/QueryService.java diff --git a/code/services-core/query-service/src/main/resources/static/public/index.html b/code/services-core/query-service/resources/static/public/index.html similarity index 100% rename from code/services-core/query-service/src/main/resources/static/public/index.html rename to code/services-core/query-service/resources/static/public/index.html diff --git a/code/services-core/query-service/src/main/resources/templates/search.hdb b/code/services-core/query-service/resources/templates/search.hdb similarity index 100% rename from code/services-core/query-service/src/main/resources/templates/search.hdb rename to code/services-core/query-service/resources/templates/search.hdb diff --git a/code/tools/crawl-data-unfcker/build.gradle b/code/tools/crawl-data-unfcker/build.gradle index 6673eab6..40ec3bcb 100644 --- a/code/tools/crawl-data-unfcker/build.gradle +++ b/code/tools/crawl-data-unfcker/build.gradle @@ -19,6 +19,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:rdrpostagger') implementation project(':third-party:porterstemmer') diff --git a/code/tools/crawl-data-unfcker/src/main/java/nu/marginalia/tools/CrawlDataUnfcker.java b/code/tools/crawl-data-unfcker/java/nu/marginalia/tools/CrawlDataUnfcker.java similarity index 100% rename from code/tools/crawl-data-unfcker/src/main/java/nu/marginalia/tools/CrawlDataUnfcker.java rename to code/tools/crawl-data-unfcker/java/nu/marginalia/tools/CrawlDataUnfcker.java diff --git a/code/tools/experiment-runner/build.gradle b/code/tools/experiment-runner/build.gradle index 44494949..36001e0b 100644 --- a/code/tools/experiment-runner/build.gradle +++ b/code/tools/experiment-runner/build.gradle @@ -19,6 +19,8 @@ application { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':third-party:rdrpostagger') implementation project(':third-party:porterstemmer') diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/Experiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/Experiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/Experiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/Experiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java b/code/tools/experiment-runner/java/nu/marginalia/tools/ExperimentRunnerMain.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/ExperimentRunnerMain.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/LegacyExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/LegacyExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/LegacyExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/LegacyExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AdblockExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/AdblockExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AdblockExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/AdblockExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AtagsExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/AtagsExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/AtagsExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/AtagsExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/DebugConverterExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/ExportExternalLinksExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/ExportExternalLinksExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/ExportExternalLinksExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/ExportExternalLinksExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/SentenceStatisticsExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SiteStatisticsExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/SiteStatisticsExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/SiteStatisticsExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/SiteStatisticsExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TestExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/TestExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TestExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/TestExperiment.java diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TopicExperiment.java b/code/tools/experiment-runner/java/nu/marginalia/tools/experiments/TopicExperiment.java similarity index 100% rename from code/tools/experiment-runner/src/main/java/nu/marginalia/tools/experiments/TopicExperiment.java rename to code/tools/experiment-runner/java/nu/marginalia/tools/experiments/TopicExperiment.java diff --git a/code/tools/load-test/build.gradle b/code/tools/load-test/build.gradle index e8b363c8..744333c8 100644 --- a/code/tools/load-test/build.gradle +++ b/code/tools/load-test/build.gradle @@ -11,6 +11,8 @@ java { } } +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:config') implementation project(':code:common:model') diff --git a/code/tools/load-test/src/main/java/nu/marginalia/load_test/LoadTestMain.java b/code/tools/load-test/java/nu/marginalia/load_test/LoadTestMain.java similarity index 100% rename from code/tools/load-test/src/main/java/nu/marginalia/load_test/LoadTestMain.java rename to code/tools/load-test/java/nu/marginalia/load_test/LoadTestMain.java diff --git a/code/tools/screenshot-capture-tool/build.gradle b/code/tools/screenshot-capture-tool/build.gradle index b821f7b7..82903746 100644 --- a/code/tools/screenshot-capture-tool/build.gradle +++ b/code/tools/screenshot-capture-tool/build.gradle @@ -35,6 +35,8 @@ jib { tasks.distZip.enabled = false +apply from: "$rootProject.projectDir/srcsets.gradle" + dependencies { implementation project(':code:common:model') implementation project(':code:features-search:screenshots') diff --git a/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java b/code/tools/screenshot-capture-tool/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java similarity index 100% rename from code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java rename to code/tools/screenshot-capture-tool/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java diff --git a/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java b/code/tools/screenshot-capture-tool/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java similarity index 100% rename from code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java rename to code/tools/screenshot-capture-tool/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java diff --git a/protobuf.gradle b/protobuf.gradle index af2f5317..d8eec78e 100644 --- a/protobuf.gradle +++ b/protobuf.gradle @@ -26,4 +26,14 @@ protobuf { } } } -} \ No newline at end of file +} + +tasks.compileJava.dependsOn('generateProto') + +sourceSets { + main { + proto { + srcDirs += 'src/main/protobuf' + } + } +} diff --git a/settings.gradle b/settings.gradle index 8fe58024..0dbc4376 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ -rootProject.name = 'marginalia.nu' +rootProject.name = 'marginalia' include 'code:services-core:index-service' include 'code:services-core:assistant-service' @@ -31,8 +31,8 @@ include 'code:index' include 'code:index:api' include 'code:index:index-journal' include 'code:index:query' -include 'code:index:forward-index' -include 'code:index:reverse-index' +include 'code:index:index-forward' +include 'code:index:index-reverse' include 'code:libraries:array' include 'code:libraries:geo-ip' diff --git a/srcsets.gradle b/srcsets.gradle new file mode 100644 index 00000000..ae1e0729 --- /dev/null +++ b/srcsets.gradle @@ -0,0 +1,51 @@ +/** Configures the source sets for the code/ subprojects. + * + * Ideally this would have been done in the root build.gradle file, but due to disagreements + * between Gradle and IntelliJ's gradle plugin about how to interpret the sourceSets block + * when applied to subprojects from the root project, this has to be done in each subproject. + * */ +apply plugin: 'java' +apply plugin: 'io.freefair.lombok' + +dependencies { + implementation libs.lombok + testImplementation libs.lombok + annotationProcessor libs.lombok + + lombok libs.lombok // prevent plugin from downgrading the version to something incompatible with '19 +} + +test { + maxHeapSize = "8G" + useJUnitPlatform() +} + +tasks.register('fastTests', Test) { + maxHeapSize = "8G" + useJUnitPlatform { + excludeTags "slow" + } +} + +sourceSets { + main { + java { + srcDirs = [ + 'java', + 'build/generated/source/proto/main/grpc', + 'build/generated/source/proto/main/java' + ] + } + resources { + srcDirs = [ 'resources' ] + } + } + test { + java { + srcDirs = [ 'test' ] + } + resources { + srcDirs = [ 'test-resources' ] + } + } +} diff --git a/third-party/encyclopedia-marginalia-nu/build.gradle b/third-party/encyclopedia-marginalia-nu/build.gradle index 992f70e2..faf794ec 100644 --- a/third-party/encyclopedia-marginalia-nu/build.gradle +++ b/third-party/encyclopedia-marginalia-nu/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java' + id 'io.freefair.lombok' } java { @@ -15,6 +16,12 @@ dependencies { implementation libs.zstd implementation libs.bundles.slf4j + implementation libs.lombok + testImplementation libs.lombok + annotationProcessor libs.lombok + + lombok libs.lombok // prevent plugin from downgrading the version to something incompatible with '19 + implementation project(':code:libraries:blocking-thread-pool') implementation project(':third-party:openzim') diff --git a/third-party/encyclopedia-marginalia-nu/src/main/java/nu/marginalia/encyclopedia/EncyclopediaConverter.java b/third-party/encyclopedia-marginalia-nu/src/main/java/nu/marginalia/encyclopedia/EncyclopediaConverter.java index 1920f536..e5274652 100644 --- a/third-party/encyclopedia-marginalia-nu/src/main/java/nu/marginalia/encyclopedia/EncyclopediaConverter.java +++ b/third-party/encyclopedia-marginalia-nu/src/main/java/nu/marginalia/encyclopedia/EncyclopediaConverter.java @@ -3,7 +3,6 @@ package nu.marginalia.encyclopedia; import nu.marginalia.encyclopedia.cleaner.WikiCleaner; import nu.marginalia.encyclopedia.store.ArticleDbProvider; import nu.marginalia.encyclopedia.store.ArticleStoreWriter; -import nu.marginalia.util.SimpleBlockingThreadPool; import org.openzim.ZIMTypes.ZIMFile; import org.openzim.ZIMTypes.ZIMReader; import org.slf4j.LoggerFactory; @@ -13,7 +12,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Predicate; @@ -26,41 +25,35 @@ public class EncyclopediaConverter { public static void convert(Path inputFile, Path outputFile) throws IOException, SQLException, InterruptedException { var wc = new WikiCleaner(); - var pool = new SimpleBlockingThreadPool("Convert ZIM", - Math.clamp(Runtime.getRuntime().availableProcessors() - 2, 1, 32), - 2); - var size = new AtomicInteger(); - if (!Files.exists(inputFile)) { - throw new IllegalStateException("ZIM file not found: " + inputFile); - } - Files.deleteIfExists(outputFile); + try (var executor = Executors.newWorkStealingPool(Math.clamp(Runtime.getRuntime().availableProcessors() - 2, 1, 32))) { - try (var asw = new ArticleStoreWriter(new ArticleDbProvider(outputFile))) { - Predicate keepGoing = (s) -> true; + var size = new AtomicInteger(); - BiConsumer handleArticle = (url, html) -> { - if (pool.isTerminated()) - return; + if (!Files.exists(inputFile)) { + throw new IllegalStateException("ZIM file not found: " + inputFile); + } + Files.deleteIfExists(outputFile); - pool.submitQuietly(() -> { - int sz = size.incrementAndGet(); - if (sz % 1000 == 0) { - System.out.printf("\u001b[2K\r%d", sz); - } - asw.add(wc.cleanWikiJunk(url, html)); - }); + try (var asw = new ArticleStoreWriter(new ArticleDbProvider(outputFile))) { + Predicate keepGoing = (s) -> true; - size.incrementAndGet(); - }; + BiConsumer handleArticle = (url, html) -> { + if (executor.isTerminated()) + return; - new ZIMReader(new ZIMFile(inputFile.toString())).forEachArticles(handleArticle, keepGoing); + executor.submit(() -> { + int sz = size.incrementAndGet(); + if (sz % 1000 == 0) { + System.out.printf("\u001b[2K\r%d", sz); + } + asw.add(wc.cleanWikiJunk(url, html)); + }); - pool.shutDown(); - logger.info("Waiting for pool to finish"); + size.incrementAndGet(); + }; - while (!pool.awaitTermination(1, TimeUnit.SECONDS)) { - // ... + new ZIMReader(new ZIMFile(inputFile.toString())).forEachArticles(handleArticle, keepGoing); } } } From ff0ef1eebcedc4c941fde1c7433b4b8ce47c6dc9 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sat, 24 Feb 2024 15:33:56 +0100 Subject: [PATCH 32/56] (cleanup) Minor cleanups --- .../client/GrpcMultiNodeChannelPool.java | 28 +++++++++++++++---- .../client/GrpcSingleNodeChannelPool.java | 21 +++++++------- .../service/discovery/ZkServiceRegistry.java | 2 +- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index c725fa0b..d4f75e66 100644 --- a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -48,20 +48,32 @@ public class GrpcMultiNodeChannelPool { } private GrpcSingleNodeChannelPool getPoolForNode(int node) { - return pools.computeIfAbsent(node, _ -> - new GrpcSingleNodeChannelPool<>( - serviceRegistryIf, - serviceKey.forPartition(ServicePartition.partition(node)), - channelConstructor, - stubConstructor)); + return pools.computeIfAbsent(node, this::newSingleChannelPool); } + private GrpcSingleNodeChannelPool newSingleChannelPool(int node) { + return new GrpcSingleNodeChannelPool<>( + serviceRegistryIf, + serviceKey.forPartition(ServicePartition.partition(node)), + channelConstructor, + stubConstructor); + } /** Get the list of nodes that are eligible for broadcast-style requests */ public List getEligibleNodes() { return nodeConfigurationWatcher.getQueryNodes(); } + /** Create a new call builder for the given method. This is a fluent-style + * method, where you can chain calls to specify how to run the method. + *

+ * Example: + * 
+     *     var results = channelPool.call(AStub:someMethod)
+     *                   .async(someExecutor)
+     *                   .runAll(argumentToSomeMethod);
+     *
+ * */ public CallBuilderBase call(BiFunction method) { return new CallBuilderBase<>(method); } @@ -73,16 +85,20 @@ public class GrpcMultiNodeChannelPool { this.method = method; } + /** Create a call for the given method on the given node */ public GrpcSingleNodeChannelPool.CallBuilderBase forNode(int node) { return getPoolForNode(node).call(method); } + /** Run the given method on each node, returning a list of results. + * This is a blocking method, where each call will be made in sequence */ public List run(I arg) { return getEligibleNodes().stream() .map(node -> getPoolForNode(node).call(method).run(arg)) .toList(); } + /** Generate an async call builder for the given method */ public CallBuilderAsync async(ExecutorService service) { return new CallBuilderAsync<>(service, method); } diff --git a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index ec3bb38a..9c777c2b 100644 --- a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -169,6 +169,9 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { throw new ServiceNotAvailableException(serviceKey); } + /** Create a call for the given method on the given node. + * This is a fluent method, so you can chain it with other + * methods to specify the node and arguments */ public CallBuilderBase call(BiFunction method) { return new CallBuilderBase<>(method); } @@ -179,21 +182,12 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { this.method = method; } + /** Execute the call in a blocking manner */ public T run(I arg) { return call(method, arg); } - public List runFor(I... args) { - return runFor(List.of(args)); - } - - public List runFor(List args) { - List results = new ArrayList<>(); - for (var arg : args) { - results.add(call(method, arg)); - } - return results; - } + /** Create an asynchronous call using the provided executor */ public CallBuilderAsync async(Executor executor) { return new CallBuilderAsync<>(executor, method); } @@ -207,9 +201,12 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { this.method = method; } + /** Execute the call in an asynchronous manner */ public CompletableFuture run(I arg) { return CompletableFuture.supplyAsync(() -> call(method, arg), executor); } + + /** Execute the call in an asynchronous manner for each of the given arguments */ public CompletableFuture> runFor(List args) { List> results = new ArrayList<>(); for (var arg : args) { @@ -218,6 +215,8 @@ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { return CompletableFuture.allOf(results.toArray(new CompletableFuture[0])) .thenApply(v -> results.stream().map(CompletableFuture::join).toList()); } + + /** Execute the call in an asynchronous manner for each of the given arguments */ public CompletableFuture> runFor(I... args) { return runFor(List.of(args)); } diff --git a/code/common/service-discovery/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/java/nu/marginalia/service/discovery/ZkServiceRegistry.java index e4eca465..0e233ced 100644 --- a/code/common/service-discovery/java/nu/marginalia/service/discovery/ZkServiceRegistry.java +++ b/code/common/service-discovery/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -79,7 +79,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf { curatorFramework.create() .creatingParentsIfNeeded() .withMode(CreateMode.PERSISTENT) - .forPath(STR."/first-boot"); + .forPath("/first-boot"); } } From 09447f2ad284248808455b55ba2d21df653abc43 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sat, 24 Feb 2024 18:32:37 +0100 Subject: [PATCH 33/56] (process service) Inherit parent's assertion status --- code/execution/java/nu/marginalia/process/ProcessService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/execution/java/nu/marginalia/process/ProcessService.java b/code/execution/java/nu/marginalia/process/ProcessService.java index 89e87e52..848f0233 100644 --- a/code/execution/java/nu/marginalia/process/ProcessService.java +++ b/code/execution/java/nu/marginalia/process/ProcessService.java @@ -89,6 +89,10 @@ public class ProcessService { args.add(STR."\{javaHome}/bin/java"); args.add("-cp"); args.add(System.getProperty("java.class.path")); + + if (getClass().desiredAssertionStatus()) args.add("-ea"); + else args.add("-da"); + args.add("--enable-preview"); args.addAll(processId.envOpts()); args.add(processId.mainClass); From fc00701a1ea1794ea260007559a345e5ce358116 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 11:05:10 +0100 Subject: [PATCH 34/56] (index) Experimental refactoring of the indexing functionality --- .../api/searchquery/QueryProtobufCodec.java | 5 +- .../results/DecoratedSearchResultItem.java | 12 +- .../model/results/SearchResultItem.java | 8 +- .../results/SearchResultKeywordScore.java | 17 +- .../results/SearchResultPreliminaryScore.java | 4 - .../marginalia/index/ReverseIndexReader.java | 5 +- ...ServicesFactory.java => IndexFactory.java} | 22 ++- ...GrpcService.java => IndexGrpcService.java} | 178 +++++++++++++++--- .../marginalia/index/ResultPriorityQueue.java | 107 +++++++++++ .../nu/marginalia/index/SearchTermsUtil.java | 88 --------- .../index/index/CombinedIndexReader.java | 134 +++++++++++-- .../index/index/IndexQueryBuilderImpl.java | 104 +--------- .../index/index/IndexQueryService.java | 143 ++++++-------- .../marginalia/index/index/StatefulIndex.java | 25 ++- .../index/model/IndexSearchTerms.java | 38 ---- ...IndexQueryParams.java => QueryParams.java} | 14 +- ...hParameters.java => SearchParameters.java} | 38 ++-- .../marginalia/index/model/SearchTerms.java | 70 +++++++ .../index/model/SearchTermsUtil.java | 33 ++++ .../index/results/IndexMetadataService.java | 141 +++----------- .../results/IndexResultValuationContext.java | 58 +++--- .../results/IndexResultValuatorService.java | 55 +++--- .../index/results/model/QuerySearchTerms.java | 23 +++ .../results/model/TermCoherenceGroupList.java | 49 +++++ .../TermMetadataForCombinedDocumentIds.java | 36 ++++ .../results/model/ids/CombinedDocIdList.java | 52 +++++ .../index/results/model/ids/DocIdList.java | 49 +++++ .../results/model/ids/DocMetadataList.java | 45 +++++ .../index/results/model/ids/TermIdList.java | 45 +++++ .../index/searchset/SearchSetsService.java | 16 +- .../ranking/results/ResultValuator.java | 47 +---- .../results/factors/PriorityTermBonus.java | 21 --- .../nu/marginalia/index/query/IndexQuery.java | 4 + .../index/query/IndexSearchBudget.java | 1 + ...IndexQueryServiceIntegrationSmokeTest.java | 4 +- .../IndexQueryServiceIntegrationTest.java | 2 +- .../ranking/results/ResultValuatorTest.java | 9 +- .../factors/TermCoherenceFactorTest.java | 2 +- .../nu/marginalia/index/IndexService.java | 2 +- 39 files changed, 1033 insertions(+), 673 deletions(-) rename code/index/java/nu/marginalia/index/{IndexServicesFactory.java => IndexFactory.java} (93%) rename code/index/java/nu/marginalia/index/{IndexQueryGrpcService.java => IndexGrpcService.java} (50%) create mode 100644 code/index/java/nu/marginalia/index/ResultPriorityQueue.java delete mode 100644 code/index/java/nu/marginalia/index/SearchTermsUtil.java delete mode 100644 code/index/java/nu/marginalia/index/model/IndexSearchTerms.java rename code/index/java/nu/marginalia/index/model/{IndexQueryParams.java => QueryParams.java} (71%) rename code/index/java/nu/marginalia/index/model/{IndexSearchParameters.java => SearchParameters.java} (72%) create mode 100644 code/index/java/nu/marginalia/index/model/SearchTerms.java create mode 100644 code/index/java/nu/marginalia/index/model/SearchTermsUtil.java create mode 100644 code/index/java/nu/marginalia/index/results/model/QuerySearchTerms.java create mode 100644 code/index/java/nu/marginalia/index/results/model/TermCoherenceGroupList.java create mode 100644 code/index/java/nu/marginalia/index/results/model/TermMetadataForCombinedDocumentIds.java create mode 100644 code/index/java/nu/marginalia/index/results/model/ids/CombinedDocIdList.java create mode 100644 code/index/java/nu/marginalia/index/results/model/ids/DocIdList.java create mode 100644 code/index/java/nu/marginalia/index/results/model/ids/DocMetadataList.java create mode 100644 code/index/java/nu/marginalia/index/results/model/ids/TermIdList.java delete mode 100644 code/index/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java index 735c4034..58550362 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java @@ -144,7 +144,7 @@ public class QueryProtobufCodec { rawItem.getCombinedId(), keywordScores, rawItem.getResultsFromDomain(), - null + Double.NaN // Not set ); } @@ -154,8 +154,7 @@ public class QueryProtobufCodec { keywordScores.getKeyword(), keywordScores.getEncodedWordMetadata(), keywordScores.getEncodedDocMetadata(), - keywordScores.getHtmlFeatures(), - keywordScores.getHasPriorityTerms() + keywordScores.getHtmlFeatures() ); } diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java index 0a3f1f3f..b099dc01 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/DecoratedSearchResultItem.java @@ -10,7 +10,7 @@ import java.util.List; @Getter @ToString -public class DecoratedSearchResultItem { +public class DecoratedSearchResultItem implements Comparable { public final SearchResultItem rawIndexResult; @NotNull @@ -79,4 +79,14 @@ public class DecoratedSearchResultItem { this.wordsTotal = wordsTotal; this.rankingScore = rankingScore; } + + @Override + public int compareTo(@NotNull DecoratedSearchResultItem o) { + int diff = Double.compare(rankingScore, o.rankingScore); + + if (diff == 0) + diff = Long.compare(documentId(), o.documentId()); + + return diff; + } } diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java index 523d2fb5..fc6633a2 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java @@ -36,11 +36,11 @@ public class SearchResultItem implements Comparable { } /* Used for evaluation */ - private transient SearchResultPreliminaryScore scoreValue = null; - public void setScore(SearchResultPreliminaryScore score) { + private transient double scoreValue = Double.MAX_VALUE; + public void setScore(double score) { scoreValue = score; } - public SearchResultPreliminaryScore getScore() { + public double getScore() { return scoreValue; } @@ -70,7 +70,7 @@ public class SearchResultItem implements Comparable { @Override public int compareTo(@NotNull SearchResultItem o) { // this looks like a bug, but we actually want this in a reversed order - int diff = o.getScore().compareTo(getScore()); + int diff = Double.compare(o.getScore(), getScore()); if (diff != 0) return diff; diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java index e4402ce0..b84dad0b 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultKeywordScore.java @@ -11,7 +11,6 @@ public final class SearchResultKeywordScore { public final String keyword; private final long encodedWordMetadata; private final long encodedDocMetadata; - private final boolean hasPriorityTerms; private final int htmlFeatures; @@ -19,14 +18,12 @@ public final class SearchResultKeywordScore { String keyword, long encodedWordMetadata, long encodedDocMetadata, - int htmlFeatures, - boolean hasPriorityTerms) { + int htmlFeatures) { this.subquery = subquery; this.keyword = keyword; this.encodedWordMetadata = encodedWordMetadata; this.encodedDocMetadata = encodedDocMetadata; this.htmlFeatures = htmlFeatures; - this.hasPriorityTerms = hasPriorityTerms; } public boolean hasTermFlag(WordFlags flag) { @@ -65,10 +62,6 @@ public final class SearchResultKeywordScore { return htmlFeatures; } - public boolean hasPriorityTerms() { - return hasPriorityTerms; - } - @Override public boolean equals(Object obj) { if (obj == this) return true; @@ -77,13 +70,12 @@ public final class SearchResultKeywordScore { return this.subquery == that.subquery && Objects.equals(this.keyword, that.keyword) && this.encodedWordMetadata == that.encodedWordMetadata && - this.encodedDocMetadata == that.encodedDocMetadata && - this.hasPriorityTerms == that.hasPriorityTerms; + this.encodedDocMetadata == that.encodedDocMetadata; } @Override public int hashCode() { - return Objects.hash(subquery, keyword, encodedWordMetadata, encodedDocMetadata, hasPriorityTerms); + return Objects.hash(subquery, keyword, encodedWordMetadata, encodedDocMetadata); } @Override @@ -92,8 +84,7 @@ public final class SearchResultKeywordScore { "set=" + subquery + ", " + "keyword=" + keyword + ", " + "encodedWordMetadata=" + new WordMetadata(encodedWordMetadata) + ", " + - "encodedDocMetadata=" + new DocumentMetadata(encodedDocMetadata) + ", " + - "hasPriorityTerms=" + hasPriorityTerms + ']'; + "encodedDocMetadata=" + new DocumentMetadata(encodedDocMetadata) + ']'; } } diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java index 287db40a..81e16474 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultPreliminaryScore.java @@ -6,7 +6,6 @@ import static java.lang.Boolean.compare; import static java.lang.Double.compare; public record SearchResultPreliminaryScore( - boolean hasPriorityTerm, double searchRankingScore) implements Comparable { @@ -18,9 +17,6 @@ public record SearchResultPreliminaryScore( public int compareTo(@NotNull SearchResultPreliminaryScore other) { int diff; - diff = PREFER_HIGH * compare(hasPriorityTerm, other.hasPriorityTerm); - if (diff != 0) return diff; - return PREFER_LOW * compare(searchRankingScore, other.searchRankingScore); } diff --git a/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexReader.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexReader.java index 1bcfee5a..f37420dd 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexReader.java +++ b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexReader.java @@ -16,6 +16,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.concurrent.Executors; public class ReverseIndexReader { @@ -123,13 +124,13 @@ public class ReverseIndexReader { return new long[docIds.length]; } - assert isSorted(docIds) : "The input array docIds is assumed to be sorted"; + assert isUniqueAndSorted(docIds) : "The input array docIds is assumed to be unique and sorted, was " + Arrays.toString(docIds); var reader = createReaderNew(offset); return reader.queryData(docIds, 1); } - private boolean isSorted(long[] ids) { + private boolean isUniqueAndSorted(long[] ids) { if (ids.length == 0) return true; long prev = ids[0]; diff --git a/code/index/java/nu/marginalia/index/IndexServicesFactory.java b/code/index/java/nu/marginalia/index/IndexFactory.java similarity index 93% rename from code/index/java/nu/marginalia/index/IndexServicesFactory.java rename to code/index/java/nu/marginalia/index/IndexFactory.java index b59e7960..48911546 100644 --- a/code/index/java/nu/marginalia/index/IndexServicesFactory.java +++ b/code/index/java/nu/marginalia/index/IndexFactory.java @@ -16,18 +16,25 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; @Singleton -public class IndexServicesFactory { - private final Logger logger = LoggerFactory.getLogger(getClass()); +public class IndexFactory { private final FileStorageService fileStorageService; private final Path liveStorage; @Inject - public IndexServicesFactory(FileStorageService fileStorageService) { + public IndexFactory(FileStorageService fileStorageService) { this.fileStorageService = fileStorageService; this.liveStorage = IndexLocations.getCurrentIndex(fileStorageService); } + public CombinedIndexReader getCombinedIndexReader() throws IOException { + return new CombinedIndexReader( + getForwardIndexReader(), + getReverseIndexReader(), + getReverseIndexPrioReader() + ); + } + public Path getSearchSetsBase() { return IndexLocations.getSearchSetsPath(fileStorageService); } @@ -54,6 +61,7 @@ public class IndexServicesFactory { ); } + /** Switches the current index to the next index */ public void switchFiles() throws IOException { for (var file : ReverseIndexFullFileNames.FileIdentifier.values()) { @@ -82,11 +90,5 @@ public class IndexServicesFactory { } } - public CombinedIndexReader getSearchIndexReader() throws IOException { - return new CombinedIndexReader( - getForwardIndexReader(), - getReverseIndexReader(), - getReverseIndexPrioReader() - ); - } + } diff --git a/code/index/java/nu/marginalia/index/IndexQueryGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java similarity index 50% rename from code/index/java/nu/marginalia/index/IndexQueryGrpcService.java rename to code/index/java/nu/marginalia/index/IndexGrpcService.java index 8524d5b4..d0d2da1f 100644 --- a/code/index/java/nu/marginalia/index/IndexQueryGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -1,5 +1,6 @@ package nu.marginalia.index; +import com.google.common.collect.MinMaxPriorityQueue; import com.google.inject.Inject; import com.google.inject.Singleton; import io.grpc.stub.StreamObserver; @@ -10,13 +11,13 @@ import lombok.SneakyThrows; import nu.marginalia.api.searchquery.*; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.query.SearchSubquery; -import nu.marginalia.api.searchquery.model.results.ResultRankingContext; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.api.searchquery.model.results.SearchResultSet; +import nu.marginalia.api.searchquery.model.results.*; import nu.marginalia.index.index.IndexQueryService; import nu.marginalia.index.index.StatefulIndex; -import nu.marginalia.index.model.IndexSearchParameters; +import nu.marginalia.index.model.SearchParameters; +import nu.marginalia.index.model.SearchTermsUtil; import nu.marginalia.index.results.IndexResultValuatorService; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; import nu.marginalia.index.searchset.SearchSetsService; import nu.marginalia.index.searchset.SmallSearchSet; import nu.marginalia.index.searchset.SearchSet; @@ -28,9 +29,14 @@ import org.slf4j.MarkerFactory; import java.sql.SQLException; import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; @Singleton -public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { +public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -66,11 +72,11 @@ public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { @Inject - public IndexQueryGrpcService(ServiceConfiguration serviceConfiguration, - StatefulIndex index, - SearchSetsService searchSetsService, - IndexQueryService indexQueryService, - IndexResultValuatorService resultValuator) + public IndexGrpcService(ServiceConfiguration serviceConfiguration, + StatefulIndex index, + SearchSetsService searchSetsService, + IndexQueryService indexQueryService, + IndexResultValuatorService resultValuator) { this.nodeId = serviceConfiguration.node(); this.index = index; @@ -85,14 +91,17 @@ public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { StreamObserver responseObserver) { try { - var params = new IndexSearchParameters(request, getSearchSet(request)); - + var params = new SearchParameters(request, getSearchSet(request)); final String nodeName = Integer.toString(nodeId); SearchResultSet results = wmsa_query_time .labels(nodeName, "GRPC") - .time(() -> executeSearch(params)); + .time(() -> { + // Perform the search + return executeSearch(params); + }); + // Prometheus bookkeeping wmsa_query_cost .labels(nodeName, "GRPC") .set(params.getDataCost()); @@ -103,6 +112,7 @@ public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { .inc(); } + // Send the results back to the client for (var result : results.results) { var rawResult = result.rawIndexResult; @@ -118,7 +128,6 @@ public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { .setEncodedWordMetadata(score.encodedWordMetadata()) .setKeyword(score.keyword) .setHtmlFeatures(score.htmlFeatures()) - .setHasPriorityTerms(score.hasPriorityTerms()) .setSubquery(score.subquery) ); } @@ -151,7 +160,7 @@ public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { // exists for test access @SneakyThrows SearchResultSet justQuery(SearchSpecification specsSet) { - return executeSearch(new IndexSearchParameters(specsSet, getSearchSet(specsSet))); + return executeSearch(new SearchParameters(specsSet, getSearchSet(specsSet))); } private SearchSet getSearchSet(SearchSpecification specsSet) { @@ -171,23 +180,148 @@ public class IndexQueryGrpcService extends IndexApiGrpc.IndexApiImplBase { return searchSetsService.getSearchSetByName(request.getSearchSetIdentifier()); } - private SearchResultSet executeSearch(IndexSearchParameters params) throws SQLException { + + private SearchResultSet executeSearch(SearchParameters params) throws SQLException { if (!index.isLoaded()) { // Short-circuit if the index is not loaded, as we trivially know that there can be no results return new SearchResultSet(List.of()); } - var rankingContext = createRankingContext(params.rankingParams, params.subqueries); + ResultRankingContext rankingContext = createRankingContext(params.rankingParams, params.subqueries); logger.info(queryMarker, "{}", params.queryParams); - var resultIds = indexQueryService.evaluateSubqueries(params); - var resultItems = resultValuator.findBestResults(params, - rankingContext, - resultIds); + return new QueryExecution(rankingContext, params.fetchSize) + .run(params); + } + + private class QueryExecution { + private static final Executor queryExecutor = Executors.newCachedThreadPool(); + private static final Executor rankingExecutor = Executors.newCachedThreadPool(); + private final ArrayBlockingQueue resultQueue = new ArrayBlockingQueue<>(8); + private final ResultPriorityQueue resultHeap; + private final ResultRankingContext resultRankingContext; + + private final AtomicInteger remainingIndexTasks = new AtomicInteger(0); + private final AtomicInteger remainingValuationTasks = new AtomicInteger(0); + + private QueryExecution(ResultRankingContext resultRankingContext, int maxResults) { + this.resultRankingContext = resultRankingContext; + this.resultHeap = new ResultPriorityQueue(maxResults); + } + + public SearchResultSet run(SearchParameters parameters) throws SQLException { + for (var subquery : parameters.subqueries) { + queryExecutor.execute(new IndexLookup(subquery, parameters)); + } + + for (int i = 0; i < 16; i++) { + rankingExecutor.execute(new ResultRanker(parameters, resultRankingContext)); + } + + // Wait for all tasks to complete + synchronized (remainingValuationTasks) { + while (remainingValuationTasks.get() > 0) { + try { + remainingValuationTasks.wait(20); + } + catch (InterruptedException e) { + logger.warn("Interrupted while waiting for tasks to complete", e); + } + } + } + + return new SearchResultSet(resultValuator.selectBestResults(parameters, resultRankingContext, resultHeap)); + } + + class IndexLookup implements Runnable { + private final SearchSubquery subquery; + private final SearchParameters parameters; + + IndexLookup(SearchSubquery subquery, SearchParameters parameters) { + this.subquery = subquery; + this.parameters = parameters; + + logger.info("Starting index task"); + + remainingIndexTasks.incrementAndGet(); + } + + public void run() { + try { + indexQueryService.evaluateSubquery( + subquery, + parameters.queryParams, + parameters.budget, + parameters.fetchSize, + this::drain + ); + } + finally { + synchronized (remainingIndexTasks) { + if (remainingIndexTasks.decrementAndGet() == 0) { + remainingIndexTasks.notifyAll(); + } + } + logger.info("Terminating index task"); + } + } + + private void drain(CombinedDocIdList resultIds) { + long remainingTime = parameters.budget.timeLeft(); + + try { + resultQueue.offer(resultIds, remainingTime, TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + logger.warn("Interrupted while waiting to offer resultIds to queue", e); + } + } + } + + class ResultRanker implements Runnable { + private final SearchParameters parameters; + private final ResultRankingContext rankingContext; + + ResultRanker(SearchParameters parameters, ResultRankingContext rankingContext) { + this.parameters = parameters; + this.rankingContext = rankingContext; + + remainingValuationTasks.incrementAndGet(); + } + + public void run() { + try { + while (parameters.budget.timeLeft() > 0) { + + CombinedDocIdList resultIds = resultQueue.poll( + Math.clamp(parameters.budget.timeLeft(), 1, 25), + TimeUnit.MILLISECONDS); + if (resultIds == null) { + if (remainingIndexTasks.get() == 0 && resultQueue.isEmpty()) + break; + else + continue; + } + + var bestResults = resultValuator.rankResults(parameters, rankingContext, resultIds); + + resultHeap.addAll(bestResults); + } + } + catch (Exception e) { + logger.warn("Interrupted while waiting to poll resultIds from queue", e); + } + finally { + synchronized (remainingValuationTasks) { + if (remainingValuationTasks.decrementAndGet() == 0) + remainingValuationTasks.notifyAll(); + } + } + } + } - return new SearchResultSet(resultItems); } private ResultRankingContext createRankingContext(ResultRankingParameters rankingParams, List subqueries) { diff --git a/code/index/java/nu/marginalia/index/ResultPriorityQueue.java b/code/index/java/nu/marginalia/index/ResultPriorityQueue.java new file mode 100644 index 00000000..d84c43d0 --- /dev/null +++ b/code/index/java/nu/marginalia/index/ResultPriorityQueue.java @@ -0,0 +1,107 @@ +package nu.marginalia.index; + +import it.unimi.dsi.fastutil.longs.LongOpenHashSet; +import nu.marginalia.api.searchquery.model.results.SearchResultItem; +import org.jetbrains.annotations.NotNull; + +import java.util.*; + +public class ResultPriorityQueue implements Iterable, + Collection { + private final int limit; + private final ArrayList backingList = new ArrayList<>(); + private final LongOpenHashSet idsInSet = new LongOpenHashSet(); + + public ResultPriorityQueue(int limit) { + this.limit = limit; + } + + public Iterator iterator() { + return backingList.iterator(); + } + + @NotNull + @Override + public Object[] toArray() { + return backingList.toArray(); + } + + @NotNull + @Override + public T[] toArray(@NotNull T[] a) { + return backingList.toArray(a); + } + + @Override + public boolean add(SearchResultItem searchResultItem) { + throw new UnsupportedOperationException("Use addAll instead ya dingus"); + } + + @Override + public boolean remove(Object o) { + if (o instanceof SearchResultItem sri) { + idsInSet.remove(sri.getDocumentId()); + return idsInSet.remove(sri.getDocumentId()); + } + throw new IllegalArgumentException("Object is not a SearchResultItem"); + } + + @Override + public boolean containsAll(@NotNull Collection c) { + return idsInSet.containsAll(c); + } + + /** Adds all items to the queue, and returns true if any items were added. + * This is a thread-safe operation. + */ + @Override + public synchronized boolean addAll(@NotNull Collection items) { + boolean itemsAdded = false; + for (var item: items) { + if (idsInSet.add(item.getDocumentId())) { + backingList.add(item); + itemsAdded = true; + } + } + if (!itemsAdded) { + return false; + } + + backingList.sort(Comparator.naturalOrder()); + if (backingList.size() > limit) { + backingList.subList(limit, backingList.size()).clear(); + } + + return true; + } + + @Override + public boolean removeAll(@NotNull Collection c) { + return backingList.removeAll(c); + } + + @Override + public boolean retainAll(@NotNull Collection c) { + return backingList.retainAll(c); + } + + @Override + public void clear() { + + } + + public int size() { + return backingList.size(); + } + + @Override + public boolean isEmpty() { + return backingList.isEmpty(); + } + + @Override + public boolean contains(Object o) { + return backingList.contains(o); + } + +} diff --git a/code/index/java/nu/marginalia/index/SearchTermsUtil.java b/code/index/java/nu/marginalia/index/SearchTermsUtil.java deleted file mode 100644 index 6aa4592e..00000000 --- a/code/index/java/nu/marginalia/index/SearchTermsUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -package nu.marginalia.index; - -import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.longs.LongList; -import nu.marginalia.api.searchquery.model.query.SearchSubquery; -import nu.marginalia.index.model.IndexSearchTerms; -import nu.marginalia.hash.MurmurHash3_128; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class SearchTermsUtil { - - /** Extract the search terms from the specified subquery. */ - public static IndexSearchTerms extractSearchTerms(SearchSubquery request) { - final LongList excludes = new LongArrayList(); - final LongList includes = new LongArrayList(); - final LongList priority = new LongArrayList(); - final List coherences = new ArrayList<>(); - - if (!addEachTerm(includes, request.searchTermsInclude)) { - return new IndexSearchTerms(); - } - - // This looks like a bug, but it's not - // v--- ----v - if (!addEachTerm(includes, request.searchTermsAdvice)) { - return new IndexSearchTerms(); - } - - for (var coherence : request.searchTermCoherences) { - LongList parts = new LongArrayList(coherence.size()); - - if (!addEachTerm(parts, coherence)) { - return new IndexSearchTerms(); - } - - coherences.add(parts); - } - - // we don't care if we can't find these: - addEachNonMandatoryTerm(excludes, request.searchTermsExclude); - addEachNonMandatoryTerm(priority, request.searchTermsPriority); - - return new IndexSearchTerms(includes, excludes, priority, coherences); - } - - private static boolean addEachTerm(LongList ret, List words) { - boolean success = true; - - for (var word : words) { - ret.add(getWordId(word)); - } - - return success; - } - - private static void addEachNonMandatoryTerm(LongList ret, List words) { - for (var word : words) { - ret.add(getWordId(word)); - } - } - - - /** Extract all include-terms from the specified subqueries, - * and a return a map of the terms and their termIds. - */ - public static Map getAllIncludeTerms(List subqueries) { - Map ret = new HashMap<>(); - - for (var subquery : subqueries) { - for (var include : subquery.searchTermsInclude) { - ret.computeIfAbsent(include, i -> getWordId(include)); - } - } - - return ret; - } - - private static final MurmurHash3_128 hasher = new MurmurHash3_128(); - - /** Translate the word to a unique id. */ - public static long getWordId(String s) { - return hasher.hashKeyword(s); - } -} diff --git a/code/index/java/nu/marginalia/index/index/CombinedIndexReader.java b/code/index/java/nu/marginalia/index/index/CombinedIndexReader.java index aaed1269..01a94362 100644 --- a/code/index/java/nu/marginalia/index/index/CombinedIndexReader.java +++ b/code/index/java/nu/marginalia/index/index/CombinedIndexReader.java @@ -1,10 +1,17 @@ package nu.marginalia.index.index; -import nu.marginalia.index.model.IndexQueryParams; import nu.marginalia.index.ReverseIndexReader; import nu.marginalia.index.forward.ForwardIndexReader; -import nu.marginalia.index.query.*; +import nu.marginalia.index.model.QueryParams; +import nu.marginalia.index.query.IndexQuery; +import nu.marginalia.index.query.IndexQueryBuilder; +import nu.marginalia.index.query.IndexQueryPriority; import nu.marginalia.index.query.filter.QueryFilterStepIf; +import nu.marginalia.index.query.limit.SpecificationLimitType; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; +import nu.marginalia.index.results.model.ids.DocMetadataList; +import nu.marginalia.model.id.UrlIdCodec; +import nu.marginalia.model.idx.DocumentMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,28 +36,34 @@ public class CombinedIndexReader { this.reverseIndexPriorityReader = reverseIndexPriorityReader; } + public IndexQueryBuilderImpl newQueryBuilder(IndexQuery query) { + return new IndexQueryBuilderImpl(reverseIndexFullReader, reverseIndexPriorityReader, query); + } + /** Creates a query builder for terms in the priority index */ - public IndexQueryBuilder findPriorityWord(IndexQueryPriority priority, long wordId, int fetchSizeMultiplier) { - return new IndexQueryBuilderImpl(reverseIndexFullReader, reverseIndexPriorityReader, - new IndexQuery( + public IndexQueryBuilder findPriorityWord(IndexQueryPriority priority, + long wordId, + int fetchSizeMultiplier) { + return newQueryBuilder(new IndexQuery( List.of(reverseIndexPriorityReader.documents(wordId)), priority, - fetchSizeMultiplier), wordId); + fetchSizeMultiplier)) + .withSourceTerms(wordId); } /** Creates a query builder for terms in the full index */ public IndexQueryBuilder findFullWord(IndexQueryPriority priority, long wordId, int fetchSizeMultiplier) { - return new IndexQueryBuilderImpl(reverseIndexFullReader, reverseIndexPriorityReader, + return newQueryBuilder( new IndexQuery(List.of(reverseIndexFullReader.documents(wordId)), priority, - fetchSizeMultiplier), - wordId); + fetchSizeMultiplier)) + .withSourceTerms(wordId); } /** Creates a parameter matching filter step for the provided parameters */ - public QueryFilterStepIf filterForParams(IndexQueryParams params) { - return new IndexQueryBuilderImpl.ParamMatchingQueryFilter(params, forwardIndexReader); + public QueryFilterStepIf filterForParams(QueryParams params) { + return new ParamMatchingQueryFilter(params, forwardIndexReader); } /** Returns the number of occurrences of the word in the full index */ @@ -64,8 +77,8 @@ public class CombinedIndexReader { } /** Retrieves the term metadata for the specified word for the provided documents */ - public long[] getMetadata(long wordId, long[] docIds) { - return reverseIndexFullReader.getTermMeta(wordId, docIds); + public DocMetadataList getMetadata(long wordId, CombinedDocIdList docIds) { + return new DocMetadataList(reverseIndexFullReader.getTermMeta(wordId, docIds.array())); } /** Retrieves the document metadata for the specified document */ @@ -117,3 +130,98 @@ public class CombinedIndexReader { return forwardIndexReader.isLoaded(); } } + +class ParamMatchingQueryFilter implements QueryFilterStepIf { + private final QueryParams params; + private final ForwardIndexReader forwardIndexReader; + + public ParamMatchingQueryFilter(QueryParams params, + ForwardIndexReader forwardIndexReader) + { + this.params = params; + this.forwardIndexReader = forwardIndexReader; + } + + @Override + public boolean test(long combinedId) { + long docId = UrlIdCodec.removeRank(combinedId); + int domainId = UrlIdCodec.getDomainId(docId); + + long meta = forwardIndexReader.getDocMeta(docId); + + if (!validateDomain(domainId, meta)) { + return false; + } + + if (!validateQuality(meta)) { + return false; + } + + if (!validateYear(meta)) { + return false; + } + + if (!validateSize(meta)) { + return false; + } + + if (!validateRank(meta)) { + return false; + } + + return true; + } + + private boolean validateDomain(int domainId, long meta) { + return params.searchSet().contains(domainId, meta); + } + + private boolean validateQuality(long meta) { + final var limit = params.qualityLimit(); + + if (limit.type() == SpecificationLimitType.NONE) { + return true; + } + + final int quality = DocumentMetadata.decodeQuality(meta); + + return limit.test(quality); + } + + private boolean validateYear(long meta) { + if (params.year().type() == SpecificationLimitType.NONE) + return true; + + int postVal = DocumentMetadata.decodeYear(meta); + + return params.year().test(postVal); + } + + private boolean validateSize(long meta) { + if (params.size().type() == SpecificationLimitType.NONE) + return true; + + int postVal = DocumentMetadata.decodeSize(meta); + + return params.size().test(postVal); + } + + private boolean validateRank(long meta) { + if (params.rank().type() == SpecificationLimitType.NONE) + return true; + + int postVal = DocumentMetadata.decodeRank(meta); + + return params.rank().test(postVal); + } + + @Override + public double cost() { + return 32; + } + + @Override + public String describe() { + return getClass().getSimpleName(); + } +} \ No newline at end of file diff --git a/code/index/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java b/code/index/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java index e159adc2..825728ae 100644 --- a/code/index/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java +++ b/code/index/java/nu/marginalia/index/index/IndexQueryBuilderImpl.java @@ -1,15 +1,10 @@ package nu.marginalia.index.index; import gnu.trove.set.hash.TLongHashSet; -import nu.marginalia.index.model.IndexQueryParams; import nu.marginalia.index.ReverseIndexReader; -import nu.marginalia.index.forward.ForwardIndexReader; import nu.marginalia.index.query.IndexQuery; import nu.marginalia.index.query.IndexQueryBuilder; import nu.marginalia.index.query.filter.QueryFilterStepIf; -import nu.marginalia.index.query.limit.SpecificationLimitType; -import nu.marginalia.model.id.UrlIdCodec; -import nu.marginalia.model.idx.DocumentMetadata; public class IndexQueryBuilderImpl implements IndexQueryBuilder { private final IndexQuery query; @@ -26,13 +21,17 @@ public class IndexQueryBuilderImpl implements IndexQueryBuilder { IndexQueryBuilderImpl(ReverseIndexReader reverseIndexFullReader, ReverseIndexReader reverseIndexPrioReader, - IndexQuery query, long... sourceTerms) + IndexQuery query) { this.query = query; this.reverseIndexFullReader = reverseIndexFullReader; this.reverseIndexPrioReader = reverseIndexPrioReader; + } + public IndexQueryBuilder withSourceTerms(long... sourceTerms) { alreadyConsideredTerms.addAll(sourceTerms); + + return this; } public IndexQueryBuilder alsoFull(long termId) { @@ -71,98 +70,5 @@ public class IndexQueryBuilderImpl implements IndexQueryBuilder { return query; } - public static class ParamMatchingQueryFilter implements QueryFilterStepIf { - private final IndexQueryParams params; - private final ForwardIndexReader forwardIndexReader; - public ParamMatchingQueryFilter(IndexQueryParams params, - ForwardIndexReader forwardIndexReader) - { - this.params = params; - this.forwardIndexReader = forwardIndexReader; - } - - @Override - public boolean test(long combinedId) { - long docId = UrlIdCodec.removeRank(combinedId); - int domainId = UrlIdCodec.getDomainId(docId); - - long meta = forwardIndexReader.getDocMeta(docId); - - if (!validateDomain(domainId, meta)) { - return false; - } - - if (!validateQuality(meta)) { - return false; - } - - if (!validateYear(meta)) { - return false; - } - - if (!validateSize(meta)) { - return false; - } - - if (!validateRank(meta)) { - return false; - } - - return true; - } - - private boolean validateDomain(int domainId, long meta) { - return params.searchSet().contains(domainId, meta); - } - - private boolean validateQuality(long meta) { - final var limit = params.qualityLimit(); - - if (limit.type() == SpecificationLimitType.NONE) { - return true; - } - - final int quality = DocumentMetadata.decodeQuality(meta); - - return limit.test(quality); - } - - private boolean validateYear(long meta) { - if (params.year().type() == SpecificationLimitType.NONE) - return true; - - int postVal = DocumentMetadata.decodeYear(meta); - - return params.year().test(postVal); - } - - private boolean validateSize(long meta) { - if (params.size().type() == SpecificationLimitType.NONE) - return true; - - int postVal = DocumentMetadata.decodeSize(meta); - - return params.size().test(postVal); - } - - private boolean validateRank(long meta) { - if (params.rank().type() == SpecificationLimitType.NONE) - return true; - - int postVal = DocumentMetadata.decodeRank(meta); - - return params.rank().test(postVal); - } - - @Override - public double cost() { - return 32; - } - - @Override - public String describe() { - return getClass().getSimpleName(); - } - } } diff --git a/code/index/java/nu/marginalia/index/index/IndexQueryService.java b/code/index/java/nu/marginalia/index/index/IndexQueryService.java index 49417539..53eb0bb4 100644 --- a/code/index/java/nu/marginalia/index/index/IndexQueryService.java +++ b/code/index/java/nu/marginalia/index/index/IndexQueryService.java @@ -2,32 +2,27 @@ package nu.marginalia.index.index; import com.google.inject.Inject; import com.google.inject.Singleton; -import gnu.trove.list.TLongList; -import gnu.trove.list.array.TLongArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.array.buffer.LongQueryBuffer; -import nu.marginalia.index.model.IndexSearchTerms; -import nu.marginalia.index.model.IndexSearchParameters; -import nu.marginalia.index.SearchTermsUtil; +import nu.marginalia.index.model.QueryParams; +import nu.marginalia.index.model.SearchTerms; import nu.marginalia.index.query.IndexQuery; -import nu.marginalia.index.query.IndexQueryPriority; +import nu.marginalia.index.query.IndexSearchBudget; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; +import org.roaringbitmap.longlong.Roaring64Bitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; import org.slf4j.MarkerFactory; import java.util.List; +import java.util.function.Consumer; @Singleton public class IndexQueryService { private final Marker queryMarker = MarkerFactory.getMarker("QUERY"); - /** Execute subqueries and return a list of document ids. The index is queried for each subquery, - * at different priorty depths until timeout is reached or the results are all visited. - *
- * Then the results are combined. - * */ - private final ThreadLocal resultsArrayListPool = ThreadLocal.withInitial(TLongArrayList::new); private static final Logger logger = LoggerFactory.getLogger(IndexQueryService.class); private final StatefulIndex index; @@ -40,84 +35,66 @@ public class IndexQueryService { * at different priorty depths until timeout is reached or the results are all visited. * Then the results are combined. * */ - public TLongList evaluateSubqueries(IndexSearchParameters params) { - final TLongArrayList results = resultsArrayListPool.get(); - results.resetQuick(); - results.ensureCapacity(params.fetchSize); - + public void evaluateSubquery(SearchSubquery subquery, + QueryParams queryParams, + IndexSearchBudget timeout, + int fetchSize, + Consumer drain) + { // These queries are various term combinations - for (var subquery : params.subqueries) { - if (!params.hasTimeLeft()) { - logger.info("Query timed out {}, ({}), -{}", - subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); - break; - } + if (!timeout.hasTimeLeft()) { + logger.info("Query timed out {}, ({}), -{}", + subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); + return; + } + logger.info(queryMarker, "{}", subquery); - logger.info(queryMarker, "{}", subquery); - - final IndexSearchTerms searchTerms = SearchTermsUtil.extractSearchTerms(subquery); - - if (searchTerms.isEmpty()) { - logger.info(queryMarker, "empty"); - continue; - } - - // logSearchTerms(subquery, searchTerms); - - // These queries are different indices for one subquery - List queries = params.createIndexQueries(index, searchTerms); - for (var query : queries) { - - if (!params.hasTimeLeft()) - break; - - if (shouldOmitQuery(params, query, results.size())) { - logger.info(queryMarker, "Omitting {}", query); - continue; - } - - final int fetchSize = params.fetchSize * query.fetchSizeMultiplier; - final LongQueryBuffer buffer = new LongQueryBuffer(fetchSize); - - int cnt = 0; - - while (query.hasMore() - && results.size() < fetchSize - && params.budget.hasTimeLeft()) - { - buffer.reset(); - query.getMoreResults(buffer); - - for (int i = 0; i < buffer.size() && results.size() < fetchSize; i++) { - results.add(buffer.data[i]); - cnt++; - } - } - - params.dataCost += query.dataCost(); - - - logger.info(queryMarker, "{} from {}", cnt, query); - } + final SearchTerms searchTerms = new SearchTerms(subquery); + if (searchTerms.isEmpty()) { + logger.info(queryMarker, "empty"); + return; } - return results; + final Roaring64Bitmap results = new Roaring64Bitmap(); + + // logSearchTerms(subquery, searchTerms); + + // These queries are different indices for one subquery + List queries = index.createQueries(searchTerms, queryParams); + for (var query : queries) { + + if (!timeout.hasTimeLeft()) + break; + + final LongQueryBuffer buffer = new LongQueryBuffer(512); + + while (query.hasMore() + && results.getIntCardinality() < fetchSize * query.fetchSizeMultiplier + && timeout.hasTimeLeft()) + { + buffer.reset(); + query.getMoreResults(buffer); + + for (int i = 0; i < buffer.size(); i++) { + results.add(buffer.data[i]); + } + + if (results.getIntCardinality() > 512) { + drain.accept(new CombinedDocIdList(results)); + results.clear(); + } + } + + logger.info(queryMarker, "{} from {}", results.getIntCardinality(), query); + } + + if (!results.isEmpty()) { + drain.accept(new CombinedDocIdList(results)); + } } - /** @see IndexQueryPriority */ - private boolean shouldOmitQuery(IndexSearchParameters params, IndexQuery query, int resultCount) { - - var priority = query.queryPriority; - - return switch (priority) { - case IndexQueryPriority.BEST -> false; - case IndexQueryPriority.GOOD -> resultCount > params.fetchSize / 4; - case IndexQueryPriority.FALLBACK -> resultCount > params.fetchSize / 8; - }; - } - - private void logSearchTerms(SearchSubquery subquery, IndexSearchTerms searchTerms) { + private void logSearchTerms(SearchSubquery subquery, SearchTerms searchTerms) { // This logging should only be enabled in testing, as it is very verbose // and contains sensitive information diff --git a/code/index/java/nu/marginalia/index/index/StatefulIndex.java b/code/index/java/nu/marginalia/index/index/StatefulIndex.java index f2e006d0..ae2a8f6e 100644 --- a/code/index/java/nu/marginalia/index/index/StatefulIndex.java +++ b/code/index/java/nu/marginalia/index/index/StatefulIndex.java @@ -2,11 +2,14 @@ package nu.marginalia.index.index; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.index.model.IndexQueryParams; -import nu.marginalia.index.IndexServicesFactory; -import nu.marginalia.index.model.IndexSearchTerms; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; +import nu.marginalia.index.results.model.ids.DocMetadataList; +import nu.marginalia.index.model.QueryParams; +import nu.marginalia.index.IndexFactory; +import nu.marginalia.index.model.SearchTerms; import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepFromPredicate; +import nu.marginalia.index.results.model.ids.TermIdList; import nu.marginalia.service.control.ServiceEventLog; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -32,13 +35,13 @@ public class StatefulIndex { private final ReadWriteLock indexReplacementLock = new ReentrantReadWriteLock(); @NotNull - private final IndexServicesFactory servicesFactory; + private final IndexFactory servicesFactory; private final ServiceEventLog eventLog; private volatile CombinedIndexReader combinedIndexReader; @Inject - public StatefulIndex(@NotNull IndexServicesFactory servicesFactory, + public StatefulIndex(@NotNull IndexFactory servicesFactory, ServiceEventLog eventLog) { this.servicesFactory = servicesFactory; this.eventLog = eventLog; @@ -52,7 +55,7 @@ public class StatefulIndex { logger.info("Initializing index"); if (combinedIndexReader == null) { - combinedIndexReader = servicesFactory.getSearchIndexReader(); + combinedIndexReader = servicesFactory.getCombinedIndexReader(); eventLog.logEvent("INDEX-INIT", "Index loaded"); } else { @@ -78,7 +81,7 @@ public class StatefulIndex { servicesFactory.switchFiles(); - combinedIndexReader = servicesFactory.getSearchIndexReader(); + combinedIndexReader = servicesFactory.getCombinedIndexReader(); eventLog.logEvent("INDEX-SWITCH-OK", ""); } @@ -106,7 +109,7 @@ public class StatefulIndex { } - public List createQueries(IndexSearchTerms terms, IndexQueryParams params, LongPredicate includePred) { + public List createQueries(SearchTerms terms, QueryParams params) { if (!isLoaded()) { logger.warn("Index reader not ready"); @@ -160,10 +163,6 @@ public class StatefulIndex { query = query.notFull(term); } - // This filtering step needs to happen only on terms that have passed all term-based filtering steps, - // it's essentially a memoization of the params filtering job which is relatively expensive - query = query.addInclusionFilter(new QueryFilterStepFromPredicate(includePred)); - // Run these last, as they'll worst-case cause as many page faults as there are // items in the buffer queries.add(query.addInclusionFilter(combinedIndexReader.filterForParams(params)).build()); @@ -190,7 +189,7 @@ public class StatefulIndex { * document identifiers provided; with metadata for termId. The input array * docs[] *must* be sorted. */ - public long[] getTermMetadata(long termId, long[] docs) { + public DocMetadataList getTermMetadata(long termId, CombinedDocIdList docs) { return combinedIndexReader.getMetadata(termId, docs); } public long getDocumentMetadata(long docId) { diff --git a/code/index/java/nu/marginalia/index/model/IndexSearchTerms.java b/code/index/java/nu/marginalia/index/model/IndexSearchTerms.java deleted file mode 100644 index 51110333..00000000 --- a/code/index/java/nu/marginalia/index/model/IndexSearchTerms.java +++ /dev/null @@ -1,38 +0,0 @@ -package nu.marginalia.index.model; - -import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.longs.LongComparator; -import it.unimi.dsi.fastutil.longs.LongList; -import it.unimi.dsi.fastutil.longs.LongOpenHashSet; - -import java.util.Collections; -import java.util.List; - -public record IndexSearchTerms( - LongList includes, - LongList excludes, - LongList priority, - List coherences - ) -{ - public IndexSearchTerms() { - this(LongList.of(), LongList.of(), LongList.of(), Collections.emptyList()); - } - - public boolean isEmpty() { - return includes.isEmpty(); - } - - public long[] sortedDistinctIncludes(LongComparator comparator) { - if (includes.isEmpty()) - return includes.toLongArray(); - - LongList list = new LongArrayList(new LongOpenHashSet(includes)); - list.sort(comparator); - return list.toLongArray(); - } - - public int size() { - return includes.size() + excludes.size() + priority.size(); - } -} diff --git a/code/index/java/nu/marginalia/index/model/IndexQueryParams.java b/code/index/java/nu/marginalia/index/model/QueryParams.java similarity index 71% rename from code/index/java/nu/marginalia/index/model/IndexQueryParams.java rename to code/index/java/nu/marginalia/index/model/QueryParams.java index a8bd5026..87774347 100644 --- a/code/index/java/nu/marginalia/index/model/IndexQueryParams.java +++ b/code/index/java/nu/marginalia/index/model/QueryParams.java @@ -16,13 +16,13 @@ import nu.marginalia.index.query.limit.SpecificationLimit; * @param queryStrategy The query strategy. May impose additional constraints on the query, such as requiring * the keywords to appear in the title, or in the domain. */ -public record IndexQueryParams(SpecificationLimit qualityLimit, - SpecificationLimit year, - SpecificationLimit size, - SpecificationLimit rank, - SpecificationLimit domainCount, - SearchSet searchSet, - QueryStrategy queryStrategy +public record QueryParams(SpecificationLimit qualityLimit, + SpecificationLimit year, + SpecificationLimit size, + SpecificationLimit rank, + SpecificationLimit domainCount, + SearchSet searchSet, + QueryStrategy queryStrategy ) { diff --git a/code/index/java/nu/marginalia/index/model/IndexSearchParameters.java b/code/index/java/nu/marginalia/index/model/SearchParameters.java similarity index 72% rename from code/index/java/nu/marginalia/index/model/IndexSearchParameters.java rename to code/index/java/nu/marginalia/index/model/SearchParameters.java index b4fe8cb3..097abf10 100644 --- a/code/index/java/nu/marginalia/index/model/IndexSearchParameters.java +++ b/code/index/java/nu/marginalia/index/model/SearchParameters.java @@ -15,7 +15,9 @@ import nu.marginalia.index.searchset.SearchSet; import java.util.ArrayList; import java.util.List; -public class IndexSearchParameters { +import static nu.marginalia.api.searchquery.IndexProtobufCodec.convertSpecLimit; + +public class SearchParameters { /** * This is how many results matching the keywords we'll try to get * before evaluating them for the best result. @@ -23,7 +25,7 @@ public class IndexSearchParameters { public final int fetchSize; public final IndexSearchBudget budget; public final List subqueries; - public final IndexQueryParams queryParams; + public final QueryParams queryParams; public final ResultRankingParameters rankingParams; public final int limitByDomain; @@ -36,13 +38,7 @@ public class IndexSearchParameters { */ public long dataCost = 0; - /** - * A set of id:s considered during each subquery, - * for deduplication - */ - public final TLongHashSet consideredUrlIds; - - public IndexSearchParameters(SearchSpecification specsSet, SearchSet searchSet) { + public SearchParameters(SearchSpecification specsSet, SearchSet searchSet) { var limits = specsSet.queryLimits; this.fetchSize = limits.fetchSize(); @@ -51,9 +47,7 @@ public class IndexSearchParameters { this.limitByDomain = limits.resultsByDomain(); this.limitTotal = limits.resultsTotal(); - this.consideredUrlIds = CachedObjects.getConsideredUrlsMap(); - - queryParams = new IndexQueryParams( + queryParams = new QueryParams( specsSet.quality, specsSet.year, specsSet.size, @@ -65,7 +59,7 @@ public class IndexSearchParameters { rankingParams = specsSet.rankingParams; } - public IndexSearchParameters(RpcIndexQuery request, SearchSet searchSet) { + public SearchParameters(RpcIndexQuery request, SearchSet searchSet) { var limits = IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()); this.fetchSize = limits.fetchSize(); @@ -77,24 +71,18 @@ public class IndexSearchParameters { this.limitByDomain = limits.resultsByDomain(); this.limitTotal = limits.resultsTotal(); - this.consideredUrlIds = CachedObjects.getConsideredUrlsMap(); - - queryParams = new IndexQueryParams( - IndexProtobufCodec.convertSpecLimit(request.getQuality()), - IndexProtobufCodec.convertSpecLimit(request.getYear()), - IndexProtobufCodec.convertSpecLimit(request.getSize()), - IndexProtobufCodec.convertSpecLimit(request.getRank()), - IndexProtobufCodec.convertSpecLimit(request.getDomainCount()), + queryParams = new QueryParams( + convertSpecLimit(request.getQuality()), + convertSpecLimit(request.getYear()), + convertSpecLimit(request.getSize()), + convertSpecLimit(request.getRank()), + convertSpecLimit(request.getDomainCount()), searchSet, QueryStrategy.valueOf(request.getQueryStrategy())); rankingParams = IndexProtobufCodec.convertRankingParameterss(request.getParameters()); } - public List createIndexQueries(StatefulIndex index, IndexSearchTerms terms) { - return index.createQueries(terms, queryParams, consideredUrlIds::add); - } - public boolean hasTimeLeft() { return budget.hasTimeLeft(); } diff --git a/code/index/java/nu/marginalia/index/model/SearchTerms.java b/code/index/java/nu/marginalia/index/model/SearchTerms.java new file mode 100644 index 00000000..dc62ae11 --- /dev/null +++ b/code/index/java/nu/marginalia/index/model/SearchTerms.java @@ -0,0 +1,70 @@ +package nu.marginalia.index.model; + +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongComparator; +import it.unimi.dsi.fastutil.longs.LongList; +import it.unimi.dsi.fastutil.longs.LongOpenHashSet; +import nu.marginalia.api.searchquery.model.query.SearchSubquery; + +import java.util.ArrayList; +import java.util.List; + +import static nu.marginalia.index.model.SearchTermsUtil.getWordId; + +public record SearchTerms( + LongList includes, + LongList excludes, + LongList priority, + List coherences + ) +{ + public SearchTerms(SearchSubquery subquery) { + this(new LongArrayList(), + new LongArrayList(), + new LongArrayList(), + new ArrayList<>()); + + for (var word : subquery.searchTermsInclude) { + includes.add(getWordId(word)); + } + for (var word : subquery.searchTermsAdvice) { + // This looks like a bug, but it's not + includes.add(getWordId(word)); + } + + + for (var coherence : subquery.searchTermCoherences) { + LongList parts = new LongArrayList(coherence.size()); + + for (var word : coherence) { + parts.add(getWordId(word)); + } + + coherences.add(parts); + } + + for (var word : subquery.searchTermsExclude) { + excludes.add(getWordId(word)); + } + for (var word : subquery.searchTermsPriority) { + priority.add(getWordId(word)); + } + } + + public boolean isEmpty() { + return includes.isEmpty(); + } + + public long[] sortedDistinctIncludes(LongComparator comparator) { + if (includes.isEmpty()) + return includes.toLongArray(); + + LongList list = new LongArrayList(new LongOpenHashSet(includes)); + list.sort(comparator); + return list.toLongArray(); + } + + public int size() { + return includes.size() + excludes.size() + priority.size(); + } +} diff --git a/code/index/java/nu/marginalia/index/model/SearchTermsUtil.java b/code/index/java/nu/marginalia/index/model/SearchTermsUtil.java new file mode 100644 index 00000000..9797ca95 --- /dev/null +++ b/code/index/java/nu/marginalia/index/model/SearchTermsUtil.java @@ -0,0 +1,33 @@ +package nu.marginalia.index.model; + +import nu.marginalia.api.searchquery.model.query.SearchSubquery; +import nu.marginalia.hash.MurmurHash3_128; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SearchTermsUtil { + + /** Extract all include-terms from the specified subqueries, + * and a return a map of the terms and their termIds. + */ + public static Map getAllIncludeTerms(List subqueries) { + Map ret = new HashMap<>(); + + for (var subquery : subqueries) { + for (var include : subquery.searchTermsInclude) { + ret.computeIfAbsent(include, i -> getWordId(include)); + } + } + + return ret; + } + + private static final MurmurHash3_128 hasher = new MurmurHash3_128(); + + /** Translate the word to a unique id. */ + public static long getWordId(String s) { + return hasher.hashKeyword(s); + } +} diff --git a/code/index/java/nu/marginalia/index/results/IndexMetadataService.java b/code/index/java/nu/marginalia/index/results/IndexMetadataService.java index 5b8df400..d5356f53 100644 --- a/code/index/java/nu/marginalia/index/results/IndexMetadataService.java +++ b/code/index/java/nu/marginalia/index/results/IndexMetadataService.java @@ -2,46 +2,45 @@ package nu.marginalia.index.results; import com.google.inject.Inject; import gnu.trove.map.hash.TObjectLongHashMap; -import gnu.trove.set.hash.TLongHashSet; -import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap; import it.unimi.dsi.fastutil.longs.LongArrayList; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.index.index.StatefulIndex; -import nu.marginalia.index.SearchTermsUtil; -import nu.marginalia.model.idx.WordMetadata; -import nu.marginalia.ranking.results.ResultValuator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import nu.marginalia.index.model.SearchTermsUtil; +import nu.marginalia.index.results.model.QuerySearchTerms; +import nu.marginalia.index.results.model.TermCoherenceGroupList; +import nu.marginalia.index.results.model.TermMetadataForCombinedDocumentIds; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; +import nu.marginalia.index.results.model.ids.TermIdList; import java.util.ArrayList; import java.util.List; +import static nu.marginalia.index.results.model.TermCoherenceGroupList.TermCoherenceGroup; +import static nu.marginalia.index.results.model.TermMetadataForCombinedDocumentIds.DocumentsWithMetadata; + public class IndexMetadataService { private final StatefulIndex index; - private final SearchTermsUtil searchTermsUtil; - private final ResultValuator searchResultValuator; - private static final Logger logger = LoggerFactory.getLogger(IndexMetadataService.class); + @Inject - public IndexMetadataService(StatefulIndex index, - SearchTermsUtil searchTermsUtil, - ResultValuator searchResultValuator) { + public IndexMetadataService(StatefulIndex index) { this.index = index; - this.searchTermsUtil = searchTermsUtil; - this.searchResultValuator = searchResultValuator; } - public long getDocumentMetadata(long docId) { - return index.getDocumentMetadata(docId); - } + public TermMetadataForCombinedDocumentIds getTermMetadataForDocuments(CombinedDocIdList combinedIdsAll, + TermIdList termIdsList) + { + Long2ObjectArrayMap termdocToMeta = + new Long2ObjectArrayMap<>(termIdsList.size()); - public int getHtmlFeatures(long urlId) { - return index.getHtmlFeatures(urlId); - } + for (long termId : termIdsList.array()) { + var metadata = index.getTermMetadata(termId, combinedIdsAll); + termdocToMeta.put(termId, + new DocumentsWithMetadata(combinedIdsAll, metadata)); + } - public TermMetadataForDocuments getTermMetadataForDocuments(long[] docIdsAll, long[] termIdsList) { - return new TermMetadataForDocuments(docIdsAll, termIdsList); + return new TermMetadataForCombinedDocumentIds(termdocToMeta); } public QuerySearchTerms getSearchTerms(List searchTermVariants) { @@ -63,110 +62,24 @@ public class IndexMetadataService { } return new QuerySearchTerms(termToId, - termIdsList.toLongArray(), + new TermIdList(termIdsList), getTermCoherences(searchTermVariants)); } - private TermCoherences getTermCoherences(List searchTermVariants) { - List coherences = new ArrayList<>(); + private TermCoherenceGroupList getTermCoherences(List searchTermVariants) { + List coherences = new ArrayList<>(); for (var subquery : searchTermVariants) { for (var coh : subquery.searchTermCoherences) { - long[] ids = coh.stream().mapToLong(SearchTermsUtil::getWordId).toArray(); - coherences.add(ids); + coherences.add(new TermCoherenceGroup(coh)); } // It's assumed each subquery has identical coherences break; } - return new TermCoherences(coherences); + return new TermCoherenceGroupList(coherences); } - public TLongHashSet getResultsWithPriorityTerms(List subqueries, long[] resultsArray) { - long[] priorityTermIds = - subqueries.stream() - .flatMap(sq -> sq.searchTermsPriority.stream()) - .distinct() - .mapToLong(SearchTermsUtil::getWordId) - .toArray(); - - var ret = new TLongHashSet(resultsArray.length); - - for (long priorityTerm : priorityTermIds) { - long[] metadata = index.getTermMetadata(priorityTerm, resultsArray); - for (int i = 0; i < metadata.length; i++) { - if (metadata[i] != 0) ret.add(resultsArray[i]); - } - } - - return ret; - } - - public ResultValuator getSearchResultValuator() { - return searchResultValuator; - } - - public class TermMetadataForDocuments { - private final Long2ObjectArrayMap termdocToMeta; - - public TermMetadataForDocuments(long[] docIdsAll, long[] termIdsList) { - termdocToMeta = new Long2ObjectArrayMap<>(termIdsList.length); - - for (long termId : termIdsList) { - var metadata = index.getTermMetadata(termId, docIdsAll); - termdocToMeta.put(termId, new Long2LongOpenHashMap(docIdsAll, metadata)); - } - } - - public long getTermMetadata(long termId, long docId) { - var docsForTerm = termdocToMeta.get(termId); - if (docsForTerm == null) { - logger.warn("Missing meta for term {}", termId); - return 0; - } - - return docsForTerm.getOrDefault(docId, 0); - } - - public boolean testCoherence(long docId, TermCoherences coherences) { - - for (var coherenceSet : coherences.words()) { - long overlap = 0xFF_FFFF_FFFF_FFFFL; - - for (var word : coherenceSet) { - long positions = WordMetadata.decodePositions(getTermMetadata(word, docId)); - overlap &= positions; - } - if (overlap == 0L) { - return false; - } - } - - return true; - } - } - - public static class QuerySearchTerms { - private final TObjectLongHashMap termToId; - public final long[] termIdsAll; - - public final TermCoherences coherences; - - public QuerySearchTerms(TObjectLongHashMap termToId, - long[] termIdsAll, - TermCoherences coherences) { - this.termToId = termToId; - this.termIdsAll = termIdsAll; - this.coherences = coherences; - } - - public long getIdForTerm(String searchTerm) { - return termToId.get(searchTerm); - } - } - - /** wordIds that we require to be in the same sentence */ - public record TermCoherences(List words) {} } diff --git a/code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java b/code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java index 58689d69..967a600f 100644 --- a/code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java +++ b/code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java @@ -1,13 +1,14 @@ package nu.marginalia.index.results; -import gnu.trove.list.TLongList; -import gnu.trove.set.hash.TLongHashSet; import nu.marginalia.api.searchquery.model.query.SearchSubquery; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; -import nu.marginalia.api.searchquery.model.results.SearchResultPreliminaryScore; -import nu.marginalia.index.model.IndexQueryParams; +import nu.marginalia.index.index.StatefulIndex; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; +import nu.marginalia.index.model.QueryParams; +import nu.marginalia.index.results.model.QuerySearchTerms; +import nu.marginalia.index.results.model.TermMetadataForCombinedDocumentIds; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordMetadata; @@ -15,44 +16,39 @@ import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.ranking.results.ResultValuator; import javax.annotation.Nullable; -import java.util.Arrays; import java.util.List; /** This class is responsible for calculating the score of a search result. * It holds the data required to perform the scoring, as there is strong * reasons to cache this data, and performs the calculations */ public class IndexResultValuationContext { - private final IndexMetadataService metadataService; + private final StatefulIndex statefulIndex; private final List> searchTermVariants; - private final IndexQueryParams queryParams; - private final TLongHashSet resultsWithPriorityTerms; + private final QueryParams queryParams; - private final IndexMetadataService.TermMetadataForDocuments termMetadataForDocuments; - private final IndexMetadataService.QuerySearchTerms searchTerms; + private final TermMetadataForCombinedDocumentIds termMetadataForCombinedDocumentIds; + private final QuerySearchTerms searchTerms; private final ResultRankingContext rankingContext; private final ResultValuator searchResultValuator; public IndexResultValuationContext(IndexMetadataService metadataService, - TLongList results, + ResultValuator searchResultValuator, + CombinedDocIdList ids, + StatefulIndex statefulIndex, ResultRankingContext rankingContext, List subqueries, - IndexQueryParams queryParams + QueryParams queryParams ) { + this.statefulIndex = statefulIndex; this.rankingContext = rankingContext; - this.searchResultValuator = metadataService.getSearchResultValuator(); - - final long[] ids = results.toArray(); - Arrays.sort(ids); + this.searchResultValuator = searchResultValuator; this.searchTermVariants = subqueries.stream().map(sq -> sq.searchTermsInclude).distinct().toList(); this.queryParams = queryParams; - this.metadataService = metadataService; this.searchTerms = metadataService.getSearchTerms(subqueries); - this.termMetadataForDocuments = metadataService.getTermMetadataForDocuments(ids, searchTerms.termIdsAll); - - resultsWithPriorityTerms = metadataService.getResultsWithPriorityTerms(subqueries, ids); + this.termMetadataForCombinedDocumentIds = metadataService.getTermMetadataForDocuments(ids, searchTerms.termIdsAll); } private final long flagsFilterMask = @@ -61,19 +57,19 @@ public class IndexResultValuationContext { @Nullable public SearchResultItem calculatePreliminaryScore(long combinedId) { - final long docId = UrlIdCodec.removeRank(combinedId); + long docId = UrlIdCodec.removeRank(combinedId); - if (!termMetadataForDocuments.testCoherence(combinedId, searchTerms.coherences)) + if (!searchTerms.coherences.test(termMetadataForCombinedDocumentIds, docId)) return null; - long docMetadata = metadataService.getDocumentMetadata(docId); - int htmlFeatures = metadataService.getHtmlFeatures(docId); + long docMetadata = statefulIndex.getDocumentMetadata(docId); + int htmlFeatures = statefulIndex.getHtmlFeatures(docId); int maxFlagsCount = 0; boolean anyAllSynthetic = false; int maxPositionsSet = 0; - SearchResultItem searchResult = new SearchResultItem(combinedId, + SearchResultItem searchResult = new SearchResultItem(docId, searchTermVariants.stream().mapToInt(List::size).sum()); for (int querySetId = 0; @@ -89,9 +85,9 @@ public class IndexResultValuationContext { for (int termIdx = 0; termIdx < termList.size(); termIdx++) { String searchTerm = termList.get(termIdx); - long termMetadata = termMetadataForDocuments.getTermMetadata( + long termMetadata = termMetadataForCombinedDocumentIds.getTermMetadata( searchTerms.getIdForTerm(searchTerm), - searchResult.combinedId + combinedId ); var score = new SearchResultKeywordScore( @@ -99,8 +95,7 @@ public class IndexResultValuationContext { searchTerm, termMetadata, docMetadata, - htmlFeatures, - resultsWithPriorityTerms.contains(searchResult.combinedId) + htmlFeatures ); synthetic &= WordFlags.Synthetic.isPresent(termMetadata); @@ -135,10 +130,7 @@ public class IndexResultValuationContext { 5000, // use a dummy value here as it's not present in the index rankingContext); - searchResult.setScore(new SearchResultPreliminaryScore( - resultsWithPriorityTerms.contains(combinedId), - score - )); + searchResult.setScore(score); return searchResult; } diff --git a/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java b/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java index 0a1d6cf9..9fa1713a 100644 --- a/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java +++ b/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java @@ -4,10 +4,13 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import gnu.trove.list.TLongList; import gnu.trove.list.array.TLongArrayList; +import it.unimi.dsi.fastutil.longs.LongArrayList; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.api.searchquery.model.results.SearchResultItem; -import nu.marginalia.index.model.IndexSearchParameters; +import nu.marginalia.index.index.StatefulIndex; +import nu.marginalia.index.model.SearchParameters; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; import nu.marginalia.linkdb.docs.DocumentDbReader; import nu.marginalia.linkdb.model.DocdbUrlDetail; import nu.marginalia.ranking.results.ResultValuator; @@ -25,47 +28,51 @@ public class IndexResultValuatorService { private final IndexMetadataService metadataService; private final DocumentDbReader documentDbReader; private final ResultValuator resultValuator; + private final StatefulIndex statefulIndex; @Inject public IndexResultValuatorService(IndexMetadataService metadataService, DocumentDbReader documentDbReader, - ResultValuator resultValuator) { + ResultValuator resultValuator, + StatefulIndex statefulIndex) + { this.metadataService = metadataService; this.documentDbReader = documentDbReader; this.resultValuator = resultValuator; + this.statefulIndex = statefulIndex; } - - /** From a list of candidate document IDs, find the best results and decorate them with additional information */ - public List findBestResults(IndexSearchParameters params, ResultRankingContext rankingContext, TLongList resultIds) throws SQLException { - + public List rankResults(SearchParameters params, + ResultRankingContext rankingContext, + CombinedDocIdList resultIds) + { final var evaluator = new IndexResultValuationContext(metadataService, + resultValuator, resultIds, + statefulIndex, rankingContext, params.subqueries, params.queryParams); - // Sort the ids for more favorable access patterns on disk - resultIds.sort(); + List results = new ArrayList<>(resultIds.size()); - // Parallel stream to calculate scores is a minor performance boost - var results = Arrays.stream(resultIds.toArray()) - .parallel() - .mapToObj(evaluator::calculatePreliminaryScore) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + for (long docId : resultIds.array()) { + var score = evaluator.calculatePreliminaryScore(docId); + if (score != null) { + results.add(score); + } + } - results = selectBestResults(params, results); - - return decorateAndRerank(results, rankingContext); + return results; } - private List selectBestResults(IndexSearchParameters params, List results) { + + public List selectBestResults(SearchParameters params, + ResultRankingContext rankingContext, + Collection results) throws SQLException { var domainCountFilter = new IndexResultDomainDeduplicator(params.limitByDomain); - results.sort(Comparator.naturalOrder()); - List resultsList = new ArrayList<>(results.size()); for (var item : results) { @@ -94,7 +101,12 @@ public class IndexResultValuatorService { result.resultsFromDomain = domainCountFilter.getCount(result); } - return resultsList; + LongArrayList idsList = new LongArrayList(resultsList.size()); + for (var result : resultsList) { + idsList.add(result.getCombinedId()); + } + + return decorateAndRerank(resultsList, rankingContext); } /** Decorate the result items with additional information from the link database @@ -128,6 +140,7 @@ public class IndexResultValuatorService { if (decoratedItems.size() != rawResults.size()) logger.warn("Result list shrunk during decoration?"); + decoratedItems.sort(Comparator.naturalOrder()); return decoratedItems; } diff --git a/code/index/java/nu/marginalia/index/results/model/QuerySearchTerms.java b/code/index/java/nu/marginalia/index/results/model/QuerySearchTerms.java new file mode 100644 index 00000000..d72e0ea9 --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/QuerySearchTerms.java @@ -0,0 +1,23 @@ +package nu.marginalia.index.results.model; + +import gnu.trove.map.hash.TObjectLongHashMap; +import nu.marginalia.index.results.model.ids.TermIdList; + +public class QuerySearchTerms { + private final TObjectLongHashMap termToId; + public final TermIdList termIdsAll; + + public final TermCoherenceGroupList coherences; + + public QuerySearchTerms(TObjectLongHashMap termToId, + TermIdList termIdsAll, + TermCoherenceGroupList coherences) { + this.termToId = termToId; + this.termIdsAll = termIdsAll; + this.coherences = coherences; + } + + public long getIdForTerm(String searchTerm) { + return termToId.get(searchTerm); + } +} diff --git a/code/index/java/nu/marginalia/index/results/model/TermCoherenceGroupList.java b/code/index/java/nu/marginalia/index/results/model/TermCoherenceGroupList.java new file mode 100644 index 00000000..2b6c24f5 --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/TermCoherenceGroupList.java @@ -0,0 +1,49 @@ +package nu.marginalia.index.results.model; + +import nu.marginalia.index.model.SearchTermsUtil; +import nu.marginalia.model.idx.WordMetadata; + +import java.util.Collections; +import java.util.List; + +/** + * wordIds that we require to be in the same sentence + */ +public record TermCoherenceGroupList(List words) { + + public TermCoherenceGroupList(List words) { + this.words = Collections.unmodifiableList(words); + } + + public boolean test(TermMetadataForCombinedDocumentIds documents, long docId) { + for (var coherenceSet : words()) { + if (!coherenceSet.test(documents, docId)) { + return false; + } + } + + return true; + } + + public static final class TermCoherenceGroup { + private final long[] words; + + public TermCoherenceGroup(long[] words) { + this.words = words; + } + + public TermCoherenceGroup(List coh) { + this(coh.stream().mapToLong(SearchTermsUtil::getWordId).toArray()); + } + + public boolean test(TermMetadataForCombinedDocumentIds documents, long docId) { + long overlap = 0xFF_FFFF_FFFF_FFFFL; + + for (var word : words) { + overlap &= documents.getTermMetadata(word, docId); + } + + return WordMetadata.decodePositions(overlap) != 0L; + } + } +} diff --git a/code/index/java/nu/marginalia/index/results/model/TermMetadataForCombinedDocumentIds.java b/code/index/java/nu/marginalia/index/results/model/TermMetadataForCombinedDocumentIds.java new file mode 100644 index 00000000..9068dd69 --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/TermMetadataForCombinedDocumentIds.java @@ -0,0 +1,36 @@ +package nu.marginalia.index.results.model; + +import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectArrayMap; +import nu.marginalia.index.results.model.ids.CombinedDocIdList; +import nu.marginalia.index.results.model.ids.DocMetadataList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TermMetadataForCombinedDocumentIds { + private static final Logger logger = LoggerFactory.getLogger(TermMetadataForCombinedDocumentIds.class); + private final Long2ObjectArrayMap termdocToMeta; + + public TermMetadataForCombinedDocumentIds(Long2ObjectArrayMap termdocToMeta) { + this.termdocToMeta = termdocToMeta; + } + + public long getTermMetadata(long termId, long combinedId) { + var metaByCombinedId = termdocToMeta.get(termId); + if (metaByCombinedId == null) { + logger.warn("Missing meta for term {}", termId); + return 0; + } + return metaByCombinedId.get(combinedId); + } + + public record DocumentsWithMetadata(Long2LongOpenHashMap data) { + public DocumentsWithMetadata(CombinedDocIdList combinedDocIdsAll, DocMetadataList metadata) { + this(new Long2LongOpenHashMap(combinedDocIdsAll.array(), metadata.array())); + } + + public long get(long combinedId) { + return data.getOrDefault(combinedId, 0); + } + } +} diff --git a/code/index/java/nu/marginalia/index/results/model/ids/CombinedDocIdList.java b/code/index/java/nu/marginalia/index/results/model/ids/CombinedDocIdList.java new file mode 100644 index 00000000..bec6fb8e --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/ids/CombinedDocIdList.java @@ -0,0 +1,52 @@ +package nu.marginalia.index.results.model.ids; + +import it.unimi.dsi.fastutil.longs.LongArrayList; +import org.roaringbitmap.longlong.Roaring64Bitmap; + +import java.util.Arrays; +import java.util.stream.LongStream; + +public final class CombinedDocIdList { + private final long[] data; + + public CombinedDocIdList(LongArrayList data) { + this.data = data.toLongArray(); + } + public CombinedDocIdList(Roaring64Bitmap data) { + this.data = data.toArray(); + } + public CombinedDocIdList() { + this.data = new long[0]; + } + + public int size() { + return data.length; + } + + public LongStream stream() { + return Arrays.stream(data); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (CombinedDocIdList) obj; + return Arrays.equals(this.data, that.data); + } + + @Override + public int hashCode() { + return Arrays.hashCode(data); + } + + + public long[] array() { + return data; + } + + public void sort() { + Arrays.sort(data); + } +} + diff --git a/code/index/java/nu/marginalia/index/results/model/ids/DocIdList.java b/code/index/java/nu/marginalia/index/results/model/ids/DocIdList.java new file mode 100644 index 00000000..beb07c00 --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/ids/DocIdList.java @@ -0,0 +1,49 @@ +package nu.marginalia.index.results.model.ids; + +import it.unimi.dsi.fastutil.longs.LongArrayList; + +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.LongStream; + +public final class DocIdList { + private final long[] array; + + public DocIdList(long[] array) { + this.array = array; + } + + public DocIdList(LongArrayList list) { + this(list.toLongArray()); + } + + public int size() { + return array.length; + } + + public void sort() { + Arrays.sort(array); + } + + public LongStream stream() { + return LongStream.of(array); + } + + public long[] array() { + return array; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (DocIdList) obj; + return Arrays.equals(this.array, that.array); + } + + @Override + public int hashCode() { + return Arrays.hashCode(array); + } + +} diff --git a/code/index/java/nu/marginalia/index/results/model/ids/DocMetadataList.java b/code/index/java/nu/marginalia/index/results/model/ids/DocMetadataList.java new file mode 100644 index 00000000..0104f89c --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/ids/DocMetadataList.java @@ -0,0 +1,45 @@ +package nu.marginalia.index.results.model.ids; + +import it.unimi.dsi.fastutil.longs.LongArrayList; + +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.LongStream; + +public final class DocMetadataList { + private final long[] array; + + public DocMetadataList(long[] array) { + this.array = array; + } + + public DocMetadataList(LongArrayList list) { + this(list.toLongArray()); + } + + public int size() { + return array.length; + } + + public LongStream stream() { + return LongStream.of(array); + } + + public long[] array() { + return array; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (DocMetadataList) obj; + return Arrays.equals(this.array, that.array); + } + + @Override + public int hashCode() { + return Arrays.hashCode(array); + } + +} diff --git a/code/index/java/nu/marginalia/index/results/model/ids/TermIdList.java b/code/index/java/nu/marginalia/index/results/model/ids/TermIdList.java new file mode 100644 index 00000000..f25ab1b9 --- /dev/null +++ b/code/index/java/nu/marginalia/index/results/model/ids/TermIdList.java @@ -0,0 +1,45 @@ +package nu.marginalia.index.results.model.ids; + +import it.unimi.dsi.fastutil.longs.LongArrayList; + +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.LongStream; + +public final class TermIdList { + private final long[] array; + + public TermIdList(long[] array) { + this.array = array; + } + + public TermIdList(LongArrayList list) { + this(list.toLongArray()); + } + + public int size() { + return array.length; + } + + public LongStream stream() { + return LongStream.of(array); + } + + public long[] array() { + return array; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (TermIdList) obj; + return Arrays.equals(this.array, that.array); + } + + @Override + public int hashCode() { + return Arrays.hashCode(array); + } + +} diff --git a/code/index/java/nu/marginalia/index/searchset/SearchSetsService.java b/code/index/java/nu/marginalia/index/searchset/SearchSetsService.java index 52c5ad89..4fe974fe 100644 --- a/code/index/java/nu/marginalia/index/searchset/SearchSetsService.java +++ b/code/index/java/nu/marginalia/index/searchset/SearchSetsService.java @@ -6,7 +6,7 @@ import gnu.trove.list.TIntList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import nu.marginalia.db.DomainRankingSetsService; import nu.marginalia.db.DomainTypes; -import nu.marginalia.index.IndexServicesFactory; +import nu.marginalia.index.IndexFactory; import nu.marginalia.index.domainrankings.DomainRankings; import nu.marginalia.ranking.domains.PageRankDomainRanker; import nu.marginalia.ranking.domains.accumulator.RankingResultHashMapAccumulator; @@ -29,7 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; public class SearchSetsService { private final Logger logger = LoggerFactory.getLogger(getClass()); private final DomainTypes domainTypes; - private final IndexServicesFactory indexServicesFactory; + private final IndexFactory indexFactory; private final ServiceEventLog eventLog; private final DomainRankingSetsService domainRankingSetsService; private final DbUpdateRanks dbUpdateRanks; @@ -51,13 +51,13 @@ public class SearchSetsService { ServiceConfiguration serviceConfiguration, LinkGraphSource rankingDomains, SimilarityGraphSource similarityDomains, - IndexServicesFactory indexServicesFactory, + IndexFactory indexFactory, ServiceEventLog eventLog, DomainRankingSetsService domainRankingSetsService, DbUpdateRanks dbUpdateRanks) throws IOException { this.nodeId = serviceConfiguration.node(); this.domainTypes = domainTypes; - this.indexServicesFactory = indexServicesFactory; + this.indexFactory = indexFactory; this.eventLog = eventLog; this.domainRankingSetsService = domainRankingSetsService; @@ -77,7 +77,7 @@ public class SearchSetsService { for (var rankingSet : domainRankingSetsService.getAll()) { rankingSets.put(rankingSet.name(), new RankingSearchSet(rankingSet.name(), - rankingSet.fileName(indexServicesFactory.getSearchSetsBase()) + rankingSet.fileName(indexFactory.getSearchSetsBase()) ) ); } @@ -147,7 +147,7 @@ public class SearchSetsService { .forDomainNames(source, domains) .calculate(rankingSet.depth(), RankingResultHashSetAccumulator::new); - var set = new RankingSearchSet(rankingSet.name(), rankingSet.fileName(indexServicesFactory.getSearchSetsBase()), data); + var set = new RankingSearchSet(rankingSet.name(), rankingSet.fileName(indexFactory.getSearchSetsBase()), data); rankingSets.put(rankingSet.name(), set); try { @@ -170,7 +170,7 @@ public class SearchSetsService { } synchronized (this) { - var blogSet = new RankingSearchSet(rankingSet.name(), rankingSet.fileName(indexServicesFactory.getSearchSetsBase()), new IntOpenHashSet(knownDomains.toArray())); + var blogSet = new RankingSearchSet(rankingSet.name(), rankingSet.fileName(indexFactory.getSearchSetsBase()), new IntOpenHashSet(knownDomains.toArray())); rankingSets.put(rankingSet.name(), blogSet); blogSet.write(); } @@ -197,7 +197,7 @@ public class SearchSetsService { domainRankings = new DomainRankings(ranks); } - domainRankings.save(indexServicesFactory.getSearchSetsBase()); + domainRankings.save(indexFactory.getSearchSetsBase()); if (nodeId == 1) { // The EC_DOMAIN table has a field that reflects the rank, this needs to be set for search result ordering to diff --git a/code/index/java/nu/marginalia/ranking/results/ResultValuator.java b/code/index/java/nu/marginalia/ranking/results/ResultValuator.java index 332f5326..6c67559d 100644 --- a/code/index/java/nu/marginalia/ranking/results/ResultValuator.java +++ b/code/index/java/nu/marginalia/ranking/results/ResultValuator.java @@ -24,32 +24,21 @@ public class ResultValuator { private final Bm25Factor bm25Factor; private final TermCoherenceFactor termCoherenceFactor; - private final PriorityTermBonus priorityTermBonus; - - private final ThreadLocal> listPool = - ThreadLocal.withInitial(ValuatorListPool::new); - private static final Logger logger = LoggerFactory.getLogger(ResultValuator.class); @Inject public ResultValuator(Bm25Factor bm25Factor, - TermCoherenceFactor termCoherenceFactor, - PriorityTermBonus priorityTermBonus) { - + TermCoherenceFactor termCoherenceFactor) { this.bm25Factor = bm25Factor; this.termCoherenceFactor = termCoherenceFactor; - this.priorityTermBonus = priorityTermBonus; - } public double calculateSearchResultValue(List scores, int length, ResultRankingContext ctx) { - var threadListPool = listPool.get(); int sets = numberOfSets(scores); - long documentMetadata = documentMetadata(scores); int features = htmlFeatures(scores); var rankingParams = ctx.params; @@ -84,8 +73,7 @@ public class ResultValuator { + rankingBonus + topologyBonus + temporalBias - + flagsPenalty - + priorityTermBonus.calculate(scores); + + flagsPenalty; double bestTcf = 0; double bestBM25F = 0; @@ -93,7 +81,7 @@ public class ResultValuator { double bestBM25PN = 0; for (int set = 0; set < sets; set++) { - ResultKeywordSet keywordSet = createKeywordSet(threadListPool, scores, set); + ResultKeywordSet keywordSet = createKeywordSet(scores, set); if (keywordSet.isEmpty()) continue; @@ -185,12 +173,10 @@ public class ResultValuator { return 0; } - private ResultKeywordSet createKeywordSet(ValuatorListPool listPool, - List rawScores, + private ResultKeywordSet createKeywordSet(List rawScores, int thisSet) { - List scoresList = listPool.get(thisSet); - scoresList.clear(); + List scoresList = new ArrayList<>(); for (var score : rawScores) { if (score.subquery != thisSet) @@ -225,26 +211,3 @@ public class ResultValuator { return Math.sqrt((1.0 + scalingFactor + 10 * penalty) / (1.0 + value)); } } - -/** Pool of List instances used to reduce memory churn during result ranking in the index - * where potentially tens of thousands of candidate results are ranked. - * - * @param - */ -@SuppressWarnings({"unchecked", "rawtypes"}) -class ValuatorListPool { - private final ArrayList[] items = new ArrayList[256]; - - public ValuatorListPool() { - for (int i = 0; i < items.length; i++) { - items[i] = new ArrayList(); - } - } - - public List get(int i) { - var ret = (ArrayList) items[i]; - ret.clear(); - return ret; - } - -} diff --git a/code/index/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java b/code/index/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java deleted file mode 100644 index 18d044c3..00000000 --- a/code/index/java/nu/marginalia/ranking/results/factors/PriorityTermBonus.java +++ /dev/null @@ -1,21 +0,0 @@ -package nu.marginalia.ranking.results.factors; - - -import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; - -import java.util.List; - -/** Rewards results that have a priority term */ -public class PriorityTermBonus { - public double calculate(List scores) { - - for (var result : scores) { - if (result.hasPriorityTerms()) { - return 2.0; - } - } - - return 0; - } - -} diff --git a/code/index/query/java/nu/marginalia/index/query/IndexQuery.java b/code/index/query/java/nu/marginalia/index/query/IndexQuery.java index b55a8bef..734f08dd 100644 --- a/code/index/query/java/nu/marginalia/index/query/IndexQuery.java +++ b/code/index/query/java/nu/marginalia/index/query/IndexQuery.java @@ -22,6 +22,10 @@ public class IndexQuery { public final IndexQueryPriority queryPriority; public final int fetchSizeMultiplier; + public IndexQuery(EntrySource... sources) { + this(List.of(sources), IndexQueryPriority.BEST, 1); + } + /** * Creates an IndexQuery object with the given sources, priority, and fetchSizeMultiplier. * diff --git a/code/index/query/java/nu/marginalia/index/query/IndexSearchBudget.java b/code/index/query/java/nu/marginalia/index/query/IndexSearchBudget.java index 1ed211d1..f8d1ffa3 100644 --- a/code/index/query/java/nu/marginalia/index/query/IndexSearchBudget.java +++ b/code/index/query/java/nu/marginalia/index/query/IndexSearchBudget.java @@ -10,4 +10,5 @@ public class IndexSearchBudget { } public boolean hasTimeLeft() { return System.currentTimeMillis() < timeout; } + public long timeLeft() { return timeout - System.currentTimeMillis(); } } diff --git a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java index 758c1a3d..0370898f 100644 --- a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java +++ b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java @@ -60,7 +60,7 @@ public class IndexQueryServiceIntegrationSmokeTest { IndexQueryServiceIntegrationTestModule testModule; @Inject - IndexQueryGrpcService queryService; + IndexGrpcService queryService; @Inject StatefulIndex statefulIndex; @@ -135,6 +135,8 @@ public class IndexQueryServiceIntegrationSmokeTest { .mapToLong(i -> i.rawIndexResult.getDocumentId()) .toArray(); + System.out.println(Arrays.toString(actual)); + System.out.println(Arrays.toString(ids)); Assertions.assertArrayEquals(ids, actual); } diff --git a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java index b9207437..3431c18e 100644 --- a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java +++ b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java @@ -62,7 +62,7 @@ public class IndexQueryServiceIntegrationTest { IndexQueryServiceIntegrationTestModule testModule; @Inject - IndexQueryGrpcService queryService; + IndexGrpcService queryService; @Inject StatefulIndex statefulIndex; diff --git a/code/index/test/nu/marginalia/ranking/results/ResultValuatorTest.java b/code/index/test/nu/marginalia/ranking/results/ResultValuatorTest.java index 60beead2..8f8f7eaa 100644 --- a/code/index/test/nu/marginalia/ranking/results/ResultValuatorTest.java +++ b/code/index/test/nu/marginalia/ranking/results/ResultValuatorTest.java @@ -31,8 +31,7 @@ class ResultValuatorTest { valuator = new ResultValuator( new Bm25Factor(), - new TermCoherenceFactor(), - new PriorityTermBonus() + new TermCoherenceFactor() ); } @@ -40,20 +39,20 @@ class ResultValuatorTest { new SearchResultKeywordScore(0, "bob", wordMetadata(Set.of(1), EnumSet.of(WordFlags.Title)), docMetadata(0, 2010, 5, EnumSet.noneOf(DocumentFlags.class)), - 0, false) + 0) ); List highCountNoTitleSet = List.of( new SearchResultKeywordScore(0, "bob", wordMetadata(Set.of(1,3,4,6,7,9,10,11,12,14,15,16), EnumSet.of(WordFlags.TfIdfHigh)), docMetadata(0, 2010, 5, EnumSet.noneOf(DocumentFlags.class)), - 0, false) + 0) ); List highCountSubjectSet = List.of( new SearchResultKeywordScore(0, "bob", wordMetadata(Set.of(1,3,4,6,7,9,10,11,12,14,15,16), EnumSet.of(WordFlags.TfIdfHigh, WordFlags.Subjects)), docMetadata(0, 2010, 5, EnumSet.noneOf(DocumentFlags.class)), - 0, false) + 0) ); diff --git a/code/index/test/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java b/code/index/test/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java index 6ae9c448..a5bca54e 100644 --- a/code/index/test/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java +++ b/code/index/test/nu/marginalia/ranking/results/factors/TermCoherenceFactorTest.java @@ -89,7 +89,7 @@ class TermCoherenceFactorTest { for (int i = 0; i < positionMasks.length; i++) { keywords.add(new SearchResultKeywordScore(0, "", - new WordMetadata(positionMasks[i], (byte) 0).encode(), 0, 0, false)); + new WordMetadata(positionMasks[i], (byte) 0).encode(), 0, 0)); } return new ResultKeywordSet(keywords); diff --git a/code/services-core/index-service/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/java/nu/marginalia/index/IndexService.java index 9278f699..ee96e9d4 100644 --- a/code/services-core/index-service/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/java/nu/marginalia/index/IndexService.java @@ -42,7 +42,7 @@ public class IndexService extends Service { @Inject public IndexService(BaseServiceParams params, IndexOpsService opsService, - IndexQueryGrpcService indexQueryService, + IndexGrpcService indexQueryService, StatefulIndex statefulIndex, FileStorageService fileStorageService, DocumentDbReader documentDbReader, From f7f0100174f7d2f6566ab03d52940af438dab1e8 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 11:08:49 +0100 Subject: [PATCH 35/56] (build) Make docker image registry and tag configurable in root build.gradle --- build.gradle | 2 ++ code/services-application/api-service/build.gradle | 2 +- code/services-application/dating-service/build.gradle | 2 +- code/services-application/explorer-service/build.gradle | 2 +- code/services-application/search-service/build.gradle | 2 +- code/services-core/assistant-service/build.gradle | 2 +- code/services-core/control-service/build.gradle | 2 +- code/services-core/executor-service/build.gradle | 2 +- code/services-core/index-service/build.gradle | 6 +++--- code/services-core/query-service/build.gradle | 2 +- code/tools/screenshot-capture-tool/build.gradle | 2 +- 11 files changed, 14 insertions(+), 12 deletions(-) diff --git a/build.gradle b/build.gradle index 3e4e8c99..8b76efda 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,8 @@ subprojects.forEach {it -> } ext { dockerImageBase='container-registry.oracle.com/graalvm/jdk:21@sha256:1fd33d4d4eba3a9e1a41a728e39ea217178d257694eea1214fec68d2ed4d3d9b' + dockerImageTag='latest' + dockerImageRegistry='marginalia' } idea { diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index b7b29038..c09c0c9f 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -24,7 +24,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-application/dating-service/build.gradle b/code/services-application/dating-service/build.gradle index cae0cb59..6d209102 100644 --- a/code/services-application/dating-service/build.gradle +++ b/code/services-application/dating-service/build.gradle @@ -18,7 +18,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-application/explorer-service/build.gradle b/code/services-application/explorer-service/build.gradle index cf35c610..6aba03b7 100644 --- a/code/services-application/explorer-service/build.gradle +++ b/code/services-application/explorer-service/build.gradle @@ -18,7 +18,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index cb799ab9..d290e4c4 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -13,7 +13,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index 27f89012..b6d4aa4f 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -18,7 +18,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 2ecd1f92..6d46926e 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -23,7 +23,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 62324c7f..43c0a23e 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -18,7 +18,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 69f1f38d..05aa3af9 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -15,11 +15,11 @@ tasks.distZip.enabled = false jib { from { - image = image = rootProject.ext.dockerImageBase + image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name - tags = ['latest'] + image = rootProject.ext.dockerImageRegistry+'/'+project.name + tags = [ rootProject.ext.dockerImageTag ] } container { diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 0e283788..96022efd 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -18,7 +18,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { diff --git a/code/tools/screenshot-capture-tool/build.gradle b/code/tools/screenshot-capture-tool/build.gradle index 82903746..57ea3f72 100644 --- a/code/tools/screenshot-capture-tool/build.gradle +++ b/code/tools/screenshot-capture-tool/build.gradle @@ -22,7 +22,7 @@ jib { image = image = rootProject.ext.dockerImageBase } to { - image = 'marginalia/'+project.name + image = rootProject.ext.dockerImageRegistry+'/'+project.name tags = ['latest'] } container { From 9429bf5c45fd1226340bc5abbb436c1375ef51e5 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 11:11:59 +0100 Subject: [PATCH 36/56] (index) Clean up --- .../model/results/SearchResultItem.java | 2 +- .../nu/marginalia/index/IndexGrpcService.java | 7 +-- .../index/index/IndexQueryService.java | 60 +------------------ 3 files changed, 3 insertions(+), 66 deletions(-) diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java index fc6633a2..cc02ae28 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java @@ -70,7 +70,7 @@ public class SearchResultItem implements Comparable { @Override public int compareTo(@NotNull SearchResultItem o) { // this looks like a bug, but we actually want this in a reversed order - int diff = Double.compare(o.getScore(), getScore()); + int diff = Double.compare(getScore(), o.getScore()); if (diff != 0) return diff; diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index d0d2da1f..92440796 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -1,6 +1,5 @@ package nu.marginalia.index; -import com.google.common.collect.MinMaxPriorityQueue; import com.google.inject.Inject; import com.google.inject.Singleton; import io.grpc.stub.StreamObserver; @@ -190,7 +189,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { ResultRankingContext rankingContext = createRankingContext(params.rankingParams, params.subqueries); - logger.info(queryMarker, "{}", params.queryParams); +// logger.info(queryMarker, "{}", params.queryParams); return new QueryExecution(rankingContext, params.fetchSize) .run(params); @@ -243,8 +242,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { this.subquery = subquery; this.parameters = parameters; - logger.info("Starting index task"); - remainingIndexTasks.incrementAndGet(); } @@ -254,7 +251,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { subquery, parameters.queryParams, parameters.budget, - parameters.fetchSize, this::drain ); } @@ -264,7 +260,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { remainingIndexTasks.notifyAll(); } } - logger.info("Terminating index task"); } } diff --git a/code/index/java/nu/marginalia/index/index/IndexQueryService.java b/code/index/java/nu/marginalia/index/index/IndexQueryService.java index 53eb0bb4..f430de30 100644 --- a/code/index/java/nu/marginalia/index/index/IndexQueryService.java +++ b/code/index/java/nu/marginalia/index/index/IndexQueryService.java @@ -38,28 +38,11 @@ public class IndexQueryService { public void evaluateSubquery(SearchSubquery subquery, QueryParams queryParams, IndexSearchBudget timeout, - int fetchSize, Consumer drain) { - // These queries are various term combinations - - if (!timeout.hasTimeLeft()) { - logger.info("Query timed out {}, ({}), -{}", - subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); - return; - } - logger.info(queryMarker, "{}", subquery); - final SearchTerms searchTerms = new SearchTerms(subquery); - if (searchTerms.isEmpty()) { - logger.info(queryMarker, "empty"); - return; - } - final Roaring64Bitmap results = new Roaring64Bitmap(); - // logSearchTerms(subquery, searchTerms); - // These queries are different indices for one subquery List queries = index.createQueries(searchTerms, queryParams); for (var query : queries) { @@ -69,9 +52,7 @@ public class IndexQueryService { final LongQueryBuffer buffer = new LongQueryBuffer(512); - while (query.hasMore() - && results.getIntCardinality() < fetchSize * query.fetchSizeMultiplier - && timeout.hasTimeLeft()) + while (query.hasMore() && timeout.hasTimeLeft()) { buffer.reset(); query.getMoreResults(buffer); @@ -85,8 +66,6 @@ public class IndexQueryService { results.clear(); } } - - logger.info(queryMarker, "{} from {}", results.getIntCardinality(), query); } if (!results.isEmpty()) { @@ -94,41 +73,4 @@ public class IndexQueryService { } } - private void logSearchTerms(SearchSubquery subquery, SearchTerms searchTerms) { - - // This logging should only be enabled in testing, as it is very verbose - // and contains sensitive information - - if (!logger.isInfoEnabled(queryMarker)) { - return; - } - - var includes = subquery.searchTermsInclude; - var advice = subquery.searchTermsAdvice; - var excludes = subquery.searchTermsExclude; - var priority = subquery.searchTermsPriority; - - for (int i = 0; i < includes.size(); i++) { - logger.info(queryMarker, "{} -> {} I", includes.get(i), - Long.toHexString(searchTerms.includes().getLong(i)) - ); - } - for (int i = 0; i < advice.size(); i++) { - logger.info(queryMarker, "{} -> {} A", advice.get(i), - Long.toHexString(searchTerms.includes().getLong(includes.size() + i)) - ); - } - for (int i = 0; i < excludes.size(); i++) { - logger.info(queryMarker, "{} -> {} E", excludes.get(i), - Long.toHexString(searchTerms.excludes().getLong(i)) - ); - } - for (int i = 0; i < priority.size(); i++) { - logger.info(queryMarker, "{} -> {} P", priority.get(i), - Long.toHexString(searchTerms.priority().getLong(i)) - ); - } - } - - } From b8e336e809be65c6977c82d07086d735bb63a21b Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 11:30:53 +0100 Subject: [PATCH 37/56] (index) Reduce time allocation a bit --- .../java/nu/marginalia/index/model/SearchParameters.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/index/java/nu/marginalia/index/model/SearchParameters.java b/code/index/java/nu/marginalia/index/model/SearchParameters.java index 097abf10..7d440563 100644 --- a/code/index/java/nu/marginalia/index/model/SearchParameters.java +++ b/code/index/java/nu/marginalia/index/model/SearchParameters.java @@ -63,7 +63,11 @@ public class SearchParameters { var limits = IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()); this.fetchSize = limits.fetchSize(); - this.budget = new IndexSearchBudget(limits.timeoutMs()); + + // The time budget is halved because this is the point when we start to + // wrap up the search and return the results. + this.budget = new IndexSearchBudget(limits.timeoutMs() / 2); + this.subqueries = new ArrayList<>(request.getSubqueriesCount()); for (int i = 0; i < request.getSubqueriesCount(); i++) { this.subqueries.add(IndexProtobufCodec.convertSearchSubquery(request.getSubqueries(i))); From 7fc0d4d7868e8738d034997fc5e943c5cb580d13 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 11:54:31 +0100 Subject: [PATCH 38/56] (index) Observability for query execution queues --- .../nu/marginalia/index/IndexGrpcService.java | 112 ++++++++++++++---- 1 file changed, 86 insertions(+), 26 deletions(-) diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index 92440796..b210c851 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; @Singleton public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { @@ -60,6 +61,17 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { .help("Index-side query time") .register(); + private static final Gauge wmsa_index_query_exec_stall_time = Gauge.build() + .name("wmsa_index_query_exec_stall_time") + .help("Execution stall time") + .labelNames("node") + .register(); + + private static final Gauge wmsa_index_query_exec_block_time = Gauge.build() + .name("wmsa_index_query_exec_block_time") + .help("Execution stall time") + .labelNames("node") + .register(); private final StatefulIndex index; private final SearchSetsService searchSetsService; @@ -68,7 +80,9 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { private final IndexResultValuatorService resultValuator; private final int nodeId; + private final String nodeName; + private final int indexValuationThreads = Integer.getInteger("index.valuationThreads", 8); @Inject public IndexGrpcService(ServiceConfiguration serviceConfiguration, @@ -78,6 +92,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { IndexResultValuatorService resultValuator) { this.nodeId = serviceConfiguration.node(); + this.nodeName = Integer.toString(nodeId); this.index = index; this.searchSetsService = searchSetsService; this.resultValuator = resultValuator; @@ -91,7 +106,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { try { var params = new SearchParameters(request, getSearchSet(request)); - final String nodeName = Integer.toString(nodeId); SearchResultSet results = wmsa_query_time .labels(nodeName, "GRPC") @@ -180,7 +194,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { return searchSetsService.getSearchSetByName(request.getSearchSetIdentifier()); } - private SearchResultSet executeSearch(SearchParameters params) throws SQLException { + private SearchResultSet executeSearch(SearchParameters params) throws SQLException, InterruptedException { if (!index.isLoaded()) { // Short-circuit if the index is not loaded, as we trivially know that there can be no results @@ -189,51 +203,84 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { ResultRankingContext rankingContext = createRankingContext(params.rankingParams, params.subqueries); -// logger.info(queryMarker, "{}", params.queryParams); + var queryExecution = new QueryExecution(rankingContext, params.fetchSize); - return new QueryExecution(rankingContext, params.fetchSize) - .run(params); + var ret = queryExecution.run(params); + + wmsa_index_query_exec_block_time + .labels(nodeName) + .set(queryExecution.getBlockTime() / 1000.); + wmsa_index_query_exec_stall_time + .labels(nodeName) + .set(queryExecution.getStallTime() / 1000.); + + return ret; } + /** This class is responsible for executing a search query. It uses a thread pool to + * execute the subqueries in parallel, and then uses another thread pool to rank the + * results in parallel. The results are then combined into a bounded priority queue, + * and finally the best results are returned. + */ private class QueryExecution { - private static final Executor queryExecutor = Executors.newCachedThreadPool(); - private static final Executor rankingExecutor = Executors.newCachedThreadPool(); - private final ArrayBlockingQueue resultQueue = new ArrayBlockingQueue<>(8); + private static final Executor workerPool = Executors.newCachedThreadPool(); + + private final ArrayBlockingQueue resultCandidateQueue + = new ArrayBlockingQueue<>(8); + private final ResultPriorityQueue resultHeap; private final ResultRankingContext resultRankingContext; private final AtomicInteger remainingIndexTasks = new AtomicInteger(0); private final AtomicInteger remainingValuationTasks = new AtomicInteger(0); + private final AtomicLong blockTime = new AtomicLong(0); + private final AtomicLong stallTime = new AtomicLong(0); + + public long getStallTime() { + return stallTime.get(); + } + public long getBlockTime() { + return blockTime.get(); + } + private QueryExecution(ResultRankingContext resultRankingContext, int maxResults) { this.resultRankingContext = resultRankingContext; this.resultHeap = new ResultPriorityQueue(maxResults); } - public SearchResultSet run(SearchParameters parameters) throws SQLException { + /** Execute a search query */ + public SearchResultSet run(SearchParameters parameters) throws SQLException, InterruptedException { for (var subquery : parameters.subqueries) { - queryExecutor.execute(new IndexLookup(subquery, parameters)); + workerPool.execute(new IndexLookup(subquery, parameters)); } - for (int i = 0; i < 16; i++) { - rankingExecutor.execute(new ResultRanker(parameters, resultRankingContext)); + for (int i = 0; i < indexValuationThreads; i++) { + workerPool.execute(new ResultRanker(parameters, resultRankingContext)); } // Wait for all tasks to complete - synchronized (remainingValuationTasks) { - while (remainingValuationTasks.get() > 0) { - try { - remainingValuationTasks.wait(20); - } - catch (InterruptedException e) { - logger.warn("Interrupted while waiting for tasks to complete", e); - } - } - } + awaitCompletion(); - return new SearchResultSet(resultValuator.selectBestResults(parameters, resultRankingContext, resultHeap)); + // Return the best results + return new SearchResultSet( + resultValuator.selectBestResults(parameters, + resultRankingContext, + resultHeap)); } + /** Wait for all tasks to complete */ + private void awaitCompletion() throws InterruptedException { + synchronized (remainingValuationTasks) { + while (remainingValuationTasks.get() > 0) { + remainingValuationTasks.wait(20); + } + } + } + + /** This class is responsible for executing a subquery and adding the results to the + * resultCandidateQueue, which depending on the state of the valuator threads may + * or may not block*/ class IndexLookup implements Runnable { private final SearchSubquery subquery; private final SearchParameters parameters; @@ -267,7 +314,11 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { long remainingTime = parameters.budget.timeLeft(); try { - resultQueue.offer(resultIds, remainingTime, TimeUnit.MILLISECONDS); + if (!resultCandidateQueue.offer(resultIds)) { + long start = System.currentTimeMillis(); + resultCandidateQueue.offer(resultIds, remainingTime, TimeUnit.MILLISECONDS); + blockTime.addAndGet(System.currentTimeMillis() - start); + } } catch (InterruptedException e) { logger.warn("Interrupted while waiting to offer resultIds to queue", e); @@ -275,6 +326,9 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { } } + /** This class is responsible for ranking the results and adding the best results to the + * resultHeap, which depending on the state of the indexLookup threads may or may not block + */ class ResultRanker implements Runnable { private final SearchParameters parameters; private final ResultRankingContext rankingContext; @@ -290,16 +344,22 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { try { while (parameters.budget.timeLeft() > 0) { - CombinedDocIdList resultIds = resultQueue.poll( + long start = System.currentTimeMillis(); + + CombinedDocIdList resultIds = resultCandidateQueue.poll( Math.clamp(parameters.budget.timeLeft(), 1, 25), TimeUnit.MILLISECONDS); + if (resultIds == null) { - if (remainingIndexTasks.get() == 0 && resultQueue.isEmpty()) + if (remainingIndexTasks.get() == 0 + && resultCandidateQueue.isEmpty()) break; else continue; } + stallTime.addAndGet(System.currentTimeMillis() - start); + var bestResults = resultValuator.rankResults(parameters, rankingContext, resultIds); resultHeap.addAll(bestResults); From 823ca73a3f716199db3e6a57f73be3218fa3cdca Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 12:14:51 +0100 Subject: [PATCH 39/56] (domain-ranking) Fix a crash during ranking the edges of the similarity graph doesn't quite match the vertices of the link graph. --- .../ranking/domains/data/SimilarityGraphSource.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/code/index/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java index 84f9730e..e52091b1 100644 --- a/code/index/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java +++ b/code/index/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java @@ -52,6 +52,14 @@ public class SimilarityGraphSource extends AbstractGraphSource { while (rs.next()) { int src = rs.getInt(1); int dest = rs.getInt(2); + + // Similarity data may contain domain ids that we don't have indexed, + // omit these from the graph. + if (!graph.containsVertex(src)) + continue; + if (!graph.containsVertex(dest)) + continue; + double weight = rs.getDouble(3); graph.addEdge(src, dest); From 427f3e922fb4296b89509fe72f0d1d18e9f3eb85 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 12:46:30 +0100 Subject: [PATCH 40/56] (index) Retire count operation, clean up index code. --- .../api/searchquery/QueryProtobufCodec.java | 4 -- .../searchquery/model/query/QueryParams.java | 2 - .../model/query/SearchSpecification.java | 2 - .../api/src/main/protobuf/query-api.proto | 2 - .../searchquery/query_parser/QueryParser.java | 2 - .../searchquery/query_parser/token/Token.java | 1 - .../query_parser/token/TokenType.java | 1 - .../query_parser/token/TokenVisitor.java | 1 - .../searchquery/svc/QueryFactory.java | 1 - .../svc/QueryLimitsAccumulator.java | 7 --- .../svc/QuerySearchTermsAccumulator.java | 2 - .../query/svc/QueryFactoryTest.java | 1 - .../nu/marginalia/index/IndexGrpcService.java | 7 +-- .../marginalia/index/model/QueryParams.java | 3 -- .../index/model/SearchParameters.java | 2 - .../results/IndexResultValuatorService.java | 49 ++++++------------- ...IndexQueryServiceIntegrationSmokeTest.java | 3 -- .../IndexQueryServiceIntegrationTest.java | 1 - .../search/SearchQueryParamFactory.java | 4 -- 19 files changed, 18 insertions(+), 77 deletions(-) diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java index 58550362..28d14c82 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java @@ -34,7 +34,6 @@ public class QueryProtobufCodec { builder.setYear(IndexProtobufCodec.convertSpecLimit(query.specs.year)); builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size)); builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank)); - builder.setDomainCount(IndexProtobufCodec.convertSpecLimit(query.specs.domainCount)); builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); @@ -63,7 +62,6 @@ public class QueryProtobufCodec { builder.setYear(IndexProtobufCodec.convertSpecLimit(query.specs.year)); builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size)); builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank)); - builder.setDomainCount(IndexProtobufCodec.convertSpecLimit(query.specs.domainCount)); builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); @@ -92,7 +90,6 @@ public class QueryProtobufCodec { IndexProtobufCodec.convertSpecLimit(request.getYear()), IndexProtobufCodec.convertSpecLimit(request.getSize()), IndexProtobufCodec.convertSpecLimit(request.getRank()), - IndexProtobufCodec.convertSpecLimit(request.getDomainCount()), request.getDomainIdsList(), IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()), request.getSearchSetIdentifier(), @@ -174,7 +171,6 @@ public class QueryProtobufCodec { IndexProtobufCodec.convertSpecLimit(specs.getYear()), IndexProtobufCodec.convertSpecLimit(specs.getSize()), IndexProtobufCodec.convertSpecLimit(specs.getRank()), - IndexProtobufCodec.convertSpecLimit(specs.getDomainCount()), IndexProtobufCodec.convertQueryLimits(specs.getQueryLimits()), QueryStrategy.valueOf(specs.getQueryStrategy()), IndexProtobufCodec.convertRankingParameterss(specs.getParameters()) diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java index b713217b..176b977e 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java @@ -19,7 +19,6 @@ public record QueryParams( SpecificationLimit year, SpecificationLimit size, SpecificationLimit rank, - SpecificationLimit domainCount, List domainIds, QueryLimits limits, String identifier, @@ -37,7 +36,6 @@ public record QueryParams( SpecificationLimit.none(), SpecificationLimit.none(), SpecificationLimit.none(), - SpecificationLimit.none(), List.of(), limits, identifier, diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java index 2b5d1867..be2a6895 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java @@ -24,8 +24,6 @@ public class SearchSpecification { public final SpecificationLimit size; public final SpecificationLimit rank; - public final SpecificationLimit domainCount; - public final QueryLimits queryLimits; public final QueryStrategy queryStrategy; diff --git a/code/functions/search-query/api/src/main/protobuf/query-api.proto b/code/functions/search-query/api/src/main/protobuf/query-api.proto index e508ba34..f5ec5e8d 100644 --- a/code/functions/search-query/api/src/main/protobuf/query-api.proto +++ b/code/functions/search-query/api/src/main/protobuf/query-api.proto @@ -25,7 +25,6 @@ message RpcQsQuery { RpcSpecLimit year = 8; RpcSpecLimit size = 9; RpcSpecLimit rank = 10; - RpcSpecLimit domainCount = 11; repeated int32 domainIds = 12; RpcQueryLimits queryLimits = 13; string searchSetIdentifier = 14; @@ -61,7 +60,6 @@ message RpcIndexQuery { RpcSpecLimit year = 6; RpcSpecLimit size = 7; RpcSpecLimit rank = 8; - RpcSpecLimit domainCount = 9; RpcQueryLimits queryLimits = 10; string queryStrategy = 11; // Named query configuration RpcResultRankingParameters parameters = 12; diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java index 27ad1cbc..bbaf5c87 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/QueryParser.java @@ -82,8 +82,6 @@ public class QueryParser { entity.replace(new Token(TokenType.SIZE_TERM, t.str.substring(4), t.displayStr)); } else if (t.str.startsWith("rank") && t.str.matches("rank[=><]\\d+")) { entity.replace(new Token(TokenType.RANK_TERM, t.str.substring(4), t.displayStr)); - } else if (t.str.startsWith("count") && t.str.matches("count[=><]\\d+")) { - entity.replace(new Token(TokenType.DOMAIN_COUNT_TERM, t.str.substring(5), t.displayStr)); } else if (t.str.startsWith("qs=")) { entity.replace(new Token(TokenType.QS_TERM, t.str.substring(3), t.displayStr)); } else if (t.str.contains(":")) { diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java index b973d777..06c28972 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/Token.java @@ -40,7 +40,6 @@ public class Token { case YEAR_TERM: visitor.onYearTerm(this); break; case RANK_TERM: visitor.onRankTerm(this); break; - case DOMAIN_COUNT_TERM: visitor.onDomainCountTerm(this); break; case SIZE_TERM: visitor.onSizeTerm(this); break; case QS_TERM: visitor.onQsTerm(this); break; diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java index 8c542e5a..85d55c35 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenType.java @@ -16,7 +16,6 @@ public enum TokenType implements Predicate { YEAR_TERM, SIZE_TERM, RANK_TERM, - DOMAIN_COUNT_TERM, NEAR_TERM, QS_TERM, diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java index f6a48d4d..2e14f837 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/query_parser/token/TokenVisitor.java @@ -9,7 +9,6 @@ public interface TokenVisitor { void onYearTerm(Token token); void onSizeTerm(Token token); void onRankTerm(Token token); - void onDomainCountTerm(Token token); void onQualityTerm(Token token); void onQsTerm(Token token); } diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java index 4ed97afb..f641e52f 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java @@ -127,7 +127,6 @@ public class QueryFactory { .subqueries(subqueries) .humanQuery(query) .quality(qualityLimits.qualityLimit) - .domainCount(qualityLimits.domainCount) .year(qualityLimits.year) .size(qualityLimits.size) .rank(qualityLimits.rank) diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java index 4d230650..1b49bab3 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryLimitsAccumulator.java @@ -11,7 +11,6 @@ public class QueryLimitsAccumulator implements TokenVisitor { public SpecificationLimit year; public SpecificationLimit size; public SpecificationLimit rank; - public SpecificationLimit domainCount; public QueryStrategy queryStrategy = QueryStrategy.AUTO; @@ -20,7 +19,6 @@ public class QueryLimitsAccumulator implements TokenVisitor { year = params.year(); size = params.size(); rank = params.rank(); - domainCount = params.domainCount(); } private SpecificationLimit parseSpecificationLimit(String str) { @@ -67,11 +65,6 @@ public class QueryLimitsAccumulator implements TokenVisitor { rank = parseSpecificationLimit(token.str); } - @Override - public void onDomainCountTerm(Token token) { - domainCount = parseSpecificationLimit(token.str); - } - @Override public void onQualityTerm(Token token) { qualityLimit = parseSpecificationLimit(token.str); diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java index 00991eac..e4def0d0 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QuerySearchTermsAccumulator.java @@ -103,8 +103,6 @@ public class QuerySearchTermsAccumulator implements TokenVisitor { @Override public void onRankTerm(Token token) {} @Override - public void onDomainCountTerm(Token token) {} - @Override public void onQualityTerm(Token token) {} @Override public void onQsTerm(Token token) {} diff --git a/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java b/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java index bafb929d..fe93a1f6 100644 --- a/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java +++ b/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java @@ -48,7 +48,6 @@ public class QueryFactoryTest { SpecificationLimit.none(), SpecificationLimit.none(), SpecificationLimit.none(), - SpecificationLimit.none(), null, new QueryLimits(100, 100, 100, 100), "NONE", diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index b210c851..f7821276 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -79,7 +79,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { private final IndexQueryService indexQueryService; private final IndexResultValuatorService resultValuator; - private final int nodeId; private final String nodeName; private final int indexValuationThreads = Integer.getInteger("index.valuationThreads", 8); @@ -91,7 +90,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { IndexQueryService indexQueryService, IndexResultValuatorService resultValuator) { - this.nodeId = serviceConfiguration.node(); + var nodeId = serviceConfiguration.node(); this.nodeName = Integer.toString(nodeId); this.index = index; this.searchSetsService = searchSetsService; @@ -107,6 +106,8 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { try { var params = new SearchParameters(request, getSearchSet(request)); + long endTime = System.currentTimeMillis() + request.getQueryLimits().getTimeoutMs(); + SearchResultSet results = wmsa_query_time .labels(nodeName, "GRPC") .time(() -> { @@ -119,7 +120,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { .labels(nodeName, "GRPC") .set(params.getDataCost()); - if (!params.hasTimeLeft()) { + if (System.currentTimeMillis() >= endTime) { wmsa_query_timeouts .labels(nodeName, "GRPC") .inc(); diff --git a/code/index/java/nu/marginalia/index/model/QueryParams.java b/code/index/java/nu/marginalia/index/model/QueryParams.java index 87774347..56e40551 100644 --- a/code/index/java/nu/marginalia/index/model/QueryParams.java +++ b/code/index/java/nu/marginalia/index/model/QueryParams.java @@ -10,8 +10,6 @@ import nu.marginalia.index.query.limit.SpecificationLimit; * @param year The year limit. * @param size The size limit. Eliminates results from domains that do not satisfy the size criteria. * @param rank The rank limit. Eliminates results from domains that do not satisfy the domain rank criteria. - * @param domainCount The domain count limit. Filters out results from domains that do not contain enough - * documents that match the query. * @param searchSet The search set. Limits the search to a set of domains. * @param queryStrategy The query strategy. May impose additional constraints on the query, such as requiring * the keywords to appear in the title, or in the domain. @@ -20,7 +18,6 @@ public record QueryParams(SpecificationLimit qualityLimit, SpecificationLimit year, SpecificationLimit size, SpecificationLimit rank, - SpecificationLimit domainCount, SearchSet searchSet, QueryStrategy queryStrategy ) diff --git a/code/index/java/nu/marginalia/index/model/SearchParameters.java b/code/index/java/nu/marginalia/index/model/SearchParameters.java index 7d440563..0594bd68 100644 --- a/code/index/java/nu/marginalia/index/model/SearchParameters.java +++ b/code/index/java/nu/marginalia/index/model/SearchParameters.java @@ -52,7 +52,6 @@ public class SearchParameters { specsSet.year, specsSet.size, specsSet.rank, - specsSet.domainCount, searchSet, specsSet.queryStrategy); @@ -80,7 +79,6 @@ public class SearchParameters { convertSpecLimit(request.getYear()), convertSpecLimit(request.getSize()), convertSpecLimit(request.getRank()), - convertSpecLimit(request.getDomainCount()), searchSet, QueryStrategy.valueOf(request.getQueryStrategy())); diff --git a/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java b/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java index 9fa1713a..9251a5d2 100644 --- a/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java +++ b/code/index/java/nu/marginalia/index/results/IndexResultValuatorService.java @@ -19,6 +19,7 @@ import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.util.*; +import java.util.function.Consumer; import java.util.stream.Collectors; @Singleton @@ -77,33 +78,15 @@ public class IndexResultValuatorService { for (var item : results) { if (domainCountFilter.test(item)) { - resultsList.add(item); + // It's important that this filter runs across all results, not just the top N + if (resultsList.size() < params.limitTotal) { + resultsList.add(item); + } } } - if (!params.queryParams.domainCount().isNone()) { - // Remove items that don't meet the domain count requirement - // This isn't perfect because the domain count is calculated - // after the results are sorted - resultsList.removeIf(item -> !params.queryParams.domainCount().test(domainCountFilter.getCount(item))); - } - - if (resultsList.size() > params.limitTotal) { - // This can't be made a stream limit() operation because we need domainCountFilter - // to run over the entire list to provide accurate statistics - - resultsList.subList(params.limitTotal, resultsList.size()).clear(); - } - - // populate results with the total number of results encountered from - // the same domain so this information can be presented to the user - for (var result : resultsList) { - result.resultsFromDomain = domainCountFilter.getCount(result); - } - - LongArrayList idsList = new LongArrayList(resultsList.size()); - for (var result : resultsList) { - idsList.add(result.getCombinedId()); + for (var item : resultsList) { + item.resultsFromDomain = domainCountFilter.getCount(item); } return decorateAndRerank(resultsList, rankingContext); @@ -125,23 +108,19 @@ public class IndexResultValuatorService { for (var item : documentDbReader.getUrlDetails(idsList)) urlDetailsById.put(item.urlId(), item); - List decoratedItems = new ArrayList<>(); + List resultItems = new ArrayList<>(rawResults.size()); for (var result : rawResults) { - var docData = urlDetailsById.get(result.getDocumentId()); + var id = result.getDocumentId(); + var docData = urlDetailsById.get(id); - if (null == docData) { - logger.warn("No data for document id {}", result.getDocumentId()); + if (docData == null) { + logger.warn("No document data for id {}", id); continue; } - decoratedItems.add(createCombinedItem(result, docData, rankingContext)); + resultItems.add(createCombinedItem(result, docData, rankingContext)); } - - if (decoratedItems.size() != rawResults.size()) - logger.warn("Result list shrunk during decoration?"); - - decoratedItems.sort(Comparator.naturalOrder()); - return decoratedItems; + return resultItems; } private DecoratedSearchResultItem createCombinedItem(SearchResultItem result, diff --git a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java index 0370898f..634481f4 100644 --- a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java +++ b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java @@ -120,7 +120,6 @@ public class IndexQueryServiceIntegrationSmokeTest { .quality(SpecificationLimit.none()) .size(SpecificationLimit.none()) .rank(SpecificationLimit.none()) - .domainCount(SpecificationLimit.none()) .rankingParams(ResultRankingParameters.sensibleDefaults()) .domains(new ArrayList<>()) .searchSetIdentifier("NONE") @@ -164,7 +163,6 @@ public class IndexQueryServiceIntegrationSmokeTest { .quality(SpecificationLimit.none()) .size(SpecificationLimit.none()) .rank(SpecificationLimit.none()) - .domainCount(SpecificationLimit.none()) .rankingParams(ResultRankingParameters.sensibleDefaults()) .queryStrategy(QueryStrategy.SENTENCE) .domains(List.of(2)) @@ -201,7 +199,6 @@ public class IndexQueryServiceIntegrationSmokeTest { .year(SpecificationLimit.equals(1998)) .size(SpecificationLimit.none()) .rank(SpecificationLimit.none()) - .domainCount(SpecificationLimit.none()) .queryStrategy(QueryStrategy.SENTENCE) .searchSetIdentifier("NONE") .rankingParams(ResultRankingParameters.sensibleDefaults()) diff --git a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java index 3431c18e..6def5bbc 100644 --- a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java +++ b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java @@ -422,7 +422,6 @@ public class IndexQueryServiceIntegrationTest { .quality(SpecificationLimit.none()) .size(SpecificationLimit.none()) .rank(SpecificationLimit.none()) - .domainCount(SpecificationLimit.none()) .rankingParams(ResultRankingParameters.sensibleDefaults()) .domains(new ArrayList<>()) .searchSetIdentifier("NONE") diff --git a/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java b/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java index 100e62b6..15c8567e 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java +++ b/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java @@ -33,7 +33,6 @@ public class SearchQueryParamFactory { profile.getYearLimit(), profile.getSizeLimit(), SpecificationLimit.none(), - SpecificationLimit.none(), List.of(), new QueryLimits(5, 100, 200, 8192), profile.searchSetIdentifier.name(), @@ -54,7 +53,6 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), SpecificationLimit.none(), - SpecificationLimit.none(), List.of(), new QueryLimits(count, count, 100, 512), SearchSetIdentifier.NONE.name(), @@ -74,7 +72,6 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), SpecificationLimit.none(), - SpecificationLimit.none(), List.of(), new QueryLimits(100, 100, 100, 512), SearchSetIdentifier.NONE.name(), @@ -94,7 +91,6 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), SpecificationLimit.none(), - SpecificationLimit.none(), List.of(), new QueryLimits(100, 100, 100, 512), SearchSetIdentifier.NONE.name(), From 3eb0800742bb9ff5a9c8da8c717c82359635a297 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 13:34:12 +0100 Subject: [PATCH 41/56] (index) Improve granularity of candidate queue polling --- code/index/java/nu/marginalia/index/IndexGrpcService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index f7821276..bdd2d5e4 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -348,7 +348,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { long start = System.currentTimeMillis(); CombinedDocIdList resultIds = resultCandidateQueue.poll( - Math.clamp(parameters.budget.timeLeft(), 1, 25), + Math.clamp(parameters.budget.timeLeft(), 1, 5), TimeUnit.MILLISECONDS); if (resultIds == null) { From 1a51ec2d69e0b1c94e592ebee2b4fa02b80b551f Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 14:01:58 +0100 Subject: [PATCH 42/56] (index) Index optimization --- .../nu/marginalia/index/IndexGrpcService.java | 30 ++++++++++------ .../index/index/IndexQueryService.java | 34 ++++++------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index bdd2d5e4..32fe3ca0 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -14,7 +14,10 @@ import nu.marginalia.api.searchquery.model.results.*; import nu.marginalia.index.index.IndexQueryService; import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.model.SearchParameters; +import nu.marginalia.index.model.SearchTerms; import nu.marginalia.index.model.SearchTermsUtil; +import nu.marginalia.index.query.IndexQuery; +import nu.marginalia.index.query.IndexSearchBudget; import nu.marginalia.index.results.IndexResultValuatorService; import nu.marginalia.index.results.model.ids.CombinedDocIdList; import nu.marginalia.index.searchset.SearchSetsService; @@ -252,8 +255,15 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { /** Execute a search query */ public SearchResultSet run(SearchParameters parameters) throws SQLException, InterruptedException { + for (var subquery : parameters.subqueries) { - workerPool.execute(new IndexLookup(subquery, parameters)); + var terms = new SearchTerms(subquery); + if (terms.isEmpty()) + continue; + + for (var indexQuery : index.createQueries(terms, parameters.queryParams)) { + workerPool.execute(new IndexLookup(indexQuery, parameters.budget)); + } } for (int i = 0; i < indexValuationThreads; i++) { @@ -283,12 +293,13 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { * resultCandidateQueue, which depending on the state of the valuator threads may * or may not block*/ class IndexLookup implements Runnable { - private final SearchSubquery subquery; - private final SearchParameters parameters; + private final IndexQuery query; + private final IndexSearchBudget budget; - IndexLookup(SearchSubquery subquery, SearchParameters parameters) { - this.subquery = subquery; - this.parameters = parameters; + IndexLookup(IndexQuery query, + IndexSearchBudget budget) { + this.query = query; + this.budget = budget; remainingIndexTasks.incrementAndGet(); } @@ -296,9 +307,8 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { public void run() { try { indexQueryService.evaluateSubquery( - subquery, - parameters.queryParams, - parameters.budget, + query, + budget, this::drain ); } @@ -312,7 +322,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { } private void drain(CombinedDocIdList resultIds) { - long remainingTime = parameters.budget.timeLeft(); + long remainingTime = budget.timeLeft(); try { if (!resultCandidateQueue.offer(resultIds)) { diff --git a/code/index/java/nu/marginalia/index/index/IndexQueryService.java b/code/index/java/nu/marginalia/index/index/IndexQueryService.java index f430de30..72a13910 100644 --- a/code/index/java/nu/marginalia/index/index/IndexQueryService.java +++ b/code/index/java/nu/marginalia/index/index/IndexQueryService.java @@ -16,7 +16,6 @@ import org.slf4j.LoggerFactory; import org.slf4j.Marker; import org.slf4j.MarkerFactory; -import java.util.List; import java.util.function.Consumer; @Singleton @@ -35,36 +34,25 @@ public class IndexQueryService { * at different priorty depths until timeout is reached or the results are all visited. * Then the results are combined. * */ - public void evaluateSubquery(SearchSubquery subquery, - QueryParams queryParams, + public void evaluateSubquery(IndexQuery query, IndexSearchBudget timeout, Consumer drain) { - final SearchTerms searchTerms = new SearchTerms(subquery); - final Roaring64Bitmap results = new Roaring64Bitmap(); + final LongArrayList results = new LongArrayList(512); // These queries are different indices for one subquery - List queries = index.createQueries(searchTerms, queryParams); - for (var query : queries) { + final LongQueryBuffer buffer = new LongQueryBuffer(512); - if (!timeout.hasTimeLeft()) - break; + while (query.hasMore() && timeout.hasTimeLeft()) + { + buffer.reset(); + query.getMoreResults(buffer); - final LongQueryBuffer buffer = new LongQueryBuffer(512); + results.addElements(0, buffer.data, 0, buffer.end); - while (query.hasMore() && timeout.hasTimeLeft()) - { - buffer.reset(); - query.getMoreResults(buffer); - - for (int i = 0; i < buffer.size(); i++) { - results.add(buffer.data[i]); - } - - if (results.getIntCardinality() > 512) { - drain.accept(new CombinedDocIdList(results)); - results.clear(); - } + if (results.size() < 512) { + drain.accept(new CombinedDocIdList(results)); + results.clear(); } } From 5604e9f531620e1ca836624ed669654a3aab853f Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 25 Feb 2024 16:01:45 +0100 Subject: [PATCH 43/56] (query) Bump query length, see what happens :P --- .../nu/marginalia/functions/searchquery/svc/QueryFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java index f641e52f..ac7ce2b2 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/svc/QueryFactory.java @@ -81,7 +81,7 @@ public class QueryFactory { var basicQuery = queryParser.parse(query); - if (basicQuery.size() >= 8) { + if (basicQuery.size() >= 12) { problems.add("Your search query is too long"); basicQuery.clear(); } From 67aa20ea2c3c9a2fbaab86142d1ee2501fc090f8 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Mon, 26 Feb 2024 10:32:49 +0100 Subject: [PATCH 44/56] (array) Attempting to debug strange errors --- .../ReverseIndexBTreeTransformer.java | 2 +- .../index/construction/ReversePreindex.java | 37 ++-- .../ReversePreindexWordSegments.java | 13 +- .../array/page/SegmentLongArray.java | 6 +- .../array/page/UnsafeLongArray.java | 173 ++++++++++++++---- 5 files changed, 169 insertions(+), 62 deletions(-) diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java index 2f5c05f4..4ace48a9 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java @@ -30,7 +30,7 @@ public class ReverseIndexBTreeTransformer implements LongIOTransformer { @Override public long transform(long pos, long end) throws IOException { - final int size = (int) (end - start) / entrySize; + final int size = (int) ((end - start) / entrySize); if (size == 0) { return -1; diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindex.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindex.java index 22fc1431..ac39e817 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindex.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindex.java @@ -92,7 +92,8 @@ public class ReversePreindex { LongArray wordIds = segments.wordIds; - assert offsets.size() == wordIds.size() : "Offsets and word-ids of different size"; + if (offsets.size() != wordIds.size()) + throw new IllegalStateException("Offsets and word-ids of different size"); if (offsets.size() > Integer.MAX_VALUE) { throw new IllegalStateException("offsets.size() too big!"); } @@ -137,7 +138,7 @@ public class ReversePreindex { Path docsFile = Files.createTempFile(destDir, "docs", ".dat"); - LongArray mergedDocuments = LongArrayFactory.mmapForWritingConfined(docsFile, 2 * (left.documents.size() + right.documents.size())); + LongArray mergedDocuments = LongArrayFactory.mmapForWritingConfined(docsFile, left.documents.size() + right.documents.size()); leftIter.next(); rightIter.next(); @@ -180,9 +181,15 @@ public class ReversePreindex { } - assert !leftIter.isPositionBeforeEnd() : "Left has more to go"; - assert !rightIter.isPositionBeforeEnd() : "Right has more to go"; - assert !mergingIter.canPutMore() : "Source iters ran dry before merging iter"; + if (leftIter.isPositionBeforeEnd()) + throw new IllegalStateException("Left has more to go"); + if (rightIter.isPositionBeforeEnd()) + throw new IllegalStateException("Right has more to go"); + if (mergingIter.canPutMore()) + throw new IllegalStateException("Source iters ran dry before merging iter"); + + + mergingSegment.force(); // We may have overestimated the size of the merged docs size in the case there were // duplicates in the data, so we need to shrink it to the actual size we wrote. @@ -190,8 +197,6 @@ public class ReversePreindex { mergedDocuments = shrinkMergedDocuments(mergedDocuments, docsFile, 2 * mergingSegment.totalSize()); - mergingSegment.force(); - return new ReversePreindex( mergingSegment, new ReversePreindexDocuments(mergedDocuments, docsFile) @@ -233,16 +238,15 @@ public class ReversePreindex { mergedDocuments.force(); long beforeSize = mergedDocuments.size(); - try (var bc = Files.newByteChannel(docsFile, StandardOpenOption.WRITE)) { - bc.truncate(sizeLongs * 8); - } - long afterSize = mergedDocuments.size(); - mergedDocuments.close(); - - mergedDocuments = LongArrayFactory.mmapForWritingConfined(docsFile, sizeLongs); - + long afterSize = sizeLongs * 8; if (beforeSize != afterSize) { + mergedDocuments.close(); + try (var bc = Files.newByteChannel(docsFile, StandardOpenOption.WRITE)) { + bc.truncate(sizeLongs * 8); + } + logger.info("Shrunk {} from {}b to {}b", docsFile, beforeSize, afterSize); + mergedDocuments = LongArrayFactory.mmapForWritingConfined(docsFile, sizeLongs); } return mergedDocuments; @@ -291,7 +295,8 @@ public class ReversePreindex { boolean putNext = mergingIter.putNext(size / 2); boolean iterNext = sourceIter.next(); - assert putNext || !iterNext : "Source iterator ran out before dest iterator?!"; + if (!putNext && iterNext) + throw new IllegalStateException("Source iterator ran out before dest iterator?!"); return iterNext; } diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java index bcfe486e..04303210 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java @@ -38,6 +38,9 @@ public class ReversePreindexWordSegments { * and each value is the start offset of the data. */ public Long2LongOpenHashMap asMap(int recordSize) { + if (wordIds.size() > Integer.MAX_VALUE) + throw new IllegalArgumentException("Cannot create a map with more than Integer.MAX_VALUE entries"); + Long2LongOpenHashMap ret = new Long2LongOpenHashMap((int) wordIds.size(), 0.75f); var iter = iterator(recordSize); @@ -62,7 +65,7 @@ public class ReversePreindexWordSegments { // Create the words file by iterating over the map and inserting them into // the words file in whatever bizarro hash table order they appear in - int i = 0; + long i = 0; LongIterator iter = countsMap.keySet().iterator(); while (iter.hasNext()) { words.set(i++, iter.nextLong()); @@ -120,8 +123,8 @@ public class ReversePreindexWordSegments { this.fileSize = wordIds.size(); } - private int i = -1; - public int idx() { + private long i = -1; + public long idx() { return i; } public boolean next() { @@ -166,8 +169,8 @@ public class ReversePreindexWordSegments { this.wordId = wordIds.get(0); } - private int i = 0; - public int idx() { + private long i = 0; + public long idx() { return i; } diff --git a/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java b/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java index 21044b68..5e3b8a59 100644 --- a/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java +++ b/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java @@ -154,7 +154,11 @@ public class SegmentLongArray implements PartitionPage, LongArray { @Override public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { - final int stride = 1024*1204*128; // Copy 1 GB at a time 'cause byte buffers are 'a byte buffering + final int stride = 1024*1024*128; // Copy 1 GB at a time 'cause byte buffers are 'a byte buffering + + if (source.size() / 8 < sourceStart + (arrayEnd - arrayStart)) { + throw new IndexOutOfBoundsException(STR."Source channel too small: \{source.size()} < \{sourceStart + (arrayEnd - arrayStart)}"); + } long ss = sourceStart; for (long as = arrayStart; as < arrayEnd; as += stride, ss += stride) { diff --git a/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java b/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java index 8ba7182d..4ef0da02 100644 --- a/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java +++ b/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java @@ -2,8 +2,11 @@ package nu.marginalia.array.page; import nu.marginalia.array.ArrayRangeReference; import nu.marginalia.array.LongArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import sun.misc.Unsafe; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.lang.foreign.Arena; @@ -12,7 +15,6 @@ import java.nio.ByteBuffer; import java.nio.LongBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; -import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -23,9 +25,13 @@ import static java.lang.foreign.ValueLayout.JAVA_LONG; public class UnsafeLongArray implements PartitionPage, LongArray { private static final Unsafe unsafe = UnsafeProvider.getUnsafe(); + private static final Logger logger = LoggerFactory.getLogger(UnsafeLongArray.class); @Nullable private final Arena arena; + @Nullable + private final FileChannel channel; + private final MemorySegment segment; private boolean closed; @@ -33,6 +39,15 @@ public class UnsafeLongArray implements PartitionPage, LongArray { @Nullable Arena arena) { this.segment = segment; this.arena = arena; + this.channel = null; + } + + UnsafeLongArray(MemorySegment segment, + @Nonnull FileChannel channel, + @Nullable Arena arena) { + this.segment = segment; + this.arena = arena; + this.channel = channel; } public static UnsafeLongArray onHeap(Arena arena, long size) { @@ -40,38 +55,26 @@ public class UnsafeLongArray implements PartitionPage, LongArray { } public static UnsafeLongArray fromMmapReadOnly(Arena arena, Path file, long offset, long size) throws IOException { - return new UnsafeLongArray( - mmapFile(arena, file, offset, size, FileChannel.MapMode.READ_ONLY, StandardOpenOption.READ), - arena); - } - - public static UnsafeLongArray fromMmapReadWrite(Arena arena, Path file, long offset, long size) throws IOException { - - return new UnsafeLongArray( - mmapFile(arena, file, offset, size, FileChannel.MapMode.READ_WRITE, - StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE), - arena); - } - - private static MemorySegment mmapFile(Arena arena, - Path file, - long offset, - long size, - FileChannel.MapMode mode, - OpenOption... openOptions) throws IOException - { - try (var channel = (FileChannel) Files.newByteChannel(file, openOptions)) { - - return channel.map(mode, - JAVA_LONG.byteSize() * offset, - JAVA_LONG.byteSize() * size, - arena); + try (var channel = (FileChannel) Files.newByteChannel(file, StandardOpenOption.READ)) { + return new UnsafeLongArray(channel.map(FileChannel.MapMode.READ_ONLY, + JAVA_LONG.byteSize() * offset, JAVA_LONG.byteSize() * size, + arena), arena); } catch (IOException ex) { throw new IOException("Failed to map file " + file + " (" + offset + ":" + size + ")", ex); } } + public static UnsafeLongArray fromMmapReadWrite(Arena arena, Path file, long offset, long size) throws IOException { + var channel = (FileChannel) Files.newByteChannel(file, + StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + var segment = channel.map(FileChannel.MapMode.READ_WRITE, + JAVA_LONG.byteSize() * offset, JAVA_LONG.byteSize() * size, + arena); + + return new UnsafeLongArray(segment, channel, arena); + } + @Override public LongArray range(long start, long end) { return new UnsafeLongArray( @@ -122,6 +125,15 @@ public class UnsafeLongArray implements PartitionPage, LongArray { if (arena != null && !closed) { arena.close(); } + if (channel != null && !closed) { + try { + channel.close(); + } + catch (IOException ex) { + throw new RuntimeException("Failed to close channel", ex); + } + } + closed = true; } @@ -149,6 +161,13 @@ public class UnsafeLongArray implements PartitionPage, LongArray { public void force() { if (segment.isMapped()) { segment.force(); + try { + if (channel != null) { + channel.force(false); + } + } catch (IOException e) { + throw new RuntimeException("Failed to force channel", e); + } } } @@ -156,26 +175,102 @@ public class UnsafeLongArray implements PartitionPage, LongArray { return new ArrayRangeReference<>(this, start, end); } - @Override - public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { + public void chanelChannelTransfer(FileChannel source, + long sourceStartL, + long arrayStartL, + long arrayEndL) throws IOException { - final int stride = 1024*1204*128; // Copy 1 GB at a time 'cause byte buffers are 'a byte buffering + assert channel != null; - long ss = sourceStart; - for (long as = arrayStart; as < arrayEnd; as += stride, ss += stride) { - long ae = Math.min(as + stride, arrayEnd); + final int B_per_L = (int) JAVA_LONG.byteSize(); - long index = as * JAVA_LONG.byteSize(); - long length = (ae - as) * JAVA_LONG.byteSize(); + final int strideB = 128*1024*1024; // Copy in 128 MB chunks - var bufferSlice = segment.asSlice(index, length).asByteBuffer(); + final long destStartB = arrayStartL * B_per_L; + final long destEndB = arrayEndL * B_per_L; + final long lengthB = destEndB - destStartB; - long startPos = ss * JAVA_LONG.byteSize(); - while (bufferSlice.position() < bufferSlice.capacity()) { - source.read(bufferSlice, startPos + bufferSlice.position()); + final long sourceStartB = sourceStartL * B_per_L; + final long sourceEndB = sourceStartB + lengthB; + + + if (sourceStartB > sourceEndB) + throw new IndexOutOfBoundsException("Source start after end"); + if (sourceStartB > source.size()) + throw new IndexOutOfBoundsException("Source channel too small, start " + sourceStartB + " < input size " + source.size()); + if (sourceEndB > source.size()) + throw new IndexOutOfBoundsException("Source channel too small, end " + sourceEndB + " < input size " + source.size()); + + long destIndexB = destStartB; + + source.position(sourceStartB); + + while (destIndexB < destEndB) + { + long stepSizeB = Math.min(destIndexB + strideB, destEndB); + long copyLengthB = (stepSizeB - destIndexB); + + long transferred = channel.transferFrom(source, destIndexB, copyLengthB); + if (transferred != copyLengthB) { + logger.warn("Less than {} bytes were copied: {}", copyLengthB, transferred); } + + destIndexB += copyLengthB; + } + } + + @Override + public void transferFrom(FileChannel source, + long sourceStartL, + long arrayStartL, + long arrayEndL) throws IOException { + + + if (channel != null) { + chanelChannelTransfer(source, sourceStartL, arrayStartL, arrayEndL); + return; } + final int B_per_L = (int) JAVA_LONG.byteSize(); + + final int strideB = 1024*1024*1024; // Copy 1 GB at a time + + final long arrayStartB = arrayStartL * B_per_L; + final long arrayEndB = arrayEndL * B_per_L; + final long arrayLengthB = arrayEndB - arrayStartB; + + final long sourceStartB = sourceStartL * B_per_L; + final long sourceEndB = sourceStartB + arrayLengthB; + + + if (sourceStartB > sourceEndB) + throw new IndexOutOfBoundsException("Source start after end"); + if (sourceStartB > source.size()) + throw new IndexOutOfBoundsException("Source channel too small, start " + sourceStartB + " < input size " + source.size()); + if (sourceEndB > source.size()) + throw new IndexOutOfBoundsException("Source channel too small, end " + sourceEndB + " < input size " + source.size()); + + long channelIndexB = sourceStartB; + long segmentIndexB = arrayStartB; + + while (segmentIndexB < arrayEndB) + { + long segmentEndB = Math.min(segmentIndexB + strideB, arrayEndB); + long lengthB = (segmentEndB - segmentIndexB); + + var bufferSlice = segment.asSlice(segmentIndexB, lengthB).asByteBuffer(); + + while (bufferSlice.position() < bufferSlice.capacity()) { + if (source.position() + bufferSlice.capacity() > sourceEndB) + throw new IndexOutOfBoundsException("Source channel too small"); + + if (source.read(bufferSlice, channelIndexB + bufferSlice.position()) < 0) + throw new IOException("Failed to read from source"); + } + + channelIndexB += lengthB; + segmentIndexB += lengthB; + } } } From 8d0af9548ba3ec29078c85683dac330f9289b057 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Mon, 26 Feb 2024 13:53:12 +0100 Subject: [PATCH 45/56] (search) Bot mitigation Add the ability to indicate to the search service that a request is malicious, and to poison the results by providing randomly reorered old results instead. --- .../nu/marginalia/search/SearchOperator.java | 56 +++++++++++++++---- .../search/command/SearchParameters.java | 13 +++-- .../search/svc/SearchQueryService.java | 3 +- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java index d3141159..e07186b6 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java @@ -20,6 +20,7 @@ import org.slf4j.Marker; import org.slf4j.MarkerFactory; import javax.annotation.Nullable; +import java.lang.ref.WeakReference; import java.time.Duration; import java.util.*; import java.util.concurrent.Future; @@ -87,31 +88,62 @@ public class SearchOperator { return searchQueryService.getResultsFromQuery(queryResponse); } + private volatile WeakReference> oldResults = new WeakReference<>(Collections.emptyList()); + public DecoratedSearchResults doSearch(SearchParameters userParams) { Future eval = searchUnitConversionService.tryEval(userParams.query()); - var queryParams = paramFactory.forRegularSearch(userParams); - var queryResponse = queryClient.search(queryParams); - List queryResults = searchQueryService.getResultsFromQuery(queryResponse); + List clusteredResults; + QueryResponse queryResponse; + List problems; + String evalResult; + String focusDomain; - logger.info(queryMarker, "Human terms: {}", Strings.join(queryResponse.searchTermsHuman(), ',')); - logger.info(queryMarker, "Search Result Count: {}", queryResults.size()); + if (userParams.poisonResults() && Math.random() > 0.1) { - String evalResult = getFutureOrDefault(eval, ""); + // For botnet users, we return random old query results. This is to make + // it harder for them to figure out if they are being rate limited. - List clusteredResults = SearchResultClusterer - .selectStrategy(queryResponse) - .clusterResults(queryResults, 25); + clusteredResults = new ArrayList<>(Objects.requireNonNullElse(oldResults.get(), List.of())); + + // Shuffle the results to make it harder to distinguish + Collections.shuffle(clusteredResults); + + problems = List.of(); + evalResult = ""; + focusDomain = ""; + } else { + var queryParams = paramFactory.forRegularSearch(userParams); + queryResponse = queryClient.search(queryParams); + var queryResults = searchQueryService.getResultsFromQuery(queryResponse); + + logger.info(queryMarker, "Human terms: {}", Strings.join(queryResponse.searchTermsHuman(), ',')); + logger.info(queryMarker, "Search Result Count: {}", queryResults.size()); + + evalResult = getFutureOrDefault(eval, ""); + + clusteredResults = SearchResultClusterer + .selectStrategy(queryResponse) + .clusterResults(queryResults, 25); + + focusDomain = queryResponse.domain(); + problems = getProblems(evalResult, queryResults, queryResponse); + + if (userParams.poisonResults()) { + // Save the results to feed to the botnet + oldResults = new WeakReference<>(clusteredResults); + } + } return DecoratedSearchResults.builder() .params(userParams) - .problems(getProblems(evalResult, queryResults, queryResponse)) + .problems(problems) .evalResult(evalResult) .results(clusteredResults) .filters(new SearchFilters(websiteUrl, userParams)) - .focusDomain(queryResponse.domain()) - .focusDomainId(getDomainId(queryResponse.domain())) + .focusDomain(focusDomain) + .focusDomainId(getDomainId(focusDomain)) .build(); } diff --git a/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java index fcb90f13..9c47419d 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java +++ b/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java @@ -16,29 +16,30 @@ public record SearchParameters(String query, SearchJsParameter js, SearchRecentParameter recent, SearchTitleParameter searchTitle, - SearchAdtechParameter adtech + SearchAdtechParameter adtech, + boolean poisonResults ) { public String profileStr() { return profile.filterId; } public SearchParameters withProfile(SearchProfile profile) { - return new SearchParameters(query, profile, js, recent, searchTitle, adtech); + return new SearchParameters(query, profile, js, recent, searchTitle, adtech, poisonResults); } public SearchParameters withJs(SearchJsParameter js) { - return new SearchParameters(query, profile, js, recent, searchTitle, adtech); + return new SearchParameters(query, profile, js, recent, searchTitle, adtech, poisonResults); } public SearchParameters withAdtech(SearchAdtechParameter adtech) { - return new SearchParameters(query, profile, js, recent, searchTitle, adtech); + return new SearchParameters(query, profile, js, recent, searchTitle, adtech, poisonResults); } public SearchParameters withRecent(SearchRecentParameter recent) { - return new SearchParameters(query, profile, js, recent, searchTitle, adtech); + return new SearchParameters(query, profile, js, recent, searchTitle, adtech, poisonResults); } public SearchParameters withTitle(SearchTitleParameter title) { - return new SearchParameters(query, profile, js, recent, title, adtech); + return new SearchParameters(query, profile, js, recent, title, adtech, poisonResults); } public String renderUrl(WebsiteUrl baseUrl) { diff --git a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java index 948b0af7..3cc9166d 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java +++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchQueryService.java @@ -57,7 +57,8 @@ public class SearchQueryService { SearchJsParameter.parse(request.queryParams("js")), SearchRecentParameter.parse(request.queryParams("recent")), SearchTitleParameter.parse(request.queryParams("searchTitle")), - SearchAdtechParameter.parse(request.queryParams("adtech")) + SearchAdtechParameter.parse(request.queryParams("adtech")), + "1".equals(request.headers("X-Poison-Results")) ); } catch (Exception ex) { From dbf64b09875fc93fcb4f6aea51dfff5deea05e87 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 27 Feb 2024 13:39:24 +0100 Subject: [PATCH 46/56] (logs) Add the option for json logging --- .../nu/marginalia/service/server/Service.java | 5 ++++ code/common/service/resources/log4j2-json.xml | 28 +++++++++++++++++++ .../nu/marginalia/process/ProcessService.java | 6 ++++ 3 files changed, 39 insertions(+) create mode 100644 code/common/service/resources/log4j2-json.xml diff --git a/code/common/service/java/nu/marginalia/service/server/Service.java b/code/common/service/java/nu/marginalia/service/server/Service.java index ba076ae5..c959e269 100644 --- a/code/common/service/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/java/nu/marginalia/service/server/Service.java @@ -90,6 +90,11 @@ public class Service { initialization.addCallback(() -> params.eventLog.logEvent("SVC-INIT", serviceName + ":" + config.node())); initialization.addCallback(() -> serviceRegistry.announceInstance(config.instanceUuid())); + Thread.setDefaultUncaughtExceptionHandler((t, e) -> { + logger.error("Uncaught exception", e); + request_counter_err.labels(serviceName, Integer.toString(node)).inc(); + }); + if (!initialization.isReady() && ! initialized ) { initialized = true; diff --git a/code/common/service/resources/log4j2-json.xml b/code/common/service/resources/log4j2-json.xml new file mode 100644 index 00000000..dd2fc6d2 --- /dev/null +++ b/code/common/service/resources/log4j2-json.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/code/execution/java/nu/marginalia/process/ProcessService.java b/code/execution/java/nu/marginalia/process/ProcessService.java index 848f0233..0d153c8f 100644 --- a/code/execution/java/nu/marginalia/process/ProcessService.java +++ b/code/execution/java/nu/marginalia/process/ProcessService.java @@ -94,6 +94,12 @@ public class ProcessService { else args.add("-da"); args.add("--enable-preview"); + + String loggingOpts = System.getProperty("log4j2.configurationFile"); + if (loggingOpts != null) { + args.add("-Dlog4j.configurationFile=" + loggingOpts); + } + args.addAll(processId.envOpts()); args.add(processId.mainClass); args.addAll(Arrays.asList(extraArgs)); From eaf836dc6685c7d7cc98efc64bab4e3265bb9464 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 27 Feb 2024 15:04:34 +0100 Subject: [PATCH 47/56] (service/grpc) Reduce thread count Netty and GRPC by default spawns an incredible number of threads on high-core CPUs, which amount to a fair bit of RAM usage. Add custom executors that throttle this behavior. --- .../client/GrpcChannelPoolFactory.java | 16 ++++++++++ .../nu/marginalia/service/server/Service.java | 31 ++++++++++++++++++- .../nu/marginalia/feedlot/FeedlotClient.java | 2 +- .../api/domains/DomainInfoClient.java | 8 ++--- .../nu/marginalia/api/math/MathClient.java | 13 ++++---- .../nu/marginalia/index/api/IndexClient.java | 2 +- .../nu/marginalia/index/IndexGrpcService.java | 4 +-- .../IndexResultDomainDeduplicator.java | 22 +------------ 8 files changed, 62 insertions(+), 36 deletions(-) diff --git a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java index a4d0035a..96221909 100644 --- a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java +++ b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java @@ -10,7 +10,12 @@ import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; import nu.marginalia.service.discovery.property.ServiceKey; import nu.marginalia.service.discovery.property.ServicePartition; +import org.jetbrains.annotations.NotNull; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @Singleton @@ -18,6 +23,16 @@ public class GrpcChannelPoolFactory { private final NodeConfigurationWatcher nodeConfigurationWatcher; private final ServiceRegistryIf serviceRegistryIf; + private static final Executor executor = Executors.newFixedThreadPool( + Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 16), new ThreadFactory() { + static final AtomicInteger threadNumber = new AtomicInteger(1); + @Override + public Thread newThread(@NotNull Runnable r) { + var thread = new Thread(r, STR."gRPC-Channel-Pool[\{threadNumber.getAndIncrement()}]"); + thread.setDaemon(true); + return thread; + } + }); @Inject public GrpcChannelPoolFactory(NodeConfigurationWatcher nodeConfigurationWatcher, @@ -49,6 +64,7 @@ public class GrpcChannelPoolFactory { var mc = ManagedChannelBuilder .forAddress(route.host(), route.port()) + .executor(executor) .usePlaintext() .build(); diff --git a/code/common/service/java/nu/marginalia/service/server/Service.java b/code/common/service/java/nu/marginalia/service/server/Service.java index c959e269..dcc514a6 100644 --- a/code/common/service/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/java/nu/marginalia/service/server/Service.java @@ -2,6 +2,8 @@ package nu.marginalia.service.server; import io.grpc.*; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel; import io.prometheus.client.Counter; import lombok.SneakyThrows; import nu.marginalia.mq.inbox.*; @@ -19,6 +21,11 @@ import spark.Spark; import java.net.InetSocketAddress; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; public class Service { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -121,8 +128,16 @@ public class Service { int port = params.serviceRegistry.requestPort(config.externalAddress(), new ServiceKey.Grpc<>("-", partition)); + + int nThreads = Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 8); + // Start the gRPC server - var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port)); + var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port)) + .executor(namedExecutor("nettyExecutor", nThreads)) + .workerEventLoopGroup(new NioEventLoopGroup(nThreads, namedExecutor("Worker-ELG", nThreads))) + .bossEventLoopGroup(new NioEventLoopGroup(nThreads, namedExecutor("Boss-ELG", nThreads))) + .channelType(NioServerSocketChannel.class); + for (var grpcService : grpcServices) { var svc = grpcService.bindService(); @@ -138,6 +153,20 @@ public class Service { } } + private ExecutorService namedExecutor(String name, int limit) { + return Executors.newFixedThreadPool( + limit, + new ThreadFactory() { + static final AtomicInteger threadNumber = new AtomicInteger(1); + @Override + public Thread newThread(Runnable r) { + var thread = new Thread(r, STR."\{name}[\{threadNumber.getAndIncrement()}]"); + thread.setDaemon(true); + return thread; + } + }); + } + public Service(BaseServiceParams params, ServicePartition partition, List grpcServices) { diff --git a/code/features-search/feedlot-client/java/nu/marginalia/feedlot/FeedlotClient.java b/code/features-search/feedlot-client/java/nu/marginalia/feedlot/FeedlotClient.java index 3392a8d2..d247a8e2 100644 --- a/code/features-search/feedlot-client/java/nu/marginalia/feedlot/FeedlotClient.java +++ b/code/features-search/feedlot-client/java/nu/marginalia/feedlot/FeedlotClient.java @@ -30,7 +30,7 @@ public class FeedlotClient { this.gson = gson; httpClient = HttpClient.newBuilder() - .executor(Executors.newVirtualThreadPerTaskExecutor()) + .executor(Executors.newCachedThreadPool()) .connectTimeout(connectTimeout) .build(); this.requestTimeout = requestTimeout; diff --git a/code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java b/code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java index 80ca97fb..dbd9b102 100644 --- a/code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java +++ b/code/functions/domain-info/api/java/nu/marginalia/api/domains/DomainInfoClient.java @@ -19,7 +19,7 @@ public class DomainInfoClient { private static final Logger logger = LoggerFactory.getLogger(DomainInfoClient.class); private final GrpcSingleNodeChannelPool channelPool; - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + private final ExecutorService executor = Executors.newWorkStealingPool(8); @Inject public DomainInfoClient(GrpcChannelPoolFactory factory) { @@ -30,21 +30,21 @@ public class DomainInfoClient { public Future> similarDomains(int domainId, int count) { return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getSimilarDomains) - .async(virtualExecutorService) + .async(executor) .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count)) .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse); } public Future> linkedDomains(int domainId, int count) { return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getLinkingDomains) - .async(virtualExecutorService) + .async(executor) .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count)) .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse); } public Future domainInformation(int domainId) { return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getDomainInfo) - .async(virtualExecutorService) + .async(executor) .run(DomainsProtobufCodec.DomainInfo.createRequest(domainId)) .thenApply(DomainsProtobufCodec.DomainInfo::convertResponse); } diff --git a/code/functions/math/api/java/nu/marginalia/api/math/MathClient.java b/code/functions/math/api/java/nu/marginalia/api/math/MathClient.java index 27867b6c..ee0a55cd 100644 --- a/code/functions/math/api/java/nu/marginalia/api/math/MathClient.java +++ b/code/functions/math/api/java/nu/marginalia/api/math/MathClient.java @@ -24,7 +24,8 @@ public class MathClient { private static final Logger logger = LoggerFactory.getLogger(MathClient.class); private final GrpcSingleNodeChannelPool channelPool; - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + private final ExecutorService executor = Executors.newWorkStealingPool(8); + @Inject public MathClient(GrpcChannelPoolFactory factory) { this.channelPool = factory.createSingle( @@ -35,7 +36,7 @@ public class MathClient { public Future dictionaryLookup(String word) { return channelPool.call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) - .async(virtualExecutorService) + .async(executor) .run(DictionaryLookup.createRequest(word)) .thenApply(DictionaryLookup::convertResponse); } @@ -43,7 +44,7 @@ public class MathClient { @SuppressWarnings("unchecked") public Future> spellCheck(String word) { return channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck) - .async(virtualExecutorService) + .async(executor) .run(SpellCheck.createRequest(word)) .thenApply(SpellCheck::convertResponse); } @@ -52,7 +53,7 @@ public class MathClient { List requests = words.stream().map(SpellCheck::createRequest).toList(); var future = channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck) - .async(virtualExecutorService) + .async(executor) .runFor(requests); try { @@ -70,14 +71,14 @@ public class MathClient { public Future unitConversion(String value, String from, String to) { return channelPool.call(MathApiGrpc.MathApiBlockingStub::unitConversion) - .async(virtualExecutorService) + .async(executor) .run(UnitConversion.createRequest(from, to, value)) .thenApply(UnitConversion::convertResponse); } public Future evalMath(String expression) { return channelPool.call(MathApiGrpc.MathApiBlockingStub::evalMath) - .async(virtualExecutorService) + .async(executor) .run(EvalMath.createRequest(expression)) .thenApply(EvalMath::convertResponse); } diff --git a/code/index/api/java/nu/marginalia/index/api/IndexClient.java b/code/index/api/java/nu/marginalia/index/api/IndexClient.java index 4d8b79d2..33848f06 100644 --- a/code/index/api/java/nu/marginalia/index/api/IndexClient.java +++ b/code/index/api/java/nu/marginalia/index/api/IndexClient.java @@ -22,7 +22,7 @@ import java.util.concurrent.Executors; public class IndexClient { private static final Logger logger = LoggerFactory.getLogger(IndexClient.class); private final GrpcMultiNodeChannelPool channelPool; - private static final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); + private static final ExecutorService executor = Executors.newFixedThreadPool(8); @Inject public IndexClient(GrpcChannelPoolFactory channelPoolFactory) { this.channelPool = channelPoolFactory.createMulti( diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index 32fe3ca0..957dec5c 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -84,7 +84,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { private final String nodeName; - private final int indexValuationThreads = Integer.getInteger("index.valuationThreads", 8); + private static final int indexValuationThreads = Integer.getInteger("index.valuationThreads", 8); @Inject public IndexGrpcService(ServiceConfiguration serviceConfiguration, @@ -227,7 +227,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase { * and finally the best results are returned. */ private class QueryExecution { - private static final Executor workerPool = Executors.newCachedThreadPool(); + private static final Executor workerPool = Executors.newWorkStealingPool(indexValuationThreads*4); private final ArrayBlockingQueue resultCandidateQueue = new ArrayBlockingQueue<>(8); diff --git a/code/index/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java b/code/index/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java index fac47a47..d87b39fe 100644 --- a/code/index/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java +++ b/code/index/java/nu/marginalia/index/results/IndexResultDomainDeduplicator.java @@ -5,7 +5,7 @@ import gnu.trove.map.hash.TLongIntHashMap; import nu.marginalia.api.searchquery.model.results.SearchResultItem; public class IndexResultDomainDeduplicator { - final TLongIntMap resultsByDomainId = CachedObjects.getMap(); + final TLongIntMap resultsByDomainId = new TLongIntHashMap(2048, 0.5f, -1, 0); final int limitByDomain; public IndexResultDomainDeduplicator(int limitByDomain) { @@ -23,25 +23,5 @@ public class IndexResultDomainDeduplicator { return resultsByDomainId.get(key); } - - private static class CachedObjects { - private static final ThreadLocal mapCache = ThreadLocal.withInitial(() -> - new TLongIntHashMap(2048, 0.5f, -1, 0) - ); - - private static TLongIntHashMap getMap() { - var ret = mapCache.get(); - ret.clear(); - return ret; - } - - public static void clear() { - mapCache.remove(); - } - } - - static void clearCachedObjects() { - CachedObjects.clear(); - } } From c943954bb4bcd2d3716225a040ef0b598844c22e Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 27 Feb 2024 20:25:52 +0100 Subject: [PATCH 48/56] (domain-info) Reduce memory usage --- code/functions/domain-info/build.gradle | 1 + .../domains/SimilarDomainsService.java | 62 ++++++++++--------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/code/functions/domain-info/build.gradle b/code/functions/domain-info/build.gradle index 11e3f08c..1aa2da16 100644 --- a/code/functions/domain-info/build.gradle +++ b/code/functions/domain-info/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation libs.spark implementation libs.opencsv implementation libs.trove + implementation libs.roaringbitmap implementation libs.fastutil implementation libs.bundles.gson implementation libs.bundles.mariadb diff --git a/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java b/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java index 8ce2b55c..9be481b7 100644 --- a/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java +++ b/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java @@ -8,10 +8,12 @@ import gnu.trove.map.hash.TIntDoubleHashMap; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; +import it.unimi.dsi.fastutil.ints.Int2DoubleArrayMap; import nu.marginalia.api.domains.*; import nu.marginalia.api.domains.model.SimilarDomain; import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.model.EdgeDomain; +import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,12 +34,12 @@ public class SimilarDomainsService { private volatile TIntIntHashMap domainIdToIdx = new TIntIntHashMap(100_000); private volatile int[] domainIdxToId; - public volatile TIntDoubleHashMap[] relatedDomains; + public volatile Int2DoubleArrayMap[] relatedDomains; public volatile TIntList[] domainNeighbors = null; - public volatile BitSet screenshotDomains = null; - public volatile BitSet activeDomains = null; - public volatile BitSet indexedDomains = null; - public volatile double[] domainRanks = null; + public volatile RoaringBitmap screenshotDomains = null; + public volatile RoaringBitmap activeDomains = null; + public volatile RoaringBitmap indexedDomains = null; + public volatile TIntDoubleHashMap domainRanks = null; public volatile String[] domainNames = null; volatile boolean isReady = false; @@ -69,13 +71,13 @@ public class SimilarDomainsService { domainIdxToId[idx] = id; return true; }); - domainRanks = new double[domainIdToIdx.size()]; + domainRanks = new TIntDoubleHashMap(100_000, 0.5f, -1, 0.); domainNames = new String[domainIdToIdx.size()]; domainNeighbors = new TIntList[domainIdToIdx.size()]; - screenshotDomains = new BitSet(domainIdToIdx.size()); - activeDomains = new BitSet(domainIdToIdx.size()); - indexedDomains = new BitSet(domainIdToIdx.size()); - relatedDomains = new TIntDoubleHashMap[domainIdToIdx.size()]; + screenshotDomains = new RoaringBitmap(); + activeDomains = new RoaringBitmap(); + indexedDomains = new RoaringBitmap(); + relatedDomains = new Int2DoubleArrayMap[domainIdToIdx.size()]; logger.info("Loaded {} domain IDs", domainIdToIdx.size()); @@ -94,13 +96,17 @@ public class SimilarDomainsService { int higherIndex = Math.max(didx, nidx); if (relatedDomains[lowerIndex] == null) - relatedDomains[lowerIndex] = new TIntDoubleHashMap(32); - relatedDomains[lowerIndex].put(higherIndex, Math.round(100 * rs.getDouble(3))); + relatedDomains[lowerIndex] = new Int2DoubleArrayMap(4); + + double rank = Math.round(100 * rs.getDouble(3)); + if (rank > 0.1) { + relatedDomains[lowerIndex].put(higherIndex, rank); + } if (domainNeighbors[didx] == null) - domainNeighbors[didx] = new TIntArrayList(32); + domainNeighbors[didx] = new TIntArrayList(4); if (domainNeighbors[nidx] == null) - domainNeighbors[nidx] = new TIntArrayList(32); + domainNeighbors[nidx] = new TIntArrayList(4); domainNeighbors[didx].add(nidx); domainNeighbors[nidx].add(didx); @@ -122,14 +128,14 @@ public class SimilarDomainsService { final int id = rs.getInt("ID"); final int idx = domainIdToIdx.get(id); - domainRanks[idx] = Math.round(100 * (1. - rs.getDouble("RANK"))); + domainRanks.put(idx, Math.round(100 * (1. - rs.getDouble("RANK")))); domainNames[idx] = rs.getString("DOMAIN_NAME"); if (rs.getBoolean("INDEXED")) - indexedDomains.set(idx); + indexedDomains.add(idx); if (rs.getBoolean("ACTIVE")) - activeDomains.set(idx); + activeDomains.add(idx); } @@ -142,10 +148,10 @@ public class SimilarDomainsService { final int id = rs.getInt(1); final int idx = domainIdToIdx.get(id); - screenshotDomains.set(idx); + screenshotDomains.add(idx); } - logger.info("Loaded {} domains", domainRanks.length); + logger.info("Loaded {} domains", domainRanks.size()); isReady = true; } } @@ -222,10 +228,10 @@ public class SimilarDomainsService { .setDomainId(id) .setUrl(new EdgeDomain(domainNames[idx]).toRootUrl().toString()) .setRelatedness(getRelatedness(domainId, id)) - .setRank(domainRanks[idx]) - .setIndexed(indexedDomains.get(idx)) - .setActive(activeDomains.get(idx)) - .setScreenshot(screenshotDomains.get(idx)) + .setRank(domainRanks.get(idx)) + .setIndexed(indexedDomains.contains(idx)) + .setActive(activeDomains.contains(idx)) + .setScreenshot(screenshotDomains.contains(idx)) .setLinkType(RpcSimilarDomain.LINK_TYPE.valueOf(linkType.name())) .build()); @@ -291,7 +297,7 @@ public class SimilarDomainsService { double[] ranksArray = new double[idsArray.length]; for (int i = 0; i < idxArray.length; i++) { - ranksArray[i] = this.domainRanks[idxArray[i]]; + ranksArray[i] = this.domainRanks.get(idxArray[i]); } double[] relatednessArray = new double[idsArray.length]; for (int i = 0; i < idsArray.length; i++) { @@ -337,10 +343,10 @@ public class SimilarDomainsService { .setDomainId(id) .setUrl(new EdgeDomain(domainNames[idx]).toRootUrl().toString()) .setRelatedness(getRelatedness(domainId, id)) - .setRank(domainRanks[idx]) - .setIndexed(indexedDomains.get(idx)) - .setActive(activeDomains.get(idx)) - .setScreenshot(screenshotDomains.get(idx)) + .setRank(ranksArray[id]) + .setIndexed(indexedDomains.contains(idx)) + .setActive(activeDomains.contains(idx)) + .setScreenshot(screenshotDomains.contains(idx)) .setLinkType(RpcSimilarDomain.LINK_TYPE.valueOf(linkType.name())) .build()); From e696fd9e92f30c63f9600f86e61c9b99b96f01ab Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 27 Feb 2024 21:15:49 +0100 Subject: [PATCH 49/56] (docs) Begin un-fucking the docs after refactoring --- code/common/db/readme.md | 4 ++-- code/common/linkdb/readme.md | 14 +++++++------- code/common/model/readme.md | 12 ++++++------ code/common/renderer/readme.md | 2 +- code/common/service-discovery/readme.md | 10 +++++----- code/common/service/readme.md | 4 ++-- code/features-convert/adblock/readme.md | 2 +- code/features-convert/data-extractors/readme.md | 6 +++--- .../keyword-extraction/readme.md | 4 ++-- code/features-convert/pubdate/readme.md | 2 +- .../summary-extraction/readme.md | 2 +- code/features-crawl/crawl-blocklist/readme.md | 6 +++--- code/features-crawl/link-parser/readme.md | 2 +- code/index/index-forward/readme.md | 8 ++++---- code/index/index-journal/readme.md | 10 +++++----- code/index/index-reverse/readme.md | 6 +++--- code/index/query/readme.md | 6 +++--- code/index/readme.md | 10 +++++----- code/libraries/array/readme.md | 6 +++--- .../braille-block-punch-cards/readme.md | 2 +- code/libraries/btree/readme.md | 6 +++--- code/libraries/easy-lsh/readme.md | 2 +- code/libraries/guarded-regex/readme.md | 2 +- code/libraries/language-processing/readme.md | 4 ++-- code/libraries/random-write-funnel/readme.md | 8 ++++---- code/libraries/term-frequency-dict/readme.md | 2 +- code/process-models/crawl-spec/readme.md | 6 +++--- code/process-models/crawling-model/readme.md | 16 ++++++++-------- code/process-models/processed-data/readme.md | 10 +++++----- code/processes/converting-process/readme.md | 14 +++++++------- code/processes/crawling-process/readme.md | 6 +++--- .../index-constructor-process/readme.md | 2 +- code/processes/loading-process/readme.md | 2 +- code/services-application/api-service/readme.md | 2 +- .../search-service/readme.md | 6 +++--- code/services-core/assistant-service/readme.md | 2 +- code/services-core/control-service/readme.md | 2 +- code/services-core/executor-service/readme.md | 2 +- code/services-core/query-service/readme.md | 2 +- 39 files changed, 107 insertions(+), 107 deletions(-) diff --git a/code/common/db/readme.md b/code/common/db/readme.md index ae683741..07b6191c 100644 --- a/code/common/db/readme.md +++ b/code/common/db/readme.md @@ -17,14 +17,14 @@ It's well documented and these are probably the only four tasks you'll ever need If you are not running the system via docker, you need to provide alternative connection details than the defaults (TODO: how?). -The migration files are in [resources/db/migration](src/main/resources/db/migration). The file name convention +The migration files are in [resources/db/migration](resources/db/migration). The file name convention incorporates the project's cal-ver versioning; and are applied in lexicographical order. VYY_MM_v_nnn__description.sql ## Central Paths -* [migrations](src/main/resources/db/migration) - Flyway migrations +* [migrations](resources/db/migration) - Flyway migrations ## See Also diff --git a/code/common/linkdb/readme.md b/code/common/linkdb/readme.md index ab86b931..b5a4e8fe 100644 --- a/code/common/linkdb/readme.md +++ b/code/common/linkdb/readme.md @@ -4,11 +4,11 @@ The domain link database contains information about links between domains. It is a static in-memory database loaded from a binary file. -* [DomainLinkDb](src/main/java/nu/marginalia/linkdb/DomainLinkDb.java) -* * [FileDomainLinkDb](src/main/java/nu/marginalia/linkdb/FileDomainLinkDb.java) -* * [SqlDomainLinkDb](src/main/java/nu/marginalia/linkdb/SqlDomainLinkDb.java) -* [DomainLinkDbWriter](src/main/java/nu/marginalia/linkdb/DomainLinkDbWriter.java) -* [DomainLinkDbLoader](src/main/java/nu/marginalia/linkdb/DomainLinkDbLoader.java) +* [DomainLinkDb](java/nu/marginalia/linkdb/DomainLinkDb.java) +* * [FileDomainLinkDb](java/nu/marginalia/linkdb/FileDomainLinkDb.java) +* * [SqlDomainLinkDb](java/nu/marginalia/linkdb/SqlDomainLinkDb.java) +* [DomainLinkDbWriter](java/nu/marginalia/linkdb/DomainLinkDbWriter.java) +* [DomainLinkDbLoader](java/nu/marginalia/linkdb/DomainLinkDbLoader.java) ## Document Database @@ -21,8 +21,8 @@ is not in the MariaDB database is that this would make updates to this information take effect in production immediately, even before the information was searchable. -* [DocumentLinkDbWriter](src/main/java/nu/marginalia/linkdb/DocumentDbWriter.java) -* [DocumentLinkDbLoader](src/main/java/nu/marginalia/linkdb/DocumentDbReader.java) +* [DocumentLinkDbWriter](java/nu/marginalia/linkdb/DocumentDbWriter.java) +* [DocumentLinkDbLoader](java/nu/marginalia/linkdb/DocumentDbReader.java) ## See Also diff --git a/code/common/model/readme.md b/code/common/model/readme.md index 84337753..d07bb4fa 100644 --- a/code/common/model/readme.md +++ b/code/common/model/readme.md @@ -4,9 +4,9 @@ This package contains common models to the search engine ## Central Classes -* [EdgeDomain](src/main/java/nu/marginalia/model/EdgeDomain.java) -* [EdgeUrl](src/main/java/nu/marginalia/model/EdgeUrl.java) -* [DocumentMetadata](src/main/java/nu/marginalia/model/idx/DocumentMetadata.java) -* [DocumentFlags](src/main/java/nu/marginalia/model/idx/DocumentFlags.java) -* [WordMetadata](src/main/java/nu/marginalia/model/idx/WordMetadata.java) -* [WordFlags](src/main/java/nu/marginalia/model/idx/WordFlags.java) \ No newline at end of file +* [EdgeDomain](java/nu/marginalia/model/EdgeDomain.java) +* [EdgeUrl](java/nu/marginalia/model/EdgeUrl.java) +* [DocumentMetadata](java/nu/marginalia/model/idx/DocumentMetadata.java) +* [DocumentFlags](java/nu/marginalia/model/idx/DocumentFlags.java) +* [WordMetadata](java/nu/marginalia/model/idx/WordMetadata.java) +* [WordFlags](java/nu/marginalia/model/idx/WordFlags.java) \ No newline at end of file diff --git a/code/common/renderer/readme.md b/code/common/renderer/readme.md index 3c34830e..ff80af06 100644 --- a/code/common/renderer/readme.md +++ b/code/common/renderer/readme.md @@ -4,4 +4,4 @@ Renders handlebar-style templates for the user-facing services. ## Central Classes -* [Mustache Renderer](src/main/java/nu/marginalia/renderer/MustacheRenderer.java) \ No newline at end of file +* [Mustache Renderer](java/nu/marginalia/renderer/MustacheRenderer.java) \ No newline at end of file diff --git a/code/common/service-discovery/readme.md b/code/common/service-discovery/readme.md index 7e6ab016..5e9fe24a 100644 --- a/code/common/service-discovery/readme.md +++ b/code/common/service-discovery/readme.md @@ -71,11 +71,11 @@ lifecycle, listen to lifecycle notifications and so on. ## gRPC Channel Pool -From the [GrpcChannelPoolFactory](src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java), two types of channel pools can be created +From the [GrpcChannelPoolFactory](java/nu/marginalia/service/client/GrpcChannelPoolFactory.java), two types of channel pools can be created that are aware of the service registry: -* [GrpcMultiNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java) - This pool permits 1-n style communication with partitioned services -* [GrpcSingleNodeChannelPool](src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java) - This pool permits 1-1 style communication with non-partitioned services. +* [GrpcMultiNodeChannelPool](java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java) - This pool permits 1-n style communication with partitioned services +* [GrpcSingleNodeChannelPool](java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java) - This pool permits 1-1 style communication with non-partitioned services. if multiple instances are running, it will use one of them and fall back to another if the first is not available. @@ -145,5 +145,5 @@ Future> response = channelPool ### Central Classes -* [ServiceRegistryIf](src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java) -* [ZkServiceRegistry](src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java) \ No newline at end of file +* [ServiceRegistryIf](java/nu/marginalia/service/discovery/ServiceRegistryIf.java) +* [ZkServiceRegistry](java/nu/marginalia/service/discovery/ZkServiceRegistry.java) \ No newline at end of file diff --git a/code/common/service/readme.md b/code/common/service/readme.md index 04d216e2..14abfb07 100644 --- a/code/common/service/readme.md +++ b/code/common/service/readme.md @@ -50,5 +50,5 @@ Further the new service needs to be added to the `ServiceId` enum in [service-di ## Central Classes -* [MainClass](src/main/java/nu/marginalia/service/MainClass.java) bootstraps all executables -* [Service](src/main/java/nu/marginalia/service/server/Service.java) base class for all services. \ No newline at end of file +* [MainClass](java/nu/marginalia/service/MainClass.java) bootstraps all executables +* [Service](java/nu/marginalia/service/server/Service.java) base class for all services. \ No newline at end of file diff --git a/code/features-convert/adblock/readme.md b/code/features-convert/adblock/readme.md index 1df54936..32919300 100644 --- a/code/features-convert/adblock/readme.md +++ b/code/features-convert/adblock/readme.md @@ -5,4 +5,4 @@ uses it to identify if a document has ads. ## Central Classes -* [AdblockSimulator](src/main/java/nu/marginalia/adblock/AdblockSimulator.java) \ No newline at end of file +* [AdblockSimulator](java/nu/marginalia/adblock/AdblockSimulator.java) \ No newline at end of file diff --git a/code/features-convert/data-extractors/readme.md b/code/features-convert/data-extractors/readme.md index d8c9fc0d..ea318e9f 100644 --- a/code/features-convert/data-extractors/readme.md +++ b/code/features-convert/data-extractors/readme.md @@ -2,6 +2,6 @@ Contains converter-*like* extraction jobs that operate on crawled data to produc ## Important classes -* [AtagExporter](src/main/java/nu/marginalia/extractor/AtagExporter.java) - extracts anchor texts from the crawled data. -* [FeedExporter](src/main/java/nu/marginalia/extractor/FeedExporter.java) - tries to find RSS/Atom feeds within the crawled data. -* [TermFrequencyExporter](src/main/java/nu/marginalia/extractor/TermFrequencyExporter.java) - exports the 'TF' part of TF-IDF. \ No newline at end of file +* [AtagExporter](java/nu/marginalia/extractor/AtagExporter.java) - extracts anchor texts from the crawled data. +* [FeedExporter](java/nu/marginalia/extractor/FeedExporter.java) - tries to find RSS/Atom feeds within the crawled data. +* [TermFrequencyExporter](java/nu/marginalia/extractor/TermFrequencyExporter.java) - exports the 'TF' part of TF-IDF. \ No newline at end of file diff --git a/code/features-convert/keyword-extraction/readme.md b/code/features-convert/keyword-extraction/readme.md index 17ad8600..a9c04962 100644 --- a/code/features-convert/keyword-extraction/readme.md +++ b/code/features-convert/keyword-extraction/readme.md @@ -6,8 +6,8 @@ functions based on [POS tags](https://www.ling.upenn.edu/courses/Fall_2003/ling0 ## Central Classes -* [DocumentKeywordExtractor](src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java) -* [KeywordMetadata](src/main/java/nu/marginalia/keyword/KeywordMetadata.java) +* [DocumentKeywordExtractor](java/nu/marginalia/keyword/DocumentKeywordExtractor.java) +* [KeywordMetadata](java/nu/marginalia/keyword/KeywordMetadata.java) ## See Also diff --git a/code/features-convert/pubdate/readme.md b/code/features-convert/pubdate/readme.md index 40f28710..add657ee 100644 --- a/code/features-convert/pubdate/readme.md +++ b/code/features-convert/pubdate/readme.md @@ -4,4 +4,4 @@ Contains advanced haruspicy for figuring out when a document was published. ## Central Classes -* [PubDateSniffer](src/main/java/nu/marginalia/pubdate/PubDateSniffer.java) \ No newline at end of file +* [PubDateSniffer](java/nu/marginalia/pubdate/PubDateSniffer.java) \ No newline at end of file diff --git a/code/features-convert/summary-extraction/readme.md b/code/features-convert/summary-extraction/readme.md index 1aa38a42..b617d947 100644 --- a/code/features-convert/summary-extraction/readme.md +++ b/code/features-convert/summary-extraction/readme.md @@ -21,5 +21,5 @@ order of a 100,000,000 documents with a time budget of a couple of hours. ## Central Classes -* [SummaryExtractor](src/main/java/nu/marginalia/summary/SummaryExtractor.java) +* [SummaryExtractor](java/nu/marginalia/summary/SummaryExtractor.java) diff --git a/code/features-crawl/crawl-blocklist/readme.md b/code/features-crawl/crawl-blocklist/readme.md index 9db3912b..777f4260 100644 --- a/code/features-crawl/crawl-blocklist/readme.md +++ b/code/features-crawl/crawl-blocklist/readme.md @@ -4,6 +4,6 @@ Contains tools for blocking links from crawling. ## Central Classes -* [GeoIpBlocklist](src/main/java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java) - country blocking -* [IpBlocklist](src/main/java/nu/marginalia/ip_blocklist/IpBlockList.java) - CIDR-based blocking -* [UrlBlocklist](src/main/java/nu/marginalia/ip_blocklist/UrlBlocklist.java) - URL pattern blocking \ No newline at end of file +* [GeoIpBlocklist](java/nu/marginalia/ip_blocklist/GeoIpBlocklist.java) - country blocking +* [IpBlocklist](java/nu/marginalia/ip_blocklist/IpBlockList.java) - CIDR-based blocking +* [UrlBlocklist](java/nu/marginalia/ip_blocklist/UrlBlocklist.java) - URL pattern blocking \ No newline at end of file diff --git a/code/features-crawl/link-parser/readme.md b/code/features-crawl/link-parser/readme.md index 55289227..2893ba87 100644 --- a/code/features-crawl/link-parser/readme.md +++ b/code/features-crawl/link-parser/readme.md @@ -5,4 +5,4 @@ pathological links, etc. ## Central Classes -* [LinkParser](src/main/java/nu/marginalia/link_parser/LinkParser.java) \ No newline at end of file +* [LinkParser](java/nu/marginalia/link_parser/LinkParser.java) \ No newline at end of file diff --git a/code/index/index-forward/readme.md b/code/index/index-forward/readme.md index 545fbf1e..39e272e5 100644 --- a/code/index/index-forward/readme.md +++ b/code/index/index-forward/readme.md @@ -8,8 +8,8 @@ The `id` file contains a list of sorted document ids, and the `data` file contai metadata for each document id, in the same order as the `id` file, with a fixed size record containing data associated with each document id. -Each record contains a binary encoded [DocumentMetadata](../../common/model/src/main/java/nu/marginalia/model/idx/DocumentMetadata.java) object, -as well as a [HtmlFeatures](../../common/model/src/main/java/nu/marginalia/model/crawl/HtmlFeature.java) bitmask. +Each record contains a binary encoded [DocumentMetadata](../../common/model/java/nu/marginalia/model/idx/DocumentMetadata.java) object, +as well as a [HtmlFeatures](../../common/model/java/nu/marginalia/model/crawl/HtmlFeature.java) bitmask. Unlike the reverse index, the forward index is not split into two tiers, and the data is in the same order as it is in the source data, and the cardinality of the document IDs is assumed to fit in memory, @@ -17,5 +17,5 @@ so it's relatively easy to construct. ## Central Classes -* [ForwardIndexConverter](src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java) constructs the index. -* [ForwardIndexReader](src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java) interrogates the index. \ No newline at end of file +* [ForwardIndexConverter](java/nu/marginalia/index/forward/ForwardIndexConverter.java) constructs the index. +* [ForwardIndexReader](java/nu/marginalia/index/forward/ForwardIndexReader.java) interrogates the index. \ No newline at end of file diff --git a/code/index/index-journal/readme.md b/code/index/index-journal/readme.md index 24ed9c43..af7059b3 100644 --- a/code/index/index-journal/readme.md +++ b/code/index/index-journal/readme.md @@ -16,9 +16,9 @@ are designed to handle this transparently via their *Paging* implementation. ## Central Classes ### Model -* [IndexJournalEntry](src/main/java/nu/marginalia/index/journal/model/IndexJournalEntry.java) -* [IndexJournalEntryHeader](src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java) -* [IndexJournalEntryData](src/main/java/nu/marginalia/index/journal/model/IndexJournalEntryData.java) +* [IndexJournalEntry](java/nu/marginalia/index/journal/model/IndexJournalEntry.java) +* [IndexJournalEntryHeader](java/nu/marginalia/index/journal/model/IndexJournalEntryHeader.java) +* [IndexJournalEntryData](java/nu/marginalia/index/journal/model/IndexJournalEntryData.java) ### I/O -* [IndexJournalReader](src/main/java/nu/marginalia/index/journal/reader/IndexJournalReader.java) -* [IndexJournalWriter](src/main/java/nu/marginalia/index/journal/writer/IndexJournalWriter.java) \ No newline at end of file +* [IndexJournalReader](java/nu/marginalia/index/journal/reader/IndexJournalReader.java) +* [IndexJournalWriter](java/nu/marginalia/index/journal/writer/IndexJournalWriter.java) \ No newline at end of file diff --git a/code/index/index-reverse/readme.md b/code/index/index-reverse/readme.md index a27371d6..fcc4fcfc 100644 --- a/code/index/index-reverse/readme.md +++ b/code/index/index-reverse/readme.md @@ -34,9 +34,9 @@ to form a finalized reverse index. ![Illustration of the data layout of the finalized index](index.svg) ## Central Classes -* [ReversePreindex](src/main/java/nu/marginalia/index/construction/ReversePreindex.java) intermediate reverse index state. -* [ReverseIndexConstructor](src/main/java/nu/marginalia/index/construction/ReverseIndexConstructor.java) constructs the index. -* [ReverseIndexReader](src/main/java/nu/marginalia/index/ReverseIndexReader.java) interrogates the index. +* [ReversePreindex](java/nu/marginalia/index/construction/ReversePreindex.java) intermediate reverse index state. +* [ReverseIndexConstructor](java/nu/marginalia/index/construction/ReverseIndexConstructor.java) constructs the index. +* [ReverseIndexReader](java/nu/marginalia/index/ReverseIndexReader.java) interrogates the index. ## See Also diff --git a/code/index/query/readme.md b/code/index/query/readme.md index 3334cada..7386339c 100644 --- a/code/index/query/readme.md +++ b/code/index/query/readme.md @@ -12,11 +12,11 @@ interfaces are implemented within the index-service module. ## Central Classes -* [IndexQuery](src/main/java/nu/marginalia/index/query/IndexQuery.java) -* [query/filter](src/main/java/nu/marginalia/index/query/filter/) +* [IndexQuery](java/nu/marginalia/index/query/IndexQuery.java) +* [query/filter](java/nu/marginalia/index/query/filter/) ## See Also * [index/index-reverse](../index-reverse) implements many of these interfaces. * [libraries/array](../../libraries/array) -* [libraries/array/.../LongQueryBuffer](../../libraries/array/src/main/java/nu/marginalia/array/buffer/LongQueryBuffer.java) \ No newline at end of file +* [libraries/array/.../LongQueryBuffer](../../libraries/array/java/nu/marginalia/array/buffer/LongQueryBuffer.java) \ No newline at end of file diff --git a/code/index/readme.md b/code/index/readme.md index a16c8515..bc44c7d8 100644 --- a/code/index/readme.md +++ b/code/index/readme.md @@ -29,7 +29,7 @@ results higher. ## Central Classes -* [ResultValuator](src/main/java/nu/marginalia/ranking/results/ResultValuator.java) +* [ResultValuator](java/nu/marginalia/ranking/results/ResultValuator.java) --- @@ -53,14 +53,14 @@ for creating a ranking algorithm that is focused on a particular segment of the ## Central Classes -* [PageRankDomainRanker](src/main/java/nu/marginalia/ranking/domains/PageRankDomainRanker.java) - Ranks domains using the +* [PageRankDomainRanker](java/nu/marginalia/ranking/domains/PageRankDomainRanker.java) - Ranks domains using the PageRank or Personalized PageRank algorithm depending on whether a list of influence domains is provided. ### Data sources -* [LinkGraphSource](src/main/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java) - fetches the link graph -* [InvertedLinkGraphSource](src/main/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java) - fetches the inverted link graph -* [SimilarityGraphSource](src/main/java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java) - fetches the similarity graph from the database +* [LinkGraphSource](java/nu/marginalia/ranking/domains/data/LinkGraphSource.java) - fetches the link graph +* [InvertedLinkGraphSource](java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java) - fetches the inverted link graph +* [SimilarityGraphSource](java/nu/marginalia/ranking/domains/data/SimilarityGraphSource.java) - fetches the similarity graph from the database Note that the similarity graph needs to be precomputed and stored in the database for the similarity graph source to be available. diff --git a/code/libraries/array/readme.md b/code/libraries/array/readme.md index 42417b42..7e44b3c6 100644 --- a/code/libraries/array/readme.md +++ b/code/libraries/array/readme.md @@ -32,8 +32,8 @@ try (var array = LongArrayFactory.mmapForWritingConfined(Path.of("/tmp/test"), 1 ## Query Buffers -The classes [IntQueryBuffer](src/main/java/nu/marginalia/array/buffer/IntQueryBuffer.java) -and [LongQueryBuffer](src/main/java/nu/marginalia/array/buffer/LongQueryBuffer.java) are used +The classes [IntQueryBuffer](java/nu/marginalia/array/buffer/IntQueryBuffer.java) +and [LongQueryBuffer](java/nu/marginalia/array/buffer/LongQueryBuffer.java) are used heavily in the search engine's query processing. They are dual-pointer buffers that offer tools for filtering data. @@ -75,7 +75,7 @@ buffer.finalizeFiltering(); Especially noteworthy are the operations `retain()` and `reject()` in -[IntArraySearch](src/main/java/nu/marginalia/array/algo/IntArraySearch.java) and [LongArraySearch](src/main/java/nu/marginalia/array/algo/LongArraySearch.java). +[IntArraySearch](java/nu/marginalia/array/algo/IntArraySearch.java) and [LongArraySearch](java/nu/marginalia/array/algo/LongArraySearch.java). They keep or remove all items in the buffer that exist in the referenced range of the array, which must be sorted. diff --git a/code/libraries/braille-block-punch-cards/readme.md b/code/libraries/braille-block-punch-cards/readme.md index 1785a2fc..2923ef6b 100644 --- a/code/libraries/braille-block-punch-cards/readme.md +++ b/code/libraries/braille-block-punch-cards/readme.md @@ -6,4 +6,4 @@ This is The Way when it comes to representing bit masks to humans. ## Central Classes -* [BrailleBlockPunchCards](src/main/java/nu/marginalia/bbpc/BrailleBlockPunchCards.java) \ No newline at end of file +* [BrailleBlockPunchCards](java/nu/marginalia/bbpc/BrailleBlockPunchCards.java) \ No newline at end of file diff --git a/code/libraries/btree/readme.md b/code/libraries/btree/readme.md index 446195a8..95a10e7f 100644 --- a/code/libraries/btree/readme.md +++ b/code/libraries/btree/readme.md @@ -4,11 +4,11 @@ This package contains a small library for creating and reading a static b-tree i Both binary indices (i.e. sets) are supported, as well as arbitrary multiple-of-keysize key-value mappings where the data is interlaced with the keys in the leaf nodes. This is a fairly low-level datastructure. -The b-trees are specified through a [BTreeContext](src/main/java/nu/marginalia/btree/model/BTreeContext.java) +The b-trees are specified through a [BTreeContext](java/nu/marginalia/btree/model/BTreeContext.java) which contains information about the data and index layout. -The b-trees are written through a [BTreeWriter](src/main/java/nu/marginalia/btree/BTreeWriter.java) and -read with a [BTreeReader](src/main/java/nu/marginalia/btree/BTreeReader.java). +The b-trees are written through a [BTreeWriter](java/nu/marginalia/btree/BTreeWriter.java) and +read with a [BTreeReader](java/nu/marginalia/btree/BTreeReader.java). ## Demo diff --git a/code/libraries/easy-lsh/readme.md b/code/libraries/easy-lsh/readme.md index 7bae8da9..2b2409e8 100644 --- a/code/libraries/easy-lsh/readme.md +++ b/code/libraries/easy-lsh/readme.md @@ -5,7 +5,7 @@ for document deduplication. Hashes are compared using their hamming distance. ## Central Classes -* [EasyLSH](src/main/java/nu/marginalia/lsh/EasyLSH.java) +* [EasyLSH](java/nu/marginalia/lsh/EasyLSH.java) ## Demo diff --git a/code/libraries/guarded-regex/readme.md b/code/libraries/guarded-regex/readme.md index 42d0ca08..ddef661c 100644 --- a/code/libraries/guarded-regex/readme.md +++ b/code/libraries/guarded-regex/readme.md @@ -34,4 +34,4 @@ void ifTheThingDoTheThing(String str) { ## Central Classes -* [GuardedRegexFactory](src/main/java/nu/marginalia/gregex/GuardedRegexFactory.java) \ No newline at end of file +* [GuardedRegexFactory](java/nu/marginalia/gregex/GuardedRegexFactory.java) \ No newline at end of file diff --git a/code/libraries/language-processing/readme.md b/code/libraries/language-processing/readme.md index 08965755..5b12a27d 100644 --- a/code/libraries/language-processing/readme.md +++ b/code/libraries/language-processing/readme.md @@ -4,8 +4,8 @@ This library contains various tools used in language processing. ## Central Classes -* [SentenceExtractor](src/main/java/nu/marginalia/language/sentence/SentenceExtractor.java) - -Creates a [DocumentLanguageData](src/main/java/nu/marginalia/language/model/DocumentLanguageData.java) from a text, containing +* [SentenceExtractor](java/nu/marginalia/language/sentence/SentenceExtractor.java) - +Creates a [DocumentLanguageData](java/nu/marginalia/language/model/DocumentLanguageData.java) from a text, containing its words, how they stem, POS tags, and so on. ## See Also diff --git a/code/libraries/random-write-funnel/readme.md b/code/libraries/random-write-funnel/readme.md index fc02b955..219e1439 100644 --- a/code/libraries/random-write-funnel/readme.md +++ b/code/libraries/random-write-funnel/readme.md @@ -2,12 +2,12 @@ This micro-library with strategies for solving the problem of [write amplificati writing large files out of order to disk. It offers a simple API to write data to a file in a random order, while localizing the writes. -Several strategies are available from the [RandomFileAssembler](src/main/java/nu/marginalia/rwf/RandomFileAssembler.java) +Several strategies are available from the [RandomFileAssembler](java/nu/marginalia/rwf/RandomFileAssembler.java) interface. * Writing to a memory mapped file (non-solution, for small files) * Writing to a memory buffer (for systems with enough memory) -* [RandomWriteFunnel](src/main/java/nu/marginalia/rwf/RandomWriteFunnel.java) - Not bound by memory. +* [RandomWriteFunnel](java/nu/marginalia/rwf/RandomWriteFunnel.java) - Not bound by memory. The data is written in a native byte order. @@ -41,5 +41,5 @@ catch (IOException ex) { ## Central Classes -* [RandomFileAssembler](src/main/java/nu/marginalia/rwf/RandomFileAssembler.java) -* [RandomWriteFunnel](src/main/java/nu/marginalia/rwf/RandomWriteFunnel.java) \ No newline at end of file +* [RandomFileAssembler](java/nu/marginalia/rwf/RandomFileAssembler.java) +* [RandomWriteFunnel](java/nu/marginalia/rwf/RandomWriteFunnel.java) \ No newline at end of file diff --git a/code/libraries/term-frequency-dict/readme.md b/code/libraries/term-frequency-dict/readme.md index 32912f0d..810c3751 100644 --- a/code/libraries/term-frequency-dict/readme.md +++ b/code/libraries/term-frequency-dict/readme.md @@ -5,7 +5,7 @@ the TF-IDF score of a keyword. ## Central Classes -* [TermFrequencyDict](src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java) +* [TermFrequencyDict](java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java) ## See Also diff --git a/code/process-models/crawl-spec/readme.md b/code/process-models/crawl-spec/readme.md index 63bcec96..cd59f23c 100644 --- a/code/process-models/crawl-spec/readme.md +++ b/code/process-models/crawl-spec/readme.md @@ -8,9 +8,9 @@ A crawl spec is a list of domains to be crawled. It is a parquet file with the Crawl specs are used to define the scope of a crawl in the absence of known domains. -The [CrawlSpecRecord](src/main/java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java) class is +The [CrawlSpecRecord](java/nu/marginalia/model/crawlspec/CrawlSpecRecord.java) class is used to represent a record in the crawl spec. -The [CrawlSpecRecordParquetFileReader](src/main/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java) -and [CrawlSpecRecordParquetFileWriter](src/main/java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java) +The [CrawlSpecRecordParquetFileReader](java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileReader.java) +and [CrawlSpecRecordParquetFileWriter](java/nu/marginalia/io/crawlspec/CrawlSpecRecordParquetFileWriter.java) classes are used to read and write the crawl spec parquet files. diff --git a/code/process-models/crawling-model/readme.md b/code/process-models/crawling-model/readme.md index ac0d0906..3bb9cb58 100644 --- a/code/process-models/crawling-model/readme.md +++ b/code/process-models/crawling-model/readme.md @@ -15,27 +15,27 @@ removed in the future. ## Central Classes -* [CrawledDocument](src/main/java/nu/marginalia/crawling/model/CrawledDocument.java) -* [CrawledDomain](src/main/java/nu/marginalia/crawling/model/CrawledDomain.java) +* [CrawledDocument](java/nu/marginalia/crawling/model/CrawledDocument.java) +* [CrawledDomain](java/nu/marginalia/crawling/model/CrawledDomain.java) ### Serialization These serialization classes automatically negotiate the serialization format based on the file extension. -Data is accessed through a [SerializableCrawlDataStream](src/main/java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java), +Data is accessed through a [SerializableCrawlDataStream](java/nu/marginalia/crawling/io/SerializableCrawlDataStream.java), which is a somewhat enhanced Iterator that can be used to read data. -* [CrawledDomainReader](src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java) -* [CrawledDomainWriter](src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java) +* [CrawledDomainReader](java/nu/marginalia/crawling/io/CrawledDomainReader.java) +* [CrawledDomainWriter](java/nu/marginalia/crawling/io/CrawledDomainWriter.java) ### Parquet Serialization -The parquet serialization is done using the [CrawledDocumentParquetRecordFileReader](src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java) -and [CrawledDocumentParquetRecordFileWriter](src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java) classes, +The parquet serialization is done using the [CrawledDocumentParquetRecordFileReader](java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileReader.java) +and [CrawledDocumentParquetRecordFileWriter](java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecordFileWriter.java) classes, which read and write parquet files respectively. -The model classes are serialized to parquet using the [CrawledDocumentParquetRecord](src/main/java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java) +The model classes are serialized to parquet using the [CrawledDocumentParquetRecord](java/nu/marginalia/crawling/parquet/CrawledDocumentParquetRecord.java) The record has the following fields: diff --git a/code/process-models/processed-data/readme.md b/code/process-models/processed-data/readme.md index 4bc8c857..e7f5cebb 100644 --- a/code/process-models/processed-data/readme.md +++ b/code/process-models/processed-data/readme.md @@ -4,11 +4,11 @@ reading and writing parquet files with the output from the Main models: -* [DocumentRecord](src/main/java/nu/marginalia/model/processed/DocumentRecord.java) -* * [DocumentRecordKeywordsProjection](src/main/java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java) -* * [DocumentRecordMetadataProjection](src/main/java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java) -* [DomainLinkRecord](src/main/java/nu/marginalia/model/processed/DomainLinkRecord.java) -* [DomainRecord](src/main/java/nu/marginalia/model/processed/DomainRecord.java) +* [DocumentRecord](java/nu/marginalia/model/processed/DocumentRecord.java) +* * [DocumentRecordKeywordsProjection](java/nu/marginalia/model/processed/DocumentRecordKeywordsProjection.java) +* * [DocumentRecordMetadataProjection](java/nu/marginalia/model/processed/DocumentRecordMetadataProjection.java) +* [DomainLinkRecord](java/nu/marginalia/model/processed/DomainLinkRecord.java) +* [DomainRecord](java/nu/marginalia/model/processed/DomainRecord.java) Since parquet is a column based format, some of the readable models are projections that only read parts of the input file. diff --git a/code/processes/converting-process/readme.md b/code/processes/converting-process/readme.md index 3a79c481..936ca7fe 100644 --- a/code/processes/converting-process/readme.md +++ b/code/processes/converting-process/readme.md @@ -38,16 +38,16 @@ https://www.marginalia.nu/log/93_atags/ ## Central Classes -* [ConverterMain](src/main/java/nu/marginalia/converting/ConverterMain.java) orchestrates the conversion process. -* [DocumentProcessor](src/main/java/nu/marginalia/converting/processor/DocumentProcessor.java) converts a single document. -* - [HtmlDocumentProcessorPlugin](src/main/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java) +* [ConverterMain](java/nu/marginalia/converting/ConverterMain.java) orchestrates the conversion process. +* [DocumentProcessor](java/nu/marginalia/converting/processor/DocumentProcessor.java) converts a single document. +* - [HtmlDocumentProcessorPlugin](java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java) has HTML-specific logic related to a document, keywords and identifies features such as whether it has javascript. -* * - [HtmlProcessorSpecializations](src/main/java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java) -* * - [XenForoSpecialization](src/main/java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java) ... -* - [PlainTextDocumentProcessorPlugin](src/main/java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java) +* * - [HtmlProcessorSpecializations](java/nu/marginalia/converting/processor/plugin/specialization/HtmlProcessorSpecializations.java) +* * - [XenForoSpecialization](java/nu/marginalia/converting/processor/plugin/specialization/XenForoSpecialization.java) ... +* - [PlainTextDocumentProcessorPlugin](java/nu/marginalia/converting/processor/plugin/PlainTextDocumentProcessorPlugin.java) has plain text-specific logic related to a document... -* [DomainProcessor](src/main/java/nu/marginalia/converting/processor/DomainProcessor.java) converts each document and +* [DomainProcessor](java/nu/marginalia/converting/processor/DomainProcessor.java) converts each document and generates domain-wide metadata such as link graphs. ## See Also diff --git a/code/processes/crawling-process/readme.md b/code/processes/crawling-process/readme.md index a595bf1d..0f72cb87 100644 --- a/code/processes/crawling-process/readme.md +++ b/code/processes/crawling-process/readme.md @@ -31,10 +31,10 @@ On top of organic links, the crawler can use sitemaps and rss-feeds to discover ## Central Classes -* [CrawlerMain](src/main/java/nu/marginalia/crawl/CrawlerMain.java) orchestrates the crawling. -* [CrawlerRetreiver](src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java) +* [CrawlerMain](java/nu/marginalia/crawl/CrawlerMain.java) orchestrates the crawling. +* [CrawlerRetreiver](java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java) visits known addresses from a domain and downloads each document. -* [HttpFetcher](src/main/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java) +* [HttpFetcher](java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java) fetches URLs. ## See Also diff --git a/code/processes/index-constructor-process/readme.md b/code/processes/index-constructor-process/readme.md index 9457551b..ecf791b8 100644 --- a/code/processes/index-constructor-process/readme.md +++ b/code/processes/index-constructor-process/readme.md @@ -16,5 +16,5 @@ This is a very light-weight module that delegates the actual work to the modules Their respective readme files contain more information about the indexes themselves and how they are constructed. -The process is glued together within [IndexConstructorMain](src/main/java/nu/marginalia/index/IndexConstructorMain.java), +The process is glued together within [IndexConstructorMain](java/nu/marginalia/index/IndexConstructorMain.java), which is the only class of interest in this module. diff --git a/code/processes/loading-process/readme.md b/code/processes/loading-process/readme.md index ec0c12fd..4a5cf735 100644 --- a/code/processes/loading-process/readme.md +++ b/code/processes/loading-process/readme.md @@ -6,4 +6,4 @@ the index-service. ## Central Classes -* [LoaderMain](src/main/java/nu/marginalia/loading/LoaderMain.java) main class. \ No newline at end of file +* [LoaderMain](java/nu/marginalia/loading/LoaderMain.java) main class. \ No newline at end of file diff --git a/code/services-application/api-service/readme.md b/code/services-application/api-service/readme.md index 33b36b08..8e48c9bb 100644 --- a/code/services-application/api-service/readme.md +++ b/code/services-application/api-service/readme.md @@ -4,4 +4,4 @@ The API service acts as a gateway for public API requests, it deals with API key ## Central Classes -* [ApiService](src/main/java/nu/marginalia/api/ApiService.java) handles REST requests and delegates to the appropriate handling classes. \ No newline at end of file +* [ApiService](java/nu/marginalia/api/ApiService.java) handles REST requests and delegates to the appropriate handling classes. \ No newline at end of file diff --git a/code/services-application/search-service/readme.md b/code/services-application/search-service/readme.md index 02362a55..d7afe4b5 100644 --- a/code/services-application/search-service/readme.md +++ b/code/services-application/search-service/readme.md @@ -14,13 +14,13 @@ to the user. ## Central classes -* [SearchService](src/main/java/nu/marginalia/search/SearchService.java) receives requests and delegates to the +* [SearchService](java/nu/marginalia/search/SearchService.java) receives requests and delegates to the appropriate services. -* [CommandEvaluator](src/main/java/nu/marginalia/search/command/CommandEvaluator.java) interprets a user query and acts +* [CommandEvaluator](java/nu/marginalia/search/command/CommandEvaluator.java) interprets a user query and acts upon it, dealing with special operations like `browse:` or `site:`. -* [SearchQueryIndexService](src/main/java/nu/marginalia/search/svc/SearchQueryIndexService.java) passes a parsed search query to the index service, and +* [SearchQueryIndexService](java/nu/marginalia/search/svc/SearchQueryIndexService.java) passes a parsed search query to the index service, and then decorates the search results so that they can be rendered. ## See Also diff --git a/code/services-core/assistant-service/readme.md b/code/services-core/assistant-service/readme.md index 899ac8fc..1c387c83 100644 --- a/code/services-core/assistant-service/readme.md +++ b/code/services-core/assistant-service/readme.md @@ -4,4 +4,4 @@ The assistant service helps the search service by offering various peripheral fu ## Central Classes -* [AssistantService](src/main/java/nu/marginalia/assistant/AssistantService.java) handles REST requests and delegates to the appropriate handling classes. \ No newline at end of file +* [AssistantService](java/nu/marginalia/assistant/AssistantService.java) handles REST requests and delegates to the appropriate handling classes. \ No newline at end of file diff --git a/code/services-core/control-service/readme.md b/code/services-core/control-service/readme.md index 73d2742e..5da87273 100644 --- a/code/services-core/control-service/readme.md +++ b/code/services-core/control-service/readme.md @@ -15,7 +15,7 @@ Conceptually the application is broken into three parts: ## Central Classes -* [ControlService](src/main/java/nu/marginalia/control/ControlService.java) +* [ControlService](java/nu/marginalia/control/ControlService.java) ## See Also diff --git a/code/services-core/executor-service/readme.md b/code/services-core/executor-service/readme.md index 33e612df..1f05c3a4 100644 --- a/code/services-core/executor-service/readme.md +++ b/code/services-core/executor-service/readme.md @@ -9,7 +9,7 @@ much of the executor's functionality. ## Central Classes -* [ExecutorActorControlService](src/main/java/nu/marginalia/actor/ExecutorActorControlService.java) +* [ExecutorActorControlService](java/nu/marginalia/actor/ExecutorActorControlService.java) ## See Also diff --git a/code/services-core/query-service/readme.md b/code/services-core/query-service/readme.md index d2ba1961..0aa07c05 100644 --- a/code/services-core/query-service/readme.md +++ b/code/services-core/query-service/readme.md @@ -15,7 +15,7 @@ The web interface also offers a JSON API for machine-based queries. ## Central Classes -This module is almost entirely boilerplate, except the [QueryBasicInterface](src/main/java/nu/marginalia/query/QueryBasicInterface.java) +This module is almost entirely boilerplate, except the [QueryBasicInterface](java/nu/marginalia/query/QueryBasicInterface.java) class, which offers a REST API for querying the index. Much of the guts of the query service are in the [query-service](../../functions/search-query) From 99a6e56e993c3621e698a2e934593c8aff63ed43 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 27 Feb 2024 22:00:29 +0100 Subject: [PATCH 50/56] (index-client) Increase thread count in index client This should be a fair bit larger than the number of index nodes --- code/index/api/java/nu/marginalia/index/api/IndexClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/index/api/java/nu/marginalia/index/api/IndexClient.java b/code/index/api/java/nu/marginalia/index/api/IndexClient.java index 33848f06..3a83b5de 100644 --- a/code/index/api/java/nu/marginalia/index/api/IndexClient.java +++ b/code/index/api/java/nu/marginalia/index/api/IndexClient.java @@ -22,7 +22,7 @@ import java.util.concurrent.Executors; public class IndexClient { private static final Logger logger = LoggerFactory.getLogger(IndexClient.class); private final GrpcMultiNodeChannelPool channelPool; - private static final ExecutorService executor = Executors.newFixedThreadPool(8); + private static final ExecutorService executor = Executors.newFixedThreadPool(32); @Inject public IndexClient(GrpcChannelPoolFactory channelPoolFactory) { this.channelPool = channelPoolFactory.createMulti( From 3a65fe89170f3e9586cef6c13afbb7c32ea86176 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 27 Feb 2024 22:08:39 +0100 Subject: [PATCH 51/56] Add offload executor to GrpcChannelPoolFactory --- .../service/client/GrpcChannelPoolFactory.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java index 96221909..9633b0f7 100644 --- a/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java +++ b/code/common/service-discovery/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java @@ -15,6 +15,7 @@ import org.jetbrains.annotations.NotNull; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -33,6 +34,16 @@ public class GrpcChannelPoolFactory { return thread; } }); + private static final Executor offloadExecutor = Executors.newFixedThreadPool( + Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 16), new ThreadFactory() { + static final AtomicInteger threadNumber = new AtomicInteger(1); + @Override + public Thread newThread(@NotNull Runnable r) { + var thread = new Thread(r, STR."gRPC-Offload-Executor[\{threadNumber.getAndIncrement()}]"); + thread.setDaemon(true); + return thread; + } + }); @Inject public GrpcChannelPoolFactory(NodeConfigurationWatcher nodeConfigurationWatcher, @@ -65,6 +76,7 @@ public class GrpcChannelPoolFactory { var mc = ManagedChannelBuilder .forAddress(route.host(), route.port()) .executor(executor) + .offloadExecutor(offloadExecutor) .usePlaintext() .build(); From 9f1649636e076d71d824568a389508a74948f20a Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 28 Feb 2024 11:40:11 +0100 Subject: [PATCH 52/56] Clean up documentation and rename `domain-links` to `link-graph` --- code/common/linkdb/readme.md | 18 ++--------- code/common/readme.md | 3 +- code/execution/build.gradle | 2 +- .../actor/task/ExportDataActor.java | 10 +++--- code/execution/readme.md | 12 +++++++ .../stackexchange-xml/readme.md | 1 - code/functions/domain-info/build.gradle | 2 +- .../domains/DomainInformationService.java | 12 +++---- .../domains/SimilarDomainsService.java | 13 ++++---- .../PartitionDomainLinksClient.java | 30 ----------------- .../aggregate/build.gradle | 2 +- .../linkgraph/AggregateLinkGraphService.java} | 29 ++++++++++------- code/functions/link-graph/aggregate/readme.md | 3 ++ .../api/build.gradle | 2 +- .../linkgraph/AggregateLinkGraphClient.java} | 32 +++++++++---------- .../linkgraph/PartitionLinkGraphClient.java | 29 +++++++++++++++++ .../api/src/main/protobuf/link-graph.proto} | 6 ++-- .../partition/build.gradle | 2 +- .../nu/marginalia/linkgraph/DomainLinks.java} | 8 ++--- .../linkgraph/PartitionLinkGraphService.java} | 29 +++++++++-------- .../linkgraph/impl/DelayingDomainLinks.java} | 15 +++++---- .../linkgraph/impl/FileDomainLinks.java} | 12 ++++--- .../linkgraph/io/DomainLinksLoader.java} | 6 ++-- .../linkgraph/io/DomainLinksWriter.java} | 6 ++-- code/functions/link-graph/partition/readme.md | 11 +++++++ .../linkgraph}/DomainLinkDbTest.java | 10 +++--- code/functions/link-graph/readme.md | 9 ++++++ .../marginalia/util/ngrams/DenseBitMap.java | 3 ++ code/functions/search-query/readme.md | 4 +++ code/index/build.gradle | 2 +- .../domains/data/InvertedLinkGraphSource.java | 10 +++--- .../ranking/domains/data/LinkGraphSource.java | 10 +++--- code/index/readme.md | 6 ++-- .../RankingAlgorithmsContainerTest.java | 10 +++--- code/libraries/language-processing/readme.md | 4 +-- code/libraries/term-frequency-dict/readme.md | 4 --- .../index-constructor-process/readme.md | 4 +-- code/processes/loading-process/build.gradle | 2 ++ .../nu/marginalia/loading/LoaderModule.java | 6 ++-- .../links/DomainLinksLoaderService.java | 6 ++-- code/processes/readme.md | 2 +- .../build.gradle | 2 +- .../adjacencies/AdjacenciesData.java | 4 +-- .../WebsiteAdjacenciesCalculator.java | 6 ++-- code/readme.md | 29 ++++++++++++++--- code/services-core/control-service/readme.md | 3 +- .../executor-service/build.gradle | 2 +- code/services-core/executor-service/readme.md | 18 ++++------- code/services-core/index-service/build.gradle | 4 +-- .../java/nu/marginalia/index/IndexModule.java | 8 ++--- .../nu/marginalia/index/IndexService.java | 16 +++++----- code/services-core/query-service/build.gradle | 4 +-- .../nu/marginalia/query/QueryService.java | 4 +-- settings.gradle | 6 ++-- 54 files changed, 269 insertions(+), 224 deletions(-) create mode 100644 code/execution/readme.md delete mode 100644 code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java rename code/functions/{domain-links => link-graph}/aggregate/build.gradle (92%) rename code/functions/{domain-links/aggregate/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java => link-graph/aggregate/java/nu/marginalia/linkgraph/AggregateLinkGraphService.java} (71%) create mode 100644 code/functions/link-graph/aggregate/readme.md rename code/functions/{domain-links => link-graph}/api/build.gradle (95%) rename code/functions/{domain-links/api/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java => link-graph/api/java/nu/marginalia/api/linkgraph/AggregateLinkGraphClient.java} (76%) create mode 100644 code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/PartitionLinkGraphClient.java rename code/functions/{domain-links/api/src/main/protobuf/domain-links.proto => link-graph/api/src/main/protobuf/link-graph.proto} (85%) rename code/functions/{domain-links => link-graph}/partition/build.gradle (94%) rename code/{common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java => functions/link-graph/partition/java/nu/marginalia/linkgraph/DomainLinks.java} (78%) rename code/functions/{domain-links/partition/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java => link-graph/partition/java/nu/marginalia/linkgraph/PartitionLinkGraphService.java} (73%) rename code/{common/linkdb/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java => functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/DelayingDomainLinks.java} (85%) rename code/{common/linkdb/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java => functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/FileDomainLinks.java} (90%) rename code/{common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java => functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksLoader.java} (84%) rename code/{common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java => functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksWriter.java} (82%) create mode 100644 code/functions/link-graph/partition/readme.md rename code/{common/linkdb/test/nu/marginalia/linkdb => functions/link-graph/partition/test/nu/marginalia/linkgraph}/DomainLinkDbTest.java (84%) create mode 100644 code/functions/link-graph/readme.md create mode 100644 code/functions/search-query/readme.md diff --git a/code/common/linkdb/readme.md b/code/common/linkdb/readme.md index b5a4e8fe..9b3a82a0 100644 --- a/code/common/linkdb/readme.md +++ b/code/common/linkdb/readme.md @@ -1,15 +1,3 @@ -## Domain Link Database - -The domain link database contains information about links -between domains. It is a static in-memory database loaded -from a binary file. - -* [DomainLinkDb](java/nu/marginalia/linkdb/DomainLinkDb.java) -* * [FileDomainLinkDb](java/nu/marginalia/linkdb/FileDomainLinkDb.java) -* * [SqlDomainLinkDb](java/nu/marginalia/linkdb/SqlDomainLinkDb.java) -* [DomainLinkDbWriter](java/nu/marginalia/linkdb/DomainLinkDbWriter.java) -* [DomainLinkDbLoader](java/nu/marginalia/linkdb/DomainLinkDbLoader.java) - ## Document Database The document database contains information about links, @@ -21,10 +9,10 @@ is not in the MariaDB database is that this would make updates to this information take effect in production immediately, even before the information was searchable. -* [DocumentLinkDbWriter](java/nu/marginalia/linkdb/DocumentDbWriter.java) -* [DocumentLinkDbLoader](java/nu/marginalia/linkdb/DocumentDbReader.java) +* [DocumentLinkDbWriter](java/nu/marginalia/linkdb/docs/DocumentDbWriter.java) +* [DocumentLinkDbLoader](java/nu/marginalia/linkdb/docs/DocumentDbReader.java) ## See Also -These databases are constructed by the [loading-process](../../processes/loading-process), and consumed by the [index-service](../../services-core/index-service). \ No newline at end of file +The database is constructed by the [loading-process](../../processes/loading-process), and consumed by the [index-service](../../services-core/index-service). \ No newline at end of file diff --git a/code/common/readme.md b/code/common/readme.md index 120d55f9..b6329457 100644 --- a/code/common/readme.md +++ b/code/common/readme.md @@ -7,6 +7,5 @@ as shared models. * [config](config/) contains some `@Inject`ables. * [renderer](renderer/) contains utility code for rendering website templates. * [service](service/) is the shared base classes for main methods and web services. -* [service-client](service-client/) is the shared base class for RPC. -* [service-discovery](service-discovery) contains tools that lets the services find each other. +* [service-discovery](service-discovery) contains tools that lets the services find each other and communicate. * [process](process/) contains boiler plate for batch processes. diff --git a/code/execution/build.gradle b/code/execution/build.gradle index 842bcab7..a3cec39f 100644 --- a/code/execution/build.gradle +++ b/code/execution/build.gradle @@ -34,7 +34,7 @@ dependencies { implementation project(':code:libraries:message-queue') - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation project(':code:execution:api') implementation project(':code:process-models:crawl-spec') diff --git a/code/execution/java/nu/marginalia/actor/task/ExportDataActor.java b/code/execution/java/nu/marginalia/actor/task/ExportDataActor.java index b3d9b0a2..0a5d1056 100644 --- a/code/execution/java/nu/marginalia/actor/task/ExportDataActor.java +++ b/code/execution/java/nu/marginalia/actor/task/ExportDataActor.java @@ -6,7 +6,7 @@ import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.prototype.RecordActorPrototype; import nu.marginalia.actor.state.ActorStep; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; import nu.marginalia.storage.model.FileStorageType; @@ -32,7 +32,7 @@ public class ExportDataActor extends RecordActorPrototype { private final FileStorageService storageService; private final HikariDataSource dataSource; private final Logger logger = LoggerFactory.getLogger(getClass()); - private final AggregateDomainLinksClient domainLinksClient; + private final AggregateLinkGraphClient linkGraphClient; public record Export() implements ActorStep {} public record ExportBlacklist(FileStorageId fid) implements ActorStep {} @@ -114,7 +114,7 @@ public class ExportDataActor extends RecordActorPrototype { var tmpFile = Files.createTempFile(storage.asPath(), "export", ".csv.gz", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--"))); - var allLinks = domainLinksClient.getAllDomainLinks(); + var allLinks = linkGraphClient.getAllDomainLinks(); try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))))) { @@ -155,12 +155,12 @@ public class ExportDataActor extends RecordActorPrototype { public ExportDataActor(Gson gson, FileStorageService storageService, HikariDataSource dataSource, - AggregateDomainLinksClient domainLinksClient) + AggregateLinkGraphClient linkGraphClient) { super(gson); this.storageService = storageService; this.dataSource = dataSource; - this.domainLinksClient = domainLinksClient; + this.linkGraphClient = linkGraphClient; } } diff --git a/code/execution/readme.md b/code/execution/readme.md new file mode 100644 index 00000000..d819c023 --- /dev/null +++ b/code/execution/readme.md @@ -0,0 +1,12 @@ +The execution subsystem is responsible for the execution of long running tasks on each +index node. It lives in the [executor-service](../services-core/executor-service) module. + +It accomplishes this using the [message queue and actor library](../libraries/message-queue/), +which permits program state to survive crashes and reboots. + +The subsystem exposes four [APIs](api/src/main/protobuf/executor-api.proto): + +* Execution API - for starting and stopping tasks, also contains miscellaneous commands +* Crawl API - for managing the crawl workflow +* Sideload API - for sideloading data +* Export API - for exporting data \ No newline at end of file diff --git a/code/features-convert/stackexchange-xml/readme.md b/code/features-convert/stackexchange-xml/readme.md index 8af6d05a..1701ad7f 100644 --- a/code/features-convert/stackexchange-xml/readme.md +++ b/code/features-convert/stackexchange-xml/readme.md @@ -16,4 +16,3 @@ holistically, not by question or answer, it is necessary to re-arrange the data (which is very large). SQLite does a decent job of enabling this task. -See [tools/stackexchange-converter](../../tools/stackexchange-converter). \ No newline at end of file diff --git a/code/functions/domain-info/build.gradle b/code/functions/domain-info/build.gradle index 1aa2da16..4858e935 100644 --- a/code/functions/domain-info/build.gradle +++ b/code/functions/domain-info/build.gradle @@ -15,7 +15,7 @@ apply from: "$rootProject.projectDir/srcsets.gradle" dependencies { implementation project(':code:functions:domain-info:api') - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation project(':code:common:config') implementation project(':code:common:service') diff --git a/code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInformationService.java b/code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInformationService.java index bf8ad9df..1aeffae6 100644 --- a/code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInformationService.java +++ b/code/functions/domain-info/java/nu/marginalia/functions/domains/DomainInformationService.java @@ -2,7 +2,7 @@ package nu.marginalia.functions.domains; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.api.domains.RpcDomainInfoResponse; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import nu.marginalia.geoip.GeoIpDictionary; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; @@ -21,7 +21,7 @@ public class DomainInformationService { private final GeoIpDictionary geoIpDictionary; private DbDomainQueries dbDomainQueries; - private final AggregateDomainLinksClient domainLinksClient; + private final AggregateLinkGraphClient linkGraphClient; private HikariDataSource dataSource; private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -29,11 +29,11 @@ public class DomainInformationService { public DomainInformationService( DbDomainQueries dbDomainQueries, GeoIpDictionary geoIpDictionary, - AggregateDomainLinksClient domainLinksClient, + AggregateLinkGraphClient linkGraphClient, HikariDataSource dataSource) { this.dbDomainQueries = dbDomainQueries; this.geoIpDictionary = geoIpDictionary; - this.domainLinksClient = domainLinksClient; + this.linkGraphClient = linkGraphClient; this.dataSource = dataSource; } @@ -84,8 +84,8 @@ public class DomainInformationService { inCrawlQueue = rs.next(); builder.setInCrawlQueue(inCrawlQueue); - builder.setIncomingLinks(domainLinksClient.countLinksToDomain(domainId)); - builder.setOutboundLinks(domainLinksClient.countLinksFromDomain(domainId)); + builder.setIncomingLinks(linkGraphClient.countLinksToDomain(domainId)); + builder.setOutboundLinks(linkGraphClient.countLinksFromDomain(domainId)); rs = stmt.executeQuery(STR.""" SELECT KNOWN_URLS, GOOD_URLS, VISITED_URLS FROM DOMAIN_METADATA WHERE ID=\{domainId} diff --git a/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java b/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java index 9be481b7..8d6cd70e 100644 --- a/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java +++ b/code/functions/domain-info/java/nu/marginalia/functions/domains/SimilarDomainsService.java @@ -11,7 +11,7 @@ import gnu.trove.set.hash.TIntHashSet; import it.unimi.dsi.fastutil.ints.Int2DoubleArrayMap; import nu.marginalia.api.domains.*; import nu.marginalia.api.domains.model.SimilarDomain; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import nu.marginalia.model.EdgeDomain; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; @@ -20,7 +20,6 @@ import org.slf4j.LoggerFactory; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import java.util.BitSet; import java.util.List; import java.util.concurrent.Executors; import java.util.stream.IntStream; @@ -29,7 +28,7 @@ public class SimilarDomainsService { private static final Logger logger = LoggerFactory.getLogger(SimilarDomainsService.class); private final HikariDataSource dataSource; - private final AggregateDomainLinksClient domainLinksClient; + private final AggregateLinkGraphClient linkGraphClient; private volatile TIntIntHashMap domainIdToIdx = new TIntIntHashMap(100_000); private volatile int[] domainIdxToId; @@ -45,9 +44,9 @@ public class SimilarDomainsService { volatile boolean isReady = false; @Inject - public SimilarDomainsService(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) { + public SimilarDomainsService(HikariDataSource dataSource, AggregateLinkGraphClient linkGraphClient) { this.dataSource = dataSource; - this.domainLinksClient = domainLinksClient; + this.linkGraphClient = linkGraphClient; Executors.newSingleThreadExecutor().submit(this::init); } @@ -262,7 +261,7 @@ public class SimilarDomainsService { private TIntSet getLinkingIdsDToS(int domainIdx) { var items = new TIntHashSet(); - for (int id : domainLinksClient.getLinksFromDomain(domainIdxToId[domainIdx])) { + for (int id : linkGraphClient.getLinksFromDomain(domainIdxToId[domainIdx])) { items.add(domainIdToIdx.get(id)); } @@ -272,7 +271,7 @@ public class SimilarDomainsService { private TIntSet getLinkingIdsSToD(int domainIdx) { var items = new TIntHashSet(); - for (int id : domainLinksClient.getLinksToDomain(domainIdxToId[domainIdx])) { + for (int id : linkGraphClient.getLinksToDomain(domainIdxToId[domainIdx])) { items.add(domainIdToIdx.get(id)); } diff --git a/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java b/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java deleted file mode 100644 index 6e561b12..00000000 --- a/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java +++ /dev/null @@ -1,30 +0,0 @@ -package nu.marginalia.api.indexdomainlinks; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.api.domainlink.DomainLinksApiGrpc; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; -import nu.marginalia.service.discovery.property.ServiceKey; -import nu.marginalia.service.discovery.property.ServicePartition; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Singleton -public class PartitionDomainLinksClient { - private static final Logger logger = LoggerFactory.getLogger(PartitionDomainLinksClient.class); - - private final GrpcMultiNodeChannelPool channelPool; - - @Inject - public PartitionDomainLinksClient(GrpcChannelPoolFactory factory) { - this.channelPool = factory.createMulti( - ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.multi()), - DomainLinksApiGrpc::newBlockingStub); - } - - public GrpcMultiNodeChannelPool getChannelPool() { - return channelPool; - } - -} diff --git a/code/functions/domain-links/aggregate/build.gradle b/code/functions/link-graph/aggregate/build.gradle similarity index 92% rename from code/functions/domain-links/aggregate/build.gradle rename to code/functions/link-graph/aggregate/build.gradle index 11260f1f..52be585f 100644 --- a/code/functions/domain-links/aggregate/build.gradle +++ b/code/functions/link-graph/aggregate/build.gradle @@ -14,7 +14,7 @@ java { apply from: "$rootProject.projectDir/srcsets.gradle" dependencies { - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation project(':code:common:config') implementation project(':code:common:service') diff --git a/code/functions/domain-links/aggregate/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java b/code/functions/link-graph/aggregate/java/nu/marginalia/linkgraph/AggregateLinkGraphService.java similarity index 71% rename from code/functions/domain-links/aggregate/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java rename to code/functions/link-graph/aggregate/java/nu/marginalia/linkgraph/AggregateLinkGraphService.java index ff842075..7731b335 100644 --- a/code/functions/domain-links/aggregate/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java +++ b/code/functions/link-graph/aggregate/java/nu/marginalia/linkgraph/AggregateLinkGraphService.java @@ -1,20 +1,25 @@ -package nu.marginalia.functions.domainlinks; +package nu.marginalia.linkgraph; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.api.domainlink.*; -import nu.marginalia.api.indexdomainlinks.PartitionDomainLinksClient; +import nu.marginalia.api.linkgraph.*; +import nu.marginalia.api.linkgraph.PartitionLinkGraphClient; +import nu.marginalia.api.linkgraph.LinkGraphApiGrpc; +import nu.marginalia.api.linkgraph.LinkGraphApiGrpc.LinkGraphApiBlockingStub; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; -public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase { - private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksService.class); - private final PartitionDomainLinksClient client; +/** This class is responsible for aggregating the link graph data from the partitioned link graph + * services. + */ +public class AggregateLinkGraphService extends LinkGraphApiGrpc.LinkGraphApiImplBase { + private static final Logger logger = LoggerFactory.getLogger(AggregateLinkGraphService.class); + private final PartitionLinkGraphClient client; @Inject - public AggregateDomainLinksService(PartitionDomainLinksClient client) { + public AggregateLinkGraphService(PartitionLinkGraphClient client) { this.client = client; } @@ -22,7 +27,7 @@ public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksA public void getAllLinks(Empty request, StreamObserver responseObserver) { - client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks) + client.getChannelPool().call(LinkGraphApiBlockingStub::getAllLinks) .run(Empty.getDefaultInstance()) .forEach(iter -> iter.forEachRemaining(responseObserver::onNext)); @@ -34,7 +39,7 @@ public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksA StreamObserver responseObserver) { var rspBuilder = RpcDomainIdList.newBuilder(); - client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain) + client.getChannelPool().call(LinkGraphApiBlockingStub::getLinksFromDomain) .run(request) .stream() .map(RpcDomainIdList::getDomainIdList) @@ -51,7 +56,7 @@ public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksA var rspBuilder = RpcDomainIdList.newBuilder(); - client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain) + client.getChannelPool().call(LinkGraphApiBlockingStub::getLinksToDomain) .run(request) .stream() .map(RpcDomainIdList::getDomainIdList) @@ -65,7 +70,7 @@ public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksA @Override public void countLinksFromDomain(RpcDomainId request, StreamObserver responseObserver) { - int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain) + int sum = client.getChannelPool().call(LinkGraphApiBlockingStub::countLinksFromDomain) .run(request) .stream() .mapToInt(RpcDomainIdCount::getIdCount) @@ -81,7 +86,7 @@ public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksA public void countLinksToDomain(RpcDomainId request, StreamObserver responseObserver) { - int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain) + int sum = client.getChannelPool().call(LinkGraphApiBlockingStub::countLinksToDomain) .run(request) .stream() .mapToInt(RpcDomainIdCount::getIdCount) diff --git a/code/functions/link-graph/aggregate/readme.md b/code/functions/link-graph/aggregate/readme.md new file mode 100644 index 00000000..a9429ca5 --- /dev/null +++ b/code/functions/link-graph/aggregate/readme.md @@ -0,0 +1,3 @@ +This module is responsible for aggregating the link graph from the partitioned services, and exposing a unified +view of the link graph. It does not keep any data or state, but instead delegates to the partitioned +services. \ No newline at end of file diff --git a/code/functions/domain-links/api/build.gradle b/code/functions/link-graph/api/build.gradle similarity index 95% rename from code/functions/domain-links/api/build.gradle rename to code/functions/link-graph/api/build.gradle index 3232a623..ffeab7f8 100644 --- a/code/functions/domain-links/api/build.gradle +++ b/code/functions/link-graph/api/build.gradle @@ -11,7 +11,7 @@ java { } } -jar.archiveBaseName = 'index-domain-links-api' +jar.archiveBaseName = 'link-graph-api' apply from: "$rootProject.projectDir/protobuf.gradle" apply from: "$rootProject.projectDir/srcsets.gradle" diff --git a/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java b/code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/AggregateLinkGraphClient.java similarity index 76% rename from code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java rename to code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/AggregateLinkGraphClient.java index c8f5c5ec..4e9c9a3d 100644 --- a/code/functions/domain-links/api/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java +++ b/code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/AggregateLinkGraphClient.java @@ -1,10 +1,8 @@ -package nu.marginalia.api.indexdomainlinks; +package nu.marginalia.api.linkgraph; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.api.domainlink.DomainLinksApiGrpc; -import nu.marginalia.api.domainlink.Empty; -import nu.marginalia.api.domainlink.RpcDomainId; +import nu.marginalia.api.linkgraph.LinkGraphApiGrpc; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.client.GrpcSingleNodeChannelPool; import nu.marginalia.service.discovery.property.ServiceKey; @@ -17,24 +15,26 @@ import org.slf4j.LoggerFactory; import java.time.Duration; import java.util.List; -@Singleton -public class AggregateDomainLinksClient { - private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksClient.class); +import static nu.marginalia.api.linkgraph.LinkGraphApiGrpc.*; - private final GrpcSingleNodeChannelPool channelPool; +@Singleton +public class AggregateLinkGraphClient { + private static final Logger logger = LoggerFactory.getLogger(AggregateLinkGraphClient.class); + + private final GrpcSingleNodeChannelPool channelPool; @Inject - public AggregateDomainLinksClient(GrpcChannelPoolFactory factory) { + public AggregateLinkGraphClient(GrpcChannelPoolFactory factory) { this.channelPool = factory.createSingle( - ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.any()), - DomainLinksApiGrpc::newBlockingStub); + ServiceKey.forGrpcApi(LinkGraphApiGrpc.class, ServicePartition.any()), + LinkGraphApiGrpc::newBlockingStub); } public AllLinks getAllDomainLinks() { AllLinks links = new AllLinks(); - channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks) + channelPool.call(LinkGraphApiBlockingStub::getAllLinks) .run(Empty.getDefaultInstance()) .forEachRemaining(pairs -> { for (int i = 0; i < pairs.getDestIdsCount(); i++) { @@ -47,7 +47,7 @@ public class AggregateDomainLinksClient { public List getLinksToDomain(int domainId) { try { - return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain) + return channelPool.call(LinkGraphApiBlockingStub::getLinksToDomain) .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) .getDomainIdList() .stream() @@ -62,7 +62,7 @@ public class AggregateDomainLinksClient { public List getLinksFromDomain(int domainId) { try { - return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain) + return channelPool.call(LinkGraphApiBlockingStub::getLinksFromDomain) .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) .getDomainIdList() .stream() @@ -78,7 +78,7 @@ public class AggregateDomainLinksClient { public int countLinksToDomain(int domainId) { try { - return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain) + return channelPool.call(LinkGraphApiBlockingStub::countLinksToDomain) .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) .getIdCount(); @@ -91,7 +91,7 @@ public class AggregateDomainLinksClient { public int countLinksFromDomain(int domainId) { try { - return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain) + return channelPool.call(LinkGraphApiBlockingStub::countLinksFromDomain) .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) .getIdCount(); } diff --git a/code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/PartitionLinkGraphClient.java b/code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/PartitionLinkGraphClient.java new file mode 100644 index 00000000..e6fa5ce8 --- /dev/null +++ b/code/functions/link-graph/api/java/nu/marginalia/api/linkgraph/PartitionLinkGraphClient.java @@ -0,0 +1,29 @@ +package nu.marginalia.api.linkgraph; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class PartitionLinkGraphClient { + private static final Logger logger = LoggerFactory.getLogger(PartitionLinkGraphClient.class); + + private final GrpcMultiNodeChannelPool channelPool; + + @Inject + public PartitionLinkGraphClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createMulti( + ServiceKey.forGrpcApi(LinkGraphApiGrpc.class, ServicePartition.multi()), + LinkGraphApiGrpc::newBlockingStub); + } + + public GrpcMultiNodeChannelPool getChannelPool() { + return channelPool; + } + +} diff --git a/code/functions/domain-links/api/src/main/protobuf/domain-links.proto b/code/functions/link-graph/api/src/main/protobuf/link-graph.proto similarity index 85% rename from code/functions/domain-links/api/src/main/protobuf/domain-links.proto rename to code/functions/link-graph/api/src/main/protobuf/link-graph.proto index 2a31bbf7..753c8e1a 100644 --- a/code/functions/domain-links/api/src/main/protobuf/domain-links.proto +++ b/code/functions/link-graph/api/src/main/protobuf/link-graph.proto @@ -1,10 +1,10 @@ syntax="proto3"; -package nu.marginalia.api.domainlinks; +package nu.marginalia.api.linkgraph; -option java_package="nu.marginalia.api.domainlink"; +option java_package="nu.marginalia.api.linkgraph"; option java_multiple_files=true; -service DomainLinksApi { +service LinkGraphApi { rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {} rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {} rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {} diff --git a/code/functions/domain-links/partition/build.gradle b/code/functions/link-graph/partition/build.gradle similarity index 94% rename from code/functions/domain-links/partition/build.gradle rename to code/functions/link-graph/partition/build.gradle index 940cecfe..43a2e654 100644 --- a/code/functions/domain-links/partition/build.gradle +++ b/code/functions/link-graph/partition/build.gradle @@ -14,7 +14,7 @@ java { apply from: "$rootProject.projectDir/srcsets.gradle" dependencies { - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation project(':code:common:config') implementation project(':code:common:service') diff --git a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/DomainLinks.java similarity index 78% rename from code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java rename to code/functions/link-graph/partition/java/nu/marginalia/linkgraph/DomainLinks.java index ed6f0a1d..ed0be39e 100644 --- a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDb.java +++ b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/DomainLinks.java @@ -1,13 +1,13 @@ -package nu.marginalia.linkdb.dlinks; +package nu.marginalia.linkgraph; import gnu.trove.list.array.TIntArrayList; import java.nio.file.Path; -/** A database of source-destination pairs of domain IDs. The database is loaded into memory from - * a source. The database is then kept in memory, reloading it upon switchInput(). +/** A repository of source-destination pairs of domain IDs. The database is loaded into memory from + * a source. The data is then kept in memory, reloading it upon switchInput(). */ -public interface DomainLinkDb { +public interface DomainLinks { /** Replace the current db file with the provided file. The provided file will be deleted. * The in-memory database MAY be updated to reflect the change. * */ diff --git a/code/functions/domain-links/partition/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/PartitionLinkGraphService.java similarity index 73% rename from code/functions/domain-links/partition/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java rename to code/functions/link-graph/partition/java/nu/marginalia/linkgraph/PartitionLinkGraphService.java index bb89fc6b..814f7ffb 100644 --- a/code/functions/domain-links/partition/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java +++ b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/PartitionLinkGraphService.java @@ -1,25 +1,28 @@ -package nu.marginalia.functions.domainlinks; +package nu.marginalia.linkgraph; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.api.domainlink.*; -import nu.marginalia.linkdb.dlinks.DomainLinkDb; +import nu.marginalia.api.linkgraph.*; +import nu.marginalia.api.linkgraph.Empty; +import nu.marginalia.api.linkgraph.LinkGraphApiGrpc; -/** GRPC service for interrogating domain links +/** GRPC service for interrogating domain links for a single partition. For accessing the data + * in the application, the AggregateLinkGraphService should be used instead via the + * AggregateLinkGraphClient. */ -public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase { - private final DomainLinkDb domainLinkDb; +public class PartitionLinkGraphService extends LinkGraphApiGrpc.LinkGraphApiImplBase { + private final DomainLinks domainLinks; @Inject - public PartitionDomainLinksService(DomainLinkDb domainLinkDb) { - this.domainLinkDb = domainLinkDb; + public PartitionLinkGraphService(DomainLinks domainLinks) { + this.domainLinks = domainLinks; } public void getAllLinks(Empty request, io.grpc.stub.StreamObserver responseObserver) { try (var idsConverter = new AllIdsResponseConverter(responseObserver)) { - domainLinkDb.forEach(idsConverter::accept); + domainLinks.forEach(idsConverter::accept); } responseObserver.onCompleted(); @@ -58,7 +61,7 @@ public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksA public void getLinksFromDomain(RpcDomainId request, StreamObserver responseObserver) { - var links = domainLinkDb.findDestinations(request.getDomainId()); + var links = domainLinks.findDestinations(request.getDomainId()); var rspBuilder = RpcDomainIdList.newBuilder(); for (int i = 0; i < links.size(); i++) { @@ -73,7 +76,7 @@ public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksA public void getLinksToDomain(RpcDomainId request, StreamObserver responseObserver) { - var links = domainLinkDb.findSources(request.getDomainId()); + var links = domainLinks.findSources(request.getDomainId()); var rspBuilder = RpcDomainIdList.newBuilder(); for (int i = 0; i < links.size(); i++) { @@ -87,7 +90,7 @@ public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksA public void countLinksFromDomain(RpcDomainId request, StreamObserver responseObserver) { responseObserver.onNext(RpcDomainIdCount.newBuilder() - .setIdCount(domainLinkDb.countDestinations(request.getDomainId())) + .setIdCount(domainLinks.countDestinations(request.getDomainId())) .build()); responseObserver.onCompleted(); } @@ -95,7 +98,7 @@ public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksA public void countLinksToDomain(RpcDomainId request, StreamObserver responseObserver) { responseObserver.onNext(RpcDomainIdCount.newBuilder() - .setIdCount(domainLinkDb.countSources(request.getDomainId())) + .setIdCount(domainLinks.countSources(request.getDomainId())) .build()); responseObserver.onCompleted(); } diff --git a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/DelayingDomainLinks.java similarity index 85% rename from code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java rename to code/functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/DelayingDomainLinks.java index 3d2c7270..cf0ad162 100644 --- a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DelayingDomainLinkDb.java +++ b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/DelayingDomainLinks.java @@ -1,7 +1,8 @@ -package nu.marginalia.linkdb.dlinks; +package nu.marginalia.linkgraph.impl; import com.google.inject.name.Named; import gnu.trove.list.array.TIntArrayList; +import nu.marginalia.linkgraph.DomainLinks; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,13 +15,13 @@ import java.nio.file.StandardCopyOption; * is not yet loaded. This speeds up the startup of the index service, as the database is * loaded in a separate thread. */ -public class DelayingDomainLinkDb implements DomainLinkDb { - private final static Logger logger = LoggerFactory.getLogger(DelayingDomainLinkDb.class); +public class DelayingDomainLinks implements DomainLinks { + private final static Logger logger = LoggerFactory.getLogger(DelayingDomainLinks.class); - private volatile DomainLinkDb currentDb; + private volatile DomainLinks currentDb; private final Path filename; - public DelayingDomainLinkDb(@Named("domain-linkdb-file") Path filename) { + public DelayingDomainLinks(@Named("domain-linkdb-file") Path filename) { this.filename = filename; // Load the database in a separate thread, so that the constructor can return @@ -29,7 +30,7 @@ public class DelayingDomainLinkDb implements DomainLinkDb { Thread.ofPlatform().start(() -> { try { - currentDb = new FileDomainLinkDb(filename); + currentDb = new FileDomainLinks(filename); logger.info("Loaded linkdb"); } catch (Exception e) { logger.error("Failed to load linkdb", e); @@ -43,7 +44,7 @@ public class DelayingDomainLinkDb implements DomainLinkDb { Thread.ofPlatform().start(() -> { try { - currentDb = new FileDomainLinkDb(filename); + currentDb = new FileDomainLinks(filename); } catch (IOException e) { logger.error("Failed to load linkdb", e); } diff --git a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/FileDomainLinks.java similarity index 90% rename from code/common/linkdb/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java rename to code/functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/FileDomainLinks.java index 0fda3467..68281229 100644 --- a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/FileDomainLinkDb.java +++ b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/impl/FileDomainLinks.java @@ -1,7 +1,9 @@ -package nu.marginalia.linkdb.dlinks; +package nu.marginalia.linkgraph.impl; import com.google.inject.name.Named; import gnu.trove.list.array.TIntArrayList; +import nu.marginalia.linkgraph.DomainLinks; +import nu.marginalia.linkgraph.io.DomainLinksLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,13 +16,13 @@ import java.util.Arrays; /** Canonical DomainLinkDb implementation. The database is loaded into memory from * a file. The database is then kept in memory, reloading it upon switchInput(). */ -public class FileDomainLinkDb implements DomainLinkDb { - private static final Logger logger = LoggerFactory.getLogger(FileDomainLinkDb.class); +public class FileDomainLinks implements DomainLinks { + private static final Logger logger = LoggerFactory.getLogger(FileDomainLinks.class); private final Path filename; private volatile long[] sourceToDest = new long[0]; private volatile long[] destToSource = new long[0]; - public FileDomainLinkDb(@Named("domain-linkdb-file") Path filename) throws IOException { + public FileDomainLinks(@Named("domain-linkdb-file") Path filename) throws IOException { this.filename = filename; if (Files.exists(filename)) { @@ -35,7 +37,7 @@ public class FileDomainLinkDb implements DomainLinkDb { } public void loadInput(Path filename) throws IOException { - try (var loader = new DomainLinkDbLoader(filename)) { + try (var loader = new DomainLinksLoader(filename)) { int size = loader.size(); var newSourceToDest = new long[size]; diff --git a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksLoader.java similarity index 84% rename from code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java rename to code/functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksLoader.java index 83af733d..4ab10414 100644 --- a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbLoader.java +++ b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksLoader.java @@ -1,17 +1,17 @@ -package nu.marginalia.linkdb.dlinks; +package nu.marginalia.linkgraph.io; import java.io.DataInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -public class DomainLinkDbLoader implements AutoCloseable { +public class DomainLinksLoader implements AutoCloseable { private final DataInputStream stream; private final Path filename; private long nextVal; - public DomainLinkDbLoader(Path filename) throws IOException { + public DomainLinksLoader(Path filename) throws IOException { this.stream = new DataInputStream(Files.newInputStream(filename)); this.filename = filename; } diff --git a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksWriter.java similarity index 82% rename from code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java rename to code/functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksWriter.java index 99830443..0744bcf6 100644 --- a/code/common/linkdb/java/nu/marginalia/linkdb/dlinks/DomainLinkDbWriter.java +++ b/code/functions/link-graph/partition/java/nu/marginalia/linkgraph/io/DomainLinksWriter.java @@ -1,4 +1,4 @@ -package nu.marginalia.linkdb.dlinks; +package nu.marginalia.linkgraph.io; import java.io.DataOutputStream; import java.io.IOException; @@ -6,10 +6,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -public class DomainLinkDbWriter implements AutoCloseable { +public class DomainLinksWriter implements AutoCloseable { private final DataOutputStream stream; - public DomainLinkDbWriter(Path fileName) throws IOException { + public DomainLinksWriter(Path fileName) throws IOException { this.stream = new DataOutputStream(Files.newOutputStream(fileName, StandardOpenOption.CREATE, StandardOpenOption.WRITE, diff --git a/code/functions/link-graph/partition/readme.md b/code/functions/link-graph/partition/readme.md new file mode 100644 index 00000000..cb116bc0 --- /dev/null +++ b/code/functions/link-graph/partition/readme.md @@ -0,0 +1,11 @@ +The link graph partition module is responsible for knowledge about the link graph +for a single index node. It's based on in-memory data structures, and is updated +atomically from file. + +## Central Classes + +* [PartitionLinkGraphService](java/nu/marginalia/linkgraph/PartitionLinkGraphService.java) +* [DomainLink](java/nu/marginalia/linkgraph/DomainLinks.java) +* * [FileDomainLinks](java/nu/marginalia/linkgraph/impl/FileDomainLinks.java) +* [DomainLinksWriter](java/nu/marginalia/linkgraph/io/DomainLinksWriter.java) +* [DomainLinksLoader](java/nu/marginalia/linkgraph/io/DomainLinksLoader.java) \ No newline at end of file diff --git a/code/common/linkdb/test/nu/marginalia/linkdb/DomainLinkDbTest.java b/code/functions/link-graph/partition/test/nu/marginalia/linkgraph/DomainLinkDbTest.java similarity index 84% rename from code/common/linkdb/test/nu/marginalia/linkdb/DomainLinkDbTest.java rename to code/functions/link-graph/partition/test/nu/marginalia/linkgraph/DomainLinkDbTest.java index 6db4a8cf..e8b0d3d3 100644 --- a/code/common/linkdb/test/nu/marginalia/linkdb/DomainLinkDbTest.java +++ b/code/functions/link-graph/partition/test/nu/marginalia/linkgraph/DomainLinkDbTest.java @@ -1,7 +1,7 @@ -package nu.marginalia.linkdb; +package nu.marginalia.linkgraph; -import nu.marginalia.linkdb.dlinks.DomainLinkDbLoader; -import nu.marginalia.linkdb.dlinks.DomainLinkDbWriter; +import nu.marginalia.linkgraph.io.DomainLinksLoader; +import nu.marginalia.linkgraph.io.DomainLinksWriter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -24,7 +24,7 @@ public class DomainLinkDbTest { @Test public void testWriteRead() { - try (var writer = new DomainLinkDbWriter(fileName)) { + try (var writer = new DomainLinksWriter(fileName)) { writer.write(1, 2); writer.write(2, 3); writer.write(3, 4); @@ -33,7 +33,7 @@ public class DomainLinkDbTest { throw new RuntimeException(ex); } - try (var reader = new DomainLinkDbLoader(fileName)) { + try (var reader = new DomainLinksLoader(fileName)) { Assertions.assertTrue(reader.next()); Assertions.assertEquals(1, reader.getSource()); Assertions.assertEquals(2, reader.getDest()); diff --git a/code/functions/link-graph/readme.md b/code/functions/link-graph/readme.md new file mode 100644 index 00000000..b906978d --- /dev/null +++ b/code/functions/link-graph/readme.md @@ -0,0 +1,9 @@ +The link graph subsystem is responsible for knowledge about the link graph. + +A SQL database is not very well suited for this, principally it's too slow to update, +instead the link graph is stored in memory, and atomically updated from file. The storage +aspect is handled by the [common/linkdb](../../common/linkdb/) component. + +The link graph subsystem has two components, one which injects into the partitioned services, +e.g. index or execution, and one which aggregates the results from the partitioned services, +and exposes a unified view of the link graph. \ No newline at end of file diff --git a/code/functions/search-query/java/nu/marginalia/util/ngrams/DenseBitMap.java b/code/functions/search-query/java/nu/marginalia/util/ngrams/DenseBitMap.java index 5234b62d..008b17b3 100644 --- a/code/functions/search-query/java/nu/marginalia/util/ngrams/DenseBitMap.java +++ b/code/functions/search-query/java/nu/marginalia/util/ngrams/DenseBitMap.java @@ -5,7 +5,10 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.BitSet; +// It's unclear why this exists, we should probably use a BitSet instead? +// Chesterton's fence? public class DenseBitMap { public static final long MAX_CAPACITY_2GB_16BN_ITEMS=(1L<<34)-8; diff --git a/code/functions/search-query/readme.md b/code/functions/search-query/readme.md new file mode 100644 index 00000000..54022655 --- /dev/null +++ b/code/functions/search-query/readme.md @@ -0,0 +1,4 @@ +The search query subsystem is responsible for parsing a query, +translating it to a request, and then dispatching it to the +appropriate index nodes and translating the responses back again. + diff --git a/code/index/build.gradle b/code/index/build.gradle index 403da54a..7d52facc 100644 --- a/code/index/build.gradle +++ b/code/index/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':third-party:commons-codec') implementation project(':code:index:api') - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation project(':code:libraries:array') implementation project(':code:libraries:btree') diff --git a/code/index/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java index a3863f6b..8562cdce 100644 --- a/code/index/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java +++ b/code/index/java/nu/marginalia/ranking/domains/data/InvertedLinkGraphSource.java @@ -3,7 +3,7 @@ package nu.marginalia.ranking.domains.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; @@ -13,12 +13,12 @@ import org.jgrapht.graph.DefaultEdge; * which is the same as the regular graph except * the direction of the links have been inverted */ public class InvertedLinkGraphSource extends AbstractGraphSource { - private final AggregateDomainLinksClient queryClient; + private final AggregateLinkGraphClient graphClient; @Inject - public InvertedLinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient queryClient) { + public InvertedLinkGraphSource(HikariDataSource dataSource, AggregateLinkGraphClient graphClient) { super(dataSource); - this.queryClient = queryClient; + this.graphClient = graphClient; } @SneakyThrows @Override @@ -27,7 +27,7 @@ public class InvertedLinkGraphSource extends AbstractGraphSource { addVertices(graph); - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = graphClient.getAllDomainLinks(); var iter = allLinks.iterator(); while (iter.advance()) { if (!graph.containsVertex(iter.dest())) { diff --git a/code/index/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java b/code/index/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java index 7c4bcb55..341a77df 100644 --- a/code/index/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java +++ b/code/index/java/nu/marginalia/ranking/domains/data/LinkGraphSource.java @@ -3,19 +3,19 @@ package nu.marginalia.ranking.domains.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; /** A source for the regular link graph. */ public class LinkGraphSource extends AbstractGraphSource { - private final AggregateDomainLinksClient domainLinksClient; + private final AggregateLinkGraphClient graphClient; @Inject - public LinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) { + public LinkGraphSource(HikariDataSource dataSource, AggregateLinkGraphClient graphClient) { super(dataSource); - this.domainLinksClient = domainLinksClient; + this.graphClient = graphClient; } @SneakyThrows @@ -25,7 +25,7 @@ public class LinkGraphSource extends AbstractGraphSource { addVertices(graph); - var allLinks = domainLinksClient.getAllDomainLinks(); + var allLinks = graphClient.getAllDomainLinks(); var iter = allLinks.iterator(); while (iter.advance()) { if (!graph.containsVertex(iter.dest())) { diff --git a/code/index/readme.md b/code/index/readme.md index bc44c7d8..2254c2a2 100644 --- a/code/index/readme.md +++ b/code/index/readme.md @@ -1,6 +1,6 @@ # Index -This module contains the components that make up the search index. +This index subsystem contains the components that make up the search index. It exposes an API for querying the index, and contains the logic for ranking search results. It does not parse the query, that is @@ -10,9 +10,9 @@ the responsibility of the [search-query](../functions/search-query) module. There are two indexes with accompanying tools for constructing them. -* [index-reverse](reverse-index/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. +* [index-reverse](index-reverse/) is code for `word->document` indexes. There are two such indexes, one containing only document-word pairs that are flagged as important, e.g. the word appears in the title or has a high TF-IDF. This allows good results to be discovered quickly without having to sift through ten thousand bad ones first. -* [index-forward](forward-index/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. +* [index-forward](index-forward/) is the `document->word` index containing metadata about each word, such as its position. It is used after identifying candidate search results via the reverse index to fetch metadata and rank the results. Additionally, the [index-journal](index-journal/) contains code for constructing a journal of the index, which is used to keep the index up to date. diff --git a/code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java b/code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java index c25b818a..f748465a 100644 --- a/code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java +++ b/code/index/test/nu/marginalia/ranking/domains/RankingAlgorithmsContainerTest.java @@ -3,7 +3,7 @@ package nu.marginalia.ranking.domains; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import nu.marginalia.ranking.domains.data.InvertedLinkGraphSource; import nu.marginalia.ranking.domains.data.LinkGraphSource; import nu.marginalia.ranking.domains.data.SimilarityGraphSource; @@ -36,8 +36,8 @@ public class RankingAlgorithmsContainerTest { static HikariDataSource dataSource; - AggregateDomainLinksClient domainLinksClient; - AggregateDomainLinksClient.AllLinks allLinks; + AggregateLinkGraphClient domainLinksClient; + AggregateLinkGraphClient.AllLinks allLinks; @BeforeAll public static void setup() { @@ -66,8 +66,8 @@ public class RankingAlgorithmsContainerTest { @BeforeEach public void setupQueryClient() { - domainLinksClient = Mockito.mock(AggregateDomainLinksClient.class); - allLinks = new AggregateDomainLinksClient.AllLinks(); + domainLinksClient = Mockito.mock(AggregateLinkGraphClient.class); + allLinks = new AggregateLinkGraphClient.AllLinks(); when(domainLinksClient.getAllDomainLinks()).thenReturn(allLinks); try (var conn = dataSource.getConnection(); diff --git a/code/libraries/language-processing/readme.md b/code/libraries/language-processing/readme.md index 5b12a27d..7b8ee049 100644 --- a/code/libraries/language-processing/readme.md +++ b/code/libraries/language-processing/readme.md @@ -11,6 +11,4 @@ its words, how they stem, POS tags, and so on. ## See Also [features-convert/keyword-extraction](../../features-convert/keyword-extraction) uses this code to identify which keywords -are important. - -[features-qs/query-parser](../../features-qs/query-parser) also does some language processing. \ No newline at end of file +are important. \ No newline at end of file diff --git a/code/libraries/term-frequency-dict/readme.md b/code/libraries/term-frequency-dict/readme.md index 810c3751..1c1e9c67 100644 --- a/code/libraries/term-frequency-dict/readme.md +++ b/code/libraries/term-frequency-dict/readme.md @@ -6,7 +6,3 @@ the TF-IDF score of a keyword. ## Central Classes * [TermFrequencyDict](java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java) - -## See Also - -* [tools/term-frequency-extractor](../../tools/term-frequency-extractor) constructs this file \ No newline at end of file diff --git a/code/processes/index-constructor-process/readme.md b/code/processes/index-constructor-process/readme.md index ecf791b8..6e7a46ee 100644 --- a/code/processes/index-constructor-process/readme.md +++ b/code/processes/index-constructor-process/readme.md @@ -10,8 +10,8 @@ There are three types of indexes: This is a very light-weight module that delegates the actual work to the modules: -* [features-index/index-reverse](../../features-index/index-reverse) -* [features-index/index-forward](../../features-index/index-forward) +* [features-index/index-reverse](../../index/index-reverse) +* [features-index/index-forward](../../index/index-forward) Their respective readme files contain more information about the indexes themselves and how they are constructed. diff --git a/code/processes/loading-process/build.gradle b/code/processes/loading-process/build.gradle index cf3ef16b..4a2afc68 100644 --- a/code/processes/loading-process/build.gradle +++ b/code/processes/loading-process/build.gradle @@ -40,6 +40,8 @@ dependencies { implementation project(':code:process-models:work-log') implementation project(':code:features-convert:keyword-extraction') + implementation project(':code:functions:link-graph:partition') + implementation libs.bundles.slf4j implementation libs.guice diff --git a/code/processes/loading-process/java/nu/marginalia/loading/LoaderModule.java b/code/processes/loading-process/java/nu/marginalia/loading/LoaderModule.java index 4f9765a4..8b4719e6 100644 --- a/code/processes/loading-process/java/nu/marginalia/loading/LoaderModule.java +++ b/code/processes/loading-process/java/nu/marginalia/loading/LoaderModule.java @@ -9,7 +9,7 @@ import com.google.inject.name.Names; import nu.marginalia.LanguageModels; import nu.marginalia.WmsaHome; import nu.marginalia.IndexLocations; -import nu.marginalia.linkdb.dlinks.DomainLinkDbWriter; +import nu.marginalia.linkgraph.io.DomainLinksWriter; import nu.marginalia.storage.FileStorageService; import nu.marginalia.linkdb.docs.DocumentDbWriter; import nu.marginalia.model.gson.GsonFactory; @@ -45,7 +45,7 @@ public class LoaderModule extends AbstractModule { } @Inject @Provides @Singleton - private DomainLinkDbWriter createDomainLinkdbWriter(FileStorageService service) throws SQLException, IOException { + private DomainLinksWriter createDomainLinkdbWriter(FileStorageService service) throws SQLException, IOException { Path dbPath = IndexLocations.getLinkdbWritePath(service).resolve(DOMAIN_LINKS_FILE_NAME); @@ -53,7 +53,7 @@ public class LoaderModule extends AbstractModule { Files.delete(dbPath); } - return new DomainLinkDbWriter(dbPath); + return new DomainLinksWriter(dbPath); } private Gson createGson() { diff --git a/code/processes/loading-process/java/nu/marginalia/loading/links/DomainLinksLoaderService.java b/code/processes/loading-process/java/nu/marginalia/loading/links/DomainLinksLoaderService.java index 8cf42218..06bf4c95 100644 --- a/code/processes/loading-process/java/nu/marginalia/loading/links/DomainLinksLoaderService.java +++ b/code/processes/loading-process/java/nu/marginalia/loading/links/DomainLinksLoaderService.java @@ -4,7 +4,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; import nu.marginalia.io.processed.DomainLinkRecordParquetFileReader; -import nu.marginalia.linkdb.dlinks.DomainLinkDbWriter; +import nu.marginalia.linkgraph.io.DomainLinksWriter; import nu.marginalia.loading.LoaderInputData; import nu.marginalia.loading.domains.DomainIdRegistry; import nu.marginalia.model.processed.DomainLinkRecord; @@ -20,10 +20,10 @@ public class DomainLinksLoaderService { private static final Logger logger = LoggerFactory.getLogger(DomainLinksLoaderService.class); - private final DomainLinkDbWriter domainLinkDbWriter; + private final DomainLinksWriter domainLinkDbWriter; @Inject - public DomainLinksLoaderService(DomainLinkDbWriter domainLinkDbWriter) { + public DomainLinksLoaderService(DomainLinksWriter domainLinkDbWriter) { this.domainLinkDbWriter = domainLinkDbWriter; } diff --git a/code/processes/readme.md b/code/processes/readme.md index acfe5a39..3bdc0970 100644 --- a/code/processes/readme.md +++ b/code/processes/readme.md @@ -17,7 +17,7 @@ described in [processed-data](../process-models/processed-data/). The [loading-process](loading-process/) reads the processed data. -It has creates an [index journal](../features-index/index-journal), +It has creates an [index journal](../index/index-journal), a [link database](../common/linkdb), and loads domains and domain-links into the [MariaDB database](../common/db). diff --git a/code/processes/website-adjacencies-calculator/build.gradle b/code/processes/website-adjacencies-calculator/build.gradle index 7c4b9623..6019d1dd 100644 --- a/code/processes/website-adjacencies-calculator/build.gradle +++ b/code/processes/website-adjacencies-calculator/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation project(':code:common:process') implementation project(':code:common:service-discovery') implementation project(':code:common:service') - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation libs.bundles.slf4j diff --git a/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesData.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesData.java index d0886b9f..f52a3161 100644 --- a/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesData.java +++ b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/AdjacenciesData.java @@ -4,7 +4,7 @@ import gnu.trove.list.TIntList; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +35,7 @@ public class AdjacenciesData { return ret; } - public AdjacenciesData(AggregateDomainLinksClient linksClient, + public AdjacenciesData(AggregateLinkGraphClient linksClient, DomainAliases aliases) { logger.info("Loading adjacency data"); diff --git a/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index 1a40c354..92aae06a 100644 --- a/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -4,7 +4,7 @@ import com.google.inject.Guice; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.ProcessConfiguration; -import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; +import nu.marginalia.api.linkgraph.AggregateLinkGraphClient; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.model.EdgeDomain; import nu.marginalia.process.control.ProcessHeartbeat; @@ -32,7 +32,7 @@ public class WebsiteAdjacenciesCalculator extends ProcessMainClass { private static final Logger logger = LoggerFactory.getLogger(WebsiteAdjacenciesCalculator.class); float[] weights; - public WebsiteAdjacenciesCalculator(AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) throws SQLException { + public WebsiteAdjacenciesCalculator(AggregateLinkGraphClient domainLinksClient, HikariDataSource dataSource) throws SQLException { this.dataSource = dataSource; domainAliases = new DomainAliases(dataSource); @@ -154,7 +154,7 @@ public class WebsiteAdjacenciesCalculator extends ProcessMainClass { var dataSource = injector.getInstance(HikariDataSource.class); - var lc = injector.getInstance(AggregateDomainLinksClient.class); + var lc = injector.getInstance(AggregateLinkGraphClient.class); if (!lc.waitReady(Duration.ofSeconds(30))) { throw new IllegalStateException("Failed to connect to domain-links"); diff --git a/code/readme.md b/code/readme.md index 1792ec5f..d75912b2 100644 --- a/code/readme.md +++ b/code/readme.md @@ -23,18 +23,38 @@ eligible index services. The control service is responsible for distributing co service, and for monitoring the health of the system. It also offers a web interface for operating the system. ### Services + * [core services](services-core/) Most of these services are stateful, memory hungry, and doing heavy lifting. * * [control](services-core/control-service) * * [query](services-core/query-service) +* * * Exposes the [functions/link-graph](functions/link-graph) subsystem +* * * Exposes the [functions/search-query](functions/search-query) subsystem * * [index](services-core/index-service) +* * * Exposes the [index](index) subsystem +* * * Exposes the [functions/link-graph](functions/link-graph) subsystem * * [executor](services-core/executor-service) +* * * Exposes the [execution](execution) subsystem * * [assistant](services-core/assistant-service) +* * * Exposes the [functions/math](functions/math) subsystem +* * * Exposes the [functions/domain-info](functions/domain-info) subsystem * [application services](services-application/) Mostly stateless gateways providing access to the core services. -* * [api](services-application/api-service) - public API +* * [api](services-application/api-service) - public API gateway * * [search](services-application/search-service) - marginalia search application -* * [dating](services-application/dating-service) - [https://explore.marginalia.nu/](https://explore.marginalia.nu/) -* * [explorer](services-application/explorer-service) - [https://explore2.marginalia.nu/](https://explore2.marginalia.nu/) -* an [internal API](api/) +* * [dating](services-application/dating-service) - [https://explore.marginalia.nu/](https://explore.marginalia.nu/) +* * [explorer](services-application/explorer-service) - [https://explore2.marginalia.nu/](https://explore2.marginalia.nu/) + +The system uses a service registry to find the services. The service registry is based on zookeeper, +and is a separate service. The registry doesn't keep track of processes, but APIs. This means that +the system is flexible to reconfiguration. The same code can in principle be run as a micro-service +mesh or as a monolith. + +This is an unusual architecture, but it has the benefit that you don't need to think too much about +the layout of the system. You can just request an API and talk to it. Because of this, several of the +services have almost no code of their own. They merely import a library and expose it as a service. + +These skeleton services are marked with (S). + +Services that expose HTTP endpoints tend to have more code. They are marked with (G). ### Processes @@ -55,7 +75,6 @@ but isolated. * [features-search](features-search) * [features-crawl](features-crawl) * [features-convert](features-convert) -* [features-index](features-index) ### Libraries and primitives diff --git a/code/services-core/control-service/readme.md b/code/services-core/control-service/readme.md index 5da87273..24958d95 100644 --- a/code/services-core/control-service/readme.md +++ b/code/services-core/control-service/readme.md @@ -4,8 +4,7 @@ The control service provides an operator's user interface. By default, this int exposed on port 8081. It does not offer any sort of access control or authentication. The control service will itself execute tasks that affect the entire system, but delegate -node-specific tasks to the corresponding [executor-service](../executor-service) via the -[executor-api](../../api/executor-api). +node-specific tasks to the corresponding to the [execution subsystem](../../execution). Conceptually the application is broken into three parts: diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 43c0a23e..a6a88251 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation project(':code:libraries:message-queue') - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:api') implementation project(':code:process-models:crawl-spec') implementation project(':code:process-models:crawling-model') diff --git a/code/services-core/executor-service/readme.md b/code/services-core/executor-service/readme.md index 1f05c3a4..280defd1 100644 --- a/code/services-core/executor-service/readme.md +++ b/code/services-core/executor-service/readme.md @@ -1,16 +1,10 @@ The executor service is a partitioned service responsible for executing and keeping -track of long running maintenance and operational tasks, such as crawling or data -processing. +track of long-running maintenance and operational tasks, such as crawling or data +processing. -It accomplishes this using the [message queue and actor library](../../libraries/message-queue/), -which permits program state to survive crashes and reboots. The executor service is closely -linked to the [control-service](../control-service), which provides a user interface for -much of the executor's functionality. +The executor service is closely linked to the [control-service](../control-service), +which provides a user interface for much of the executor's functionality. -## Central Classes +The service it itself relatively bare of code, but imports and exposes the [execution subsystem](../../execution), +which is responsible for the actual execution of tasks. -* [ExecutorActorControlService](java/nu/marginalia/actor/ExecutorActorControlService.java) - -## See Also - -* [api/executor-api](../../api/executor-api) \ No newline at end of file diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 05aa3af9..f68d093a 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -46,8 +46,8 @@ dependencies { implementation project(':code:common:linkdb') implementation project(':code:index') - implementation project(':code:functions:domain-links:partition') - implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:link-graph:partition') + implementation project(':code:functions:link-graph:api') implementation project(':code:functions:search-query:api') implementation project(':code:index:api') diff --git a/code/services-core/index-service/java/nu/marginalia/index/IndexModule.java b/code/services-core/index-service/java/nu/marginalia/index/IndexModule.java index 526e34bd..f998dfcf 100644 --- a/code/services-core/index-service/java/nu/marginalia/index/IndexModule.java +++ b/code/services-core/index-service/java/nu/marginalia/index/IndexModule.java @@ -4,8 +4,8 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; import com.google.inject.name.Named; -import nu.marginalia.linkdb.dlinks.DomainLinkDb; -import nu.marginalia.linkdb.dlinks.DelayingDomainLinkDb; +import nu.marginalia.linkgraph.DomainLinks; +import nu.marginalia.linkgraph.impl.DelayingDomainLinks; import nu.marginalia.storage.FileStorageService; import nu.marginalia.IndexLocations; import org.slf4j.Logger; @@ -26,13 +26,13 @@ public class IndexModule extends AbstractModule { @Provides @Singleton - public DomainLinkDb domainLinkDb ( + public DomainLinks domainLinkDb ( FileStorageService storageService ) { Path path = IndexLocations.getLinkdbLivePath(storageService).resolve(DOMAIN_LINKS_FILE_NAME); - return new DelayingDomainLinkDb(path); + return new DelayingDomainLinks(path); } @Provides diff --git a/code/services-core/index-service/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/java/nu/marginalia/index/IndexService.java index ee96e9d4..cf9187d0 100644 --- a/code/services-core/index-service/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/java/nu/marginalia/index/IndexService.java @@ -3,9 +3,9 @@ package nu.marginalia.index; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; -import nu.marginalia.functions.domainlinks.PartitionDomainLinksService; +import nu.marginalia.linkgraph.PartitionLinkGraphService; import nu.marginalia.index.index.StatefulIndex; -import nu.marginalia.linkdb.dlinks.DomainLinkDb; +import nu.marginalia.linkgraph.DomainLinks; import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.api.IndexMqEndpoints; @@ -34,7 +34,7 @@ public class IndexService extends Service { private final FileStorageService fileStorageService; private final DocumentDbReader documentDbReader; - private final DomainLinkDb domainLinkDb; + private final DomainLinks domainLinks; private final ServiceEventLog eventLog; @@ -46,21 +46,21 @@ public class IndexService extends Service { StatefulIndex statefulIndex, FileStorageService fileStorageService, DocumentDbReader documentDbReader, - DomainLinkDb domainLinkDb, - PartitionDomainLinksService partitionDomainLinksService, + DomainLinks domainLinks, + PartitionLinkGraphService partitionLinkGraphService, ServiceEventLog eventLog) { super(params, ServicePartition.partition(params.configuration.node()), List.of(indexQueryService, - partitionDomainLinksService) + partitionLinkGraphService) ); this.opsService = opsService; this.statefulIndex = statefulIndex; this.fileStorageService = fileStorageService; this.documentDbReader = documentDbReader; - this.domainLinkDb = domainLinkDb; + this.domainLinks = domainLinks; this.eventLog = eventLog; this.init = params.initialization; @@ -106,7 +106,7 @@ public class IndexService extends Service { if (Files.exists(newPathDomains)) { eventLog.logEvent("INDEX-SWITCH-DOMAIN-LINKDB", ""); - domainLinkDb.switchInput(newPathDomains); + domainLinks.switchInput(newPathDomains); } } diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 96022efd..591ef884 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -50,8 +50,8 @@ dependencies { implementation project(':code:functions:search-query') implementation project(':code:functions:search-query:api') - implementation project(':code:functions:domain-links:api') - implementation project(':code:functions:domain-links:aggregate') + implementation project(':code:functions:link-graph:api') + implementation project(':code:functions:link-graph:aggregate') implementation libs.bundles.slf4j diff --git a/code/services-core/query-service/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/java/nu/marginalia/query/QueryService.java index b8a4c6c5..b7dcc04c 100644 --- a/code/services-core/query-service/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/java/nu/marginalia/query/QueryService.java @@ -2,7 +2,7 @@ package nu.marginalia.query; import com.google.inject.Inject; import lombok.SneakyThrows; -import nu.marginalia.functions.domainlinks.AggregateDomainLinksService; +import nu.marginalia.linkgraph.AggregateLinkGraphService; import nu.marginalia.functions.searchquery.QueryGRPCService; import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; @@ -17,7 +17,7 @@ public class QueryService extends Service { @SneakyThrows @Inject public QueryService(BaseServiceParams params, - AggregateDomainLinksService domainLinksService, + AggregateLinkGraphService domainLinksService, QueryGRPCService queryGRPCService, QueryBasicInterface queryBasicInterface) { diff --git a/settings.gradle b/settings.gradle index 0dbc4376..fca2091c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,9 +17,9 @@ include 'code:functions:math:api' include 'code:functions:domain-info' include 'code:functions:domain-info:api' -include 'code:functions:domain-links:partition' -include 'code:functions:domain-links:aggregate' -include 'code:functions:domain-links:api' +include 'code:functions:link-graph:partition' +include 'code:functions:link-graph:aggregate' +include 'code:functions:link-graph:api' include 'code:functions:search-query' include 'code:functions:search-query:api' From 37ae8cb33c6af8ab9f8be1d2db2d8838ce0c3eb8 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 28 Feb 2024 11:48:16 +0100 Subject: [PATCH 53/56] Migrate the docker compose files --- run/env/service.env | 4 +- .../docker-compose-barebones-1.yml.template | 27 +++------- .../docker-compose-barebones-2.yml.template | 48 ++++++----------- .../docker-compose-marginalia.yml.template | 51 ++++--------------- 4 files changed, 31 insertions(+), 99 deletions(-) diff --git a/run/env/service.env b/run/env/service.env index 5ccabc86..f14a93cc 100644 --- a/run/env/service.env +++ b/run/env/service.env @@ -1,7 +1,5 @@ -WMSA_HOME=run/ -JAVA_OPTS="--enable-preview -da -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" +JDK_JAVA_OPTIONS="--enable-preview -da -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" JAVA_TOOL_OPTIONS="--enable-preview -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5000" -EXECUTOR_SERVICE_OPTS="" CONVERTER_PROCESS_OPTS="-Dservice-name=converter" CRAWLER_PROCESS_OPTS="-Dservice-name=crawler" LOADER_PROCESS_OPTS="-Dservice-name=loader" diff --git a/run/install/docker-compose-barebones-1.yml.template b/run/install/docker-compose-barebones-1.yml.template index 11dc8dcf..8b36d64e 100644 --- a/run/install/docker-compose-barebones-1.yml.template +++ b/run/install/docker-compose-barebones-1.yml.template @@ -8,21 +8,12 @@ x-svc: &service - logs:/var/log/wmsa networks: - wmsa - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 + depends_on: + - mariadb + - zookeeper x-p1: &partition-1 env_file: - "${INSTALL_DIR}/env/service.env" - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 volumes: - conf:/wmsa/conf:ro - model:/wmsa/model @@ -37,28 +28,22 @@ x-p1: &partition-1 - wmsa environment: - "WMSA_SERVICE_NODE=1" + depends_on: + - mariadb + - zookeeper services: index-service-1: <<: *partition-1 image: "marginalia/index-service" container_name: "index-service-1" - depends_on: - control-service: - condition: service_healthy executor-service-1: <<: *partition-1 image: "marginalia/executor-service" container_name: "executor-service-1" - depends_on: - control-service: - condition: service_healthy query-service: <<: *service image: "marginalia/query-service" container_name: "query-service" - depends_on: - control-service: - condition: service_healthy expose: - 80 labels: diff --git a/run/install/docker-compose-barebones-2.yml.template b/run/install/docker-compose-barebones-2.yml.template index 94e35d34..6232cf01 100644 --- a/run/install/docker-compose-barebones-2.yml.template +++ b/run/install/docker-compose-barebones-2.yml.template @@ -8,21 +8,12 @@ x-svc: &service - logs:/var/log/wmsa networks: - wmsa - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 + depends_on: + - mariadb + - zookeeper x-p1: &partition-1 env_file: - "${INSTALL_DIR}/env/service.env" - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 volumes: - conf:/wmsa/conf:ro - model:/wmsa/model @@ -35,17 +26,14 @@ x-p1: &partition-1 - uploads-1:/uploads networks: - wmsa + depends_on: + - mariadb + - zookeeper environment: - "WMSA_SERVICE_NODE=1" x-p2: &partition-2 env_file: - "${INSTALL_DIR}/env/service.env" - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 volumes: - conf:/wmsa/conf:ro - model:/wmsa/model @@ -58,6 +46,9 @@ x-p2: &partition-2 - uploads-2:/uploads networks: - wmsa + depends_on: + - mariadb + - zookeeper environment: - "WMSA_SERVICE_NODE=2" services: @@ -65,37 +56,22 @@ services: <<: *partition-1 image: "marginalia/index-service" container_name: "index-service-1" - depends_on: - control-service: - condition: service_healthy executor-service-1: <<: *partition-1 image: "marginalia/executor-service" container_name: "executor-service-1" - depends_on: - control-service: - condition: service_healthy index-service-2: <<: *partition-2 image: "marginalia/index-service" container_name: "index-service-2" - depends_on: - control-service: - condition: service_healthy executor-service-2: <<: *partition-2 image: "marginalia/executor-service" container_name: "executor-service-2" - depends_on: - control-service: - condition: service_healthy query-service: <<: *service image: "marginalia/query-service" container_name: "query-service" - depends_on: - control-service: - condition: service_healthy expose: - 80 labels: @@ -140,6 +116,12 @@ services: - db:/var/lib/mysql networks: - wmsa + zookeeper: + image: zookeeper + container_name: "zookeeper" + restart: always + ports: + - "127.0.0.1:2181:2181" traefik: image: "traefik:v2.10" container_name: "traefik" diff --git a/run/install/docker-compose-marginalia.yml.template b/run/install/docker-compose-marginalia.yml.template index 2adef02d..b92543e7 100644 --- a/run/install/docker-compose-marginalia.yml.template +++ b/run/install/docker-compose-marginalia.yml.template @@ -8,21 +8,12 @@ x-svc: &service - logs:/var/log/wmsa networks: - wmsa - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 + depends_on: + - mariadb + - zookeeper x-p1: &partition-1 env_file: - "${INSTALL_DIR}/env/service.env" - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 volumes: - conf:/wmsa/conf:ro - model:/wmsa/model @@ -35,17 +26,14 @@ x-p1: &partition-1 - uploads-1:/uploads networks: - wmsa + depends_on: + - mariadb + - zookeeper environment: - "WMSA_SERVICE_NODE=1" x-p2: &partition-2 env_file: - "${INSTALL_DIR}/env/service.env" - healthcheck: - test: curl -f http://localhost:80/internal/ping || exit 1 - start_period: 1s - interval: 5s - timeout: 5s - retries: 60 volumes: - conf:/wmsa/conf:ro - model:/wmsa/model @@ -58,6 +46,9 @@ x-p2: &partition-2 - uploads-2:/uploads networks: - wmsa + depends_on: + - mariadb + - zookeeper environment: - "WMSA_SERVICE_NODE=2" services: @@ -65,44 +56,26 @@ services: <<: *partition-1 image: "marginalia/index-service" container_name: "index-service-1" - depends_on: - control-service: - condition: service_healthy executor-service-1: <<: *partition-1 image: "marginalia/executor-service" container_name: "executor-service-1" - depends_on: - control-service: - condition: service_healthy index-service-2: <<: *partition-2 image: "marginalia/index-service" container_name: "index-service-2" - depends_on: - control-service: - condition: service_healthy executor-service-2: <<: *partition-2 image: "marginalia/executor-service" container_name: "executor-service-2" - depends_on: - control-service: - condition: service_healthy query-service: <<: *service image: "marginalia/query-service" container_name: "query-service" - depends_on: - control-service: - condition: service_healthy search-service: <<: *service image: "marginalia/search-service" container_name: "search-service" - depends_on: - control-service: - condition: service_healthy expose: - 80 labels: @@ -117,9 +90,6 @@ services: <<: *service image: "marginalia/assistant-service" container_name: "assistant-service" - depends_on: - control-service: - condition: service_healthy expose: - 80 labels: @@ -138,9 +108,6 @@ services: <<: *service image: "marginalia/api-service" container_name: "api-service" - depends_on: - control-service: - condition: service_healthy expose: - "80" labels: From 20fc0ef13cd72b22e72e023ceaa9dc432929d6e2 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 28 Feb 2024 11:59:15 +0100 Subject: [PATCH 54/56] (gradle) Add task alias 'docker' for 'jibDockerBuild' The change also moves the jib boilerplate to an include. --- .../api-service/build.gradle | 17 +---------------- .../dating-service/build.gradle | 16 +--------------- .../explorer-service/build.gradle | 16 +--------------- .../search-service/build.gradle | 16 +--------------- .../assistant-service/build.gradle | 18 +----------------- .../services-core/control-service/build.gradle | 16 +--------------- .../executor-service/build.gradle | 17 ++--------------- code/services-core/index-service/build.gradle | 16 +--------------- code/services-core/query-service/build.gradle | 16 +--------------- .../tools/screenshot-capture-tool/build.gradle | 16 +--------------- docker.gradle | 18 ++++++++++++++++++ 11 files changed, 29 insertions(+), 153 deletions(-) create mode 100644 docker.gradle diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index c09c0c9f..ff27b83e 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -19,23 +19,8 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} - - apply from: "$rootProject.projectDir/srcsets.gradle" +apply from: "$rootProject.projectDir/docker.gradle" dependencies { implementation project(':code:common:db') diff --git a/code/services-application/dating-service/build.gradle b/code/services-application/dating-service/build.gradle index 6d209102..72a3057a 100644 --- a/code/services-application/dating-service/build.gradle +++ b/code/services-application/dating-service/build.gradle @@ -13,21 +13,7 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} +apply from: "$rootProject.projectDir/docker.gradle" java { diff --git a/code/services-application/explorer-service/build.gradle b/code/services-application/explorer-service/build.gradle index 6aba03b7..6d2e56ac 100644 --- a/code/services-application/explorer-service/build.gradle +++ b/code/services-application/explorer-service/build.gradle @@ -13,21 +13,7 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} +apply from: "$rootProject.projectDir/docker.gradle" java { diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index d290e4c4..d5285334 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -8,21 +8,6 @@ plugins { id 'com.google.cloud.tools.jib' version '3.4.0' } -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} - application { mainClass = 'nu.marginalia.search.SearchMain' applicationName = 'search-service' @@ -43,6 +28,7 @@ sass { } apply from: "$rootProject.projectDir/srcsets.gradle" +apply from: "$rootProject.projectDir/docker.gradle" dependencies { implementation project(':code:common:db') diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index b6d4aa4f..278bbcdf 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -13,23 +13,6 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} - - java { toolchain { languageVersion.set(JavaLanguageVersion.of(21)) @@ -37,6 +20,7 @@ java { } apply from: "$rootProject.projectDir/srcsets.gradle" +apply from: "$rootProject.projectDir/docker.gradle" dependencies { implementation project(':third-party:symspell') diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 6d46926e..a14aec08 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -18,21 +18,7 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} +apply from: "$rootProject.projectDir/docker.gradle" apply from: "$rootProject.projectDir/srcsets.gradle" diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index a6a88251..69ad7d1f 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -13,20 +13,7 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} + java { toolchain { @@ -35,7 +22,7 @@ java { } apply from: "$rootProject.projectDir/srcsets.gradle" - +apply from: "$rootProject.projectDir/docker.gradle" dependencies { // These look weird but they're needed to be able to spawn the processes // from the executor service diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index f68d093a..f29a1494 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -13,21 +13,7 @@ application { tasks.distZip.enabled = false -jib { - from { - image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = [ rootProject.ext.dockerImageTag ] - } - container { - - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} +apply from: "$rootProject.projectDir/docker.gradle" java { diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 591ef884..b4e2a179 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -13,21 +13,7 @@ application { tasks.distZip.enabled = false -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} +apply from: "$rootProject.projectDir/docker.gradle" java { diff --git a/code/tools/screenshot-capture-tool/build.gradle b/code/tools/screenshot-capture-tool/build.gradle index 57ea3f72..75aef932 100644 --- a/code/tools/screenshot-capture-tool/build.gradle +++ b/code/tools/screenshot-capture-tool/build.gradle @@ -17,25 +17,11 @@ application { applicationName = 'screenshot-capture-tool' } -jib { - from { - image = image = rootProject.ext.dockerImageBase - } - to { - image = rootProject.ext.dockerImageRegistry+'/'+project.name - tags = ['latest'] - } - container { - mainClass = application.mainClass - jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] - volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] - } -} - tasks.distZip.enabled = false apply from: "$rootProject.projectDir/srcsets.gradle" +apply from: "$rootProject.projectDir/docker.gradle" dependencies { implementation project(':code:common:model') diff --git a/docker.gradle b/docker.gradle new file mode 100644 index 00000000..b3feccb5 --- /dev/null +++ b/docker.gradle @@ -0,0 +1,18 @@ +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = rootProject.ext.dockerImageRegistry+'/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + +tasks.register('docker', { + dependsOn('jibDockerBuild') +}) From a8ec59eb75c4113733cc615263ed8333d09be9a1 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 28 Feb 2024 12:09:01 +0100 Subject: [PATCH 55/56] (conf) Add migration warning when ZOOKEEPER_HOSTS is not set. --- .../service/ServiceDiscoveryModule.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/code/common/service-discovery/java/nu/marginalia/service/ServiceDiscoveryModule.java b/code/common/service-discovery/java/nu/marginalia/service/ServiceDiscoveryModule.java index cc783c72..6daa084c 100644 --- a/code/common/service-discovery/java/nu/marginalia/service/ServiceDiscoveryModule.java +++ b/code/common/service-discovery/java/nu/marginalia/service/ServiceDiscoveryModule.java @@ -17,7 +17,7 @@ public class ServiceDiscoveryModule extends AbstractModule { private static final Logger logger = LoggerFactory.getLogger(ServiceDiscoveryModule.class); public void configure() { - var hosts = getZookeeperHosts().orElseThrow(() -> new IllegalStateException("Zookeeper hosts not set")); + var hosts = getZookeeperHosts(); logger.info("Using Zookeeper service registry at {}", hosts); CuratorFramework client = CuratorFrameworkFactory .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000)); @@ -26,11 +26,21 @@ public class ServiceDiscoveryModule extends AbstractModule { bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); } - private Optional getZookeeperHosts() { + private String getZookeeperHosts() { if (System.getProperty("zookeeper-hosts") != null) { - return Optional.of(System.getProperty("zookeeper-hosts")); + return System.getProperty("zookeeper-hosts"); } - return Optional.ofNullable(System.getenv("ZOOKEEPER_HOSTS")); + String env = System.getenv("ZOOKEEPER_HOSTS"); + if (null == env) { + System.err.println(""" + ZOOKEEPER_HOSTS not set. This probably means that you are running an old installation, + or that the environment is not set up correctly. + + See the 2024-03+ migration notes, https://docs.marginalia.nu/6_notes/6_1__migrate_2024_03_plus + + """); + } + return env; } } From d78e9e715f0ebe5223719b8aade80e42b57348a3 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 28 Feb 2024 12:12:43 +0100 Subject: [PATCH 56/56] (misc) Fix broken tests --- .../index/results/IndexResultDomainDeduplicatorTest.java | 5 ----- .../marginalia/search/command/commands/BangCommandTest.java | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/code/index/test/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java b/code/index/test/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java index 7bc88ce4..4f5a12cd 100644 --- a/code/index/test/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java +++ b/code/index/test/nu/marginalia/index/results/IndexResultDomainDeduplicatorTest.java @@ -9,11 +9,6 @@ import static org.junit.jupiter.api.Assertions.*; class IndexResultDomainDeduplicatorTest { - @AfterEach - public void clear() { - IndexResultDomainDeduplicator.clearCachedObjects(); - } - @Test public void testDeduplicator() { diff --git a/code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java b/code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java index 3912a51c..f5dd9816 100644 --- a/code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java +++ b/code/services-application/search-service/test/nu/marginalia/search/command/commands/BangCommandTest.java @@ -15,7 +15,7 @@ class BangCommandTest { try { bangCommand.process(null, new SearchParameters(" !g test", - null, null, null, null, null) + null, null, null, null, null, false) ); Assertions.fail("Should have thrown RedirectException"); }