161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(zk-registry) epic jak shaving WIP

Browse Source 
Cleaning out a lot of old junk from the code, and one thing lead to another...

* Build is improved, now constructing docker images with 'jib'.  Clean build went from 3 minutes to 50 seconds.
* The ProcessService's spawning is smarter.  Will now just spawn a java process instead of relying on the application plugin's generated outputs.
* Project is migrated to GraalVM
* gRPC clients are re-written with a neat fluent/functional style. e.g.
```channelPool.call(grpcStub::method)
              .async(executor) // <-- optional
              .run(argument);
```
This change is primarily to allow handling ManagedChannel errors, but it turned out to be a pretty clean API overall.
* For now the project is all in on zookeeper
* Service discovery is now based on APIs and not services.  Theoretically means we could ship the same code either a monolith or a service mesh.
* To this end, began modularizing a few of the APIs so that they aren't strongly "living" in a service.  WIP!

Missing is documentation and testing, and some more breaking apart of code.
This commit is contained in:
Viktor Lofgren 2024-02-22 14:01:23 +01:00
parent 73947d9eca
commit 66c1281301
106 changed files with 2082 additions and 1663 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build.gradle
View File

@ -3,6 +3,10 @@ plugins {
id("org.jetbrains.gradle.plugin.idea-ext") version "1.0"
id "io.freefair.lombok" version "8.3"
id "me.champeau.jmh" version "0.6.6"
// This is a workaround for a bug in the Jib plugin that causes it to stall randomly
// https://github.com/GoogleContainerTools/jib/issues/3347
id 'com.google.cloud.tools.jib' version '3.4.0' apply(false)
}
group 'marginalia'
@ -29,11 +33,12 @@ subprojects.forEach {it ->
reproducibleFileOrder = true
}
}
ext {
dockerImageBase='container-registry.oracle.com/graalvm/jdk:21@sha256:1fd33d4d4eba3a9e1a41a728e39ea217178d257694eea1214fec68d2ed4d3d9b'
}
allprojects {
apply plugin: 'java'
apply plugin: 'io.freefair.lombok'
dependencies {
implementation libs.lombok
testImplementation libs.lombok
@ -77,3 +82,4 @@ java {
languageVersion.set(JavaLanguageVersion.of(21))
}
}

8
code/api/assistant-api/readme.md
View File

@ -1,8 +0,0 @@
# Assistant API
Client and models for talking to the [assistant-service](../../services-core/assistant-service),
implemented with the base client from [service-client](../../common/service-client).
## Central Classes
* [AssistantClient](src/main/java/nu/marginalia/assistant/client/AssistantClient.java)

159
code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java
View File

@ -1,159 +0,0 @@
package nu.marginalia.assistant.client;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.assistant.api.AssistantApiGrpc;
import nu.marginalia.assistant.client.model.DictionaryResponse;
import nu.marginalia.assistant.client.model.DomainInformation;
import nu.marginalia.assistant.client.model.SimilarDomain;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
import nu.marginalia.service.id.ServiceId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import static nu.marginalia.assistant.client.AssistantProtobufCodec.*;
@Singleton
public class AssistantClient {
private static final Logger logger = LoggerFactory.getLogger(AssistantClient.class);
private final GrpcSingleNodeChannelPool<AssistantApiGrpc.AssistantApiBlockingStub> channelPool;
private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
@Inject
public AssistantClient(GrpcChannelPoolFactory factory) {
this.channelPool = factory.createSingle(ServiceId.Assistant, AssistantApiGrpc::newBlockingStub);
}
public Future<DictionaryResponse> dictionaryLookup(String word) {
return virtualExecutorService.submit(() -> {
var rsp = channelPool.api().dictionaryLookup(
DictionaryLookup.createRequest(word)
);
return DictionaryLookup.convertResponse(rsp);
});
}
@SuppressWarnings("unchecked")
public Future<List<String>> spellCheck(String word) {
return virtualExecutorService.submit(() -> {
var rsp = channelPool.api().spellCheck(
SpellCheck.createRequest(word)
);
return SpellCheck.convertResponse(rsp);
});
}
public Map<String, List<String>> spellCheck(List<String> words, Duration timeout) throws InterruptedException {
List<Callable<Map.Entry<String, List<String>>>> tasks = new ArrayList<>();
for (String w : words) {
tasks.add(() -> {
var rsp = channelPool.api().spellCheck(
SpellCheck.createRequest(w)
);
return Map.entry(w, SpellCheck.convertResponse(rsp));
});
}
var futures = virtualExecutorService.invokeAll(tasks, timeout.toMillis(), TimeUnit.MILLISECONDS);
Map<String, List<String>> results = new HashMap<>();
for (var f : futures) {
if (!f.isDone())
continue;
var entry = f.resultNow();
results.put(entry.getKey(), entry.getValue());
}
return results;
}
public Future<String> unitConversion(String value, String from, String to) {
return virtualExecutorService.submit(() -> {
var rsp = channelPool.api().unitConversion(
UnitConversion.createRequest(from, to, value)
);
return UnitConversion.convertResponse(rsp);
});
}
public Future<String> evalMath(String expression) {
return virtualExecutorService.submit(() -> {
var rsp = channelPool.api().evalMath(
EvalMath.createRequest(expression)
);
return EvalMath.convertResponse(rsp);
});
}
public Future<List<SimilarDomain>> similarDomains(int domainId, int count) {
return virtualExecutorService.submit(() -> {
try {
var rsp = channelPool.api().getSimilarDomains(
DomainQueries.createRequest(domainId, count)
);
return DomainQueries.convertResponse(rsp);
}
catch (Exception e) {
logger.warn("Failed to get similar domains", e);
throw e;
}
});
}
public Future<List<SimilarDomain>> linkedDomains(int domainId, int count) {
return virtualExecutorService.submit(() -> {
try {
var rsp = channelPool.api().getLinkingDomains(
DomainQueries.createRequest(domainId, count)
);
return DomainQueries.convertResponse(rsp);
}
catch (Exception e) {
logger.warn("Failed to get linked domains", e);
throw e;
}
});
}
public Future<DomainInformation> domainInformation(int domainId) {
return virtualExecutorService.submit(() -> {
try {
var rsp = channelPool.api().getDomainInfo(
DomainInfo.createRequest(domainId)
);
return DomainInfo.convertResponse(rsp);
}
catch (Exception e) {
logger.warn("Failed to get domain information", e);
throw e;
}
});
}
public boolean isAccepting() {
return channelPool.hasChannel();
}
}

197
code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java
View File

@ -13,7 +13,8 @@ import nu.marginalia.executor.upload.UploadDirContents;
import nu.marginalia.executor.upload.UploadDirItem;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.storage.model.FileStorageId;
@ -22,6 +23,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
@ -39,180 +41,190 @@ public class ExecutorClient {
{
this.registry = registry;
this.channelPool = grpcChannelPoolFactory
.createMulti(ServiceId.Executor, ExecutorApiGrpc::newBlockingStub);
.createMulti(
ServiceKey.forGrpcApi(ExecutorApiGrpc.class, ServicePartition.multi()),
ExecutorApiGrpc::newBlockingStub);
}
public void startFsm(int node, String actorName) {
channelPool.apiForNode(node).startFsm(
RpcFsmName.newBuilder()
channelPool.call(ExecutorApiBlockingStub::startFsm)
.forNode(node)
.run(RpcFsmName.newBuilder()
.setActorName(actorName)
.build()
);
.build());
}
public void stopFsm(int node, String actorName) {
channelPool.apiForNode(node).stopFsm(
RpcFsmName.newBuilder()
channelPool.call(ExecutorApiBlockingStub::stopFsm)
.forNode(node)
.run(RpcFsmName.newBuilder()
.setActorName(actorName)
.build()
);
.build());
}
public void stopProcess(int node, String id) {
channelPool.apiForNode(node).stopProcess(
RpcProcessId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::stopProcess)
.forNode(node)
.run(RpcProcessId.newBuilder()
.setProcessId(id)
.build()
);
.build());
}
public void triggerCrawl(int node, FileStorageId fid) {
channelPool.apiForNode(node).triggerCrawl(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::triggerCrawl)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void triggerRecrawl(int node, FileStorageId fid) {
channelPool.apiForNode(node).triggerRecrawl(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::triggerRecrawl)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void triggerConvert(int node, FileStorageId fid) {
channelPool.apiForNode(node).triggerConvert(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::triggerConvert)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void triggerConvertAndLoad(int node, FileStorageId fid) {
channelPool.apiForNode(node).triggerConvertAndLoad(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::triggerConvertAndLoad)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void loadProcessedData(int node, List<FileStorageId> ids) {
channelPool.apiForNode(node).loadProcessedData(
RpcFileStorageIds.newBuilder()
channelPool.call(ExecutorApiBlockingStub::loadProcessedData)
.forNode(node)
.run(RpcFileStorageIds.newBuilder()
.addAllFileStorageIds(ids.stream().map(FileStorageId::id).toList())
.build()
);
.build());
}
public void calculateAdjacencies(int node) {
channelPool.apiForNode(node).calculateAdjacencies(Empty.getDefaultInstance());
channelPool.call(ExecutorApiBlockingStub::calculateAdjacencies)
.forNode(node)
.run(Empty.getDefaultInstance());
}
public void sideloadEncyclopedia(int node, Path sourcePath, String baseUrl) {
channelPool.apiForNode(node).sideloadEncyclopedia(
RpcSideloadEncyclopedia.newBuilder()
channelPool.call(ExecutorApiBlockingStub::sideloadEncyclopedia)
.forNode(node)
.run(RpcSideloadEncyclopedia.newBuilder()
.setBaseUrl(baseUrl)
.setSourcePath(sourcePath.toString())
.build()
);
.build());
}
public void sideloadDirtree(int node, Path sourcePath) {
channelPool.apiForNode(node).sideloadDirtree(
RpcSideloadDirtree.newBuilder()
channelPool.call(ExecutorApiBlockingStub::sideloadDirtree)
.forNode(node)
.run(RpcSideloadDirtree.newBuilder()
.setSourcePath(sourcePath.toString())
.build()
);
.build());
}
public void sideloadReddit(int node, Path sourcePath) {
channelPool.apiForNode(node).sideloadReddit(
RpcSideloadReddit.newBuilder()
channelPool.call(ExecutorApiBlockingStub::sideloadReddit)
.forNode(node)
.run(RpcSideloadReddit.newBuilder()
.setSourcePath(sourcePath.toString())
.build()
);
.build());
}
public void sideloadWarc(int node, Path sourcePath) {
channelPool.apiForNode(node).sideloadWarc(
RpcSideloadWarc.newBuilder()
channelPool.call(ExecutorApiBlockingStub::sideloadWarc)
.forNode(node)
.run(RpcSideloadWarc.newBuilder()
.setSourcePath(sourcePath.toString())
.build()
);
.build());
}
public void sideloadStackexchange(int node, Path sourcePath) {
channelPool.apiForNode(node).sideloadStackexchange(
RpcSideloadStackexchange.newBuilder()
channelPool.call(ExecutorApiBlockingStub::sideloadStackexchange)
.forNode(node)
.run(RpcSideloadStackexchange.newBuilder()
.setSourcePath(sourcePath.toString())
.build()
);
.build());
}
public void createCrawlSpecFromDownload(int node, String description, String url) {
channelPool.apiForNode(node).createCrawlSpecFromDownload(
RpcCrawlSpecFromDownload.newBuilder()
channelPool.call(ExecutorApiBlockingStub::createCrawlSpecFromDownload)
.forNode(node)
.run(RpcCrawlSpecFromDownload.newBuilder()
.setDescription(description)
.setUrl(url)
.build()
);
.build());
}
public void exportAtags(int node, FileStorageId fid) {
channelPool.apiForNode(node).exportAtags(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::exportAtags)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void exportSampleData(int node, FileStorageId fid, int size, String name) {
channelPool.apiForNode(node).exportSampleData(
RpcExportSampleData.newBuilder()
channelPool.call(ExecutorApiBlockingStub::exportSampleData)
.forNode(node)
.run(RpcExportSampleData.newBuilder()
.setFileStorageId(fid.id())
.setSize(size)
.setName(name)
.build()
);
.build());
}
public void exportRssFeeds(int node, FileStorageId fid) {
channelPool.apiForNode(node).exportRssFeeds(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::exportRssFeeds)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void exportTermFrequencies(int node, FileStorageId fid) {
channelPool.apiForNode(node).exportTermFrequencies(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::exportTermFrequencies)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public void downloadSampleData(int node, String sampleSet) {
channelPool.apiForNode(node).downloadSampleData(
RpcDownloadSampleData.newBuilder()
channelPool.call(ExecutorApiBlockingStub::downloadSampleData)
.forNode(node)
.run(RpcDownloadSampleData.newBuilder()
.setSampleSet(sampleSet)
.build()
);
.build());
}
public void exportData(int node) {
channelPool.apiForNode(node).exportData(Empty.getDefaultInstance());
channelPool.call(ExecutorApiBlockingStub::exportData)
.forNode(node)
.run(Empty.getDefaultInstance());
}
public void restoreBackup(int node, FileStorageId fid) {
channelPool.apiForNode(node).restoreBackup(
RpcFileStorageId.newBuilder()
channelPool.call(ExecutorApiBlockingStub::restoreBackup)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fid.id())
.build()
);
.build());
}
public ActorRunStates getActorStates(int node) {
try {
var rs = channelPool.apiForNode(node).getActorStates(Empty.getDefaultInstance());
var rs = channelPool.call(ExecutorApiBlockingStub::getActorStates)
.forNode(node)
.run(Empty.getDefaultInstance());
var states = rs.getActorRunStatesList().stream()
.map(r -> new ActorRunState(
r.getActorName(),
@ -236,7 +248,9 @@ public class ExecutorClient {
public UploadDirContents listSideloadDir(int node) {
try {
var rs = channelPool.apiForNode(node).listSideloadDir(Empty.getDefaultInstance());
var rs = channelPool.call(ExecutorApiBlockingStub::listSideloadDir)
.forNode(node)
.run(Empty.getDefaultInstance());
var items = rs.getEntriesList().stream()
.map(i -> new UploadDirItem(i.getName(), i.getLastModifiedTime(), i.getIsDirectory(), i.getSize()))
.toList();
@ -252,11 +266,12 @@ public class ExecutorClient {
public FileStorageContent listFileStorage(int node, FileStorageId fileId) {
try {
var rs = channelPool.apiForNode(node).listFileStorage(
RpcFileStorageId.newBuilder()
var rs = channelPool.call(ExecutorApiBlockingStub::listFileStorage)
.forNode(node)
.run(RpcFileStorageId.newBuilder()
.setFileStorageId(fileId.id())
.build()
);
);
return new FileStorageContent(rs.getEntriesList().stream()
.map(e -> new FileStorageFile(e.getName(), e.getSize(), e.getLastModifiedTime()))
@ -274,13 +289,13 @@ public class ExecutorClient {
String uriPath = STR."/transfer/file/\{fileId.id()}";
String uriQuery = STR."path=\{URLEncoder.encode(path, StandardCharsets.UTF_8)}";
var service = registry.getEndpoints(ApiSchema.REST, ServiceId.Executor, node)
var service = registry.getEndpoints(ServiceKey.forRest(ServiceId.Executor, node))
.stream().findFirst().orElseThrow();
try (var urlStream = service.endpoint().toURL(uriPath, uriQuery).openStream()) {
urlStream.transferTo(destOutputStream);
}
catch (IOException ex) {
catch (IOException | URISyntaxException ex) {
throw new RuntimeException(ex);
}
}

22
code/api/index-api/src/main/protobuf/index-api.proto
View File

@ -4,28 +4,6 @@ package actorapi;
option java_package="nu.marginalia.index.api";
option java_multiple_files=true;
service IndexDomainLinksApi {
rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {}
rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {}
rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {}
rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {}
rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {}
}
message RpcDomainId {
int32 domainId = 1;
}
message RpcDomainIdList {
repeated int32 domainId = 1 [packed=true];
}
message RpcDomainIdCount {
int32 idCount = 1;
}
message RpcDomainIdPairs {
repeated int32 sourceIds = 1 [packed=true];
repeated int32 destIds = 2 [packed=true];
}
service QueryApi {
rpc query(RpcQsQuery) returns (RpcQsResponse) {}
}

127
code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java
View File

@ -9,14 +9,12 @@ import nu.marginalia.query.model.QueryParams;
import nu.marginalia.query.model.QueryResponse;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
import nu.marginalia.service.id.ServiceId;
import org.roaringbitmap.longlong.PeekableLongIterator;
import org.roaringbitmap.longlong.Roaring64Bitmap;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.CheckReturnValue;
import java.util.List;
@Singleton
public class QueryClient {
@ -27,134 +25,25 @@ public class QueryClient {
.register();
private final GrpcSingleNodeChannelPool<QueryApiGrpc.QueryApiBlockingStub> queryApiPool;
private final GrpcSingleNodeChannelPool<IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub> domainLinkApiPool;
private final Logger logger = LoggerFactory.getLogger(getClass());
@Inject
public QueryClient(GrpcChannelPoolFactory channelPoolFactory) {
this.queryApiPool = channelPoolFactory.createSingle(ServiceId.Query, QueryApiGrpc::newBlockingStub);
this.domainLinkApiPool = channelPoolFactory.createSingle(ServiceId.Query, IndexDomainLinksApiGrpc::newBlockingStub);
this.queryApiPool = channelPoolFactory.createSingle(
ServiceKey.forGrpcApi(QueryApiGrpc.class, ServicePartition.any()),
QueryApiGrpc::newBlockingStub);
}
@CheckReturnValue
public QueryResponse search(QueryParams params) {
var query = QueryProtobufCodec.convertQueryParams(params);
return wmsa_qs_api_search_time.time(
() -> QueryProtobufCodec.convertQueryResponse(queryApiPool
.importantCall((api) -> api.query(query))
return wmsa_qs_api_search_time.time(() ->
QueryProtobufCodec.convertQueryResponse(
queryApiPool.call(QueryApiGrpc.QueryApiBlockingStub::query).run(query)
)
);
}
public AllLinks getAllDomainLinks() {
AllLinks links = new AllLinks();
domainLinkApiPool.api()
.getAllLinks(Empty.getDefaultInstance())
.forEachRemaining(pairs -> {
for (int i = 0; i < pairs.getDestIdsCount(); i++) {
links.add(pairs.getSourceIds(i), pairs.getDestIds(i));
}
});
return links;
}
public List<Integer> getLinksToDomain(int domainId) {
try {
return domainLinkApiPool.api()
.getLinksToDomain(RpcDomainId
.newBuilder()
.setDomainId(domainId)
.build())
.getDomainIdList()
.stream()
.sorted()
.toList();
}
catch (Exception e) {
logger.error("API Exception", e);
return List.of();
}
}
public List<Integer> getLinksFromDomain(int domainId) {
try {
return domainLinkApiPool.api()
.getLinksFromDomain(RpcDomainId
.newBuilder()
.setDomainId(domainId)
.build())
.getDomainIdList()
.stream()
.sorted()
.toList();
}
catch (Exception e) {
logger.error("API Exception", e);
return List.of();
}
}
public int countLinksToDomain(int domainId) {
try {
return domainLinkApiPool.api()
.countLinksToDomain(RpcDomainId
.newBuilder()
.setDomainId(domainId)
.build())
.getIdCount();
}
catch (Exception e) {
logger.error("API Exception", e);
return 0;
}
}
public int countLinksFromDomain(int domainId) {
try {
return domainLinkApiPool.api()
.countLinksFromDomain(RpcDomainId
.newBuilder()
.setDomainId(domainId)
.build())
.getIdCount();
}
catch (Exception e) {
logger.error("API Exception", e);
return 0;
}
}
public static class AllLinks {
private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap();
public void add(int source, int dest) {
sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest));
}
public Iterator iterator() {
return new Iterator();
}
public class Iterator {
private final PeekableLongIterator base = sourceToDest.getLongIterator();
long val = Long.MIN_VALUE;
public boolean advance() {
if (base.hasNext()) {
val = base.next();
return true;
}
return false;
}
public int source() {
return (int) (val >>> 32);
}
public int dest() {
return (int) (val & 0xFFFF_FFFFL);
}
}
}
}

44
code/common/config/src/main/java/nu/marginalia/WmsaHome.java
View File

@ -5,8 +5,7 @@ import nu.marginalia.service.ServiceHomeNotConfiguredException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import java.util.Objects;
import java.util.stream.Stream;
public class WmsaHome {
@ -26,15 +25,28 @@ public class WmsaHome {
}
public static Path getHomePath() {
var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElseGet(WmsaHome::findDefaultHomePath);
String[] possibleLocations = new String[] {
System.getenv("WMSA_HOME"),
System.getProperty("system.homePath"),
"/var/lib/wmsa",
"/wmsa"
};
String retStr = Stream.of(possibleLocations)
.filter(Objects::nonNull)
.map(Path::of)
.filter(Files::isDirectory)
.map(Path::toString)
.findFirst()
.orElseThrow(() ->
new ServiceHomeNotConfiguredException("""
Could not find $WMSA_HOME, either set environment
variable, the 'system.homePath' property,
or ensure either /wmssa or /var/lib/wmsa exists
"""));
var ret = Path.of(retStr);
if (!Files.isDirectory(ret)) {
throw new ServiceHomeNotConfiguredException("Could not find $WMSA_HOME, either set environment variable or ensure " + retStr + " exists");
}
if (!Files.isDirectory(ret.resolve("model"))) {
throw new ServiceHomeNotConfiguredException("You need to run 'run/setup.sh' to download models to run/ before this will work!");
}
@ -42,22 +54,6 @@ public class WmsaHome {
return ret;
}
private static String findDefaultHomePath() {
// Assume this is a local developer and not a production system, since it would have WMSA_HOME set.
// Developers probably have a "run/" somewhere upstream from cwd.
//
return Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent)
.filter(p -> Files.exists(p.resolve("run/env")))
.filter(p -> Files.exists(p.resolve("run/setup.sh")))
.map(p -> p.resolve("run"))
.findAny()
.orElse(Path.of("/var/lib/wmsa"))
.toString();
}
public static Path getAdsDefinition() {
return getHomePath().resolve("data").resolve("adblock.txt");
}

7
code/common/service-discovery/build.gradle
View File

@ -6,6 +6,11 @@ plugins {
repositories {
mavenLocal()
mavenCentral()
repositories {
mavenCentral()
maven { url 'https://jitpack.io' }
}
}
java {
@ -18,6 +23,7 @@ dependencies {
implementation libs.bundles.curator
implementation libs.guice
implementation libs.bundles.gson
implementation libs.bundles.mariadb
implementation libs.bundles.grpc
implementation libs.notnull
@ -29,4 +35,5 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:functions:math:api')
}

18
code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java
View File

@ -1,7 +1,6 @@
package nu.marginalia.service;
import com.google.inject.AbstractModule;
import nu.marginalia.service.discovery.FixedServiceRegistry;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.ZkServiceRegistry;
import org.apache.curator.framework.CuratorFramework;
@ -18,18 +17,13 @@ public class ServiceDiscoveryModule extends AbstractModule {
private static final Logger logger = LoggerFactory.getLogger(ServiceDiscoveryModule.class);
public void configure() {
getZookeeperHosts().ifPresentOrElse((hosts) -> {
logger.info("Using Zookeeper service registry at {}", hosts);
CuratorFramework client = CuratorFrameworkFactory
.newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000));
var hosts = getZookeeperHosts().orElseThrow(() -> new IllegalStateException("Zookeeper hosts not set"));
logger.info("Using Zookeeper service registry at {}", hosts);
CuratorFramework client = CuratorFrameworkFactory
.newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000));
bind(CuratorFramework.class).toInstance(client);
bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class);
},
() -> {
logger.info("Using fixed service registry");
bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class);
});
bind(CuratorFramework.class).toInstance(client);
bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class);
}
private Optional<String> getZookeeperHosts() {

4
code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java
View File

@ -1,10 +1,6 @@
package nu.marginalia.service;
public class ServiceHomeNotConfiguredException extends RuntimeException {
public ServiceHomeNotConfiguredException() {
super("WMSA_HOME environment variable not set");
}
public ServiceHomeNotConfiguredException(String message) {
super(message);
}

24
code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java
View File

@ -6,8 +6,10 @@ import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import nu.marginalia.service.NodeConfigurationWatcher;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.property.PartitionTraits;
import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import java.util.function.Function;
@ -26,30 +28,32 @@ public class GrpcChannelPoolFactory {
}
/** Create a new multi-node channel pool for the given service. */
public <STUB> GrpcMultiNodeChannelPool<STUB> createMulti(ServiceId serviceId,
public <STUB> GrpcMultiNodeChannelPool<STUB> createMulti(ServiceKey<ServicePartition.Multi> key,
Function<ManagedChannel, STUB> stubConstructor)
{
return new GrpcMultiNodeChannelPool<>(serviceRegistryIf,
serviceId,
key,
this::createChannel,
stubConstructor,
nodeConfigurationWatcher);
}
/** Create a new single-node channel pool for the given service. */
public <STUB> GrpcSingleNodeChannelPool<STUB> createSingle(ServiceId serviceId,
public <STUB> GrpcSingleNodeChannelPool<STUB> createSingle(ServiceKey<? extends PartitionTraits.Unicast> key,
Function<ManagedChannel, STUB> stubConstructor)
{
return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, serviceId,
new NodeSelectionStrategy.Any(),
this::createChannel,
stubConstructor);
return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, key, this::createChannel, stubConstructor);
}
private ManagedChannel createChannel(InstanceAddress<?> route) {
return ManagedChannelBuilder
private ManagedChannel createChannel(InstanceAddress route) {
var mc = ManagedChannelBuilder
.forAddress(route.host(), route.port())
.usePlaintext()
.build();
mc.getState(true);
return mc;
}
}

114
code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java
View File

@ -4,13 +4,17 @@ import io.grpc.ManagedChannel;
import lombok.SneakyThrows;
import nu.marginalia.service.NodeConfigurationWatcher;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.property.PartitionTraits;
import nu.marginalia.service.discovery.property.ServiceEndpoint;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Stream;
@ -22,21 +26,20 @@ public class GrpcMultiNodeChannelPool<STUB> {
private final ConcurrentHashMap<Integer, GrpcSingleNodeChannelPool<STUB>> pools =
new ConcurrentHashMap<>();
private static final Logger logger = LoggerFactory.getLogger(GrpcMultiNodeChannelPool.class);
private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
private final ServiceRegistryIf serviceRegistryIf;
private final ServiceId serviceId;
private final Function<ServiceEndpoint.InstanceAddress<?>, ManagedChannel> channelConstructor;
private final ServiceKey<? extends PartitionTraits.Multicast> serviceKey;
private final Function<ServiceEndpoint.InstanceAddress, ManagedChannel> channelConstructor;
private final Function<ManagedChannel, STUB> stubConstructor;
private final NodeConfigurationWatcher nodeConfigurationWatcher;
@SneakyThrows
public GrpcMultiNodeChannelPool(ServiceRegistryIf serviceRegistryIf,
ServiceId serviceId,
Function<ServiceEndpoint.InstanceAddress<?>, ManagedChannel> channelConstructor,
ServiceKey<ServicePartition.Multi> serviceKey,
Function<ServiceEndpoint.InstanceAddress, ManagedChannel> channelConstructor,
Function<ManagedChannel, STUB> stubConstructor,
NodeConfigurationWatcher nodeConfigurationWatcher) {
this.serviceRegistryIf = serviceRegistryIf;
this.serviceId = serviceId;
this.serviceKey = serviceKey;
this.channelConstructor = channelConstructor;
this.stubConstructor = stubConstructor;
this.nodeConfigurationWatcher = nodeConfigurationWatcher;
@ -51,51 +54,74 @@ public class GrpcMultiNodeChannelPool<STUB> {
return pools.computeIfAbsent(node, _ ->
new GrpcSingleNodeChannelPool<>(
serviceRegistryIf,
serviceId,
new NodeSelectionStrategy.Just(node),
serviceKey.forPartition(ServicePartition.partition(node)),
channelConstructor,
stubConstructor));
}
/** Get an API stub for the given node */
public STUB apiForNode(int node) {
return pools.computeIfAbsent(node, this::getPoolForNode).api();
}
/** Invoke a function on each node, returning a list of futures in a terminal state, as per
* ExecutorService$invokeAll */
public <T> List<Future<T>> invokeAll(Function<STUB, Callable<T>> callF) throws InterruptedException {
List<Callable<T>> calls = getEligibleNodes().stream()
.mapMulti(this::passNodeIfOk)
.map(callF)
.toList();
return virtualExecutorService.invokeAll(calls);
}
/** Invoke a function on each node, returning a stream of results */
public <T> Stream<T> callEachSequential(Function<STUB, T> call) {
return getEligibleNodes().stream()
.mapMulti(this::passNodeIfOk)
.map(call);
}
// Eat connectivity exceptions and log them when doing a broadcast-style calls
private void passNodeIfOk(Integer nodeId, Consumer<STUB> consumer) {
try {
consumer.accept(apiForNode(nodeId));
}
catch (Exception ex) {
logger.error("Error calling node {}", nodeId, ex);
}
}
/** Get the list of nodes that are eligible for broadcast-style requests */
public List<Integer> getEligibleNodes() {
return nodeConfigurationWatcher.getQueryNodes();
}
public <T, I> CallBuilderBase<T, I> call(BiFunction<STUB, I, T> method) {
return new CallBuilderBase<>(method);
}
public class CallBuilderBase<T, I> {
private final BiFunction<STUB, I, T> method;
private CallBuilderBase(BiFunction<STUB, I, T> method) {
this.method = method;
}
public GrpcSingleNodeChannelPool<STUB>.CallBuilderBase<T, I> forNode(int node) {
return getPoolForNode(node).call(method);
}
public List<T> run(I arg) {
return getEligibleNodes().stream()
.map(node -> getPoolForNode(node).call(method).run(arg))
.toList();
}
public CallBuilderAsync<T, I> async(ExecutorService service) {
return new CallBuilderAsync<>(service, method);
}
}
public class CallBuilderAsync<T, I> {
private final Executor executor;
private final BiFunction<STUB, I, T> method;
public CallBuilderAsync(Executor executor, BiFunction<STUB, I, T> method) {
this.executor = executor;
this.method = method;
}
public CompletableFuture<List<T>> runAll(I arg) {
var futures = getEligibleNodes().stream()
.map(GrpcMultiNodeChannelPool.this::getPoolForNode)
.map(pool ->
pool.call(method)
.async(executor)
.run(arg)
).toList();
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.thenApply(v -> futures.stream().map(CompletableFuture::join).toList());
}
public List<CompletableFuture<T>> runEach(I arg) {
return getEligibleNodes().stream()
.map(GrpcMultiNodeChannelPool.this::getPoolForNode)
.map(pool ->
pool.call(method)
.async(executor)
.run(arg)
).toList();
}
}
}

230
code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java
View File

@ -1,142 +1,232 @@
package nu.marginalia.service.client;
import com.google.common.collect.Sets;
import io.grpc.ConnectivityState;
import io.grpc.ManagedChannel;
import lombok.SneakyThrows;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.service.discovery.property.PartitionTraits;
import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.Function;
/** A pool of gRPC channels for a service, with a separate channel for each node.
* <p></p>
* Manages unicast-style requests */
public class GrpcSingleNodeChannelPool<STUB> extends ServiceChangeMonitor {
private final Map<InstanceAddress<?>, ManagedChannel> channels = new ConcurrentHashMap<>();
private final Map<Integer, Set<InstanceAddress<?>>> routes = new ConcurrentHashMap<>();
private final Map<InstanceAddress, ConnectionHolder> channels = new ConcurrentHashMap<>();
private static final Logger logger = LoggerFactory.getLogger(GrpcSingleNodeChannelPool.class);
private final ServiceRegistryIf serviceRegistryIf;
private final ServiceId serviceId;
private final NodeSelectionStrategy nodeSelectionStrategy;
private final Function<InstanceAddress<?>, ManagedChannel> channelConstructor;
private final Function<InstanceAddress, ManagedChannel> channelConstructor;
private final Function<ManagedChannel, STUB> stubConstructor;
@SneakyThrows
public GrpcSingleNodeChannelPool(ServiceRegistryIf serviceRegistryIf,
ServiceId serviceId,
NodeSelectionStrategy nodeSelectionStrategy,
Function<InstanceAddress<?>, ManagedChannel> channelConstructor,
ServiceKey<? extends PartitionTraits.Unicast> serviceKey,
Function<InstanceAddress, ManagedChannel> channelConstructor,
Function<ManagedChannel, STUB> stubConstructor) {
super(serviceId);
super(serviceKey);
this.serviceRegistryIf = serviceRegistryIf;
this.serviceId = serviceId;
this.nodeSelectionStrategy = nodeSelectionStrategy;
this.channelConstructor = channelConstructor;
this.stubConstructor = stubConstructor;
serviceRegistryIf.registerMonitor(this);
onChange();
awaitChannel(Duration.ofSeconds(5));
}
@Override
public boolean onChange() {
switch (nodeSelectionStrategy) {
case NodeSelectionStrategy.Any() ->
serviceRegistryIf
.getServiceNodes(serviceId)
.forEach(this::refreshNode);
case NodeSelectionStrategy.Just(int node) ->
refreshNode(node);
public synchronized boolean onChange() {
Set<InstanceAddress> newRoutes = serviceRegistryIf.getEndpoints(serviceKey);
Set<InstanceAddress> oldRoutes = new HashSet<>(channels.keySet());
// Find the routes that have been added or removed
for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) {
ConnectionHolder oldChannel;
if (newRoutes.contains(route)) {
logger.info("Adding route {}", route);
oldChannel = channels.put(route, new ConnectionHolder(route));
} else {
logger.info("Expelling route {}", route);
oldChannel = channels.remove(route);
}
if (oldChannel != null) {
oldChannel.close();
}
}
return true;
}
private void refreshNode(int node) {
private class ConnectionHolder implements Comparable<ConnectionHolder> {
private final AtomicReference<ManagedChannel> channel = new AtomicReference<>();
private final InstanceAddress address;
Set<InstanceAddress<?>> newRoutes = serviceRegistryIf.getEndpoints(ApiSchema.GRPC, serviceId, node);
Set<InstanceAddress<?>> oldRoutes = routes.getOrDefault(node, Set.of());
// Find the routes that have been added or removed
for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) {
ManagedChannel oldChannel;
if (newRoutes.contains(route)) {
var newChannel = channelConstructor.apply(route);
oldChannel = channels.put(route, newChannel);
} else {
oldChannel = channels.remove(route);
}
if (oldChannel != null)
oldChannel.shutdown();
ConnectionHolder(InstanceAddress address) {
this.address = address;
}
routes.put(node, newRoutes);
public ManagedChannel get() {
var value = channel.get();
if (value != null) {
return value;
}
try {
logger.info("Creating channel for {}:{}", serviceKey, address);
value = channelConstructor.apply(address);
if (channel.compareAndSet(null, value)) {
return value;
}
else {
value.shutdown();
return channel.get();
}
}
catch (Exception e) {
logger.error(STR."Failed to get channel for \{address}", e);
return null;
}
}
public void close() {
ManagedChannel mc = channel.getAndSet(null);
if (mc != null) {
mc.shutdown();
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ConnectionHolder that = (ConnectionHolder) o;
return Objects.equals(address, that.address);
}
@Override
public int hashCode() {
return Objects.hash(address);
}
@Override
public int compareTo(@NotNull GrpcSingleNodeChannelPool<STUB>.ConnectionHolder o) {
return -Long.compare(address.cxTime(), o.address.cxTime()); // Reverse order
}
}
public boolean hasChannel() {
return !channels.isEmpty();
}
/** Get an API stub for the given node */
public STUB api() {
return stubConstructor.apply(getChannel());
public synchronized boolean awaitChannel(Duration timeout) throws InterruptedException {
if (hasChannel()) return true;
final long endTime = System.currentTimeMillis() + timeout.toMillis();
while (!hasChannel()) {
long timeLeft = endTime - System.currentTimeMillis();
if (timeLeft <= 0) return false;
this.wait(timeLeft);
}
return hasChannel();
}
/** Try to make the call go through. The function will cycle through
* available routes until exhaustion, and only then will it give up
*/
public <T> T importantCall(Function<STUB, T> function) {
for (int i = 0; i < channels.size(); i++) {
private <T, I> T call(BiFunction<STUB, I, T> call, I arg) throws RuntimeException {
final List<Exception> exceptions = new ArrayList<>();
final List<ConnectionHolder> connectionHolders = new ArrayList<>(channels.values());
// Randomize the order of the connection holders to spread out the load
Collections.shuffle(connectionHolders);
for (var channel : connectionHolders) {
try {
return function.apply(api());
return call.apply(stubConstructor.apply(channel.get()), arg);
}
catch (Exception e) {
logger.error("API Exception", e);
exceptions.add(e);
}
}
throw new ServiceNotAvailableException(serviceId);
for (var e : exceptions) {
logger.error("Failed to call service {}", serviceKey, e);
}
throw new ServiceNotAvailableException(serviceKey);
}
/** Get the channel that is most ready to use */
public ManagedChannel getChannel() {
return channels
.values()
.stream()
.min(this::compareChannelsByState)
.orElseThrow(() -> new ServiceNotAvailableException(serviceId));
public <T, I> CallBuilderBase<T, I> call(BiFunction<STUB, I, T> method) {
return new CallBuilderBase<>(method);
}
/** Sort the channels by how ready they are to use */
private int compareChannelsByState(ManagedChannel a, ManagedChannel b) {
var aState = a.getState(true);
var bState = b.getState(true);
public class CallBuilderBase<T, I> {
private final BiFunction<STUB, I, T> method;
private CallBuilderBase(BiFunction<STUB, I, T> method) {
this.method = method;
}
if (aState == ConnectivityState.READY) return -1;
if (bState == ConnectivityState.READY) return 1;
if (aState == ConnectivityState.CONNECTING) return -1;
if (bState == ConnectivityState.CONNECTING) return 1;
if (aState == ConnectivityState.IDLE) return -1;
if (bState == ConnectivityState.IDLE) return 1;
public T run(I arg) {
return call(method, arg);
}
return 0;
public List<T> runFor(I... args) {
return runFor(List.of(args));
}
public List<T> runFor(List<I> args) {
List<T> results = new ArrayList<>();
for (var arg : args) {
results.add(call(method, arg));
}
return results;
}
public CallBuilderAsync<T, I> async(Executor executor) {
return new CallBuilderAsync<>(executor, method);
}
}
public class CallBuilderAsync<T, I> {
private final Executor executor;
private final BiFunction<STUB, I, T> method;
public CallBuilderAsync(Executor executor, BiFunction<STUB, I, T> method) {
this.executor = executor;
this.method = method;
}
public CompletableFuture<T> run(I arg) {
return CompletableFuture.supplyAsync(() -> call(method, arg), executor);
}
public CompletableFuture<List<T>> runFor(List<I> args) {
List<CompletableFuture<T>> results = new ArrayList<>();
for (var arg : args) {
results.add(CompletableFuture.supplyAsync(() -> call(method, arg), executor));
}
return CompletableFuture.allOf(results.toArray(new CompletableFuture[0]))
.thenApply(v -> results.stream().map(CompletableFuture::join).toList());
}
public CompletableFuture<List<T>> runFor(I... args) {
return runFor(List.of(args));
}
}
}

17
code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java
View File

@ -1,17 +0,0 @@
package nu.marginalia.service.client;
public sealed interface NodeSelectionStrategy {
boolean test(int node);
record Any() implements NodeSelectionStrategy {
@Override
public boolean test(int node) {
return true;
}
}
record Just(int node) implements NodeSelectionStrategy {
@Override
public boolean test(int node) {
return this.node == node;
}
}
}

9
code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java
View File

@ -1,12 +1,9 @@
package nu.marginalia.service.client;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
public class ServiceNotAvailableException extends RuntimeException {
public ServiceNotAvailableException(ServiceId id, int node) {
super(STR."Service \{id} not available on node \{node}");
}
public ServiceNotAvailableException(ServiceId id) {
super(STR."Service \{id} not available");
public ServiceNotAvailableException(ServiceKey<?> key) {
super(STR."Service \{key} not available");
}
}

122
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java
View File

@ -1,122 +0,0 @@
package nu.marginalia.service.discovery;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.service.discovery.monitor.*;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.service.discovery.property.ServiceEndpoint;
import nu.marginalia.service.discovery.property.ServiceEndpoint.GrpcEndpoint;
import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
import nu.marginalia.service.discovery.property.ServiceEndpoint.RestEndpoint;
import nu.marginalia.service.id.ServiceId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
/** A service registry that returns fixed endpoints for all services.
* <p></p>
* This is for backwards-compatibility with old docker-compose files with no
* ZooKeeper configured.
* */
public class FixedServiceRegistry implements ServiceRegistryIf {
private static final Logger logger = LoggerFactory.getLogger(FixedServiceRegistry.class);
private final HikariDataSource dataSource;
@Inject
public FixedServiceRegistry(HikariDataSource dataSource) {
this.dataSource = dataSource;
}
@Override
public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, int node, UUID instanceUUID, String externalAddress) throws Exception {
return switch (schema) {
case REST -> new ServiceEndpoint.RestEndpoint(externalAddress, 80);
case GRPC -> new ServiceEndpoint.GrpcEndpoint(externalAddress, 81);
};
}
@Override
public void announceInstance(ServiceId id, int node, UUID instanceUUID) {
// No-op
}
@Override
public Set<Integer> getServiceNodes(ServiceId id) {
if (id == ServiceId.Executor || id == ServiceId.Index) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("SELECT ID FROM NODE_CONFIGURATION")) {
Set<Integer> ret = new HashSet<>();
var rs = stmt.executeQuery();
while (rs.next()) {
ret.add(rs.getInt(1));
}
return ret;
}
catch (SQLException ex) {
return Set.of();
}
}
else return Set.of(0);
}
@Override
public int requestPort(String externalHost, ApiSchema schema, ServiceId id, int node) {
return switch(schema) {
case REST -> 80;
case GRPC -> 81;
};
}
@Override
public Set<InstanceAddress<? extends ServiceEndpoint>> getEndpoints(ApiSchema schema, ServiceId id, int node) {
return switch (schema) {
case REST -> Set.of(new InstanceAddress<>(
new RestEndpoint(id.serviceName + "-" + node, 80),
UUID.randomUUID()));
case GRPC -> Set.of(new InstanceAddress<>(
new GrpcEndpoint(id.serviceName + "-" + node, 81),
UUID.randomUUID()));
};
}
public void registerMonitor(ServiceMonitorIf monitor) throws Exception {
// We don't have any notification mechanism, so we just periodically
// invoke the monitor's onChange method to simulate it.
periodicallyInvoke(monitor, Duration.ofSeconds(15));
}
void periodicallyInvoke(ServiceMonitorIf monitor, Duration d) {
Thread.ofPlatform().name("PeriodicInvoker").start(() -> {
for (;;) {
try {
Thread.sleep(d);
} catch (InterruptedException e) {
break;
}
boolean reRegister;
try {
reRegister = monitor.onChange();
}
catch (Exception ex) {
logger.error("Monitor failed", ex);
reRegister = true;
}
if (!reRegister) {
break;
}
}
});
}
}

38
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java
View File

@ -1,10 +1,10 @@
package nu.marginalia.service.discovery;
import nu.marginalia.service.discovery.monitor.*;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.service.discovery.property.ServiceEndpoint;
import static nu.marginalia.service.discovery.property.ServiceEndpoint.*;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
import java.util.Set;
import java.util.UUID;
@ -16,40 +16,33 @@ public interface ServiceRegistryIf {
/**
* Register a service with the registry.
* <p></p>
* Once the instance has announced itself with {@link #announceInstance(ServiceId id, int node, UUID instanceUUID) announceInstance(...)},
* the service will be available for discovery with {@link #getEndpoints(ApiSchema schema, ServiceId id, int node) getEndpoints(...)}.
* Once the instance has announced itself with {@link #announceInstance(UUID instanceUUID) announceInstance(...)},
* the service will be available for discovery with {@link #getEndpoints(ServiceKey key) getEndpoints(...)}.
*
* @param schema the API schema
* @param id the service identifier
* @param node the node number
* @param key the key identifying the service
* @param instanceUUID the unique UUID of the instance
* @param externalAddress the public address of the service
*/
ServiceEndpoint registerService(ApiSchema schema,
ServiceId id,
int node,
ServiceEndpoint registerService(ServiceKey<?> key,
UUID instanceUUID,
String externalAddress) throws Exception;
void declareFirstBoot();
void waitForFirstBoot() throws InterruptedException;
/** Let the world know that the service is running
* and ready to accept requests. */
void announceInstance(ServiceId id, int node, UUID instanceUUID);
/** Return all nodes that are running for the specified service. */
Set<Integer> getServiceNodes(ServiceId id);
void announceInstance(UUID instanceUUID);
/** At the discretion of the implementation, provide a port that is unique
* across (externalHost, serviceId, schema, node). It may be randomly selected
* across (host, api-schema). It may be randomly selected
* or hard-coded or some combination of behaviors.
*/
int requestPort(String externalHost,
ApiSchema schema,
ServiceId id,
int node);
int requestPort(String externalHost, ServiceKey<?> key);
/** Get all endpoints for the service on the specified node and schema. */
Set<InstanceAddress<? extends ServiceEndpoint>>
getEndpoints(ApiSchema schema, ServiceId id, int node);
Set<InstanceAddress> getEndpoints(ServiceKey<?> schema);
/** Register a monitor to be notified when the service registry changes.
* <p></p>
@ -61,9 +54,6 @@ public interface ServiceRegistryIf {
* monitor type.
* <ul>
* <li>{@link ServiceChangeMonitor} is notified when any node for the service changes.</li>
* <li>{@link ServiceNodeChangeMonitor} is notified when a specific node for the service changes.</li>
* <li>{@link ServiceRestEndpointChangeMonitor} is notified when the REST endpoints for the specified node service changes.</li>
* <li>{@link ServiceGrpcEndpointChangeMonitor} is notified when the gRPC endpoints for the specified node service changes.</li>
* </ul>
* */
void registerMonitor(ServiceMonitorIf monitor) throws Exception;

235
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java
View File

@ -4,10 +4,10 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import lombok.SneakyThrows;
import nu.marginalia.service.discovery.monitor.*;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.service.discovery.property.ServiceEndpoint;
import static nu.marginalia.service.discovery.property.ServiceEndpoint.*;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.CuratorWatcher;
import org.apache.curator.utils.ZKPaths;
@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/** A versatile service registry that uses ZooKeeper to store service endpoints.
* It is used to register services and to look up the endpoints of other services.
@ -35,6 +34,8 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
private static final Logger logger = LoggerFactory.getLogger(ZkServiceRegistry.class);
private volatile boolean stopped = false;
private final List<String> livenessPaths = new ArrayList<>();
@Inject
@SneakyThrows
public ZkServiceRegistry(CuratorFramework curatorFramework) {
@ -51,93 +52,98 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
}
@Override
public ServiceEndpoint registerService(ApiSchema schema, ServiceId id,
int node,
public ServiceEndpoint registerService(ServiceKey<?> key,
UUID instanceUUID,
String externalAddress)
throws Exception
{
var ephemeralProperty = curatorFramework.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.EPHEMERAL);
var endpoint = new ServiceEndpoint(externalAddress, requestPort(externalAddress, key));
var endpoint = ServiceEndpoint.forSchema(schema, externalAddress,
requestPort(externalAddress, schema, id, node)
);
String path;
byte[] payload;
switch (endpoint) {
case ServiceEndpoint.GrpcEndpoint(String host, int port) -> {
path = STR."/services/\{id.serviceName}/\{node}/grpc/\{instanceUUID.toString()}";
payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8);
}
case ServiceEndpoint.RestEndpoint(String host, int port) -> {
path = STR."/services/\{id.serviceName}/\{node}/rest/\{instanceUUID.toString()}";
payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8);
}
}
String path = STR."\{key.toPath()}/\{instanceUUID.toString()}";
byte[] payload = STR."\{endpoint.host()}:\{endpoint.port()}".getBytes(StandardCharsets.UTF_8);
logger.info("Registering {} -> {}", path, endpoint);
ephemeralProperty.forPath(path, payload);
curatorFramework.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.EPHEMERAL)
.forPath(path, payload);
return endpoint;
}
@SneakyThrows
@Override
public void announceInstance(ServiceId id, int node, UUID instanceUUID) {
public void declareFirstBoot() {
if (!isFirstBoot()) {
curatorFramework.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(STR."/first-boot");
}
}
@Override
public void waitForFirstBoot() throws InterruptedException {
if (!isFirstBoot())
logger.info("Waiting for first-boot flag");
while (true) {
if (isFirstBoot())
return;
Thread.sleep(1000);
}
}
private boolean isFirstBoot() {
try {
String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}";
return curatorFramework.checkExists().forPath("/first-boot") != null;
}
catch (Exception ex) {
logger.error("Failed to check first-boot", ex);
return false;
}
}
@Override
public void announceInstance(UUID instanceUUID) {
try {
String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}";
livenessPaths.add(serviceRoot);
curatorFramework.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.EPHEMERAL)
.forPath(serviceRoot);
}
catch (Exception ex) {
logger.error("Failed to create service root for {}", id.serviceName);
logger.error("Failed to create service root for {}", instanceUUID);
}
}
/**
* Returns true if the service has announced itself as up and running.
*/
public boolean isInstanceRunning(ServiceId id, int node, UUID instanceUUID) {
public boolean isInstanceRunning(UUID instanceUUID) {
try {
String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}";
String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}";
return null != curatorFramework.checkExists().forPath(serviceRoot);
}
catch (Exception ex) {
logger.error("Failed to check if service is running {}", id.serviceName);
logger.error("Failed to check if instance is running {}", instanceUUID);
return false;
}
}
@Override
public Set<Integer> getServiceNodes(ServiceId id) {
try {
String serviceRoot = STR."/services/\{id.serviceName}";
return curatorFramework.getChildren().forPath(serviceRoot)
.stream().map(Integer::parseInt)
.collect(Collectors.toSet());
}
catch (Exception ex) {
logger.error("Failed to get nodes for service {}", id.serviceName);
return Set.of();
}
}
@Override
public int requestPort(String externalHost,
ApiSchema schema,
ServiceId id,
int node)
{
ServiceKey<?> key) {
if (!Boolean.getBoolean("service.random-port")) {
return switch(schema) {
case REST -> 80;
case GRPC -> 81;
return switch (key) {
case ServiceKey.Rest rest -> 80;
case ServiceKey.Grpc<?> grpc -> 81;
};
}
@ -146,9 +152,9 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
var random = new Random();
String host = STR."\{id.serviceName}-\{node}";
String identifier = key.toPath();
byte[] payload = STR."\{schema}://\{host}".getBytes(StandardCharsets.UTF_8);
byte[] payload = identifier.getBytes();
for (int iter = 0; iter < 1000; iter++) {
try {
@ -161,7 +167,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
return port;
}
catch (Exception ex) {
logger.error(STR."Still negotiating port for \{schema}://\{id.serviceName}:\{node}");
logger.error(STR."Still negotiating port for \{identifier}");
}
}
@ -169,81 +175,30 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
}
@Override
public Set<InstanceAddress<? extends ServiceEndpoint>> getEndpoints(ApiSchema schema, ServiceId id, int node) {
return switch (schema) {
case REST -> getRestEndpoints(id, node);
case GRPC -> getGrpcEndpoints(id, node);
};
}
public Set<InstanceAddress<? extends ServiceEndpoint>> getRestEndpoints(ServiceId id, int node) {
public Set<InstanceAddress> getEndpoints(ServiceKey<?> key) {
try {
Set<InstanceAddress<? extends ServiceEndpoint>> ret = new HashSet<>();
String restRoot = STR."/services/\{id.serviceName}/\{node}/rest";
Set<InstanceAddress> ret = new HashSet<>();
for (var uuid : curatorFramework
.getChildren()
.forPath(restRoot)) {
.forPath(key.toPath())) {
if (!isInstanceRunning(id, node, UUID.fromString(uuid))) {
if (!isInstanceRunning(UUID.fromString(uuid))) {
continue;
}
var path = ZKPaths.makePath(key.toPath(), uuid);
byte[] data = curatorFramework
.getData()
.forPath(ZKPaths.makePath(restRoot, uuid));
String hostAndPort = new String(data);
var address = RestEndpoint
.parse(hostAndPort)
.asInstance(UUID.fromString(uuid));
.forPath(path);
// Ensure that the address is resolvable
// (this reduces the risk of exceptions when trying to connect to the service)
if (!address.endpoint().validateHost()) {
logger.warn("Omitting stale address {}, address does not resolve", address);
continue;
}
ret.add(address);
}
return ret;
}
catch (Exception ex) {
return Set.of();
}
}
public Set<InstanceAddress<? extends ServiceEndpoint>> getGrpcEndpoints(ServiceId id, int node) {
try {
Set<InstanceAddress<? extends ServiceEndpoint>> ret = new HashSet<>();
String restRoot = STR."/services/\{id.serviceName}/\{node}/grpc";
for (var uuid : curatorFramework
.getChildren()
.forPath(restRoot)) {
if (!isInstanceRunning(id, node, UUID.fromString(uuid))) {
continue;
}
byte[] data = curatorFramework
.getData()
.forPath(ZKPaths.makePath(restRoot, uuid));
long cxTime = curatorFramework.checkExists().forPath(path).getMzxid();
String hostAndPort = new String(data);
var address = GrpcEndpoint
var address = ServiceEndpoint
.parse(hostAndPort)
.asInstance(UUID.fromString(uuid));
// Ensure that the address is resolvable
// (this reduces the risk of exceptions when trying to connect to the service)
if (!address.endpoint().validateHost()) {
logger.warn("Omitting stale address {}, address does not resolve", address);
continue;
}
.asInstance(UUID.fromString(uuid), cxTime);
ret.add(address);
}
return ret;
@ -254,29 +209,13 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
}
public void registerMonitor(ServiceMonitorIf monitor) throws Exception {
monitor.register(this);
}
if (stopped)
logger.info("Not registering monitor for {} because the registry is stopped", monitor.getKey());
public void registerMonitor(ServiceChangeMonitor monitor) throws Exception {
installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}");
}
String path = monitor.getKey().toPath();
public void registerMonitor(ServiceNodeChangeMonitor monitor) throws Exception {
installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}");
}
public void registerMonitor(ServiceRestEndpointChangeMonitor monitor) throws Exception {
installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/rest");
}
public void registerMonitor(ServiceGrpcEndpointChangeMonitor monitor) throws Exception {
installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/grpc");
}
private void installMonitor(ServiceMonitorIf monitor, String path) throws Exception {
CuratorWatcher watcher = _ -> {
CuratorWatcher watcher = change -> {
boolean reRegister;
try {
reRegister = monitor.onChange();
}
@ -293,13 +232,31 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
curatorFramework.watchers().add()
.usingWatcher(watcher)
.forPath(path);
// Also register for updates to the running-instances list,
// as this will have an effect on the result of getEndpoints()
curatorFramework.watchers().add()
.usingWatcher(watcher)
.forPath("/running-instances");
}
/* Exposed for tests */
public synchronized void shutDown() {
if (!stopped) {
curatorFramework.close();
stopped = true;
if (!stopped)
return;
stopped = true;
// Delete all liveness paths
for (var path : livenessPaths) {
logger.info("Cleaning up {}", path);
try {
curatorFramework.delete().forPath(path);
}
catch (Exception ex) {
logger.error("Failed to delete path {}", path, ex);
}
}
}
}

20
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java
View File

@ -1,23 +1,17 @@
package nu.marginalia.service.discovery.monitor;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.ZkServiceRegistry;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.discovery.property.ServiceKey;
public abstract class ServiceChangeMonitor implements ServiceMonitorIf {
public final ServiceId serviceId;
public final ServiceKey<?> serviceKey;
public ServiceChangeMonitor(ServiceId serviceId) {
this.serviceId = serviceId;
public ServiceChangeMonitor(ServiceKey<?> key) {
this.serviceKey = key;
}
public abstract boolean onChange();
public void register(ServiceRegistryIf registry) throws Exception {
if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
zkServiceRegistry.registerMonitor(this);
}
else {
registry.registerMonitor(this);
}
public ServiceKey<?> getKey() {
return serviceKey;
}
}

25
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java
View File

@ -1,25 +0,0 @@
package nu.marginalia.service.discovery.monitor;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.ZkServiceRegistry;
import nu.marginalia.service.id.ServiceId;
public abstract class ServiceGrpcEndpointChangeMonitor implements ServiceMonitorIf {
public final ServiceId serviceId;
public final int node;
public ServiceGrpcEndpointChangeMonitor(ServiceId serviceId, int node) {
this.serviceId = serviceId;
this.node = node;
}
public abstract boolean onChange();
public void register(ServiceRegistryIf registry) throws Exception {
if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
zkServiceRegistry.registerMonitor(this);
}
else {
registry.registerMonitor(this);
}
}
}

9
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java
View File

@ -1,16 +1,13 @@
package nu.marginalia.service.discovery.monitor;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.property.ServiceKey;
public interface ServiceMonitorIf {
/** Called when the monitored service has changed.
* @return true if the monitor is to be refreshed
*/
boolean onChange();
ServiceKey<?> getKey();
/** Register this monitor with the given registry.
* It is preferred to use {@link ServiceRegistryIf}'s
* registerMonitor function.
* */
void register(ServiceRegistryIf registry) throws Exception;
}

25
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java
View File

@ -1,25 +0,0 @@
package nu.marginalia.service.discovery.monitor;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.ZkServiceRegistry;
import nu.marginalia.service.id.ServiceId;
public abstract class ServiceNodeChangeMonitor implements ServiceMonitorIf {
public final ServiceId serviceId;
public final int node;
public ServiceNodeChangeMonitor(ServiceId serviceId, int node) {
this.serviceId = serviceId;
this.node = node;
}
public abstract boolean onChange();
public void register(ServiceRegistryIf registry) throws Exception {
if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
zkServiceRegistry.registerMonitor(this);
}
else {
registry.registerMonitor(this);
}
}
}

25
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java
View File

@ -1,25 +0,0 @@
package nu.marginalia.service.discovery.monitor;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.discovery.ZkServiceRegistry;
import nu.marginalia.service.id.ServiceId;
public abstract class ServiceRestEndpointChangeMonitor implements ServiceMonitorIf {
public final ServiceId serviceId;
public final int node;
public ServiceRestEndpointChangeMonitor(ServiceId serviceId, int node) {
this.serviceId = serviceId;
this.node = node;
}
public abstract boolean onChange();
public void register(ServiceRegistryIf registry) throws Exception {
if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
zkServiceRegistry.registerMonitor(this);
}
else {
registry.registerMonitor(this);
}
}
}

6
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java
View File

@ -1,6 +0,0 @@
package nu.marginalia.service.discovery.property;
public enum ApiSchema {
REST,
GRPC
}

8
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java Normal file
View File

@ -0,0 +1,8 @@
package nu.marginalia.service.discovery.property;
public interface PartitionTraits {
interface Grpc {};
interface Unicast {};
interface Multicast {};
interface NoGrpc {};
}

82
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java
View File

@ -1,17 +1,23 @@
package nu.marginalia.service.discovery.property;
import lombok.SneakyThrows;
import java.net.*;
import java.util.UUID;
public sealed interface ServiceEndpoint {
String host();
int port();
public record ServiceEndpoint(String host, int port) {
URL toURL(String endpoint, String query);
default InetSocketAddress toInetSocketAddress() {
public static ServiceEndpoint parse(String hostAndPort) {
var parts = hostAndPort.split(":");
if (parts.length != 2) {
throw new IllegalArgumentException("Invalid host:port string: " + hostAndPort);
}
return new ServiceEndpoint(parts[0], Integer.parseInt(parts[1]));
}
public URL toURL(String endpoint, String query) throws URISyntaxException, MalformedURLException {
return new URI("http", null, host, port, endpoint, query, null)
.toURL();
}
public InetSocketAddress toInetSocketAddress() {
return new InetSocketAddress(host(), port());
}
@ -19,7 +25,7 @@ public sealed interface ServiceEndpoint {
*
* @return true if the host is a valid
*/
default boolean validateHost() {
public boolean validateHost() {
try {
// Throws UnknownHostException if the host is not a valid IP address or hostname
// (this should not be slow since the DNS lookup should be local, and if it isn't;
@ -31,63 +37,11 @@ public sealed interface ServiceEndpoint {
}
}
static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) {
return switch (schema) {
case REST -> new RestEndpoint(host, port);
case GRPC -> new GrpcEndpoint(host, port);
};
public InstanceAddress asInstance(UUID instance, long cxTime) {
return new InstanceAddress(this, instance, cxTime);
}
record RestEndpoint(String host, int port) implements ServiceEndpoint {
public static RestEndpoint parse(String hostColonPort) {
String[] parts = hostColonPort.split(":");
if (parts.length != 2) {
throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'");
}
return new RestEndpoint(
parts[0],
Integer.parseInt(parts[1])
);
}
@SneakyThrows
public URL toURL(String endpoint, String query) {
return new URI("http", null, host, port, endpoint, query, null)
.toURL();
}
public InstanceAddress<RestEndpoint> asInstance(UUID uuid) {
return new InstanceAddress<>(this, uuid);
}
}
record GrpcEndpoint(String host, int port) implements ServiceEndpoint {
public static GrpcEndpoint parse(String hostColonPort) {
String[] parts = hostColonPort.split(":");
if (parts.length != 2) {
throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'");
}
return new GrpcEndpoint(
parts[0],
Integer.parseInt(parts[1])
);
}
public InstanceAddress<GrpcEndpoint> asInstance(UUID uuid) {
return new InstanceAddress<>(this, uuid);
}
@Override
public URL toURL(String endpoint, String query) {
throw new UnsupportedOperationException();
}
}
record InstanceAddress<T extends ServiceEndpoint>(T endpoint, UUID instance) {
public record InstanceAddress(ServiceEndpoint endpoint, UUID instance, long cxTime) {
public String host() {
return endpoint.host();
}

69
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java Normal file
View File

@ -0,0 +1,69 @@
package nu.marginalia.service.discovery.property;
import io.grpc.ServiceDescriptor;
import nu.marginalia.service.id.ServiceId;
public sealed interface ServiceKey<P extends ServicePartition> {
String toPath();
static ServiceKey<ServicePartition.None> forRest(ServiceId id) {
return new Rest(id.serviceName);
}
static ServiceKey<ServicePartition.None> forRest(ServiceId id, int node) {
if (node == 0) {
return forRest(id);
}
return new Rest(id.serviceName + "-" + node);
}
static Grpc<ServicePartition> forServiceDescriptor(ServiceDescriptor descriptor, ServicePartition partition) {
return new Grpc<>(descriptor.getName(), partition);
}
static <P2 extends ServicePartition & PartitionTraits.Grpc> Grpc<P2> forGrpcApi(Class<?> apiClass, P2 partition) {
try {
var name = apiClass.getField("SERVICE_NAME").get(null);
return new Grpc<P2>(name.toString(), partition);
}
catch (Exception e) {
throw new IllegalArgumentException("Could not get SERVICE_NAME from " + apiClass.getSimpleName(), e);
}
}
<P2 extends ServicePartition & PartitionTraits.Grpc & PartitionTraits.Unicast>
Grpc<P2> forPartition(P2 partition);
record Rest(String name) implements ServiceKey<ServicePartition.None> {
public String toPath() {
return STR."/services/rest/\{name}";
}
@Override
public
<P2 extends ServicePartition & PartitionTraits.Grpc & PartitionTraits.Unicast>
Grpc<P2> forPartition(P2 partition)
{
throw new UnsupportedOperationException();
}
}
record Grpc<P extends ServicePartition>(String name, P partition) implements ServiceKey<P> {
public String baseName() {
return STR."/services/grpc/\{name}";
}
public String toPath() {
return STR."/services/grpc/\{name}/\{partition.identifier()}";
}
@Override
public
<P2 extends ServicePartition & PartitionTraits.Grpc & PartitionTraits.Unicast>
Grpc<P2> forPartition(P2 partition)
{
return new Grpc<>(name, partition);
}
}
}

29
code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java Normal file
View File

@ -0,0 +1,29 @@
package nu.marginalia.service.discovery.property;
public sealed interface ServicePartition {
String identifier();
static Any any() { return new Any(); }
static Multi multi() { return new Multi(); }
static Partition partition(int node) { return new Partition(node); }
static None none() { return new None(); }
record Any() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast {
public String identifier() { return "*"; }
}
record Multi() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Multicast {
public String identifier() { return "*"; }
}
record Partition(int node) implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast {
public String identifier() {
return Integer.toString(node);
}
}
record None() implements ServicePartition, PartitionTraits.NoGrpc {
public String identifier() { return ""; }
}
}

93
code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java
View File

@ -1,6 +1,8 @@
package nu.marginalia.service.discovery;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.api.math.MathApiGrpc;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.service.id.ServiceId;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
@ -13,7 +15,6 @@ import org.testcontainers.junit.jupiter.Testcontainers;
import java.util.*;
import static nu.marginalia.service.discovery.property.ServiceEndpoint.*;
import static org.junit.jupiter.api.Assertions.*;
@Testcontainers
@ -58,15 +59,18 @@ class ZkServiceRegistryTest {
List<Integer> ports = new ArrayList<>();
Set<Integer> portsSet = new HashSet<>();
var key = ServiceKey.forRest(ServiceId.Search, 0);
for (int i = 0; i < 500; i++) {
int port = registry1.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0);
int port = registry1.requestPort("127.0.0.1", key);
ports.add(port);
// Ensure we get unique ports
assertTrue(portsSet.add(port));
}
for (int i = 0; i < 50; i++) {
int port = registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0);
int port = registry2.requestPort("127.0.0.1", key);
ports.add(port);
// Ensure we get unique ports
@ -75,39 +79,86 @@ class ZkServiceRegistryTest {
registry1.shutDown();
for (int i = 0; i < 500; i++) {
// Verify we can reclaim ports
ports.add(registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0));
ports.add(registry2.requestPort("127.0.0.1", key));
}
assertEquals(1050, ports.size());
}
@Test
void getInstances() throws Exception {
void getInstancesRestgRPC() throws Exception {
var uuid1 = UUID.randomUUID();
var uuid2 = UUID.randomUUID();
var registry1 = createRegistry();
var registry2 = createRegistry();
var endpoint1 = (RestEndpoint) registry1.registerService(ApiSchema.REST, ServiceId.Search, 0, uuid1, "127.0.0.1");
var endpoint2 = (GrpcEndpoint) registry2.registerService(ApiSchema.GRPC, ServiceId.Search, 0, uuid2, "127.0.0.2");
var key1 = ServiceKey.forRest(ServiceId.Search, 0);
var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any());
registry1.announceInstance(ServiceId.Search, 0, uuid1);
registry2.announceInstance(ServiceId.Search, 0, uuid2);
var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1");
var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2");
assertEquals(Set.of(endpoint1.asInstance(uuid1)),
registry1.getRestEndpoints(ServiceId.Search, 0));
registry1.announceInstance(uuid1);
registry2.announceInstance(uuid2);
assertEquals(Set.of(endpoint2.asInstance(uuid2)),
registry1.getGrpcEndpoints(ServiceId.Search, 0));
assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)),
registry1.getEndpoints(key1));
assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)),
registry1.getEndpoints(key2));
registry1.shutDown();
Thread.sleep(100);
assertEquals(Set.of(),
registry2.getRestEndpoints(ServiceId.Search, 0));
assertEquals(Set.of(endpoint2.asInstance(uuid2)),
registry2.getGrpcEndpoints(ServiceId.Search, 0));
assertEquals(Set.of(), registry2.getEndpoints(key1));
assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key2));
}
@Test
void testInstancesTwoAny() throws Exception {
var uuid1 = UUID.randomUUID();
var uuid2 = UUID.randomUUID();
var registry1 = createRegistry();
var registry2 = createRegistry();
var key = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any());
var endpoint1 = registry1.registerService(key, uuid1, "127.0.0.1");
var endpoint2 = registry2.registerService(key, uuid2, "127.0.0.2");
registry1.announceInstance(uuid1);
registry2.announceInstance(uuid2);
assertEquals(Set.of(endpoint1.asInstance(uuid1, 0),
endpoint2.asInstance(uuid2, 0)),
registry1.getEndpoints(key));
registry1.shutDown();
Thread.sleep(100);
assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key));
}
@Test
void testInstancesTwoPartitions() throws Exception {
var uuid1 = UUID.randomUUID();
var uuid2 = UUID.randomUUID();
var registry1 = createRegistry();
var registry2 = createRegistry();
var key1 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(1));
var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(2));
var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1");
var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2");
registry1.announceInstance(uuid1);
registry2.announceInstance(uuid2);
assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), registry1.getEndpoints(key1));
assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry1.getEndpoints(key2));
}
@Test
@ -115,9 +166,9 @@ class ZkServiceRegistryTest {
var registry1 = createRegistry();
var uuid1 = UUID.randomUUID();
assertFalse(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1));
registry1.announceInstance(ServiceId.Search, 0, uuid1);
assertTrue(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1));
assertFalse(registry1.isInstanceRunning(uuid1));
registry1.announceInstance(uuid1);
assertTrue(registry1.isInstanceRunning(uuid1));
registry1.shutDown();
}

2
code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java
View File

@ -156,6 +156,8 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat {
stmt.executeUpdate();
}
}
dataSource.close();
}
}

15
code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java
View File

@ -6,6 +6,7 @@ import com.google.inject.Singleton;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows;
import nu.marginalia.WmsaHome;
import nu.marginalia.service.ServiceHomeNotConfiguredException;
import org.flywaydb.core.Flyway;
import org.mariadb.jdbc.Driver;
@ -51,7 +52,7 @@ public class DatabaseModule extends AbstractModule {
}
private Properties loadDbProperties() {
Path propDir = getHomePath().resolve("conf/db.properties");
Path propDir = WmsaHome.getHomePath().resolve("conf/db.properties");
if (!Files.isRegularFile(propDir)) {
throw new IllegalStateException("Database properties file " + propDir + " does not exist");
}
@ -72,17 +73,6 @@ public class DatabaseModule extends AbstractModule {
}
public static Path getHomePath() {
var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse("/var/lib/wmsa");
var ret = Path.of(retStr);
if (!Files.isDirectory(ret)) {
throw new ServiceHomeNotConfiguredException("Could not find WMSA_HOME, either set environment variable or ensure /var/lib/wmsa exists");
}
return ret;
}
@SneakyThrows
@Singleton
@Provides
@ -97,7 +87,6 @@ public class DatabaseModule extends AbstractModule {
try {
HikariConfig config = new HikariConfig();
config.setJdbcUrl(connStr);
config.setUsername(dbProperties.getProperty(DB_USER_KEY));
config.setPassword(dbProperties.getProperty(DB_PASS_KEY));

1
code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java
View File

@ -1,5 +1,6 @@
package nu.marginalia.service.module;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.service.id.ServiceId;
import java.util.UUID;

11
code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java
View File

@ -65,7 +65,7 @@ public class ServiceConfigurationModule extends AbstractModule {
}
// If we're in docker, we'll use the hostname
if (isDocker()) {
if (Boolean.getBoolean("service.useDockerHostname")) {
return System.getenv("HOSTNAME");
}
@ -82,14 +82,7 @@ public class ServiceConfigurationModule extends AbstractModule {
return configuredValue;
}
// If we're in docker, we'll bind to all interfaces
if (isDocker())
return "0.0.0.0";
else // If we're not in docker, we'll default to binding to localhost to avoid exposing services
return "127.0.0.1";
return "127.0.0.1";
}
boolean isDocker() {
return System.getenv("WMSA_IN_DOCKER") != null;
}
}

53
code/common/service/src/main/java/nu/marginalia/service/server/Service.java
View File

@ -5,8 +5,8 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
import io.prometheus.client.Counter;
import lombok.SneakyThrows;
import nu.marginalia.mq.inbox.*;
import nu.marginalia.service.discovery.property.ApiSchema;
import nu.marginalia.service.discovery.property.ServiceEndpoint;
import nu.marginalia.service.discovery.property.*;
import nu.marginalia.service.id.ServiceId;
import nu.marginalia.service.server.mq.ServiceMqSubscription;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -16,6 +16,7 @@ import spark.Request;
import spark.Response;
import spark.Spark;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.Optional;
@ -48,11 +49,24 @@ public class Service {
@SneakyThrows
public Service(BaseServiceParams params,
Runnable configureStaticFiles,
ServicePartition partition,
List<BindableService> grpcServices) {
this.initialization = params.initialization;
var config = params.configuration;
node = config.node();
if (config.serviceId() == ServiceId.Control) {
// Special case for first boot, since the control service
// owns database migrations and so on, we need other processes
// to wait for this to be done before they start. This is
// only needed once.
params.serviceRegistry.declareFirstBoot();
}
else {
params.serviceRegistry.waitForFirstBoot();
}
String inboxName = config.serviceName();
logger.info("Inbox name: {}", inboxName);
@ -60,8 +74,7 @@ public class Service {
var restEndpoint =
serviceRegistry.registerService(
ApiSchema.REST, config.serviceId(),
config.node(),
ServiceKey.forRest(config.serviceId(), config.node()),
config.instanceUuid(),
config.externalAddress()
);
@ -75,7 +88,7 @@ public class Service {
initialization.addCallback(params.heartbeat::start);
initialization.addCallback(messageQueueInbox::start);
initialization.addCallback(() -> params.eventLog.logEvent("SVC-INIT", serviceName + ":" + config.node()));
initialization.addCallback(() -> serviceRegistry.announceInstance(config.serviceId(), config.node(), config.instanceUuid()));
initialization.addCallback(() -> serviceRegistry.announceInstance(config.instanceUuid()));
if (!initialization.isReady() && ! initialized ) {
initialized = true;
@ -101,29 +114,39 @@ public class Service {
Spark.get("/internal/started", this::isInitialized);
Spark.get("/internal/ready", this::isReady);
ServiceEndpoint.GrpcEndpoint grpcEndpoint = (ServiceEndpoint.GrpcEndpoint) params.serviceRegistry.registerService(
ApiSchema.GRPC, config.serviceId(),
config.node(),
config.instanceUuid(),
config.externalAddress()
);
int port = params.serviceRegistry.requestPort(config.externalAddress(), new ServiceKey.Grpc<>("-", partition));
// Start the gRPC server
var grpcServerBuilder = NettyServerBuilder.forAddress(grpcEndpoint.toInetSocketAddress());
var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port));
for (var grpcService : grpcServices) {
grpcServerBuilder.addService(grpcService);
var svc = grpcService.bindService();
params.serviceRegistry.registerService(
ServiceKey.forServiceDescriptor(svc.getServiceDescriptor(), partition),
config.instanceUuid(),
config.externalAddress()
);
grpcServerBuilder.addService(svc);
}
grpcServerBuilder.build().start();
}
}
public Service(BaseServiceParams params,
ServicePartition partition,
List<BindableService> grpcServices) {
this(params, Service::defaultSparkConfig, grpcServices);
this(params,
Service::defaultSparkConfig,
partition,
grpcServices);
}
public Service(BaseServiceParams params) {
this(params, Service::defaultSparkConfig, List.of());
this(params,
Service::defaultSparkConfig,
ServicePartition.any(),
List.of());
}
private static void defaultSparkConfig() {

2
code/common/service/src/main/resources/log4j2-prod.xml
View File

@ -21,6 +21,8 @@
</RollingFile>
</Appenders>
<Loggers>
<Logger name="org.apache.zookeeper" level="WARN" />
<Root level="info">
<AppenderRef ref="Console"/>
<AppenderRef ref="LogToFile"/>

2
code/common/service/src/main/resources/log4j2-test.xml
View File

@ -20,6 +20,8 @@
</RollingFile>
</Appenders>
<Loggers>
<Logger name="org.apache.zookeeper" level="WARN" />
<Root level="info">
<AppenderRef ref="Console"/>
<AppenderRef ref="LogToFile"/>

2
code/features-index/domain-ranking/build.gradle
View File

@ -17,7 +17,7 @@ dependencies {
implementation project(':code:common:db')
implementation project(':code:common:model')
implementation project(':code:common:service')
implementation project(':code:api:query-api')
implementation project(':code:functions:domain-links:api')
implementation 'org.jgrapht:jgrapht-core:1.5.2'

9
code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java
View File

@ -3,23 +3,20 @@ package nu.marginalia.ranking.data;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import org.jgrapht.Graph;
import org.jgrapht.graph.DefaultDirectedGraph;
import org.jgrapht.graph.DefaultEdge;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
/** A source for the inverted link graph,
* which is the same as the regular graph except
* the direction of the links have been inverted */
public class InvertedLinkGraphSource extends AbstractGraphSource {
private final QueryClient queryClient;
private final AggregateDomainLinksClient queryClient;
@Inject
public InvertedLinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) {
public InvertedLinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient queryClient) {
super(dataSource);
this.queryClient = queryClient;
}

10
code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java
View File

@ -3,19 +3,19 @@ package nu.marginalia.ranking.data;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import org.jgrapht.Graph;
import org.jgrapht.graph.DefaultDirectedGraph;
import org.jgrapht.graph.DefaultEdge;
/** A source for the regular link graph. */
public class LinkGraphSource extends AbstractGraphSource {
private final QueryClient queryClient;
private final AggregateDomainLinksClient domainLinksClient;
@Inject
public LinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) {
public LinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) {
super(dataSource);
this.queryClient = queryClient;
this.domainLinksClient = domainLinksClient;
}
@SneakyThrows
@ -25,7 +25,7 @@ public class LinkGraphSource extends AbstractGraphSource {
addVertices(graph);
var allLinks = queryClient.getAllDomainLinks();
var allLinks = domainLinksClient.getAllDomainLinks();
var iter = allLinks.iterator();
while (iter.advance()) {
if (!graph.containsVertex(iter.dest())) {

19
code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java
View File

@ -3,7 +3,7 @@ package nu.marginalia.ranking;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import nu.marginalia.ranking.data.InvertedLinkGraphSource;
import nu.marginalia.ranking.data.LinkGraphSource;
import nu.marginalia.ranking.data.SimilarityGraphSource;
@ -37,8 +37,9 @@ public class RankingAlgorithmsContainerTest {
static HikariDataSource dataSource;
QueryClient queryClient;
QueryClient.AllLinks allLinks;
AggregateDomainLinksClient domainLinksClient;
AggregateDomainLinksClient.AllLinks allLinks;
@BeforeAll
public static void setup() {
HikariConfig config = new HikariConfig();
@ -66,9 +67,9 @@ public class RankingAlgorithmsContainerTest {
@BeforeEach
public void setupQueryClient() {
queryClient = Mockito.mock(QueryClient.class);
allLinks = new QueryClient.AllLinks();
when(queryClient.getAllDomainLinks()).thenReturn(allLinks);
domainLinksClient = Mockito.mock(AggregateDomainLinksClient.class);
allLinks = new AggregateDomainLinksClient.AllLinks();
when(domainLinksClient.getAllDomainLinks()).thenReturn(allLinks);
try (var conn = dataSource.getConnection();
var stmt = conn.createStatement()) {
@ -97,7 +98,7 @@ public class RankingAlgorithmsContainerTest {
@Test
public void testGetDomains() {
// should all be the same, doesn't matter which one we use
var source = new LinkGraphSource(dataSource, queryClient);
var source = new LinkGraphSource(dataSource, domainLinksClient);
Assertions.assertEquals(List.of(1),
source.domainIds(List.of("memex.marginalia.nu")));
@ -111,7 +112,7 @@ public class RankingAlgorithmsContainerTest {
public void testLinkGraphSource() {
allLinks.add(1, 3);
var graph = new LinkGraphSource(dataSource, queryClient).getGraph();
var graph = new LinkGraphSource(dataSource, domainLinksClient).getGraph();
Assertions.assertTrue(graph.containsVertex(1));
Assertions.assertTrue(graph.containsVertex(2));
@ -127,7 +128,7 @@ public class RankingAlgorithmsContainerTest {
public void testInvertedLinkGraphSource() {
allLinks.add(1, 3);
var graph = new InvertedLinkGraphSource(dataSource, queryClient).getGraph();
var graph = new InvertedLinkGraphSource(dataSource, domainLinksClient).getGraph();
Assertions.assertTrue(graph.containsVertex(1));
Assertions.assertTrue(graph.containsVertex(2));

45
code/functions/domain-info/api/build.gradle Normal file
View File

@ -0,0 +1,45 @@
plugins {
id 'java'
id "com.google.protobuf" version "0.9.4"
id 'jvm-test-suite'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
jar.archiveBaseName = 'domain-info-api'
sourceSets {
main {
proto {
srcDir 'src/main/protobuf'
}
}
}
apply from: "$rootProject.projectDir/protobuf.gradle"
dependencies {
implementation project(':code:common:model')
implementation project(':code:common:config')
implementation project(':code:common:service-discovery')
implementation libs.bundles.slf4j
implementation libs.prometheus
implementation libs.notnull
implementation libs.guice
implementation libs.gson
implementation libs.protobuf
implementation libs.javax.annotation
implementation libs.bundles.grpc
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

56
code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java Normal file
View File

@ -0,0 +1,56 @@
package nu.marginalia.api.domains;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.api.domains.model.SimilarDomain;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.*;
import nu.marginalia.api.domains.model.*;
@Singleton
public class DomainInfoClient {
private static final Logger logger = LoggerFactory.getLogger(DomainInfoClient.class);
private final GrpcSingleNodeChannelPool<DomainInfoAPIGrpc.DomainInfoAPIBlockingStub> channelPool;
private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
@Inject
public DomainInfoClient(GrpcChannelPoolFactory factory) {
this.channelPool = factory.createSingle(
ServiceKey.forGrpcApi(DomainInfoAPIGrpc.class, ServicePartition.any()),
DomainInfoAPIGrpc::newBlockingStub);
}
public Future<List<SimilarDomain>> similarDomains(int domainId, int count) {
return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getSimilarDomains)
.async(virtualExecutorService)
.run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count))
.thenApply(DomainsProtobufCodec.DomainQueries::convertResponse);
}
public Future<List<SimilarDomain>> linkedDomains(int domainId, int count) {
return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getLinkingDomains)
.async(virtualExecutorService)
.run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count))
.thenApply(DomainsProtobufCodec.DomainQueries::convertResponse);
}
public Future<DomainInformation> domainInformation(int domainId) {
return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getDomainInfo)
.async(virtualExecutorService)
.run(DomainsProtobufCodec.DomainInfo.createRequest(domainId))
.thenApply(DomainsProtobufCodec.DomainInfo::convertResponse);
}
public boolean isAccepting() {
return channelPool.hasChannel();
}
}

66
code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java → code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java
View File

@ -1,74 +1,14 @@
package nu.marginalia.assistant.client;
package nu.marginalia.api.domains;
import lombok.SneakyThrows;
import nu.marginalia.assistant.api.*;
import nu.marginalia.assistant.client.model.DictionaryEntry;
import nu.marginalia.assistant.client.model.DictionaryResponse;
import nu.marginalia.assistant.client.model.DomainInformation;
import nu.marginalia.assistant.client.model.SimilarDomain;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.api.domains.model.*;
import java.util.ArrayList;
import java.util.List;
public class AssistantProtobufCodec {
public static class DictionaryLookup {
public static RpcDictionaryLookupRequest createRequest(String word) {
return RpcDictionaryLookupRequest.newBuilder()
.setWord(word)
.build();
}
public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) {
return new DictionaryResponse(
rsp.getWord(),
rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList()
);
}
private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) {
return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition());
}
}
public static class SpellCheck {
public static RpcSpellCheckRequest createRequest(String text) {
return RpcSpellCheckRequest.newBuilder()
.setText(text)
.build();
}
public static List<String> convertResponse(RpcSpellCheckResponse rsp) {
return rsp.getSuggestionsList();
}
}
public static class UnitConversion {
public static RpcUnitConversionRequest createRequest(String from, String to, String unit) {
return RpcUnitConversionRequest.newBuilder()
.setFrom(from)
.setTo(to)
.setUnit(unit)
.build();
}
public static String convertResponse(RpcUnitConversionResponse rsp) {
return rsp.getResult();
}
}
public static class EvalMath {
public static RpcEvalMathRequest createRequest(String expression) {
return RpcEvalMathRequest.newBuilder()
.setExpression(expression)
.build();
}
public static String convertResponse(RpcEvalMathResponse rsp) {
return rsp.getResult();
}
}
public class DomainsProtobufCodec {
public static class DomainQueries {
public static RpcDomainLinksRequest createRequest(int domainId, int count) {

2
code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java → code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.client.model;
package nu.marginalia.api.domains.model;
import lombok.*;
import nu.marginalia.model.EdgeDomain;

2
code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java → code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.client.model;
package nu.marginalia.api.domains.model;
import nu.marginalia.model.EdgeUrl;

14
code/api/assistant-api/src/main/protobuf/assistant-api.proto → code/functions/domain-info/api/src/main/protobuf/domain-info.proto
View File

@ -1,18 +1,10 @@
syntax="proto3";
package assistantapi;
package marginalia.api.domain;
option java_package="nu.marginalia.assistant.api";
option java_package="nu.marginalia.api.domains";
option java_multiple_files=true;
service AssistantApi {
/** Looks up a word in the dictionary. */
rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {}
/** Checks the spelling of a text. */
rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {}
/** Converts a unit from one to another. */
rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {}
/** Evaluates a mathematical expression. */
rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {}
service DomainInfoAPI {
/** Fetches information about a domain. */
rpc getDomainInfo(RpcDomainId) returns (RpcDomainInfoResponse) {}

44
code/functions/domain-info/build.gradle Normal file
View File

@ -0,0 +1,44 @@
plugins {
id 'java'
id 'application'
id 'jvm-test-suite'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
dependencies {
implementation project(':code:functions:domain-info:api')
implementation project(':code:functions:domain-links:api')
implementation project(':code:common:config')
implementation project(':code:common:service')
implementation project(':code:common:model')
implementation project(':code:common:db')
implementation project(':code:common:service-discovery')
implementation project(':code:libraries:geo-ip')
implementation libs.bundles.slf4j
implementation libs.prometheus
implementation libs.bundles.grpc
implementation libs.notnull
implementation libs.guice
implementation libs.spark
implementation libs.opencsv
implementation libs.trove
implementation libs.fastutil
implementation libs.bundles.gson
implementation libs.bundles.mariadb
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

53
code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java Normal file
View File

@ -0,0 +1,53 @@
package nu.marginalia.functions.domains;
import com.google.inject.Inject;
import io.grpc.stub.StreamObserver;
import nu.marginalia.api.domains.DomainInfoAPIGrpc;
import nu.marginalia.api.domains.*;
public class DomainInfoGrpcService extends DomainInfoAPIGrpc.DomainInfoAPIImplBase {
private final DomainInformationService domainInformationService;
private final SimilarDomainsService similarDomainsService;
@Inject
public DomainInfoGrpcService(DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService)
{
this.domainInformationService = domainInformationService;
this.similarDomainsService = similarDomainsService;
}
@Override
public void getDomainInfo(RpcDomainId request, StreamObserver<RpcDomainInfoResponse> responseObserver) {
var ret = domainInformationService.domainInfo(request.getDomainId());
ret.ifPresent(responseObserver::onNext);
responseObserver.onCompleted();
}
@Override
public void getSimilarDomains(RpcDomainLinksRequest request,
StreamObserver<RpcSimilarDomains> responseObserver) {
var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount());
var responseBuilder = RpcSimilarDomains
.newBuilder()
.addAllDomains(ret);
responseObserver.onNext(responseBuilder.build());
responseObserver.onCompleted();
}
@Override
public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver<RpcSimilarDomains> responseObserver) {
var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount());
var responseBuilder = RpcSimilarDomains
.newBuilder()
.addAllDomains(ret);
responseObserver.onNext(responseBuilder.build());
responseObserver.onCompleted();
}
}

16
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java → code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java
View File

@ -1,11 +1,11 @@
package nu.marginalia.assistant.domains;
package nu.marginalia.functions.domains;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.assistant.api.RpcDomainInfoResponse;
import nu.marginalia.api.domains.RpcDomainInfoResponse;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import nu.marginalia.geoip.GeoIpDictionary;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.db.DbDomainQueries;
import nu.marginalia.query.client.QueryClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -21,7 +21,7 @@ public class DomainInformationService {
private final GeoIpDictionary geoIpDictionary;
private DbDomainQueries dbDomainQueries;
private final QueryClient queryClient;
private final AggregateDomainLinksClient domainLinksClient;
private HikariDataSource dataSource;
private final Logger logger = LoggerFactory.getLogger(getClass());
@ -29,11 +29,11 @@ public class DomainInformationService {
public DomainInformationService(
DbDomainQueries dbDomainQueries,
GeoIpDictionary geoIpDictionary,
QueryClient queryClient,
AggregateDomainLinksClient domainLinksClient,
HikariDataSource dataSource) {
this.dbDomainQueries = dbDomainQueries;
this.geoIpDictionary = geoIpDictionary;
this.queryClient = queryClient;
this.domainLinksClient = domainLinksClient;
this.dataSource = dataSource;
}
@ -84,8 +84,8 @@ public class DomainInformationService {
inCrawlQueue = rs.next();
builder.setInCrawlQueue(inCrawlQueue);
builder.setIncomingLinks(queryClient.countLinksToDomain(domainId));
builder.setOutboundLinks(queryClient.countLinksFromDomain(domainId));
builder.setIncomingLinks(domainLinksClient.countLinksToDomain(domainId));
builder.setOutboundLinks(domainLinksClient.countLinksFromDomain(domainId));
rs = stmt.executeQuery(STR."""
SELECT KNOWN_URLS, GOOD_URLS, VISITED_URLS FROM DOMAIN_METADATA WHERE ID=\{domainId}

18
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java → code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.domains;
package nu.marginalia.functions.domains;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
@ -8,10 +8,10 @@ import gnu.trove.map.hash.TIntDoubleHashMap;
import gnu.trove.map.hash.TIntIntHashMap;
import gnu.trove.set.TIntSet;
import gnu.trove.set.hash.TIntHashSet;
import nu.marginalia.assistant.api.RpcSimilarDomain;
import nu.marginalia.assistant.client.model.SimilarDomain;
import nu.marginalia.api.domains.*;
import nu.marginalia.api.domains.model.SimilarDomain;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.query.client.QueryClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -27,7 +27,7 @@ public class SimilarDomainsService {
private static final Logger logger = LoggerFactory.getLogger(SimilarDomainsService.class);
private final HikariDataSource dataSource;
private final QueryClient queryClient;
private final AggregateDomainLinksClient domainLinksClient;
private volatile TIntIntHashMap domainIdToIdx = new TIntIntHashMap(100_000);
private volatile int[] domainIdxToId;
@ -43,9 +43,9 @@ public class SimilarDomainsService {
volatile boolean isReady = false;
@Inject
public SimilarDomainsService(HikariDataSource dataSource, QueryClient queryClient) {
public SimilarDomainsService(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) {
this.dataSource = dataSource;
this.queryClient = queryClient;
this.domainLinksClient = domainLinksClient;
Executors.newSingleThreadExecutor().submit(this::init);
}
@ -256,7 +256,7 @@ public class SimilarDomainsService {
private TIntSet getLinkingIdsDToS(int domainIdx) {
var items = new TIntHashSet();
for (int id : queryClient.getLinksFromDomain(domainIdxToId[domainIdx])) {
for (int id : domainLinksClient.getLinksFromDomain(domainIdxToId[domainIdx])) {
items.add(domainIdToIdx.get(id));
}
@ -266,7 +266,7 @@ public class SimilarDomainsService {
private TIntSet getLinkingIdsSToD(int domainIdx) {
var items = new TIntHashSet();
for (int id : queryClient.getLinksToDomain(domainIdxToId[domainIdx])) {
for (int id : domainLinksClient.getLinksToDomain(domainIdxToId[domainIdx])) {
items.add(domainIdToIdx.get(id));
}

36
code/functions/domain-links/aggregate/build.gradle Normal file
View File

@ -0,0 +1,36 @@
plugins {
id 'java'
id 'application'
id 'jvm-test-suite'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
dependencies {
implementation project(':code:functions:domain-links:api')
implementation project(':code:common:config')
implementation project(':code:common:service')
implementation project(':code:common:model')
implementation project(':code:common:service-discovery')
implementation libs.bundles.slf4j
implementation libs.prometheus
implementation libs.bundles.grpc
implementation libs.notnull
implementation libs.guice
implementation libs.fastutil
implementation libs.bundles.mariadb
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

96
code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java Normal file
View File

@ -0,0 +1,96 @@
package nu.marginalia.functions.domainlinks;
import com.google.inject.Inject;
import io.grpc.stub.StreamObserver;
import nu.marginalia.api.domainlink.*;
import nu.marginalia.api.indexdomainlinks.PartitionDomainLinksClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase {
private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksService.class);
private final PartitionDomainLinksClient client;
@Inject
public AggregateDomainLinksService(PartitionDomainLinksClient client) {
this.client = client;
}
@Override
public void getAllLinks(Empty request,
StreamObserver<RpcDomainIdPairs> responseObserver) {
client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks)
.run(Empty.getDefaultInstance())
.forEach(iter -> iter.forEachRemaining(responseObserver::onNext));
responseObserver.onCompleted();
}
@Override
public void getLinksFromDomain(RpcDomainId request,
StreamObserver<RpcDomainIdList> responseObserver) {
var rspBuilder = RpcDomainIdList.newBuilder();
client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain)
.run(request)
.stream()
.map(RpcDomainIdList::getDomainIdList)
.flatMap(List::stream)
.forEach(rspBuilder::addDomainId);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
@Override
public void getLinksToDomain(RpcDomainId request,
StreamObserver<RpcDomainIdList> responseObserver) {
var rspBuilder = RpcDomainIdList.newBuilder();
client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain)
.run(request)
.stream()
.map(RpcDomainIdList::getDomainIdList)
.flatMap(List::stream)
.forEach(rspBuilder::addDomainId);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
@Override
public void countLinksFromDomain(RpcDomainId request,
StreamObserver<RpcDomainIdCount> responseObserver) {
int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain)
.run(request)
.stream()
.mapToInt(RpcDomainIdCount::getIdCount)
.sum();
var rspBuilder = RpcDomainIdCount.newBuilder();
rspBuilder.setIdCount(sum);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
@Override
public void countLinksToDomain(RpcDomainId request,
StreamObserver<RpcDomainIdCount> responseObserver) {
int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain)
.run(request)
.stream()
.mapToInt(RpcDomainIdCount::getIdCount)
.sum();
var rspBuilder = RpcDomainIdCount.newBuilder();
rspBuilder.setIdCount(sum);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
}

46
code/functions/domain-links/api/build.gradle Normal file
View File

@ -0,0 +1,46 @@
plugins {
id 'java'
id "com.google.protobuf" version "0.9.4"
id 'jvm-test-suite'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
jar.archiveBaseName = 'index-domain-links-api'
sourceSets {
main {
proto {
srcDir 'src/main/protobuf'
}
}
}
apply from: "$rootProject.projectDir/protobuf.gradle"
dependencies {
implementation project(':code:common:model')
implementation project(':code:common:config')
implementation project(':code:common:service-discovery')
implementation libs.bundles.slf4j
implementation libs.prometheus
implementation libs.notnull
implementation libs.guice
implementation libs.gson
implementation libs.protobuf
implementation libs.roaringbitmap
implementation libs.javax.annotation
implementation libs.bundles.grpc
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

138
code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java Normal file
View File

@ -0,0 +1,138 @@
package nu.marginalia.api.indexdomainlinks;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.api.domainlink.DomainLinksApiGrpc;
import nu.marginalia.api.domainlink.Empty;
import nu.marginalia.api.domainlink.RpcDomainId;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import org.roaringbitmap.longlong.PeekableLongIterator;
import org.roaringbitmap.longlong.Roaring64Bitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.List;
@Singleton
public class AggregateDomainLinksClient {
private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksClient.class);
private final GrpcSingleNodeChannelPool<DomainLinksApiGrpc.DomainLinksApiBlockingStub> channelPool;
@Inject
public AggregateDomainLinksClient(GrpcChannelPoolFactory factory) {
this.channelPool = factory.createSingle(
ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.any()),
DomainLinksApiGrpc::newBlockingStub);
}
public AllLinks getAllDomainLinks() {
AllLinks links = new AllLinks();
channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks)
.run(Empty.getDefaultInstance())
.forEachRemaining(pairs -> {
for (int i = 0; i < pairs.getDestIdsCount(); i++) {
links.add(pairs.getSourceIds(i), pairs.getDestIds(i));
}
});
return links;
}
public List<Integer> getLinksToDomain(int domainId) {
try {
return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain)
.run(RpcDomainId.newBuilder().setDomainId(domainId).build())
.getDomainIdList()
.stream()
.sorted()
.toList();
}
catch (Exception e) {
logger.error("API Exception", e);
return List.of();
}
}
public List<Integer> getLinksFromDomain(int domainId) {
try {
return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain)
.run(RpcDomainId.newBuilder().setDomainId(domainId).build())
.getDomainIdList()
.stream()
.sorted()
.toList();
}
catch (Exception e) {
logger.error("API Exception", e);
return List.of();
}
}
public int countLinksToDomain(int domainId) {
try {
return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain)
.run(RpcDomainId.newBuilder().setDomainId(domainId).build())
.getIdCount();
}
catch (Exception e) {
logger.error("API Exception", e);
return 0;
}
}
public int countLinksFromDomain(int domainId) {
try {
return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain)
.run(RpcDomainId.newBuilder().setDomainId(domainId).build())
.getIdCount();
}
catch (Exception e) {
logger.error("API Exception", e);
return 0;
}
}
public boolean waitReady(Duration duration) throws InterruptedException {
return channelPool.awaitChannel(duration);
}
public static class AllLinks {
private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap();
public void add(int source, int dest) {
sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest));
}
public Iterator iterator() {
return new Iterator();
}
public class Iterator {
private final PeekableLongIterator base = sourceToDest.getLongIterator();
long val = Long.MIN_VALUE;
public boolean advance() {
if (base.hasNext()) {
val = base.next();
return true;
}
return false;
}
public int source() {
return (int) (val >>> 32);
}
public int dest() {
return (int) (val & 0xFFFF_FFFFL);
}
}
}
}

30
code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java Normal file
View File

@ -0,0 +1,30 @@
package nu.marginalia.api.indexdomainlinks;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.api.domainlink.DomainLinksApiGrpc;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class PartitionDomainLinksClient {
private static final Logger logger = LoggerFactory.getLogger(PartitionDomainLinksClient.class);
private final GrpcMultiNodeChannelPool<DomainLinksApiGrpc.DomainLinksApiBlockingStub> channelPool;
@Inject
public PartitionDomainLinksClient(GrpcChannelPoolFactory factory) {
this.channelPool = factory.createMulti(
ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.multi()),
DomainLinksApiGrpc::newBlockingStub);
}
public GrpcMultiNodeChannelPool<DomainLinksApiGrpc.DomainLinksApiBlockingStub> getChannelPool() {
return channelPool;
}
}

29
code/functions/domain-links/api/src/main/protobuf/domain-links.proto Normal file
View File

@ -0,0 +1,29 @@
syntax="proto3";
package nu.marginalia.api.domainlinks;
option java_package="nu.marginalia.api.domainlink";
option java_multiple_files=true;
service DomainLinksApi {
rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {}
rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {}
rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {}
rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {}
rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {}
}
message RpcDomainId {
int32 domainId = 1;
}
message RpcDomainIdList {
repeated int32 domainId = 1 [packed=true];
}
message RpcDomainIdCount {
int32 idCount = 1;
}
message RpcDomainIdPairs {
repeated int32 sourceIds = 1 [packed=true];
repeated int32 destIds = 2 [packed=true];
}
message Empty {}

42
code/functions/domain-links/partition/build.gradle Normal file
View File

@ -0,0 +1,42 @@
plugins {
id 'java'
id 'application'
id 'jvm-test-suite'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
dependencies {
implementation project(':code:functions:domain-links:api')
implementation project(':code:common:config')
implementation project(':code:common:service')
implementation project(':code:common:model')
implementation project(':code:common:linkdb')
implementation project(':code:common:db')
implementation project(':code:common:service-discovery')
implementation libs.bundles.slf4j
implementation libs.prometheus
implementation libs.bundles.grpc
implementation libs.notnull
implementation libs.guice
implementation libs.spark
implementation libs.opencsv
implementation libs.trove
implementation libs.fastutil
implementation libs.bundles.gson
implementation libs.bundles.mariadb
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

13
code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java → code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java
View File

@ -1,22 +1,23 @@
package nu.marginalia.index.svc;
package nu.marginalia.functions.domainlinks;
import com.google.inject.Inject;
import io.grpc.stub.StreamObserver;
import nu.marginalia.index.api.*;
import nu.marginalia.api.domainlink.Empty;
import nu.marginalia.api.domainlink.*;
import nu.marginalia.linkdb.dlinks.DomainLinkDb;
/** GRPC service for interrogating domain links
*/
public class IndexDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase {
public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase {
private final DomainLinkDb domainLinkDb;
@Inject
public IndexDomainLinksService(DomainLinkDb domainLinkDb) {
public PartitionDomainLinksService(DomainLinkDb domainLinkDb) {
this.domainLinkDb = domainLinkDb;
}
public void getAllLinks(nu.marginalia.index.api.Empty request,
io.grpc.stub.StreamObserver<nu.marginalia.index.api.RpcDomainIdPairs> responseObserver) {
public void getAllLinks(Empty request,
io.grpc.stub.StreamObserver<RpcDomainIdPairs> responseObserver) {
try (var idsConverter = new AllIdsResponseConverter(responseObserver)) {
domainLinkDb.forEach(idsConverter::accept);

2
code/api/assistant-api/build.gradle → code/functions/math/api/build.gradle
View File

@ -11,6 +11,8 @@ java {
}
}
jar.archiveBaseName = 'math-api'
sourceSets {
main {
proto {

90
code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java Normal file
View File

@ -0,0 +1,90 @@
package nu.marginalia.api.math;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
import nu.marginalia.service.discovery.property.ServiceKey;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.service.id.ServiceId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import nu.marginalia.api.math.model.*;
import nu.marginalia.api.math.MathProtobufCodec.*;
@Singleton
public class MathClient {
private static final Logger logger = LoggerFactory.getLogger(MathClient.class);
private final GrpcSingleNodeChannelPool<MathApiGrpc.MathApiBlockingStub> channelPool;
private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
@Inject
public MathClient(GrpcChannelPoolFactory factory) {
this.channelPool = factory.createSingle(
ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()),
MathApiGrpc::newBlockingStub);
}
public Future<DictionaryResponse> dictionaryLookup(String word) {
return channelPool.call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup)
.async(virtualExecutorService)
.run(DictionaryLookup.createRequest(word))
.thenApply(DictionaryLookup::convertResponse);
}
@SuppressWarnings("unchecked")
public Future<List<String>> spellCheck(String word) {
return channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck)
.async(virtualExecutorService)
.run(SpellCheck.createRequest(word))
.thenApply(SpellCheck::convertResponse);
}
public Map<String, List<String>> spellCheck(List<String> words, Duration timeout) throws InterruptedException {
List<RpcSpellCheckRequest> requests = words.stream().map(SpellCheck::createRequest).toList();
var future = channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck)
.async(virtualExecutorService)
.runFor(requests);
try {
var results = future.get();
Map<String, List<String>> map = new HashMap<>();
for (int i = 0; i < words.size(); i++) {
map.put(words.get(i), SpellCheck.convertResponse(results.get(i)));
}
return map;
}
catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
public Future<String> unitConversion(String value, String from, String to) {
return channelPool.call(MathApiGrpc.MathApiBlockingStub::unitConversion)
.async(virtualExecutorService)
.run(UnitConversion.createRequest(from, to, value))
.thenApply(UnitConversion::convertResponse);
}
public Future<String> evalMath(String expression) {
return channelPool.call(MathApiGrpc.MathApiBlockingStub::evalMath)
.async(virtualExecutorService)
.run(EvalMath.createRequest(expression))
.thenApply(EvalMath::convertResponse);
}
public boolean isAccepting() {
return channelPool.hasChannel();
}
}

66
code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java Normal file
View File

@ -0,0 +1,66 @@
package nu.marginalia.api.math;
import nu.marginalia.api.math.model.DictionaryEntry;
import nu.marginalia.api.math.model.DictionaryResponse;
import java.util.List;
public class MathProtobufCodec {
public static class DictionaryLookup {
public static RpcDictionaryLookupRequest createRequest(String word) {
return RpcDictionaryLookupRequest.newBuilder()
.setWord(word)
.build();
}
public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) {
return new DictionaryResponse(
rsp.getWord(),
rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList()
);
}
private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) {
return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition());
}
}
public static class SpellCheck {
public static RpcSpellCheckRequest createRequest(String text) {
return RpcSpellCheckRequest.newBuilder()
.setText(text)
.build();
}
public static List<String> convertResponse(RpcSpellCheckResponse rsp) {
return rsp.getSuggestionsList();
}
}
public static class UnitConversion {
public static RpcUnitConversionRequest createRequest(String from, String to, String unit) {
return RpcUnitConversionRequest.newBuilder()
.setFrom(from)
.setTo(to)
.setUnit(unit)
.build();
}
public static String convertResponse(RpcUnitConversionResponse rsp) {
return rsp.getResult();
}
}
public static class EvalMath {
public static RpcEvalMathRequest createRequest(String expression) {
return RpcEvalMathRequest.newBuilder()
.setExpression(expression)
.build();
}
public static String convertResponse(RpcEvalMathResponse rsp) {
return rsp.getResult();
}
}
}

2
code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java → code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.client.model;
package nu.marginalia.api.math.model;
import lombok.AllArgsConstructor;
import lombok.Getter;

2
code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java → code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.client.model;
package nu.marginalia.api.math.model;
import lombok.AllArgsConstructor;
import lombok.Getter;

57
code/functions/math/api/src/main/protobuf/math-api.proto Normal file
View File

@ -0,0 +1,57 @@
syntax="proto3";
package nu.marginalia.api.math;
option java_package="nu.marginalia.api.math";
option java_multiple_files=true;
service MathApi {
/** Looks up a word in the dictionary. */
rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {}
/** Checks the spelling of a text. */
rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {}
/** Converts a unit from one to another. */
rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {}
/** Evaluates a mathematical expression. */
rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {}
}
message RpcDictionaryLookupRequest {
string word = 1;
}
message RpcDictionaryLookupResponse {
string word = 1;
repeated RpcDictionaryEntry entries = 2;
}
message RpcDictionaryEntry {
string type = 1;
string word = 2;
string definition = 3;
}
message RpcSpellCheckRequest {
string text = 1;
}
message RpcSpellCheckResponse {
repeated string suggestions = 1;
}
message RpcUnitConversionRequest {
string unit = 1;
string from = 2;
string to = 3;
}
message RpcUnitConversionResponse {
string result = 1;
}
message RpcEvalMathRequest {
string expression = 1;
}
message RpcEvalMathResponse {
string result = 1;
}

32
code/functions/math/build.gradle Normal file
View File

@ -0,0 +1,32 @@
plugins {
id 'java'
id 'jvm-test-suite'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
dependencies {
implementation project(':third-party:symspell')
implementation project(':code:functions:math:api')
implementation libs.bundles.slf4j
implementation libs.prometheus
implementation libs.bundles.grpc
implementation libs.notnull
implementation libs.guice
implementation libs.spark
implementation libs.opencsv
implementation libs.trove
implementation libs.fastutil
implementation libs.bundles.gson
implementation libs.bundles.mariadb
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

57
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java → code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java
View File

@ -1,34 +1,28 @@
package nu.marginalia.assistant;
package nu.marginalia.functions.math;
import com.google.inject.Inject;
import io.grpc.stub.StreamObserver;
import nu.marginalia.assistant.api.*;
import nu.marginalia.assistant.dict.DictionaryService;
import nu.marginalia.assistant.dict.SpellChecker;
import nu.marginalia.assistant.domains.DomainInformationService;
import nu.marginalia.assistant.domains.SimilarDomainsService;
import nu.marginalia.assistant.eval.MathParser;
import nu.marginalia.assistant.eval.Units;
import nu.marginalia.api.math.*;
import nu.marginalia.functions.math.dict.DictionaryService;
import nu.marginalia.functions.math.dict.SpellChecker;
import nu.marginalia.functions.math.eval.MathParser;
import nu.marginalia.functions.math.eval.Units;
public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase {
public class MathGrpcService extends MathApiGrpc.MathApiImplBase {
private final DictionaryService dictionaryService;
private final SpellChecker spellChecker;
private final Units units;
private final MathParser mathParser;
private final DomainInformationService domainInformationService;
private final SimilarDomainsService similarDomainsService;
@Inject
public AssistantGrpcService(DictionaryService dictionaryService,
SpellChecker spellChecker, Units units, MathParser mathParser, DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService)
public MathGrpcService(DictionaryService dictionaryService, SpellChecker spellChecker, Units units, MathParser mathParser)
{
this.dictionaryService = dictionaryService;
this.spellChecker = spellChecker;
this.units = units;
this.mathParser = mathParser;
this.domainInformationService = domainInformationService;
this.similarDomainsService = similarDomainsService;
}
@Override
@ -95,37 +89,4 @@ public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase
responseObserver.onCompleted();
}
@Override
public void getDomainInfo(RpcDomainId request, StreamObserver<RpcDomainInfoResponse> responseObserver) {
var ret = domainInformationService.domainInfo(request.getDomainId());
ret.ifPresent(responseObserver::onNext);
responseObserver.onCompleted();
}
@Override
public void getSimilarDomains(RpcDomainLinksRequest request,
StreamObserver<RpcSimilarDomains> responseObserver) {
var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount());
var responseBuilder = RpcSimilarDomains
.newBuilder()
.addAllDomains(ret);
responseObserver.onNext(responseBuilder.build());
responseObserver.onCompleted();
}
@Override
public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver<RpcSimilarDomains> responseObserver) {
var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount());
var responseBuilder = RpcSimilarDomains
.newBuilder()
.addAllDomains(ret);
responseObserver.onNext(responseBuilder.build());
responseObserver.onCompleted();
}
}

6
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java → code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java
View File

@ -1,10 +1,10 @@
package nu.marginalia.assistant.dict;
package nu.marginalia.functions.math.dict;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.assistant.client.model.DictionaryEntry;
import nu.marginalia.assistant.client.model.DictionaryResponse;
import nu.marginalia.api.math.model.DictionaryEntry;
import nu.marginalia.api.math.model.DictionaryResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java → code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.dict;
package nu.marginalia.functions.math.dict;
import com.google.inject.Singleton;
import symspell.SymSpell;

2
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java → code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.eval;
package nu.marginalia.functions.math.eval;
import lombok.AllArgsConstructor;
import lombok.SneakyThrows;

2
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java → code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.eval;
package nu.marginalia.functions.math.eval;
public class Unit {

2
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java → code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java
View File

@ -1,4 +1,4 @@
package nu.marginalia.assistant.eval;
package nu.marginalia.functions.math.eval;
import com.opencsv.CSVReader;
import lombok.SneakyThrows;

8
code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java
View File

@ -314,6 +314,10 @@ public class MqPersistence {
*/
public Collection<MqMessage> pollInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
if (dataSource.isClosed()) {
return Collections.emptyList();
}
// Mark new messages as claimed
int expected = markInboxMessages(inboxName, instanceUUID, tick, n);
if (expected == 0) {
@ -366,6 +370,10 @@ public class MqPersistence {
*/
public Collection<MqMessage> pollReplyInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
if (dataSource.isClosed()) {
return Collections.emptyList();
}
// Mark new messages as claimed
int expected = markInboxMessages(inboxName, instanceUUID, tick, n);
if (expected == 0) {

1
code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java
View File

@ -76,6 +76,7 @@ public class IndexConstructorMain extends ProcessMainClass {
// Grace period so we don't rug pull the logger or jdbc
TimeUnit.SECONDS.sleep(5);
System.exit(0);
}

2
code/processes/website-adjacencies-calculator/build.gradle
View File

@ -23,7 +23,7 @@ dependencies {
implementation project(':code:common:process')
implementation project(':code:common:service-discovery')
implementation project(':code:common:service')
implementation project(':code:api:query-api')
implementation project(':code:functions:domain-links:api')
implementation libs.bundles.slf4j

7
code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java
View File

@ -4,7 +4,7 @@ import gnu.trove.list.TIntList;
import gnu.trove.list.array.TIntArrayList;
import gnu.trove.map.hash.TIntObjectHashMap;
import gnu.trove.set.hash.TIntHashSet;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import org.roaringbitmap.RoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -35,14 +35,15 @@ public class AdjacenciesData {
return ret;
}
public AdjacenciesData(QueryClient queryClient,
public AdjacenciesData(AggregateDomainLinksClient linksClient,
DomainAliases aliases) {
logger.info("Loading adjacency data");
Map<Integer, RoaringBitmap> tmpMapDtoS = new HashMap<>(100_000);
int count = 0;
var allLinks = queryClient.getAllDomainLinks();
var allLinks = linksClient.getAllDomainLinks();
for (var iter = allLinks.iterator();;count++) {
if (!iter.advance()) {
break;

20
code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java
View File

@ -4,18 +4,20 @@ import com.google.inject.Guice;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows;
import nu.marginalia.ProcessConfiguration;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import nu.marginalia.db.DbDomainQueries;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.process.control.ProcessHeartbeat;
import nu.marginalia.process.control.ProcessHeartbeatImpl;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.ProcessMainClass;
import nu.marginalia.service.ServiceDiscoveryModule;
import nu.marginalia.service.module.DatabaseModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
@ -24,18 +26,18 @@ import java.util.stream.IntStream;
import static nu.marginalia.adjacencies.SparseBitVector.*;
public class WebsiteAdjacenciesCalculator extends MainClass {
public class WebsiteAdjacenciesCalculator extends ProcessMainClass {
private final HikariDataSource dataSource;
public AdjacenciesData adjacenciesData;
public DomainAliases domainAliases;
private static final Logger logger = LoggerFactory.getLogger(WebsiteAdjacenciesCalculator.class);
float[] weights;
public WebsiteAdjacenciesCalculator(QueryClient queryClient, HikariDataSource dataSource) throws SQLException {
public WebsiteAdjacenciesCalculator(AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) throws SQLException {
this.dataSource = dataSource;
domainAliases = new DomainAliases(dataSource);
adjacenciesData = new AdjacenciesData(queryClient, domainAliases);
adjacenciesData = new AdjacenciesData(domainLinksClient, domainAliases);
weights = adjacenciesData.getWeights();
}
@ -146,16 +148,20 @@ public class WebsiteAdjacenciesCalculator extends MainClass {
}
public static void main(String[] args) throws SQLException {
public static void main(String[] args) throws SQLException, InterruptedException {
var injector = Guice.createInjector(
new DatabaseModule(false),
new ServiceDiscoveryModule());
var dataSource = injector.getInstance(HikariDataSource.class);
var qc = injector.getInstance(QueryClient.class);
var lc = injector.getInstance(AggregateDomainLinksClient.class);
var main = new WebsiteAdjacenciesCalculator(qc, dataSource);
if (!lc.waitReady(Duration.ofSeconds(30))) {
throw new IllegalStateException("Failed to connect to domain-links");
}
var main = new WebsiteAdjacenciesCalculator(lc, dataSource);
if (args.length == 1 && "load".equals(args[0])) {
var processHeartbeat = new ProcessHeartbeatImpl(

18
code/services-application/api-service/build.gradle
View File

@ -2,8 +2,8 @@ plugins {
id 'java'
id 'application'
id 'com.palantir.docker' version '0.35.0'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
java {
@ -19,7 +19,21 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
dependencies {
implementation project(':code:common:db')

19
code/services-application/dating-service/build.gradle
View File

@ -2,8 +2,8 @@ plugins {
id 'java'
id 'application'
id 'com.palantir.docker' version '0.35.0'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
application {
@ -13,7 +13,22 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
java {
toolchain {

19
code/services-application/explorer-service/build.gradle
View File

@ -2,8 +2,8 @@ plugins {
id 'java'
id 'application'
id 'com.palantir.docker' version '0.35.0'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
application {
@ -13,7 +13,22 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
java {
toolchain {

23
code/services-application/search-service/build.gradle
View File

@ -2,9 +2,25 @@ plugins {
id 'java'
id 'io.freefair.sass-base' version '8.4'
id 'io.freefair.sass-java' version '8.4'
id 'com.palantir.docker' version '0.35.0'
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
application {
@ -14,7 +30,6 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
java {
toolchain {
@ -26,6 +41,7 @@ sass {
sourceMapEmbed = true
outputStyle = EXPANDED
}
dependencies {
implementation project(':code:common:db')
implementation project(':code:common:model')
@ -38,7 +54,8 @@ dependencies {
implementation project(':code:libraries:braille-block-punch-cards')
implementation project(':code:libraries:term-frequency-dict')
implementation project(':code:api:assistant-api')
implementation project(':code:functions:math:api')
implementation project(':code:functions:domain-info:api')
implementation project(':code:api:query-api')
implementation project(':code:api:index-api')
implementation project(':code:common:service-discovery')

10
code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java
View File

@ -4,7 +4,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import lombok.SneakyThrows;
import nu.marginalia.WebsiteUrl;
import nu.marginalia.assistant.client.AssistantClient;
import nu.marginalia.api.math.MathClient;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.db.DbDomainQueries;
import nu.marginalia.query.client.QueryClient;
@ -34,7 +34,7 @@ public class SearchOperator {
// Marker for filtering out sensitive content from the persistent logs
private final Marker queryMarker = MarkerFactory.getMarker("QUERY");
private final AssistantClient assistantClient;
private final MathClient mathClient;
private final DbDomainQueries domainQueries;
private final QueryClient queryClient;
private final SearchQueryIndexService searchQueryService;
@ -44,7 +44,7 @@ public class SearchOperator {
@Inject
public SearchOperator(AssistantClient assistantClient,
public SearchOperator(MathClient mathClient,
DbDomainQueries domainQueries,
QueryClient queryClient,
SearchQueryIndexService searchQueryService,
@ -53,7 +53,7 @@ public class SearchOperator {
SearchUnitConversionService searchUnitConversionService)
{
this.assistantClient = assistantClient;
this.mathClient = mathClient;
this.domainQueries = domainQueries;
this.queryClient = queryClient;
@ -162,7 +162,7 @@ public class SearchOperator {
@SneakyThrows
private void spellCheckTerms(QueryResponse response) {
var suggestions = assistantClient
var suggestions = mathClient
.spellCheck(response.searchTermsHuman(), Duration.ofMillis(20));
suggestions.entrySet()

16
code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java
View File

@ -2,11 +2,11 @@
package nu.marginalia.search.command.commands;
import com.google.inject.Inject;
import nu.marginalia.assistant.client.AssistantClient;
import nu.marginalia.assistant.client.model.DictionaryResponse;
import nu.marginalia.api.math.MathClient;
import nu.marginalia.api.math.model.DictionaryResponse;
import nu.marginalia.renderer.MustacheRenderer;
import nu.marginalia.search.command.SearchCommandInterface;
import nu.marginalia.search.command.SearchParameters;
import nu.marginalia.renderer.MustacheRenderer;
import nu.marginalia.renderer.RendererFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -23,18 +23,18 @@ public class DefinitionCommand implements SearchCommandInterface {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final MustacheRenderer<DictionaryResponse> dictionaryRenderer;
private final AssistantClient assistantClient;
private final MathClient mathClient;
private final Predicate<String> queryPatternPredicate = Pattern.compile("^define:[A-Za-z\\s-0-9]+$").asPredicate();
@Inject
public DefinitionCommand(RendererFactory rendererFactory, AssistantClient assistantClient)
public DefinitionCommand(RendererFactory rendererFactory, MathClient mathClient)
throws IOException
{
dictionaryRenderer = rendererFactory.renderer("search/dictionary-results");
this.assistantClient = assistantClient;
this.mathClient = mathClient;
}
@Override
@ -57,9 +57,9 @@ public class DefinitionCommand implements SearchCommandInterface {
String word = humanQuery.substring(definePrefix.length()).toLowerCase();
try {
return assistantClient
return mathClient
.dictionaryLookup(word)
.get(100, TimeUnit.MILLISECONDS);
.get(250, TimeUnit.MILLISECONDS);
}
catch (Exception e) {
logger.error("Failed to lookup definition for word: " + word, e);

8
code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java
View File

@ -14,17 +14,13 @@ import java.io.IOException;
import java.util.Optional;
public class SearchCommand implements SearchCommandInterface {
private final DomainBlacklist blacklist;
private final SearchOperator searchOperator;
private final MustacheRenderer<DecoratedSearchResults> searchResultsRenderer;
@Inject
public SearchCommand(DomainBlacklist blacklist,
SearchOperator searchOperator,
RendererFactory rendererFactory
) throws IOException {
this.blacklist = blacklist;
public SearchCommand(SearchOperator searchOperator,
RendererFactory rendererFactory) throws IOException {
this.searchOperator = searchOperator;
searchResultsRenderer = rendererFactory.renderer("search/search-results");

14
code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java
View File

@ -1,8 +1,8 @@
package nu.marginalia.search.svc;
import com.google.inject.Inject;
import nu.marginalia.assistant.client.AssistantClient;
import nu.marginalia.assistant.client.model.SimilarDomain;
import nu.marginalia.api.domains.DomainInfoClient;
import nu.marginalia.api.domains.model.SimilarDomain;
import nu.marginalia.browse.DbBrowseDomainsRandom;
import nu.marginalia.browse.model.BrowseResult;
import nu.marginalia.browse.model.BrowseResultSet;
@ -25,20 +25,20 @@ public class SearchBrowseService {
private final DbBrowseDomainsRandom randomDomains;
private final DbDomainQueries domainQueries;
private final DomainBlacklist blacklist;
private final AssistantClient assistantClient;
private final DomainInfoClient domainInfoClient;
private final BrowseResultCleaner browseResultCleaner;
@Inject
public SearchBrowseService(DbBrowseDomainsRandom randomDomains,
DbDomainQueries domainQueries,
DomainBlacklist blacklist,
AssistantClient assistantClient,
DomainInfoClient domainInfoClient,
BrowseResultCleaner browseResultCleaner)
{
this.randomDomains = randomDomains;
this.domainQueries = domainQueries;
this.blacklist = blacklist;
this.assistantClient = assistantClient;
this.domainInfoClient = domainInfoClient;
this.browseResultCleaner = browseResultCleaner;
}
@ -53,7 +53,7 @@ public class SearchBrowseService {
public BrowseResultSet getRelatedEntries(String domainName) throws ExecutionException, InterruptedException, TimeoutException {
var domain = domainQueries.getDomainId(new EdgeDomain(domainName));
var neighbors = assistantClient.similarDomains(domain, 50)
var neighbors = domainInfoClient.similarDomains(domain, 50)
.get(100, TimeUnit.MILLISECONDS);
neighbors.removeIf(sd -> !sd.screenshot());
@ -61,7 +61,7 @@ public class SearchBrowseService {
// If the results are very few, supplement with the alternative shitty algorithm
if (neighbors.size() < 25) {
Set<SimilarDomain> allNeighbors = new HashSet<>(neighbors);
allNeighbors.addAll(assistantClient
allNeighbors.addAll(domainInfoClient
.linkedDomains(domain, 50)
.get(100, TimeUnit.MILLISECONDS)
);

27
code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java
View File

@ -1,8 +1,9 @@
package nu.marginalia.search.svc;
import com.google.inject.Inject;
import nu.marginalia.assistant.client.AssistantClient;
import nu.marginalia.assistant.client.model.SimilarDomain;
import nu.marginalia.api.domains.DomainInfoClient;
import nu.marginalia.api.domains.model.DomainInformation;
import nu.marginalia.api.domains.model.SimilarDomain;
import nu.marginalia.db.DbDomainQueries;
import nu.marginalia.feedlot.model.FeedItems;
import nu.marginalia.model.EdgeDomain;
@ -10,7 +11,6 @@ import nu.marginalia.renderer.MustacheRenderer;
import nu.marginalia.renderer.RendererFactory;
import nu.marginalia.screenshot.ScreenshotService;
import nu.marginalia.search.SearchOperator;
import nu.marginalia.assistant.client.model.DomainInformation;
import nu.marginalia.feedlot.FeedlotClient;
import nu.marginalia.search.model.UrlDetails;
import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData;
@ -32,7 +32,7 @@ public class SearchSiteInfoService {
private static final Logger logger = LoggerFactory.getLogger(SearchSiteInfoService.class);
private final SearchOperator searchOperator;
private final AssistantClient assistantClient;
private final DomainInfoClient domainInfoClient;
private final SearchFlagSiteService flagSiteService;
private final DbDomainQueries domainQueries;
private final MustacheRenderer<Object> renderer;
@ -41,7 +41,7 @@ public class SearchSiteInfoService {
@Inject
public SearchSiteInfoService(SearchOperator searchOperator,
AssistantClient assistantClient,
DomainInfoClient domainInfoClient,
RendererFactory rendererFactory,
SearchFlagSiteService flagSiteService,
DbDomainQueries domainQueries,
@ -49,7 +49,7 @@ public class SearchSiteInfoService {
ScreenshotService screenshotService) throws IOException
{
this.searchOperator = searchOperator;
this.assistantClient = assistantClient;
this.domainInfoClient = domainInfoClient;
this.flagSiteService = flagSiteService;
this.domainQueries = domainQueries;
@ -137,15 +137,20 @@ public class SearchSiteInfoService {
boolean hasScreenshot = screenshotService.hasScreenshot(domainId);
var feedItemsFuture = feedlotClient.getFeedItems(domainName);
if (domainId < 0 || !assistantClient.isAccepting()) {
if (domainId < 0) {
domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
}
else if (!domainInfoClient.isAccepting()) {
domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
}
else {
domainInfoFuture = assistantClient.domainInformation(domainId);
similarSetFuture = assistantClient.similarDomains(domainId, 25);
linkingDomainsFuture = assistantClient.linkedDomains(domainId, 25);
domainInfoFuture = domainInfoClient.domainInformation(domainId);
similarSetFuture = domainInfoClient.similarDomains(domainId, 25);
linkingDomainsFuture = domainInfoClient.linkedDomains(domainId, 25);
}
List<UrlDetails> sampleResults = searchOperator.doSiteSearch(domainName, 5);
@ -174,7 +179,7 @@ public class SearchSiteInfoService {
private <T> T waitForFuture(Future<T> future, Supplier<T> fallback) {
try {
return future.get(50, TimeUnit.MILLISECONDS);
return future.get(250, TimeUnit.MILLISECONDS);
} catch (Exception e) {
logger.info("Failed to get domain data: {}", e.getMessage());
return fallback.get();

14
code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java
View File

@ -1,6 +1,6 @@
package nu.marginalia.search.svc;
import nu.marginalia.assistant.client.AssistantClient;
import nu.marginalia.api.math.MathClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -21,11 +21,11 @@ public class SearchUnitConversionService {
private final Pattern conversionPattern = Pattern.compile("((\\d+|\\s+|[.()\\-^+%*/]|log[^a-z]|log2[^a-z]|sqrt[^a-z]|log10|cos[^a-z]|sin[^a-z]|tan[^a-z]|log2|pi[^a-z]|e[^a-z]|2pi[^a-z])+)\\s*([a-zA-Z][a-zA-Z^.0-9]*\\s?[a-zA-Z^.0-9]*)\\s+in\\s+([a-zA-Z^.0-9]+\\s?[a-zA-Z^.0-9]*)");
private final Predicate<String> evalPredicate = Pattern.compile("(\\d+|\\s+|[.()\\-^+%*/]|log|log2|sqrt|log10|cos|sin|tan|pi|e|2pi)+").asMatchPredicate();
private final AssistantClient assistantClient;
private final MathClient mathClient;
@Inject
public SearchUnitConversionService(AssistantClient assistantClient) {
this.assistantClient = assistantClient;
public SearchUnitConversionService(MathClient mathClient) {
this.mathClient = mathClient;
}
public Optional<String> tryConversion(String query) {
@ -40,9 +40,9 @@ public class SearchUnitConversionService {
logger.info("{} -> '{}' '{}' '{}'", query, value, from, to);
try {
var resultFuture = assistantClient.unitConversion(value, from, to);
var resultFuture = mathClient.unitConversion(value, from, to);
return Optional.of(
resultFuture.get(100, TimeUnit.MILLISECONDS)
resultFuture.get(250, TimeUnit.MILLISECONDS)
);
} catch (ExecutionException e) {
logger.error("Error in unit conversion", e);
@ -68,6 +68,6 @@ public class SearchUnitConversionService {
logger.info("eval({})", expr);
return assistantClient.evalMath(expr);
return mathClient.evalMath(expr);
}
}

26
code/services-core/assistant-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.palantir.docker' version '0.35.0'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
application {
@ -13,7 +13,22 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
java {
toolchain {
@ -23,7 +38,12 @@ java {
dependencies {
implementation project(':third-party:symspell')
implementation project(':code:api:assistant-api')
implementation project(':code:functions:math')
implementation project(':code:functions:math:api')
implementation project(':code:functions:domain-info')
implementation project(':code:functions:domain-info:api')
implementation project(':code:api:query-api')
implementation project(':code:common:config')
implementation project(':code:common:service')

10
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java
View File

@ -4,8 +4,11 @@ import com.google.gson.Gson;
import com.google.inject.Inject;
import lombok.SneakyThrows;
import nu.marginalia.assistant.suggest.Suggestions;
import nu.marginalia.functions.domains.DomainInfoGrpcService;
import nu.marginalia.functions.math.MathGrpcService;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.screenshot.ScreenshotService;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.service.server.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -24,10 +27,13 @@ public class AssistantService extends Service {
@Inject
public AssistantService(BaseServiceParams params,
ScreenshotService screenshotService,
AssistantGrpcService assistantGrpcService,
DomainInfoGrpcService domainInfoGrpcService,
MathGrpcService mathGrpcService,
Suggestions suggestions)
{
super(params, List.of(assistantGrpcService));
super(params,
ServicePartition.any(),
List.of(domainInfoGrpcService, mathGrpcService));
this.suggestions = suggestions;

2
code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java
View File

@ -2,9 +2,9 @@ package nu.marginalia.assistant.suggest;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import nu.marginalia.functions.math.dict.SpellChecker;
import nu.marginalia.term_frequency_dict.TermFrequencyDict;
import nu.marginalia.model.crawl.HtmlFeature;
import nu.marginalia.assistant.dict.SpellChecker;
import org.apache.commons.collections4.trie.PatriciaTrie;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;

20
code/services-core/control-service/build.gradle
View File

@ -1,9 +1,8 @@
plugins {
id 'java'
id 'application'
id 'com.palantir.docker' version '0.35.0'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
java {
@ -19,7 +18,22 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
dependencies {
implementation libs.bundles.gson

1
code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java
View File

@ -60,6 +60,7 @@ public class ControlService extends Service {
) throws IOException {
super(params);
this.monitors = monitors;
this.heartbeatService = heartbeatService;
this.eventLogService = eventLogService;

32
code/services-core/executor-service/build.gradle
View File

@ -1,9 +1,9 @@
plugins {
id 'java'
id 'com.palantir.docker' version '0.35.0'
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
application {
@ -13,7 +13,24 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service-with-dist.gradle"
clean {
delete fileTree('build/dist-extra')
}
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
java {
toolchain {
@ -22,6 +39,15 @@ java {
}
dependencies {
// These look weird but they're needed to be able to spawn the processes
// from the executor service
implementation project(':code:processes:website-adjacencies-calculator')
implementation project(':code:processes:crawling-process')
implementation project(':code:processes:loading-process')
implementation project(':code:processes:converting-process')
implementation project(':code:processes:index-constructor-process')
implementation project(':code:common:config')
implementation project(':code:common:model')
implementation project(':code:common:process')
@ -35,6 +61,8 @@ dependencies {
implementation project(':code:libraries:message-queue')
implementation project(':code:functions:domain-links:api')
implementation project(':code:process-models:crawl-spec')
implementation project(':code:process-models:crawling-model')
implementation project(':code:features-crawl:link-parser')

10
code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java
View File

@ -6,6 +6,7 @@ import com.google.inject.Singleton;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.actor.prototype.RecordActorPrototype;
import nu.marginalia.actor.state.ActorStep;
import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorageId;
@ -32,7 +33,7 @@ public class ExportDataActor extends RecordActorPrototype {
private final FileStorageService storageService;
private final HikariDataSource dataSource;
private final Logger logger = LoggerFactory.getLogger(getClass());
private final QueryClient queryClient;
private final AggregateDomainLinksClient domainLinksClient;
public record Export() implements ActorStep {}
public record ExportBlacklist(FileStorageId fid) implements ActorStep {}
@ -114,7 +115,7 @@ public class ExportDataActor extends RecordActorPrototype {
var tmpFile = Files.createTempFile(storage.asPath(), "export", ".csv.gz",
PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
var allLinks = queryClient.getAllDomainLinks();
var allLinks = domainLinksClient.getAllDomainLinks();
try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))))
{
@ -154,12 +155,13 @@ public class ExportDataActor extends RecordActorPrototype {
@Inject
public ExportDataActor(Gson gson,
FileStorageService storageService,
HikariDataSource dataSource, QueryClient queryClient)
HikariDataSource dataSource,
AggregateDomainLinksClient domainLinksClient)
{
super(gson);
this.storageService = storageService;
this.dataSource = dataSource;
this.queryClient = queryClient;
this.domainLinksClient = domainLinksClient;
}
}

4
code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java
View File

@ -2,12 +2,14 @@ package nu.marginalia.executor;
import com.google.inject.AbstractModule;
import com.google.inject.name.Names;
import nu.marginalia.WmsaHome;
import java.nio.file.Path;
public class ExecutorModule extends AbstractModule {
public void configure() {
String dist = System.getProperty("distPath", System.getProperty("WMSA_HOME", "/var/lib/wmsa") + "/dist/current");
String dist = System.getProperty("distPath", WmsaHome.getHomePath().resolve("/dist").toString());
bind(Path.class).annotatedWith(Names.named("distPath")).toInstance(Path.of(dist));
}
}

13
code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java
View File

@ -1,10 +1,10 @@
package nu.marginalia.executor;
import com.google.gson.Gson;
import com.google.inject.Inject;
import nu.marginalia.actor.ExecutorActor;
import nu.marginalia.actor.ExecutorActorControlService;
import nu.marginalia.executor.svc.TransferService;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.service.server.BaseServiceParams;
import nu.marginalia.service.server.Service;
import nu.marginalia.service.server.mq.MqRequest;
@ -16,10 +16,7 @@ import java.util.List;
// Weird name for this one to not have clashes with java.util.concurrent.ExecutorService
public class ExecutorSvc extends Service {
private final BaseServiceParams params;
private final Gson gson;
private final ExecutorActorControlService actorControlService;
private final TransferService transferService;
private static final Logger logger = LoggerFactory.getLogger(ExecutorSvc.class);
@ -28,14 +25,12 @@ public class ExecutorSvc extends Service {
public ExecutorSvc(BaseServiceParams params,
ExecutorActorControlService actorControlService,
ExecutorGrpcService executorGrpcService,
Gson gson,
TransferService transferService)
{
super(params, List.of(executorGrpcService));
this.params = params;
this.gson = gson;
super(params,
ServicePartition.partition(params.configuration.node()),
List.of(executorGrpcService));
this.actorControlService = actorControlService;
this.transferService = transferService;
Spark.get("/transfer/file/:fid", transferService::transferFile);
}

90
code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java
View File

@ -3,6 +3,14 @@ package nu.marginalia.process;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import nu.marginalia.WmsaHome;
import nu.marginalia.adjacencies.WebsiteAdjacenciesCalculator;
import nu.marginalia.converting.ConverterMain;
import nu.marginalia.crawl.CrawlerMain;
import nu.marginalia.index.IndexConstructorMain;
import nu.marginalia.loading.LoaderMain;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.ProcessMainClass;
import nu.marginalia.service.control.ServiceEventLog;
import nu.marginalia.service.server.BaseServiceParams;
import org.slf4j.Logger;
@ -43,16 +51,32 @@ public class ProcessService {
}
public enum ProcessId {
CRAWLER("crawler-process/bin/crawler-process"),
CONVERTER("converter-process/bin/converter-process"),
LOADER("loader-process/bin/loader-process"),
INDEX_CONSTRUCTOR("index-construction-process/bin/index-construction-process"),
ADJACENCIES_CALCULATOR("website-adjacencies-calculator/bin/website-adjacencies-calculator")
CRAWLER(CrawlerMain.class),
CONVERTER(ConverterMain.class),
LOADER(LoaderMain.class),
INDEX_CONSTRUCTOR(IndexConstructorMain.class),
ADJACENCIES_CALCULATOR(WebsiteAdjacenciesCalculator.class)
;
public final String path;
ProcessId(String path) {
this.path = path;
public final String mainClass;
ProcessId(Class<? extends ProcessMainClass> mainClass) {
this.mainClass = mainClass.getName();
}
List<String> envOpts() {
String variable = switch (this) {
case CRAWLER -> "CRAWLER_PROCESS_OPTS";
case CONVERTER -> "CONVERTER_PROCESS_OPTS";
case LOADER -> "LOADER_PROCESS_OPTS";
case INDEX_CONSTRUCTOR -> "INDEX_CONSTRUCTION_PROCESS_OPTS";
case ADJACENCIES_CALCULATOR -> "ADJACENCIES_CALCULATOR_PROCESS_OPTS";
};
String value = System.getenv(variable);
if (value == null)
return List.of();
else
return Arrays.asList(value.split("\\s+"));
}
}
@ -63,27 +87,27 @@ public class ProcessService {
this.distPath = distPath;
}
public boolean trigger(ProcessId processId) throws Exception {
return trigger(processId, new String[0]);
}
public boolean trigger(ProcessId processId, String... parameters) throws Exception {
final String processPath = distPath.resolve(processId.path).toString();
public boolean trigger(ProcessId processId, String... extraArgs) throws Exception {
final String[] env = createEnvironmentVariables();
final String[] args = createCommandArguments(processPath, parameters);
List<String> args = new ArrayList<>();
String javaHome = System.getProperty("java.home");
args.add(STR."\{javaHome}/bin/java");
args.add("-cp");
args.add(System.getProperty("java.class.path"));
args.add("--enable-preview");
args.addAll(processId.envOpts());
args.add(processId.mainClass);
args.addAll(Arrays.asList(extraArgs));
Process process;
if (!Files.exists(Path.of(processPath))) {
logger.error("Process not found: {}", processPath);
return false;
}
logger.info("Starting process: {}: {} // {}", processId, Arrays.toString(args), Arrays.toString(env));
logger.info("Starting process: {} {}", processId, processId.envOpts());
synchronized (processes) {
if (processes.containsKey(processId)) return false;
process = Runtime.getRuntime().exec(args, env);
process = Runtime.getRuntime().exec(args.toArray(String[]::new), env);
processes.put(processId, process);
}
@ -107,13 +131,6 @@ public class ProcessService {
}
private String[] createCommandArguments(String processPath, String[] parameters) {
final String[] args = new String[parameters.length + 1];
args[0] = processPath;
System.arraycopy(parameters, 0, args, 1, parameters.length);
return args;
}
public boolean isRunning(ProcessId processId) {
return processes.containsKey(processId);
}
@ -131,25 +148,14 @@ public class ProcessService {
/** These environment variables are propagated from the parent process to the child process,
* along with WMSA_HOME, but it has special logic */
private final List<String> propagatedEnvironmentVariables = List.of(
"JAVA_HOME",
"ZOOKEEPER_HOSTS",
"WMSA_SERVICE_NODE",
"CONVERTER_PROCESS_OPTS",
"LOADER_PROCESS_OPTS",
"INDEX_CONSTRUCTION_PROCESS_OPTS",
"CRAWLER_PROCESS_OPTS");
"WMSA_SERVICE_NODE"
);
private String[] createEnvironmentVariables() {
List<String> opts = new ArrayList<>();
String WMSA_HOME = System.getenv("WMSA_HOME");
if (WMSA_HOME == null || WMSA_HOME.isBlank()) {
WMSA_HOME = "/var/lib/wmsa";
}
opts.add(env2str("WMSA_HOME", WMSA_HOME));
opts.add(env2str("JAVA_OPTS", "--enable-preview")); //
opts.add(env2str("WMSA_HOME", WmsaHome.getHomePath().toString()));
for (String envKey : propagatedEnvironmentVariables) {
String envValue = System.getenv(envKey);

24
code/services-core/index-service/build.gradle
View File

@ -1,9 +1,9 @@
plugins {
id 'java'
id 'com.palantir.docker' version '0.35.0'
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
application {
@ -13,7 +13,22 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
java {
toolchain {
@ -25,8 +40,13 @@ dependencies {
implementation project(':code:common:model')
implementation project(':code:common:db')
implementation project(':code:common:linkdb')
implementation project(':code:functions:domain-links:partition')
implementation project(':code:functions:domain-links:api')
implementation project(':code:common:service')
implementation project(':code:api:index-api')
implementation project(':code:common:service-discovery')
implementation project(':code:libraries:array')

11
code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java
View File

@ -4,8 +4,9 @@ import com.google.gson.Gson;
import com.google.inject.Inject;
import lombok.SneakyThrows;
import nu.marginalia.IndexLocations;
import nu.marginalia.index.svc.IndexDomainLinksService;
import nu.marginalia.functions.domainlinks.PartitionDomainLinksService;
import nu.marginalia.linkdb.dlinks.DomainLinkDb;
import nu.marginalia.service.discovery.property.ServicePartition;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.index.client.IndexMqEndpoints;
import nu.marginalia.index.index.SearchIndex;
@ -51,10 +52,14 @@ public class IndexService extends Service {
FileStorageService fileStorageService,
DocumentDbReader documentDbReader,
DomainLinkDb domainLinkDb,
IndexDomainLinksService indexDomainLinksService,
PartitionDomainLinksService partitionDomainLinksService,
ServiceEventLog eventLog)
{
super(params, List.of(indexQueryService, indexDomainLinksService));
super(params,
ServicePartition.partition(params.configuration.node()),
List.of(indexQueryService, partitionDomainLinksService));
this.opsService = opsService;
this.searchIndex = searchIndex;

22
code/services-core/query-service/build.gradle
View File

@ -1,9 +1,9 @@
plugins {
id 'java'
id 'com.palantir.docker' version '0.35.0'
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
application {
@ -13,7 +13,22 @@ application {
tasks.distZip.enabled = false
apply from: "$rootProject.projectDir/docker-service.gradle"
jib {
from {
image = image = rootProject.ext.dockerImageBase
}
to {
image = 'marginalia/'+project.name
tags = ['latest']
}
container {
mainClass = application.mainClass
jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
}
}
java {
toolchain {
@ -35,6 +50,9 @@ dependencies {
implementation project(':code:libraries:language-processing')
implementation project(':code:libraries:term-frequency-dict')
implementation project(':code:functions:domain-links:api')
implementation project(':code:functions:domain-links:aggregate')
implementation libs.bundles.slf4j
implementation libs.spark

90
code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java
View File

@ -1,90 +0,0 @@
package nu.marginalia.query;
import com.google.inject.Inject;
import io.grpc.stub.StreamObserver;
import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
import nu.marginalia.index.api.IndexDomainLinksApiGrpc;
import nu.marginalia.index.api.RpcDomainIdCount;
import nu.marginalia.index.api.RpcDomainIdList;
import nu.marginalia.index.api.RpcDomainIdPairs;
import nu.marginalia.service.client.GrpcChannelPoolFactory;
import nu.marginalia.service.id.ServiceId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class QueryGRPCDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase {
private static final Logger logger = LoggerFactory.getLogger(QueryGRPCDomainLinksService.class);
private final GrpcMultiNodeChannelPool<IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub> channelPool;
@Inject
public QueryGRPCDomainLinksService(GrpcChannelPoolFactory channelPoolFactory) {
channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub);
}
@Override
public void getAllLinks(nu.marginalia.index.api.Empty request,
StreamObserver<RpcDomainIdPairs> responseObserver) {
channelPool.callEachSequential(stub -> stub.getAllLinks(request))
.forEach(
iter -> iter.forEachRemaining(responseObserver::onNext)
);
responseObserver.onCompleted();
}
@Override
public void getLinksFromDomain(nu.marginalia.index.api.RpcDomainId request,
StreamObserver<RpcDomainIdList> responseObserver) {
var rspBuilder = RpcDomainIdList.newBuilder();
channelPool.callEachSequential(stub -> stub.getLinksFromDomain(request))
.map(RpcDomainIdList::getDomainIdList)
.forEach(rspBuilder::addAllDomainId);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
@Override
public void getLinksToDomain(nu.marginalia.index.api.RpcDomainId request,
StreamObserver<RpcDomainIdList> responseObserver) {
var rspBuilder = RpcDomainIdList.newBuilder();
channelPool.callEachSequential(stub -> stub.getLinksToDomain(request))
.map(RpcDomainIdList::getDomainIdList)
.forEach(rspBuilder::addAllDomainId);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
@Override
public void countLinksFromDomain(nu.marginalia.index.api.RpcDomainId request,
StreamObserver<RpcDomainIdCount> responseObserver) {
int sum = channelPool.callEachSequential(stub -> stub.countLinksFromDomain(request))
.mapToInt(RpcDomainIdCount::getIdCount)
.sum();
var rspBuilder = RpcDomainIdCount.newBuilder();
rspBuilder.setIdCount(sum);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
@Override
public void countLinksToDomain(nu.marginalia.index.api.RpcDomainId request,
io.grpc.stub.StreamObserver<nu.marginalia.index.api.RpcDomainIdCount> responseObserver) {
int sum = channelPool.callEachSequential(stub -> stub.countLinksToDomain(request))
.mapToInt(RpcDomainIdCount::getIdCount)
.sum();
var rspBuilder = RpcDomainIdCount.newBuilder();
rspBuilder.setIdCount(sum);
responseObserver.onNext(rspBuilder.build());
responseObserver.onCompleted();
}
}

Some files were not shown because too many files have changed in this diff Show More