161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

WIP: Killing off Archive service, adding new Encyclopedia service consisting largely of what Archive was and a few features from Assistant.

Browse source
This commit is contained in:
vlofgren 2022-05-28 00:16:31 +02:00
parent e7b4ac0d34
commit ad4521da9e
10 changed files with 157 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java Normal file
View file

@ -0,0 +1,70 @@
package nu.marginalia.wmsa.edge;
import nu.marginalia.wmsa.configuration.ServiceDescriptor;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.MountableFile;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
public abstract class E2ETestBase {
public Network network = Network.newNetwork();
public GenericContainer<?> getMariaDBContainer() {
return new MariaDBContainer<>("mariadb")
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("sql/edge-crawler-cache.sql")
.withNetwork(network)
.withNetworkAliases("mariadb");
}
public GenericContainer<?> forService(ServiceDescriptor service, GenericContainer<?> mariaDB) {
return new GenericContainer<>("openjdk:17-alpine")
.dependsOn(mariaDB)
.withCopyFileToContainer(jarFile(), "/WMSA.jar")
.withCopyFileToContainer(MountableFile.forClasspathResource("init.sh"), "/init.sh")
.withExposedPorts(service.port)
.withFileSystemBind(modelsPath(), "/var/lib/wmsa/model", BindMode.READ_ONLY)
.withNetwork(network)
.withNetworkAliases(service.name)
.withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(service.name)))
.withCommand("sh", "init.sh", service.name)
.waitingFor(Wait.forHttp("/internal/ping")
.forPort(service.port)
.withReadTimeout(Duration.ofSeconds(15)))
;
}
public static MountableFile jarFile() {
Path cwd = Path.of(System.getProperty("user.dir"));
cwd = cwd.resolve("..");
var jarFile = cwd.resolve("build/libs/wmsa-SNAPSHOT-all.jar");
if (!Files.exists(jarFile)) {
System.err.println("Could not find jarFile " + jarFile);
throw new RuntimeException();
}
else {
System.out.println("jar file = " + jarFile);
}
return MountableFile.forHostPath(jarFile);
}
public static String modelsPath() {
Path modelsPath = Path.of(System.getProperty("user.dir")).resolve("data/models");
if (!Files.isDirectory(modelsPath)) {
System.err.println("Could not find models, looked in " + modelsPath.toAbsolutePath());
throw new RuntimeException();
}
return modelsPath.toString();
}
}

59
marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EdgeSearchE2ETest.java
View file

@ -32,24 +32,16 @@ import static org.testcontainers.containers.BrowserWebDriverContainer.VncRecordi
@Tag("e2e")
@Testcontainers
public class EdgeSearchE2ETest {
Network network = Network.newNetwork();
public class EdgeSearchE2ETest extends E2ETestBase {
@Container
public GenericContainer<?> mariaDB = getMariaDBContainer();
@Container
public GenericContainer<?> mariaDB = new MariaDBContainer<>("mariadb")
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("sql/edge-crawler-cache.sql")
.withNetwork(network)
.withNetworkAliases("mariadb");
public GenericContainer<?> searchContainer = forService(EDGE_SEARCH, mariaDB);
@Container
public GenericContainer<?> searchContainer = forService(EDGE_SEARCH);
public GenericContainer<?> assistantContainer = forService(EDGE_ASSISTANT, mariaDB);
@Container
public GenericContainer<?> assistantContainer = forService(EDGE_ASSISTANT);
@Container
public GenericContainer<?> indexContainer = forService(EDGE_INDEX);
public GenericContainer<?> indexContainer = forService(EDGE_INDEX, mariaDB);
@Container
public NginxContainer<?> mockWikipedia = new NginxContainer<>("nginx:stable")
@ -88,46 +80,7 @@ public class EdgeSearchE2ETest {
.withNetwork(network)
.withNetworkAliases("proxyNginx");
;
public GenericContainer<?> forService(ServiceDescriptor service) {
return new GenericContainer<>("openjdk:17-alpine")
.dependsOn(mariaDB)
.withCopyFileToContainer(jarFile(), "/WMSA.jar")
.withCopyFileToContainer(MountableFile.forClasspathResource("init.sh"), "/init.sh")
.withExposedPorts(service.port)
.withFileSystemBind(modelsPath(), "/var/lib/wmsa/model", BindMode.READ_ONLY)
.withNetwork(network)
.withNetworkAliases(service.name)
.withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(service.name)))
.withCommand("sh", "init.sh", service.name)
.waitingFor(Wait.forHttp("/internal/ping")
.forPort(service.port)
.withReadTimeout(Duration.ofSeconds(15)))
;
}
public static MountableFile jarFile() {
Path cwd = Path.of(System.getProperty("user.dir"));
cwd = cwd.resolve("..");
var jarFile = cwd.resolve("build/libs/wmsa-SNAPSHOT-all.jar");
if (!Files.exists(jarFile)) {
System.err.println("Could not find jarFile " + jarFile);
throw new RuntimeException();
}
else {
System.out.println("jar file = " + jarFile);
}
return MountableFile.forHostPath(jarFile);
}
public static String modelsPath() {
Path modelsPath = Path.of(System.getProperty("user.dir")).resolve("data/models");
if (!Files.isDirectory(modelsPath)) {
System.err.println("Could not find models, looked in " + modelsPath.toAbsolutePath());
throw new RuntimeException();
}
return modelsPath.toString();
}
public static MountableFile ipDatabasePath() {
Path modelsPath = Path.of(System.getProperty("user.dir")).resolve("data/models/IP2LOC/IP2LOCATION-LITE-DB1.CSV");
if (!Files.isRegularFile(modelsPath)) {

27
marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EncyclopediaE2ETest.java Normal file
View file

@ -0,0 +1,27 @@
package nu.marginalia.wmsa.edge;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import static nu.marginalia.wmsa.configuration.ServiceDescriptor.ENCYCLOPEDIA;
@Tag("e2e")
@Testcontainers
public class EncyclopediaE2ETest extends E2ETestBase {
@Container
public GenericContainer<?> mariaDB = getMariaDBContainer();
@Container
public GenericContainer<?> encyclopediaContainer = forService(ENCYCLOPEDIA, mariaDB);
@Test
public void run() {
}
}

2
marginalia_nu/src/e2e/resources/crawl.sh
View file

@ -3,7 +3,7 @@
mkdir -p /var/lib/wmsa/conf/
mkdir -p /var/lib/wmsa/data/
cat > /var/lib/wmsa/db.properties <<EOF
cat > /var/lib/wmsa/conf/db.properties <<EOF
db.user=wmsa
db.pass=wmsa
db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true

9
marginalia_nu/src/e2e/resources/init.sh
View file

@ -1,5 +1,6 @@
#!/bin/bash
mkdir -p /var/lib/wmsa/encyclopedia
mkdir -p /var/lib/wmsa/conf
mkdir -p /var/lib/wmsa/index/write
mkdir -p /var/lib/wmsa/index/read
@ -21,7 +22,11 @@ many
year
EOF
cat > /var/lib/wmsa/db.properties <<EOF
cat > /var/lib/wmsa/conf/disks.properties <<EOF
encyclopedia=/var/lib/wmsa/encyclopedia
EOF
cat > /var/lib/wmsa/conf/db.properties <<EOF
db.user=wmsa
db.pass=wmsa
db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true
@ -51,7 +56,7 @@ smhi-scraper smhi-scraper
podcast-scraper podcast-scraper
edge-index edge-index
edge-search edge-search
edge-archive edge-archive
encyclopedia encyclopedia
edge-assistant edge-assistant
memex memex
dating dating

24
marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java
View file

@ -3,6 +3,7 @@ package nu.marginalia.wmsa.configuration;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;
public class WmsaHome {
private static final String DEFAULT = "/var/lib/wmsa";
@ -32,4 +33,27 @@ public class WmsaHome {
public static Path getIPLocationDatabse() {
return getHomePath().resolve("data").resolve("IP2LOCATION-LITE-DB1.CSV");
}
public static Path getDisk(String name) throws IOException {
Path p = Path.of(getDiskProperties().getProperty(name));
if (!Files.isDirectory(p)) {
throw new IOException(name + " does not exist!");
}
return p;
}
public static Properties getDiskProperties() throws IOException {
Path settingsFile = getHomePath().resolve("conf/disks.properties");
if (Files.isRegularFile(settingsFile)) {
try (var is = Files.newInputStream(settingsFile)) {
var props = new Properties();
props.load(is);
return props;
}
}
else {
throw new IOException("Could not find disk settings " + settingsFile);
}
}
}

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/module/DatabaseModule.java
View file

@ -35,7 +35,7 @@ public class DatabaseModule extends AbstractModule {
}
private Properties loadDbProperties() {
Path propDir = WmsaHome.getHomePath().resolve("db.properties");
Path propDir = WmsaHome.getHomePath().resolve("conf/db.properties");
if (!Files.isRegularFile(propDir)) {
throw new IllegalStateException("Database properties file " + propDir + " does not exist");
}

1
marginalia_nu/src/main/java/nu/marginalia/wmsa/encyclopedia/EncyclopediaMain.java
View file

@ -14,6 +14,7 @@ public class EncyclopediaMain extends MainClass {
init(ServiceDescriptor.ENCYCLOPEDIA, args);
Injector injector = Guice.createInjector(
new EncyclopediaModule(),
new ConfigurationModule());
injector.getInstance(EncyclopediaMain.class);
}

18
marginalia_nu/src/main/java/nu/marginalia/wmsa/encyclopedia/EncyclopediaModule.java Normal file
View file

@ -0,0 +1,18 @@
package nu.marginalia.wmsa.encyclopedia;
import com.google.inject.AbstractModule;
import com.google.inject.name.Names;
import lombok.SneakyThrows;
import nu.marginalia.wmsa.configuration.WmsaHome;
import java.nio.file.Path;
public class EncyclopediaModule extends AbstractModule {
@SneakyThrows
@Override
public void configure() {
bind(Path.class)
.annotatedWith(Names.named("wiki-path"))
.toInstance(WmsaHome.getDisk("encyclopedia"));
}
}

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/encyclopedia/EncyclopediaService.java
View file

@ -1,5 +1,6 @@
package nu.marginalia.wmsa.encyclopedia;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import lombok.SneakyThrows;
import nu.marginalia.wmsa.configuration.server.Context;
@ -33,6 +34,7 @@ public class EncyclopediaService extends Service {
private Path wikiPath;
private EncyclopediaDao encyclopediaDao;
@Inject
public EncyclopediaService(@Named("service-host") String ip,
@Named("service-port") Integer port,
@Named("wiki-path") Path wikiPath,