161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP: Encyclopedia service

Browse Source
This commit is contained in:
vlofgren 2022-05-28 14:35:32 +02:00
parent 0acdd5b660
commit 5a1ec53a84
4 changed files with 95 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java
View File

@ -17,7 +17,7 @@ import java.time.Duration;
public abstract class E2ETestBase {
public Network network = Network.newNetwork();
public GenericContainer<?> getMariaDBContainer() {
public MariaDBContainer<?> getMariaDBContainer() {
return new MariaDBContainer<>("mariadb")
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")

95
marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/EncyclopediaE2ETest.java
View File

@ -1,31 +1,44 @@
package nu.marginalia.wmsa.edge;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import nu.marginalia.wmsa.edge.assistant.dict.WikiArticles;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.mariadb.jdbc.Driver;
import org.openqa.selenium.By;
import org.openqa.selenium.chrome.ChromeOptions;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.BrowserWebDriverContainer;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.NginxContainer;
import org.testcontainers.containers.*;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.MountableFile;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Path;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Types;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
import static nu.marginalia.wmsa.configuration.ServiceDescriptor.ENCYCLOPEDIA;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@Tag("e2e")
@Testcontainers
public class EncyclopediaE2ETest extends E2ETestBase {
@Container
public GenericContainer<?> mariaDB = getMariaDBContainer();
public MariaDBContainer<?> mariaDB = getMariaDBContainer();
@Container
public GenericContainer<?> encyclopediaContainer = forService(ENCYCLOPEDIA, mariaDB);
@ -55,16 +68,84 @@ public class EncyclopediaE2ETest extends E2ETestBase {
.withNetwork(network)
.withCapabilities(new ChromeOptions());
private Gson gson = new GsonBuilder().create();
private OkHttpClient httpClient = new OkHttpClient.Builder()
.connectTimeout(100, TimeUnit.MILLISECONDS)
.readTimeout(6000, TimeUnit.SECONDS)
.retryOnConnectionFailure(true)
.followRedirects(true)
.build();
private Path getModelData() {
return Path.of(System.getProperty("user.dir")).resolve("data/test");
}
@Test
public void run() {
public void run() throws MalformedURLException {
new Driver();
try (var conn = DriverManager.getConnection(mariaDB.getJdbcUrl(), "wmsa", "wmsa");
var stmt = conn.prepareStatement("INSERT IGNORE INTO REF_WIKI_TITLE(NAME,REF_NAME) VALUES (?,?)")) {
stmt.setString(1, "Forg");
stmt.setString(2, "Frog");
stmt.executeUpdate();
stmt.setString(1, "Frog");
stmt.setNull(2, Types.VARCHAR);
stmt.executeUpdate();
} catch (SQLException e) {
throw new RuntimeException(e);
}
var driver = chrome.getWebDriver();
driver.get("http://proxyNginx/wiki/Frog");
System.out.println(driver.getTitle());
System.out.println(driver.findElement(new By.ByXPath("//*")).getAttribute("outerHTML"));
driver.get("http://proxyNginx/wiki-search?query=Forg");
System.out.println(driver.getTitle());
assertTrue(get(encyclopediaContainer.getHost(),
encyclopediaContainer.getMappedPort(ENCYCLOPEDIA.port),
"/wiki/has?url=Frog", Boolean.class));
assertFalse(get(encyclopediaContainer.getHost(),
encyclopediaContainer.getMappedPort(ENCYCLOPEDIA.port),
"/wiki/has?url=Marginalia", Boolean.class));
assertFalse(get(encyclopediaContainer.getHost(),
encyclopediaContainer.getMappedPort(ENCYCLOPEDIA.port),
"/wiki/has?url=Marginalia", Boolean.class));
var resultsForMarginalia = get(encyclopediaContainer.getHost(),
encyclopediaContainer.getMappedPort(ENCYCLOPEDIA.port),
"/encyclopedia/Marginalia", WikiArticles.class);
Assertions.assertTrue(resultsForMarginalia.getEntries().isEmpty());
var resultsForFrog = get(encyclopediaContainer.getHost(),
encyclopediaContainer.getMappedPort(ENCYCLOPEDIA.port),
"/encyclopedia/Frog", WikiArticles.class);
Assertions.assertFalse(resultsForFrog.getEntries().isEmpty());
var resultsForFoRg = get(encyclopediaContainer.getHost(),
encyclopediaContainer.getMappedPort(ENCYCLOPEDIA.port),
"/encyclopedia/Forg", WikiArticles.class);
Assertions.assertFalse(resultsForFoRg.getEntries().isEmpty());
}
private <T> T get(String host, Integer mappedPort, String path, Class<T> clazz) throws MalformedURLException {
var req = new Request.Builder().get().url(new URL("http", host, mappedPort, path)).build();
var call = httpClient.newCall(req);
try (var rsp = call.execute()) {
return gson.fromJson(rsp.body().charStream(), clazz);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

6
marginalia_nu/src/main/java/nu/marginalia/wmsa/encyclopedia/EncyclopediaService.java
View File

@ -1,5 +1,7 @@
package nu.marginalia.wmsa.encyclopedia;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import lombok.SneakyThrows;
@ -31,6 +33,8 @@ public class EncyclopediaService extends Service {
private static final Logger logger = LoggerFactory.getLogger(EncyclopediaService.class);
private final MustacheRenderer<String> wikiErrorPageRenderer;
private final MustacheRenderer<Object> wikiSearchResultRenderer;
private final Gson gson = new GsonBuilder().create();
private Path wikiPath;
private EncyclopediaDao encyclopediaDao;
@ -62,11 +66,11 @@ public class EncyclopediaService extends Service {
Spark.get("/wiki/has", this::pathWikiHas);
Spark.post("/wiki/submit", this::pathWikiSubmit);
Spark.get("/encyclopedia/:term", (rq, rsp) -> encyclopediaDao.encyclopedia(rq.params("term")), gson::toJson);
Spark.awaitInitialization();
}
@SneakyThrows
private Object getWikiPage(Request req, Response rsp) {
final String[] splats = req.splat();

2
marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql
View File

@ -270,7 +270,7 @@ COLLATE utf8mb4_unicode_ci;
CREATE INDEX IF NOT EXISTS REF_DICTIONARY_WORD ON REF_DICTIONARY (WORD);
CREATE TABLE IF NOT EXISTS REF_WIKI_TITLE(
CREATE TABLE IF NOT EXISTS REF_WIKI_TITLE (
NAME VARCHAR(255),
NAME_LOWER VARCHAR(255) GENERATED ALWAYS AS (LOWER(NAME)),
REF_NAME VARCHAR(255)