161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(test) Clean up test usage of migrations

Browse Source 
Several tests were manually running migrations in a large copy-paste blob of code.  This makes the test less useful as it's possible to break the code while keeping the tests green by introducing a new migration that never gets run in the tests, and it's also difficult to reason about what the tests are doing.

A new test helper library is introduced with a TestMigrationLoader that can both run Flyway migrations, or load specific migrations in the cases a specific set of migrations need to be loaded.   Existing tests are migrated to use the new code.
This commit is contained in:
Viktor Lofgren 2024-01-12 15:55:50 +01:00
parent 0caef1b307
commit 708a741960
24 changed files with 102 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/common/config/build.gradle
View File

@ -27,9 +27,11 @@ dependencies {
testImplementation libs.bundles.junit
testImplementation project(':code:libraries:test-helpers')
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

2
code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java
View File

@ -20,7 +20,7 @@ public class NodeConfigurationService {
public NodeConfiguration create(int id, String description, boolean acceptQueries, boolean keepWarcs) throws SQLException {
try (var conn = dataSource.getConnection();
var is = conn.prepareStatement("""
INSERT IGNORE INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?, ?)
INSERT IGNORE INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES, KEEP_WARCS) VALUES(?, ?, ?, ?)
""")
)
{

4
code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java
View File

@ -3,6 +3,7 @@ package nu.marginalia.nodecfg;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@ -30,7 +31,6 @@ public class NodeConfigurationServiceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_11_0_003__node_configuration.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -45,6 +45,8 @@ public class NodeConfigurationServiceTest {
dataSource = new HikariDataSource(config);
TestMigrationLoader.flywayMigration(dataSource);
nodeConfigurationService = new NodeConfigurationService(dataSource);
}

29
code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java
View File

@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.storage.model.FileStorageBaseType;
import nu.marginalia.storage.model.FileStorageType;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
@ -33,7 +34,6 @@ public class FileStorageServiceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_07_0_004__file_storage.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -50,32 +50,7 @@ public class FileStorageServiceTest {
dataSource = new HikariDataSource(config);
// apply migrations
List<String> migrations = List.of(
"db/migration/V23_11_0_000__file_storage_node.sql",
"db/migration/V23_11_0_002__file_storage_state.sql",
"db/migration/V23_11_0_004__file_storage_base_type.sql"
);
for (String migration : migrations) {
try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
"Could not load migration script " + migration);
var conn = dataSource.getConnection();
var stmt = conn.createStatement()
) {
String script = new String(resource.readAllBytes());
String[] cmds = script.split("\\s*;\\s*");
for (String cmd : cmds) {
if (cmd.isBlank())
continue;
System.out.println(cmd);
stmt.executeUpdate(cmd);
}
}
catch (IOException|SQLException ex) {
}
}
TestMigrationLoader.flywayMigration(dataSource);
}

1
code/common/db/build.gradle
View File

@ -52,6 +52,7 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}
flyway {

3
code/common/db/src/test/java/nu/marginalia/db/DomainTypesTest.java
View File

@ -3,6 +3,7 @@ package nu.marginalia.db;
import com.google.common.collect.Sets;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
@ -26,7 +27,6 @@ public class DomainTypesTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_07_0_001__domain_type.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -40,6 +40,7 @@ public class DomainTypesTest {
config.setPassword("wmsa");
dataSource = new HikariDataSource(config);
TestMigrationLoader.flywayMigration(dataSource);
domainTypes = new DomainTypes(dataSource);
}

1
code/common/linkdb/build.gradle
View File

@ -38,5 +38,6 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

1
code/libraries/message-queue/build.gradle
View File

@ -29,4 +29,5 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

4
code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java
View File

@ -11,6 +11,7 @@ import nu.marginalia.actor.state.Resume;
import nu.marginalia.mq.MessageQueueFactory;
import nu.marginalia.mq.MqTestUtil;
import nu.marginalia.mq.persistence.MqPersistence;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.testcontainers.containers.MariaDBContainer;
@ -31,7 +32,6 @@ public class ActorRecordMachineTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_07_0_003__message_queue.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -51,6 +51,8 @@ public class ActorRecordMachineTest {
config.setPassword("wmsa");
dataSource = new HikariDataSource(config);
TestMigrationLoader.flywayMigration(dataSource);
persistence = new MqPersistence(dataSource);
messageQueueFactory = new MessageQueueFactory(persistence);
}

3
code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java
View File

@ -7,6 +7,7 @@ import nu.marginalia.mq.MqMessageState;
import nu.marginalia.mq.MqTestUtil;
import nu.marginalia.mq.inbox.*;
import nu.marginalia.mq.persistence.MqPersistence;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.junit.jupiter.Container;
@ -27,7 +28,6 @@ public class MqOutboxTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_07_0_003__message_queue.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -45,6 +45,7 @@ public class MqOutboxTest {
config.setPassword("wmsa");
dataSource = new HikariDataSource(config);
TestMigrationLoader.flywayMigration(dataSource);
}
@AfterAll

3
code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java
View File

@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.mq.MqMessageState;
import nu.marginalia.mq.MqTestUtil;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.junit.jupiter.Container;
@ -24,7 +25,6 @@ public class MqPersistenceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_07_0_003__message_queue.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -47,6 +47,7 @@ public class MqPersistenceTest {
dataSource = new HikariDataSource(config);
persistence = new MqPersistence(dataSource);
TestMigrationLoader.flywayMigration(dataSource);
}
@AfterAll

22
code/libraries/test-helpers/build.gradle Normal file
View File

@ -0,0 +1,22 @@
plugins {
id 'java'
}
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(21))
}
}
dependencies {
implementation libs.bundles.slf4j
implementation libs.bundles.mariadb
implementation libs.bundles.slf4j.test
implementation libs.bundles.junit
implementation libs.mockito
implementation libs.bundles.flyway
}
test {
useJUnitPlatform()
}

45
code/libraries/test-helpers/src/main/java/nu/marginalia/test/TestMigrationLoader.java Normal file
View File

@ -0,0 +1,45 @@
package nu.marginalia.test;
import com.zaxxer.hikari.HikariDataSource;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.Assertions;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Objects;
public class TestMigrationLoader {
/** Use flyway to load migrations from the classpath and run them against the database. */
public static void flywayMigration(HikariDataSource dataSource) {
Flyway.configure().dataSource(dataSource).load().migrate();
}
/** Load specific migrations from the classpath and run them against the database. */
public static void loadMigrations(HikariDataSource dataSource, String... migrations) {
for (var migration : migrations) {
loadMigration(dataSource, migration);
}
}
/** Load specific migration from the classpath and run them against the database. */
public static void loadMigration(HikariDataSource dataSource, String migration) {
try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
"Could not load migration script " + migration);
var conn = dataSource.getConnection();
var stmt = conn.createStatement()
) {
String script = new String(resource.readAllBytes());
String[] cmds = script.split("\\s*;\\s*");
for (String cmd : cmds) {
if (cmd.isBlank())
continue;
System.out.println(cmd);
stmt.executeUpdate(cmd);
}
}
catch (IOException | SQLException ex) {
Assertions.fail("Failed to load migration " + migration, ex);
}
}
}

1
code/processes/loading-process/build.gradle
View File

@ -56,5 +56,6 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

9
code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java
View File

@ -11,6 +11,7 @@ import nu.marginalia.model.processed.DomainLinkRecord;
import nu.marginalia.model.processed.DomainRecord;
import nu.marginalia.process.control.ProcessAdHocTaskHeartbeat;
import nu.marginalia.process.control.ProcessHeartbeat;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.mockito.Mockito;
import org.testcontainers.containers.MariaDBContainer;
@ -31,14 +32,6 @@ class DomainLoaderServiceTest {
List<Path> toDelete = new ArrayList<>();
ProcessHeartbeat heartbeat;
@Container
static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_06_0_000__base.sql")
.withNetworkAliases("mariadb");
@BeforeEach
public void setUp() {
heartbeat = Mockito.mock(ProcessHeartbeat.class);

1
code/processes/website-adjacencies-calculator/build.gradle
View File

@ -40,4 +40,5 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

1
code/services-application/api-service/build.gradle
View File

@ -51,5 +51,6 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

3
code/services-application/api-service/src/test/java/nu/marginalia/api/svc/LicenseServiceTest.java
View File

@ -3,6 +3,7 @@ package nu.marginalia.api.svc;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.junit.jupiter.Container;
@ -21,7 +22,6 @@ class LicenseServiceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_06_0_006__api_key.sql")
.withNetworkAliases("mariadb");
private static LicenseService service;
@ -32,6 +32,7 @@ class LicenseServiceTest {
mariaDBContainer.start();
dataSource = getConnection(mariaDBContainer.getJdbcUrl());
TestMigrationLoader.flywayMigration(dataSource);
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""

1
code/services-core/control-service/build.gradle
View File

@ -63,6 +63,7 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

3
code/services-core/control-service/src/test/java/nu/marginalia/control/svc/ApiKeyServiceTest.java
View File

@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.app.model.ApiKeyModel;
import nu.marginalia.control.app.svc.ApiKeyService;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.parallel.Execution;
import org.testcontainers.containers.MariaDBContainer;
@ -25,7 +26,6 @@ public class ApiKeyServiceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_06_0_006__api_key.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -37,6 +37,7 @@ public class ApiKeyServiceTest {
config.setPassword("wmsa");
dataSource = new HikariDataSource(config);
TestMigrationLoader.flywayMigration(dataSource);
}
@AfterAll

23
code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java
View File

@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.sys.model.TaskHeartbeat;
import nu.marginalia.control.sys.svc.HeartbeatService;
import nu.marginalia.service.control.ServiceEventLog;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
@ -32,10 +33,8 @@ class HeartbeatServiceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_07_0_007__task_status.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@BeforeAll
public static void setup() {
@ -46,25 +45,7 @@ class HeartbeatServiceTest {
dataSource = new HikariDataSource(config);
List<String> migrations = List.of("db/migration/V23_11_0_001__heartbeat_node.sql");
for (String migration : migrations) {
try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
"Could not load migration script " + migration);
var conn = dataSource.getConnection();
var stmt = conn.createStatement()
) {
String script = new String(resource.readAllBytes());
String[] cmds = script.split("\\s*;\\s*");
for (String cmd : cmds) {
if (cmd.isBlank())
continue;
System.out.println(cmd);
stmt.executeUpdate(cmd);
}
} catch (IOException | SQLException ex) {
}
}
TestMigrationLoader.flywayMigration(dataSource);
}
@AfterAll

2
code/services-core/executor-service/build.gradle
View File

@ -68,6 +68,6 @@ dependencies {
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
testImplementation 'org.testcontainers:mariadb:1.17.4'
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
testImplementation project(':code:libraries:test-helpers')
}

28
code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java
View File

@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.db.DomainTypes;
import nu.marginalia.service.module.ServiceConfiguration;
import nu.marginalia.test.TestMigrationLoader;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
@ -29,7 +30,6 @@ class DomainListRefreshServiceTest {
.withDatabaseName("WMSA_prod")
.withUsername("wmsa")
.withPassword("wmsa")
.withInitScript("db/migration/V23_06_0_000__base.sql")
.withNetworkAliases("mariadb");
static HikariDataSource dataSource;
@ -43,31 +43,7 @@ class DomainListRefreshServiceTest {
dataSource = new HikariDataSource(config);
// apply migrations
List<String> migrations = List.of(
"db/migration/V23_06_0_003__crawl-queue.sql",
"db/migration/V23_07_0_001__domain_type.sql",
"db/migration/V23_11_0_007__domain_node_affinity.sql"
);
for (String migration : migrations) {
try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
"Could not load migration script " + migration);
var conn = dataSource.getConnection();
var stmt = conn.createStatement()
) {
String script = new String(resource.readAllBytes());
String[] cmds = script.split("\\s*;\\s*");
for (String cmd : cmds) {
if (cmd.isBlank())
continue;
System.out.println(cmd);
stmt.executeUpdate(cmd);
}
} catch (IOException | SQLException ex) {
}
}
TestMigrationLoader.flywayMigration(dataSource);
}
@AfterAll

1
settings.gradle
View File

@ -23,6 +23,7 @@ include 'code:libraries:blocking-thread-pool'
include 'code:libraries:braille-block-punch-cards'
include 'code:libraries:language-processing'
include 'code:libraries:term-frequency-dict'
include 'code:libraries:test-helpers'
include 'code:libraries:message-queue'