161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(node) Nodes auto-configure on start-up instead of requiring manual configuration.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-10-16 14:46:35 +02:00
parent c98117f69d
commit 2df3e0f881
6 changed files with 49 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java
View File

@ -17,29 +17,22 @@ public class NodeConfigurationService {
this.dataSource = dataSource;
}
public NodeConfiguration create(String description, boolean acceptQueries) throws SQLException {
public NodeConfiguration create(int id, String description, boolean acceptQueries) throws SQLException {
try (var conn = dataSource.getConnection();
var is = conn.prepareStatement("""
INSERT INTO NODE_CONFIGURATION(DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?)
""");
var qs = conn.prepareStatement("""
SELECT LAST_INSERT_ID()
"""))
INSERT INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?)
""")
)
{
is.setString(1, description);
is.setBoolean(2, acceptQueries);
is.setInt(1, id);
is.setString(2, description);
is.setBoolean(3, acceptQueries);
if (is.executeUpdate() <= 0) {
throw new IllegalStateException("Failed to insert configuration");
}
var rs = qs.executeQuery();
if (rs.next()) {
return get(rs.getInt(1));
}
throw new AssertionError("No LAST_INSERT_ID()");
return get(id);
}
}

4
code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java
View File

@ -50,8 +50,8 @@ public class NodeConfigurationServiceTest {
@Test
public void test() throws SQLException {
var a = nodeConfigurationService.create("Test", false);
var b = nodeConfigurationService.create("Foo", true);
var a = nodeConfigurationService.create(1, "Test", false);
var b = nodeConfigurationService.create(2, "Foo", true);
assertEquals(1, a.node());
assertEquals("Test", a.description());

2
code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql
View File

@ -1,5 +1,5 @@
CREATE TABLE NODE_CONFIGURATION (
ID INT PRIMARY KEY AUTO_INCREMENT,
ID INT PRIMARY KEY,
DESCRIPTION VARCHAR(255),
ACCEPT_QUERIES BOOLEAN,
DISABLED BOOLEAN DEFAULT FALSE

53
code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java
View File

@ -2,10 +2,14 @@ package nu.marginalia.service.server;
import com.google.inject.name.Named;
import jakarta.inject.Inject;
import lombok.SneakyThrows;
import nu.marginalia.nodecfg.NodeConfigurationService;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorageBaseType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.file.Path;
import java.sql.SQLException;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
@ -23,18 +27,22 @@ public class NodeStatusWatcher {
private static final Logger logger = LoggerFactory.getLogger(NodeStatusWatcher.class);
private final NodeConfigurationService configurationService;
private final FileStorageService fileStorageService;
private final int nodeId;
private final Duration pollDuration = Duration.ofSeconds(15);
@Inject
public NodeStatusWatcher(NodeConfigurationService configurationService,
@Named("wmsa-system-node") Integer nodeId) throws InterruptedException {
FileStorageService fileStorageService, @Named("wmsa-system-node") Integer nodeId) {
this.configurationService = configurationService;
this.fileStorageService = fileStorageService;
this.nodeId = nodeId;
awaitConfiguration();
if (!isConfigured()) {
setupNode();
}
var watcherThread = new Thread(this::watcher, "node watcher");
@ -42,29 +50,28 @@ public class NodeStatusWatcher {
watcherThread.start();
}
/** Wait for the presence of an enabled NodeConfiguration before permitting the service to start */
private void awaitConfiguration() throws InterruptedException {
boolean complained = false;
for (;;) {
try {
var config = configurationService.get(nodeId);
if (null != config && !config.disabled()) {
return;
}
else if (!complained) {
logger.info("Waiting for node configuration, id = {}", nodeId);
complained = true;
}
}
catch (SQLException ex) {
logger.error("Error updating node status", ex);
}
TimeUnit.SECONDS.sleep(pollDuration.toSeconds());
private void setupNode() {
try {
configurationService.create(nodeId, "Node " + nodeId, nodeId == 1);
fileStorageService.createStorageBase("Index Data", Path.of("/idx"), nodeId, FileStorageBaseType.CURRENT);
fileStorageService.createStorageBase("Index Backups", Path.of("/backup"), nodeId, FileStorageBaseType.BACKUP);
fileStorageService.createStorageBase("Crawl Data", Path.of("/storage"), nodeId, FileStorageBaseType.STORAGE);
fileStorageService.createStorageBase("Work Area", Path.of("/work"), nodeId, FileStorageBaseType.WORK);
}
catch (IllegalStateException ex) {
// There is a slight chance of a race condition between the index and executor services both trying to run this,
// at the same time. Thanks to ACID, only one of them will succeed in creating the node, and the other will throw
// IllegalStateException. This is fine!
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
@SneakyThrows
private boolean isConfigured() {
var configuration = configurationService.get(nodeId);
return configuration != null;
}
/** Look for changes in the configuration and kill the service if the corresponding

13
code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java
View File

@ -77,7 +77,6 @@ public class ControlNodeService {
var newSpecsFormRenderer = rendererFactory.renderer("control/node/node-new-specs-form");
Spark.get("/public/nodes", this::nodeListModel, nodeListRenderer::render);
Spark.post("/public/nodes", this::createNode);
Spark.get("/public/nodes/:id", this::nodeOverviewModel, overviewRenderer::render);
Spark.get("/public/nodes/:id/", this::nodeOverviewModel, overviewRenderer::render);
Spark.get("/public/nodes/:id/actors", this::nodeActorsModel, actorsRenderer::render);
@ -106,18 +105,6 @@ public class ControlNodeService {
}
private Object createNode(Request request, Response response) throws SQLException, FileNotFoundException {
var newConfig = nodeConfigurationService.create(request.queryParams("description"), "on".equalsIgnoreCase(request.queryParams("acceptQueries")));
int id = newConfig.node();
fileStorageService.createStorageBase("Index Data", Path.of("/idx"), id, FileStorageBaseType.CURRENT);
fileStorageService.createStorageBase("Index Backups", Path.of("/backup"), id, FileStorageBaseType.BACKUP);
fileStorageService.createStorageBase("Crawl Data", Path.of("/storage"), id, FileStorageBaseType.STORAGE);
fileStorageService.createStorageBase("Work Area", Path.of("/work"), id, FileStorageBaseType.WORK);
return redirectToOverview(id);
}
private Object nodeListModel(Request request, Response response) throws SQLException {
var configs = nodeConfigurationService.getAll();

33
code/services-core/control-service/src/main/resources/templates/control/node/nodes-list.hdb
View File

@ -27,34 +27,17 @@
<td>{{acceptQueries}}</td>
</tr>
{{/each}}
{{/if}}
</table>
<div class="m-3 p-3 border">
<h2>Add Node</h2>
<form method="post">
<div class="mb-3">
<label for="name" class="form-label">ID</label>
<input class="form-control" type="text" name="id" id="id" value="{{nextNodeId}}" disabled aria-disabled="true" />
</div>
<div class="mb-3">
<label for="name" class="form-label">Description</label>
<input class="form-control" type="text" name="description" id="description" />
</div>
<div class="form-check form-switch mb-3">
<input class="form-check-input" type="checkbox" role="switch" name="acceptQueries">
<label class="form-check-label" for="acceptQueries">Accept queries</label>
<div class="form-text">Sets whether queries will be routed to this node. This can be modified later.</div>
</div>
<button type="submit" class="btn btn-primary">Create</button>
</form>
<div class="m-5 p-5 border bg-light">
<h2 class="my-3">Index Nodes</h2>
<p>
Index nodes are processing units. The search engine requires at least one, but more can be added
to spread the system load across multiple physical disks or even multiple servers.
</p>
</div>
</table>
{{/if}}
</div>
</body>