From c6abcd91fabbb8b2d003271bebfcf54fc7ceb3f7 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 20 Oct 2023 16:37:49 +0200 Subject: [PATCH] (control) Better use of FS states, fix bug with start/stop actors --- .../marginalia/storage/model/FileStorage.java | 11 ++++++++++- .../storage/model/FileStorageState.java | 3 +-- .../control/node/svc/ControlNodeService.java | 14 ++++++++++++++ .../control/node/node-storage-list.hdb | 12 ++++++++++-- .../marginalia/actor/task/ConvertActor.java | 19 +++++++++++++++---- .../actor/task/ConvertAndLoadActor.java | 13 ++++++++----- 6 files changed, 58 insertions(+), 14 deletions(-) diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java index 1c54ae68..f6090d56 100644 --- a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java @@ -40,7 +40,7 @@ public record FileStorage ( type, LocalDateTime.now(), override, - FileStorageState.EPHEMERAL, + FileStorageState.UNSET, "OVERRIDE:" + type.name() ); } @@ -52,6 +52,15 @@ public record FileStorage ( public boolean isActive() { return FileStorageState.ACTIVE.equals(state); } + public boolean isNoState() { + return FileStorageState.UNSET.equals(state); + } + public boolean isDelete() { + return FileStorageState.DELETE.equals(state); + } + public boolean isNew() { + return FileStorageState.NEW.equals(state); + } @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageState.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageState.java index 5c00c76b..026be1d6 100644 --- a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageState.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageState.java @@ -4,8 +4,7 @@ public enum FileStorageState { UNSET, NEW, ACTIVE, - DELETE, - EPHEMERAL; + DELETE; public static FileStorageState parse(String value) { if ("".equals(value)) { diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java index fde12b75..7af265d0 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java @@ -101,8 +101,22 @@ public class ControlNodeService { Spark.post("/public/nodes/:id/storage/:fid/disable", this::disableFileStorage); Spark.get("/public/nodes/:id/storage/:fid/transfer", this::downloadFileFromStorage); + + Spark.post("/public/nodes/:id/fsms/:fsm/start", this::startFsm); + Spark.post("/public/nodes/:id/fsms/:fsm/stop", this::stopFsm); } + public Object startFsm(Request req, Response rsp) throws Exception { + executorClient.startFsm(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fsm").toUpperCase()); + + return redirectToOverview(req); + } + + public Object stopFsm(Request req, Response rsp) throws Exception { + executorClient.stopFsm(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fsm").toUpperCase()); + + return redirectToOverview(req); + } private Object nodeListModel(Request request, Response response) throws SQLException { var configs = nodeConfigurationService.getAll(); diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb index dda40c04..0b51fb1d 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb @@ -107,12 +107,20 @@ {{timestamp}} - {{#unless storage.active}} + {{#if storage.delete}} + DELETING + {{/if}} + + {{#if storage.new}} + NEW + {{/if}} + + {{#if storage.noState}}
- {{/unless}} + {{/if}} {{#if storage.active}}
diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java index c3b3f696..14d297b0 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java @@ -55,6 +55,8 @@ public class ConvertActor extends AbstractActorPrototype { public long loaderMsgId = 0L; }; + public record WaitInstructions(long msgId, FileStorageId dest) { } + @Override public String describe() { return "Convert a set of crawl data into a format suitable for loading into the database."; @@ -98,8 +100,8 @@ public class ConvertActor extends AbstractActorPrototype { FileStorageType.PROCESSED_DATA, "processed-data", "Processed Data; " + toProcess.description()); - storageService.setFileStorageState(processedArea.id(), FileStorageState.EPHEMERAL); storageService.relateFileStorages(toProcess.id(), processedArea.id()); + storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); // Pre-send convert request var request = new ConvertRequest(ConvertAction.ConvertCrawlData, @@ -132,6 +134,8 @@ public class ConvertActor extends AbstractActorPrototype { FileStorageType.PROCESSED_DATA, "processed-data", "Processed Encylopedia Data; " + fileName); + storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); + // Pre-send convert request var request = new ConvertRequest(ConvertAction.SideloadEncyclopedia, sourcePath.toString(), @@ -164,6 +168,8 @@ public class ConvertActor extends AbstractActorPrototype { FileStorageType.PROCESSED_DATA, "processed-data", "Processed Dirtree Data; " + fileName); + storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); + // Pre-send convert request var request = new ConvertRequest(ConvertAction.SideloadDirtree, sourcePath.toString(), @@ -195,6 +201,8 @@ public class ConvertActor extends AbstractActorPrototype { FileStorageType.PROCESSED_DATA, "processed-data", "Processed Stackexchange Data; " + fileName); + storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); + // Pre-send convert request var request = new ConvertRequest(ConvertAction.SideloadStackexchange, sourcePath.toString(), @@ -212,11 +220,14 @@ public class ConvertActor extends AbstractActorPrototype { Wait for the converter to finish processing the data. """ ) - public void convertWait(Long msgId) throws Exception { - var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessService.ProcessId.CONVERTER, msgId); + public void convertWait(WaitInstructions instructions) throws Exception { + var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessService.ProcessId.CONVERTER, instructions.msgId()); - if (rsp.state() != MqMessageState.OK) + if (rsp.state() != MqMessageState.OK) { error("Converter failed"); + } + + storageService.setFileStorageState(instructions.dest, FileStorageState.UNSET); } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java index 0fc65c67..784a4ac1 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java @@ -141,7 +141,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { var processedArea = storageService.allocateTemporaryStorage(base, FileStorageType.PROCESSED_DATA, "processed-data", "Processed Data; " + toProcess.description()); - storageService.setFileStorageState(processedArea.id(), FileStorageState.EPHEMERAL); + storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); storageService.relateFileStorages(toProcess.id(), processedArea.id()); // Pre-send convert request @@ -201,12 +201,15 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { private void cleanProcessedStorage(List processedStorageId) { try { var config = nodeConfigurationService.get(nodeId); - if (!config.autoClean()) - return; for (var id : processedStorageId) { - if (FileStorageState.EPHEMERAL.equals(storageService.getStorage(id).state())) { - storageService.flagFileForDeletion(id); + if (FileStorageState.NEW.equals(storageService.getStorage(id).state())) { + if (config.autoClean()) { + storageService.flagFileForDeletion(id); + } + else { + storageService.setFileStorageState(id, FileStorageState.UNSET); + } } } }