From 958d64720edcccde6e474a7164b252fe66ba178d Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 24 Jan 2024 12:47:10 +0100 Subject: [PATCH] (control) Add a view for restarting aborted processes This will avoid having to dig in the message queue to perform this relatively common task. The control service was also refactored to extract common timestamp formatting logic out of the data objects and into the rendering. --- .../storage/FileStorageService.java | 4 +- .../ControlHandlebarsConfigurator.java | 24 ++- .../nu/marginalia/control/ControlService.java | 2 + .../control/sys/model/AbortedProcess.java | 17 +++ .../control/sys/model/MessageQueueEntry.java | 30 +--- .../sys/svc/AbortedProcessService.java | 144 ++++++++++++++++++ .../control/partials/events-table-summary.hdb | 2 +- .../control/partials/events-table.hdb | 2 +- .../control/partials/message-queue-table.hdb | 4 +- .../templates/control/partials/nav.hdb | 5 +- .../control/sys/aborted-processes.hdb | 73 +++++++++ .../templates/control/sys/view-message.hdb | 4 +- 12 files changed, 274 insertions(+), 37 deletions(-) create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java create mode 100644 code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb diff --git a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java index 04b1da25..999a51a5 100644 --- a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java @@ -337,7 +337,9 @@ public class FileStorageService { public List getStorage(List ids) throws SQLException { List ret = new ArrayList<>(); for (var id : ids) { - ret.add(getStorage(id)); + var storage = getStorage(id); + if (storage == null) continue; + ret.add(storage); } return ret; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java index 818f2ee4..059ea6a4 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java @@ -3,15 +3,37 @@ package nu.marginalia.control; import com.github.jknack.handlebars.*; import nu.marginalia.renderer.config.HandlebarsConfigurator; -import java.io.IOException; +import java.time.LocalDate; public class ControlHandlebarsConfigurator implements HandlebarsConfigurator { @Override public void configure(Handlebars handlebars) { handlebars.registerHelper("readableUUID", new UUIDHelper()); + handlebars.registerHelper("shortTimestamp", new ShortTimestampHelper()); + } + +} + +class ShortTimestampHelper implements Helper { + @Override + public Object apply(Object context, Options options) { + if (context == null) return ""; + String ts = context.toString(); + + String retDateBase = ts.replace('T', ' '); + + // if another day, return date, hour and minute + if (!ts.startsWith(LocalDate.now().toString())) { + // return hour minute and seconds + return retDateBase.substring(0, "YYYY-MM-DDTHH:MM".length()); + } + else { // return date, hour and minute but not seconds or ms + return retDateBase.substring("YYYY-MM-DDT".length(), "YYYY-MM-DDTHH:MM:SS".length()); + } } } + /** Helper for rendering UUIDs in a more readable way */ class UUIDHelper implements Helper { @Override diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index e4b5d359..91e5d3d5 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -55,6 +55,7 @@ public class ControlService extends Service { ControlNodeService controlNodeService, ControlDomainRankingSetsService controlDomainRankingSetsService, ControlActorService controlActorService, + AbortedProcessService abortedProcessService, ControlErrorHandler errorHandler ) throws IOException { @@ -69,6 +70,7 @@ public class ControlService extends Service { sysActionsService.register(); dataSetsService.register(); controlDomainRankingSetsService.register(); + abortedProcessService.register(); // node controlFileStorageService.register(); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java new file mode 100644 index 00000000..445aae19 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java @@ -0,0 +1,17 @@ +package nu.marginalia.control.sys.model; + +import nu.marginalia.storage.model.FileStorage; + +import java.util.List; + +/** A process that has been manually aborted by a user, + * ... or error? + */ +public record AbortedProcess(String name, + long msgId, + String startDateTime, + String stopDateTime, + List associatedStorages) +{ + +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java index 61a842f3..bea285a1 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java @@ -13,8 +13,8 @@ public record MessageQueueEntry ( String ownerInstanceFull, long ownerTick, String state, - String createdTimeFull, - String updatedTimeFull, + String createdTime, + String updatedTime, int ttl ) { @@ -38,30 +38,4 @@ public record MessageQueueEntry ( default -> ""; }; } - - public String getCreatedTime() { - String retDateBase = createdTimeFull.replace('T', ' '); - - // if another day, return date, hour and minute - if (!createdTimeFull.startsWith(LocalDate.now().toString())) { - // return hour minute and seconds - return retDateBase.substring(0, "YYYY-MM-DDTHH:MM".length()); - } - else { // return date, hour and minute but not seconds or ms - return retDateBase.substring("YYYY-MM-DDT".length(), "YYYY-MM-DDTHH:MM:SS".length()); - } - } - - public String getUpdatedTime() { - String retDateBase = updatedTimeFull.replace('T', ' '); - - // if another day, return date, hour and minute - if (!updatedTimeFull.startsWith(LocalDate.now().toString())) { - // return hour minute and seconds - return retDateBase.substring(0, "YYYY-MM-DDTHH:MM".length()); - } - else { // return date, hour and minute but not seconds or ms - return retDateBase.substring("YYYY-MM-DDT".length(), "YYYY-MM-DDTHH:MM:SS".length()); - } - } } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java new file mode 100644 index 00000000..8290e7dc --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java @@ -0,0 +1,144 @@ +package nu.marginalia.control.sys.svc; + +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.control.ControlRendererFactory; +import nu.marginalia.control.RedirectControl; +import nu.marginalia.control.sys.model.AbortedProcess; +import nu.marginalia.model.gson.GsonFactory; +import nu.marginalia.mq.MqMessageState; +import nu.marginalia.mq.persistence.MqPersistence; +import nu.marginalia.nodecfg.NodeConfigurationService; +import nu.marginalia.nodecfg.model.NodeConfiguration; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorage; +import nu.marginalia.storage.model.FileStorageId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Control for listing and restarting aborted processes. + * */ +public class AbortedProcessService { + private static final Logger logger = LoggerFactory.getLogger(AbortedProcessService.class); + private static final Gson gson = GsonFactory.get(); + private final HikariDataSource dataSource; + private final FileStorageService fileStorageService; + private final ControlRendererFactory rendererFactory; + private final RedirectControl redirectControl; + private final MqPersistence mqPersistence; + private final NodeConfigurationService nodeConfigurationService; + + @Inject + public AbortedProcessService(HikariDataSource dataSource, + FileStorageService fileStorageService, + ControlRendererFactory rendererFactory, + RedirectControl redirectControl, + MqPersistence mqPersistence, + NodeConfigurationService nodeConfigurationService) + { + this.dataSource = dataSource; + this.fileStorageService = fileStorageService; + this.rendererFactory = rendererFactory; + this.redirectControl = redirectControl; + this.mqPersistence = mqPersistence; + this.nodeConfigurationService = nodeConfigurationService; + } + + public void register() { + var abortedProcessesRenderer = rendererFactory.renderer("control/sys/aborted-processes"); + + Spark.get("/public/aborted-processes", this::abortedProcessesModel, abortedProcessesRenderer::render); + Spark.get("/public/aborted-processes/", this::abortedProcessesModel, abortedProcessesRenderer::render); + Spark.post("/public/aborted-processes/:id", this::restartProcess, redirectControl.renderRedirectAcknowledgement("Restarting...", "/")); + } + + private Object abortedProcessesModel(Request request, Response response) { + return Map.of("abortedProcesses", getAbortedProcesses()); + } + + private Object restartProcess(Request request, Response response) throws SQLException { + long msgId = Long.parseLong(request.params("id")); + mqPersistence.updateMessageState(msgId, MqMessageState.NEW); + return ""; + } + + + private List getAbortedProcesses() { + List allNodeIds = nodeConfigurationService.getAll().stream() + .map(NodeConfiguration::getId) + .toList(); + + // Generate all possible values for process-related inboxes + String inboxes = Stream.of("converter", "loader", "crawler") + .flatMap(s -> allNodeIds.stream().map(i -> STR."'\{s}:\{i}'")) + .collect(Collectors.joining(",", "(", ")")); + + try (var conn = dataSource.getConnection()) { + var stmt = conn.prepareStatement(STR.""" + SELECT ID, RECIPIENT_INBOX, CREATED_TIME, UPDATED_TIME, PAYLOAD FROM MESSAGE_QUEUE + WHERE STATE = 'DEAD' + AND RECIPIENT_INBOX IN \{inboxes} + """); // SQL injection safe, string is not user input + var rs = stmt.executeQuery(); + + List abortedProcesses = new ArrayList<>(); + while (rs.next()) { + var msgId = rs.getLong("ID"); + var recipientInbox = rs.getString("RECIPIENT_INBOX"); + var createdTime = rs.getString("CREATED_TIME"); + var updatedTime = rs.getString("UPDATED_TIME"); + var payload = rs.getString("PAYLOAD"); + + List associatedStorageIds = getAssociatedStoragesIds(payload); + List associatedStorages = fileStorageService.getStorage(associatedStorageIds); + + abortedProcesses.add(new AbortedProcess(recipientInbox, msgId, createdTime, updatedTime, associatedStorages)); + } + + abortedProcesses.sort(Comparator.comparing(AbortedProcess::stopDateTime).reversed()); + + return abortedProcesses; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /* Attempt to parse the JSON payload and extract the file storage ids + * from the data, not knowing exactly what the payload looks like. + * */ + private List getAssociatedStoragesIds(String payload) { + Map fields = gson.fromJson(payload, LinkedTreeMap.class); + logger.info("{}", fields); + List associatedStorageIds = new ArrayList<>(); + + // We expect a map of objects, where some objects are a map with an "id" field + // and an integer value. We want to extract the integer values. + for (Object field : fields.values()) { + if ((field instanceof Map m) && (m.get("id") instanceof Number i)) + associatedStorageIds.add(FileStorageId.of(i.intValue())); + + if (field instanceof List) { + for (Object o : (List) field) { + if ((o instanceof Map m) && (m.get("id") instanceof Number i)) + associatedStorageIds.add(FileStorageId.of(i.intValue())); + } + } + } + + return associatedStorageIds; + } + +} diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb index 32d2ec67..78a8cef2 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb @@ -8,7 +8,7 @@ {{#each events}} - {{eventTime}} + {{shortTimestamp eventDateTime}} {{eventType}} {{eventMessage}} diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb index bb526d6f..faa21447 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb @@ -46,7 +46,7 @@ {{serviceName}} {{{readableUUID instanceFull}}} - {{eventTime}} + {{shortTimestamp eventDateTime}} {{eventType}} {{eventMessage}} diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb index 2959fa49..e0cce6d2 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb @@ -45,7 +45,7 @@ {{recipientInbox}} {{function}} {{{readableUUID ownerInstanceFull}}} - {{createdTime}} + {{shortTimestamp createdTime}} {{ttl}} @@ -57,7 +57,7 @@ {{senderInbox}} {{payload}} {{ownerTick}} - {{updatedTime}} + {{shortTimestamp updatedTime}} {{/each}} diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb index 9cadd3be..97e54d0b 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb @@ -33,9 +33,12 @@
  • diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb b/code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb new file mode 100644 index 00000000..dea073df --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb @@ -0,0 +1,73 @@ + + + + Control Service + {{> control/partials/head-includes }} + + + {{> control/partials/nav}} +
    +

    Aborted Processes

    + +
    +

    + This form will let you restart a process that has + been manually aborted. If additional automatic processing + was configured, it may not be restarted. The progress + estimation of the restarted process may be inaccurate. +

    +

    + Items will vanish from this list after a few days +

    +
    + + {{#unless abortedProcesses}} + + + + +
    No aborted processes
    + {{/unless}} + + {{#each abortedProcesses}} + + + + + + + + + + + + + + + {{#if associatedStorages}} + + + {{/if}} +
    Name: {{name}} +
    + +
    +
    msgId + {{msgId}} +
    Run Time: {{shortTimestamp startDateTime}}{{shortTimestamp stopDateTime}}
    Associated storages
    + + {{#each associatedStorages}} + + + + + + {{/each}} +
    {{type}}{{path}}{{description}}
    +
    + {{/each}} + +
    + +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb b/code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb index 0cda4d5c..e21a11f8 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb @@ -31,8 +31,8 @@ payload - Created{{createdTime}} - Updated{{updatedTime}} + Created{{shortTimestamp createdTime}} + Updated{{shortTimestamp updatedTime}} {{/with}}