(control) Add a view for restarting aborted processes

This will avoid having to dig in the message queue to perform this relatively common task.

The control service was also refactored to extract common timestamp formatting logic out of the data objects and into the rendering.

code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java

@@ -337,7 +337,9 @@ public class FileStorageService {
     public List<FileStorage> getStorage(List<FileStorageId> ids) throws SQLException {
         List<FileStorage> ret = new ArrayList<>();
         for (var id : ids) {
-            ret.add(getStorage(id));
+            var storage = getStorage(id);
+            if (storage == null) continue;
+            ret.add(storage);
         }
         return ret;
     }

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java

@@ -3,15 +3,37 @@ package nu.marginalia.control;
 import com.github.jknack.handlebars.*;
 import nu.marginalia.renderer.config.HandlebarsConfigurator;
 
-import java.io.IOException;
+import java.time.LocalDate;
 
 public class ControlHandlebarsConfigurator implements HandlebarsConfigurator {
     @Override
     public void configure(Handlebars handlebars) {
         handlebars.registerHelper("readableUUID", new UUIDHelper());
+        handlebars.registerHelper("shortTimestamp", new ShortTimestampHelper());
+    }
+
+}
+
+class ShortTimestampHelper implements Helper<Object> {
+    @Override
+    public Object apply(Object context, Options options) {
+        if (context == null) return "";
+        String ts = context.toString();
+
+        String retDateBase = ts.replace('T', ' ');
+
+        // if another day, return date, hour and minute
+        if (!ts.startsWith(LocalDate.now().toString())) {
+            // return hour minute and seconds
+            return retDateBase.substring(0, "YYYY-MM-DDTHH:MM".length());
+        }
+        else { // return date, hour and minute but not seconds or ms
+            return retDateBase.substring("YYYY-MM-DDT".length(), "YYYY-MM-DDTHH:MM:SS".length());
+        }
     }
 }
 
+
 /** Helper for rendering UUIDs in a more readable way */
 class UUIDHelper implements Helper<Object> {
     @Override

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -55,6 +55,7 @@ public class ControlService extends Service {
                           ControlNodeService controlNodeService,
                           ControlDomainRankingSetsService controlDomainRankingSetsService,
                           ControlActorService controlActorService,
+                          AbortedProcessService abortedProcessService,
                           ControlErrorHandler errorHandler
                       ) throws IOException {
 
@@ -69,6 +70,7 @@ public class ControlService extends Service {
         sysActionsService.register();
         dataSetsService.register();
         controlDomainRankingSetsService.register();
+        abortedProcessService.register();
 
         // node
         controlFileStorageService.register();

code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/AbortedProcess.java

@@ -0,0 +1,17 @@
+package nu.marginalia.control.sys.model;
+
+import nu.marginalia.storage.model.FileStorage;
+
+import java.util.List;
+
+/** A process that has been manually aborted by a user,
+ * ... or error?
+ */
+public record AbortedProcess(String name,
+                             long msgId,
+                             String startDateTime,
+                             String stopDateTime,
+                             List<FileStorage> associatedStorages)
+{
+
+}

code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java

@@ -13,8 +13,8 @@ public record MessageQueueEntry (
         String ownerInstanceFull,
         long ownerTick,
         String state,
-        String createdTimeFull,
+        String createdTime,
-        String updatedTimeFull,
+        String updatedTime,
         int ttl
 )
 {
@@ -38,30 +38,4 @@ public record MessageQueueEntry (
             default -> "";
         };
     }
-
-    public String getCreatedTime() {
-        String retDateBase = createdTimeFull.replace('T', ' ');
-
-        // if another day, return date, hour and minute
-        if (!createdTimeFull.startsWith(LocalDate.now().toString())) {
-            // return hour minute and seconds
-            return retDateBase.substring(0, "YYYY-MM-DDTHH:MM".length());
-        }
-        else { // return date, hour and minute but not seconds or ms
-            return retDateBase.substring("YYYY-MM-DDT".length(), "YYYY-MM-DDTHH:MM:SS".length());
-        }
-    }
-
-    public String getUpdatedTime() {
-        String retDateBase = updatedTimeFull.replace('T', ' ');
-
-        // if another day, return date, hour and minute
-        if (!updatedTimeFull.startsWith(LocalDate.now().toString())) {
-            // return hour minute and seconds
-            return retDateBase.substring(0, "YYYY-MM-DDTHH:MM".length());
-        }
-        else { // return date, hour and minute but not seconds or ms
-            return retDateBase.substring("YYYY-MM-DDT".length(), "YYYY-MM-DDTHH:MM:SS".length());
-        }
-    }
 }

code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/AbortedProcessService.java

@@ -0,0 +1,144 @@
+package nu.marginalia.control.sys.svc;
+
+import com.google.gson.Gson;
+import com.google.gson.internal.LinkedTreeMap;
+import com.google.inject.Inject;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
+import nu.marginalia.control.RedirectControl;
+import nu.marginalia.control.sys.model.AbortedProcess;
+import nu.marginalia.model.gson.GsonFactory;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.nodecfg.NodeConfigurationService;
+import nu.marginalia.nodecfg.model.NodeConfiguration;
+import nu.marginalia.storage.FileStorageService;
+import nu.marginalia.storage.model.FileStorage;
+import nu.marginalia.storage.model.FileStorageId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import spark.Request;
+import spark.Response;
+import spark.Spark;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** Control for listing and restarting aborted processes.
+ * */
+public class AbortedProcessService {
+    private static final Logger logger = LoggerFactory.getLogger(AbortedProcessService.class);
+    private static final Gson gson = GsonFactory.get();
+    private final HikariDataSource dataSource;
+    private final FileStorageService fileStorageService;
+    private final ControlRendererFactory rendererFactory;
+    private final RedirectControl redirectControl;
+    private final MqPersistence mqPersistence;
+    private final NodeConfigurationService nodeConfigurationService;
+
+    @Inject
+    public AbortedProcessService(HikariDataSource dataSource,
+                                 FileStorageService fileStorageService,
+                                 ControlRendererFactory rendererFactory,
+                                 RedirectControl redirectControl,
+                                 MqPersistence mqPersistence,
+                                 NodeConfigurationService nodeConfigurationService)
+    {
+        this.dataSource = dataSource;
+        this.fileStorageService = fileStorageService;
+        this.rendererFactory = rendererFactory;
+        this.redirectControl = redirectControl;
+        this.mqPersistence = mqPersistence;
+        this.nodeConfigurationService = nodeConfigurationService;
+    }
+
+    public void register() {
+        var abortedProcessesRenderer = rendererFactory.renderer("control/sys/aborted-processes");
+
+        Spark.get("/public/aborted-processes", this::abortedProcessesModel, abortedProcessesRenderer::render);
+        Spark.get("/public/aborted-processes/", this::abortedProcessesModel, abortedProcessesRenderer::render);
+        Spark.post("/public/aborted-processes/:id", this::restartProcess, redirectControl.renderRedirectAcknowledgement("Restarting...", "/"));
+    }
+
+    private Object abortedProcessesModel(Request request, Response response) {
+        return Map.of("abortedProcesses", getAbortedProcesses());
+    }
+
+    private Object restartProcess(Request request, Response response) throws SQLException {
+        long msgId = Long.parseLong(request.params("id"));
+        mqPersistence.updateMessageState(msgId, MqMessageState.NEW);
+        return "";
+    }
+
+
+    private List<AbortedProcess> getAbortedProcesses() {
+        List<Integer> allNodeIds = nodeConfigurationService.getAll().stream()
+                .map(NodeConfiguration::getId)
+                .toList();
+
+        // Generate all possible values for process-related inboxes
+        String inboxes = Stream.of("converter", "loader", "crawler")
+                .flatMap(s -> allNodeIds.stream().map(i -> STR."'\{s}:\{i}'"))
+                .collect(Collectors.joining(",", "(", ")"));
+
+        try (var conn = dataSource.getConnection()) {
+            var stmt = conn.prepareStatement(STR."""
+                SELECT ID, RECIPIENT_INBOX, CREATED_TIME, UPDATED_TIME, PAYLOAD FROM MESSAGE_QUEUE
+                WHERE STATE = 'DEAD'
+                AND RECIPIENT_INBOX IN \{inboxes}
+                """); // SQL injection safe, string is not user input
+            var rs = stmt.executeQuery();
+
+            List<AbortedProcess> abortedProcesses = new ArrayList<>();
+            while (rs.next()) {
+                var msgId = rs.getLong("ID");
+                var recipientInbox = rs.getString("RECIPIENT_INBOX");
+                var createdTime = rs.getString("CREATED_TIME");
+                var updatedTime = rs.getString("UPDATED_TIME");
+                var payload = rs.getString("PAYLOAD");
+
+                List<FileStorageId> associatedStorageIds = getAssociatedStoragesIds(payload);
+                List<FileStorage> associatedStorages = fileStorageService.getStorage(associatedStorageIds);
+
+                abortedProcesses.add(new AbortedProcess(recipientInbox, msgId, createdTime, updatedTime, associatedStorages));
+            }
+
+            abortedProcesses.sort(Comparator.comparing(AbortedProcess::stopDateTime).reversed());
+
+            return abortedProcesses;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /* Attempt to parse the JSON payload and extract the file storage ids
+    * from the data, not knowing exactly what the payload looks like.
+    * */
+    private List<FileStorageId> getAssociatedStoragesIds(String payload) {
+        Map<?,?> fields = gson.fromJson(payload, LinkedTreeMap.class);
+        logger.info("{}", fields);
+        List<FileStorageId> associatedStorageIds = new ArrayList<>();
+
+        // We expect a map of objects, where some objects are a map with an "id" field
+        // and an integer value.  We want to extract the integer values.
+        for (Object field : fields.values()) {
+            if ((field instanceof Map<?,?> m) && (m.get("id") instanceof Number i))
+                associatedStorageIds.add(FileStorageId.of(i.intValue()));
+
+            if (field instanceof List) {
+                for (Object o : (List<?>) field) {
+                    if ((o instanceof Map<?,?> m) && (m.get("id") instanceof Number i))
+                        associatedStorageIds.add(FileStorageId.of(i.intValue()));
+                }
+            }
+        }
+
+        return associatedStorageIds;
+    }
+
+}

code/services-core/control-service/src/main/resources/templates/control/partials/events-table-summary.hdb

@@ -8,7 +8,7 @@
     </tr>
     {{#each events}}
     <tr>
-        <td title="{{eventDateTime}}">{{eventTime}}</td>
+        <td title="{{eventDateTime}}">{{shortTimestamp eventDateTime}}</td>
         <td>{{eventType}}</td>
         <td>{{eventMessage}}</td>
     </tr>

code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb

@@ -46,7 +46,7 @@
     <tr>
         <td>{{serviceName}}</td>
         <td>{{{readableUUID instanceFull}}}</td>
-        <td title="{{eventDateTime}}">{{eventTime}}</td>
+        <td title="{{eventDateTime}}">{{shortTimestamp eventDateTime}}</td>
         <td>{{eventType}}</td>
         <td>{{eventMessage}}</td>
     </tr>

code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb

@@ -45,7 +45,7 @@
         <td><a href="/message-queue?inbox={{recipientInbox}}">{{recipientInbox}}</a></td>
         <td>{{function}}</td>
         <td>{{{readableUUID ownerInstanceFull}}}</td>
-        <td title="{{createdTimeFull}}">{{createdTime}}</td>
+        <td title="{{createdTime}}">{{shortTimestamp createdTime}}</td>
     </tr>
     <tr>
         <td>{{ttl}}</td>
@@ -57,7 +57,7 @@
         <td><a href="/message-queue?inbox={{senderInbox}}">{{senderInbox}}</a></td>
         <td style="word-break: break-all; font-family: monospace;">{{payload}}</td>
         <td>{{ownerTick}}</td>
-        <td title="{{updatedTimeFull}}">{{updatedTime}}</td>
+        <td title="{{updatedTime}}">{{shortTimestamp updatedTime}}</td>
     </tr>
     {{/each}}
     <tfoot>

code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb

@@ -33,9 +33,12 @@
 				<li class="nav-item dropdown">
 					<a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button"  aria-expanded="false">System</a>
 					<ul class="dropdown-menu">
-                        <li><a class="dropdown-item" href="/actions" title="System actions">Actions</a></li>
                         <li><a class="dropdown-item" href="/datasets" title="View and update the data sets">Data Sets</a></li>
                         <li><a class="dropdown-item" href="/domain-ranking-sets" title="View and update domain rankings ">Domain Ranking Sets</a></li>
+                        <li><hr class="dropdown-divider"></li>
+                        <li><a class="dropdown-item" href="/aborted-processes" title="View and restart aborted processes">Aborted Processes</a></li>
+                        <li><a class="dropdown-item" href="/actions" title="System actions">Actions</a></li>
+                        <li><hr class="dropdown-divider"></li>
                         <li><a class="dropdown-item" href="/events" title="View the event log">Events</a></li>
 						<li><a class="dropdown-item" href="/message-queue" title="View or manipulate the system message queue">Message Queue</a></li>
 					</ul>

code/services-core/control-service/src/main/resources/templates/control/sys/aborted-processes.hdb

@@ -0,0 +1,73 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Control Service</title>
+    {{> control/partials/head-includes }}
+</head>
+<body>
+  {{> control/partials/nav}}
+  <div class="container">
+  <h1 class="my-3">Aborted Processes</h1>
+
+      <div class="p-3 my-3 border bg-light">
+          <p>
+            This form will let you restart a process that has
+            been manually aborted.  If additional automatic processing
+            was configured, it may not be restarted.  The progress
+            estimation of the restarted process may be inaccurate.
+          </p>
+          <p>
+              Items will vanish from this list after a few days
+          </p>
+      </div>
+
+      {{#unless abortedProcesses}}
+      <table class="table">
+        <tr>
+            <td colspan="3">No aborted processes</td>
+        </tr>
+      </table>
+      {{/unless}}
+
+      {{#each abortedProcesses}}
+      <table class="table border-bottom">
+          <tr>
+              <th>Name: </th><td>{{name}}</td>
+              <td>
+              <form method="post" action="/aborted-processes/{{msgId}}">
+                  <button type="submit" class="btn btn-danger" onclick="return confirm('Confirm restarting of {{name}}')">Restart</button>
+              </form>
+              </td>
+          </tr>
+          <tr>
+              <th>msgId</th>
+              <td colspan="2">
+                  <a href="/message-queue/{{msgId}}">{{msgId}}</a>
+              </td>
+          </tr>
+          <tr>
+              <th>Run Time: </th>
+              <td title="{{startDateTime}}">{{shortTimestamp startDateTime}}</td>
+              <td title="{{stopDateTime}}">{{shortTimestamp stopDateTime}}</td>
+          </tr>
+          {{#if associatedStorages}}
+          <tr><th colspan="3">Associated storages</th></tr>
+          <tr><td colspan="3">
+          <table class="table">
+          {{#each associatedStorages}}
+              <tr>
+                    <td>{{type}}</td>
+                    <td><a href="/nodes/{{base.node}}/storage/details?fid={{id}}">{{path}}</a></td>
+                    <td>{{description}}</td>
+              </tr>
+          {{/each}}
+          </table>
+          </td></tr>
+          {{/if}}
+      </table>
+      {{/each}}
+
+  </div>
+</body>
+{{> control/partials/foot-includes }}
+</html>

code/services-core/control-service/src/main/resources/templates/control/sys/view-message.hdb

@@ -31,8 +31,8 @@
         <tr><td>payload</td><td>
             <textarea disabled rows="6" cols="40" id="payload" name="payload">{{payload}}</textarea>
         </td><td></td></tr>
-        <tr><td>Created</td><td>{{createdTime}}</td></td><td></td></tr>
+        <tr><td>Created</td><td title="{{createdTime}}">{{shortTimestamp createdTime}}</td></td><td></td></tr>
-        <tr><td>Updated</td><td>{{updatedTime}}</td></td><td></td></tr>
+        <tr><td>Updated</td><td title="{{updatedTime}}">{{shortTimestamp updatedTime}}</td></td><td></td></tr>
         </tr>
     </table>
     {{/with}}