diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java index cf94f12e..b729bad5 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java @@ -68,7 +68,7 @@ public class ControlNodeActionsService { Spark.post("/public/nodes/:id/actions/recrawl", this::triggerAutoRecrawl, redirectControl.renderRedirectAcknowledgement("Recrawling", "..") ); - Spark.post("/public/nodes/:id/actions/process", this::triggerAutoProcess, + Spark.post("/public/nodes/:id/actions/process", this::triggerProcess, redirectControl.renderRedirectAcknowledgement("Processing", "..") ); Spark.post("/public/nodes/:id/actions/load", this::triggerLoadSelected, @@ -183,16 +183,23 @@ public class ControlNodeActionsService { return ""; } - private Object triggerAutoProcess(Request request, Response response) throws SQLException { + private Object triggerProcess(Request request, Response response) throws SQLException { int nodeId = Integer.parseInt(request.params("id")); - + boolean isAutoload = "on".equalsIgnoreCase(request.queryParams("autoload")); var toProcess = parseSourceFileStorageId(request.queryParams("source")); changeActiveStorage(nodeId, FileStorageType.PROCESSED_DATA, toProcess); - executorClient.triggerConvertAndLoad(Context.fromRequest(request), - nodeId, - toProcess); + if (isAutoload) { + executorClient.triggerConvertAndLoad(Context.fromRequest(request), + nodeId, + toProcess); + } + else { + executorClient.triggerConvert(Context.fromRequest(request), + nodeId, + toProcess); + } return ""; } diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb index 480a4327..82d1e7c5 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb @@ -7,7 +7,7 @@ {{#if allProcessedData}}
This will load processed data on Node {{node.id}}. This isn't typically necessary for crawled data, - but is needed for sideloaded data. Select the processed data to load below. + but is necessary for sideloaded data. Select the processed data to be loaded below.
diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb index f6df65b0..b9824549 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb @@ -6,9 +6,7 @@

The reprocessing will extract keywords, titles, and other metadata from the raw crawl data - and generate parquet files that can be loaded into the index. This loading will be done automatically, - and unless the node is configured to keep the processed data, - the processed data will be deleted after loading to save disk space. + and generate parquet files that can be loaded into the index.

@@ -33,6 +31,18 @@ {{/each}} +
+
+ + +
+ + If enabled, the data will be loaded into the search engine automatically after it is created. Depending on how the + node is configured, the intermediate processed data may be deleted + after loading. If disabled, the data will be processed, but loading needs to be triggered manually. + This is useful if multiple data sets are to be loaded, such as when intermingling crawled data with sideloaded data. + +