From 4baf9527d74038f172f3d24d9a97058b67bfb0d3 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sat, 14 Oct 2023 12:07:40 +0200 Subject: [PATCH] (*) WIP Control GUI redesign, executor-service, multi-node mq This turned out to be very difficult to do in small isolated steps. * Design overhaul of the control gui using bootstrap * Move the actors out of control-service into to a new executor-service, that can be run on multiple nodes * Add node-affinity to message queue --- code/api/executor-api/build.gradle | 34 ++ .../executor/client/ExecutorClient.java | 108 ++++ .../executor}/model/ActorRunState.java | 2 +- .../executor/model/ActorRunStates.java | 5 + .../model/crawl/RecrawlParameters.java | 11 + .../executor/model/load/LoadParameters.java | 10 + .../marginalia/index/client/IndexClient.java | 15 +- code/api/process-mqapi/build.gradle | 2 +- .../mqapi/converting/ConvertRequest.java | 2 +- .../mqapi/crawling/CrawlRequest.java | 6 +- .../marginalia/mqapi/loading/LoadRequest.java | 2 +- .../marginalia/query/client/QueryClient.java | 11 - code/common/config/build.gradle | 18 + .../java/nu/marginalia/IndexLocations.java | 67 +++ .../main/java/nu/marginalia/UserAgent.java | 4 +- .../nodecfg/NodeConfigurationService.java | 104 ++++ .../nodecfg/model/NodeConfiguration.java | 9 + .../storage/FileStorageManifest.java | 6 +- .../storage/FileStorageService.java | 146 +++++- .../storage/model/FileStorage.java | 14 +- .../storage/model/FileStorageBase.java | 9 +- .../storage/model/FileStorageBaseId.java | 2 +- .../storage/model/FileStorageBaseType.java | 12 + .../storage/model/FileStorageId.java | 2 +- .../storage/model/FileStorageType.java | 10 +- .../nodecfg/NodeConfigurationServiceTest.java | 70 +++ .../storage/FileStorageServiceTest.java | 51 +- .../db/storage/model/FileStorageBaseType.java | 8 - .../V23_11_0_001__heartbeat_node.sql | 3 + .../V23_11_0_002__file_storage_state.sql | 17 + .../V23_11_0_003__node_configuration.sql | 4 + .../V23_11_0_004__file_storage_base_type.sql | 10 + .../migration/V23_11_0_005__node_config.sql | 6 + .../nu/marginalia/linkdb/LinkdbReader.java | 27 +- .../ProcessAdHocTaskHeartbeatImpl.java | 15 +- .../process/control/ProcessHeartbeatImpl.java | 9 +- .../control/ProcessTaskHeartbeatImpl.java | 16 +- .../nu/marginalia/client/ServiceMonitors.java | 41 +- .../service/SearchServiceDescriptors.java | 1 + .../service/descriptor/ServiceDescriptor.java | 6 +- .../nu/marginalia/service/id/ServiceId.java | 1 + .../service/control/ServiceHeartbeatImpl.java | 9 +- .../control/ServiceTaskHeartbeatImpl.java | 16 +- .../nu/marginalia/service/server/Service.java | 4 +- code/features-control/actors/build.gradle | 41 ++ .../main/java/nu/marginalia}/actor/Actor.java | 2 +- .../java/nu/marginalia/actor/ActorApi.java | 55 +++ .../actor/ActorControlService.java} | 70 +-- .../monitor/AbstractProcessSpawnerActor.java | 16 +- .../actor/monitor/ConverterMonitorActor.java | 13 +- .../actor/monitor/CrawlerMonitorActor.java | 5 +- .../monitor/FileStorageMonitorActor.java | 14 +- .../monitor/IndexConstructorMonitorActor.java | 11 +- .../actor/monitor/LoaderMonitorActor.java | 10 +- .../monitor/MessageQueueMonitorActor.java | 6 +- .../monitor/ProcessLivenessMonitorActor.java | 217 ++++++++ .../actor/task/ActorProcessWatcher.java | 2 +- .../marginalia}/actor/task/ConvertActor.java | 26 +- .../actor/task/ConvertAndLoadActor.java | 32 +- .../nu/marginalia}/actor/task/CrawlActor.java | 22 +- .../actor/task/CrawlJobExtractorActor.java | 25 +- .../actor/task/ExportDataActor.java | 10 +- .../marginalia}/actor/task/RecrawlActor.java | 44 +- .../actor/task/RestoreBackupActor.java | 16 +- .../TriggerAdjacencyCalculationActor.java | 6 +- .../actor/task/TruncateLinkDatabase.java | 6 +- .../nu/marginalia}/svc/BackupService.java | 57 ++- .../process-execution/build.gradle | 28 ++ .../control/process/ProcessOutboxes.java | 10 +- .../control/process/ProcessService.java | 8 +- .../reader/IndexJournalReaderPagingImpl.java | 8 + .../util/SimpleBlockingThreadPool.java | 11 +- .../marginalia/actor/ActorStateMachine.java | 9 +- .../nu/marginalia/mq/MessageQueueFactory.java | 16 +- .../nu/marginalia/mq/outbox/MqOutbox.java | 6 +- .../actor/ActorStateMachineErrorTest.java | 4 +- .../actor/ActorStateMachineNullTest.java | 4 +- .../actor/ActorStateMachineResumeTest.java | 34 +- .../actor/ActorStateMachineTest.java | 14 +- .../java/nu/marginalia/mq/MqTestUtil.java | 4 +- .../nu/marginalia/mq/outbox/MqOutboxTest.java | 48 +- .../mq/persistence/MqPersistenceTest.java | 39 +- code/process-models/crawl-spec/build.gradle | 1 + .../crawlspec/CrawlSpecFileNames.java | 18 +- .../crawlspec/CrawlSpecGenerator.java | 8 +- .../marginalia/converting/ConverterMain.java | 11 +- .../java/nu/marginalia/crawl/CrawlerMain.java | 50 +- .../index-constructor-process/build.gradle | 3 + .../index/IndexConstructorMain.java | 50 +- .../loading/LoaderIndexJournalWriter.java | 10 +- .../nu/marginalia/loading/LoaderMain.java | 30 +- .../nu/marginalia/loading/LoaderModule.java | 13 +- .../loader/LoaderIndexJournalWriterTest.java | 17 +- .../nu/marginalia/search/SearchService.java | 2 - .../control-service/build.gradle | 2 + .../control/ControlProcessModule.java | 5 +- .../nu/marginalia/control/ControlService.java | 319 ++---------- .../nu/marginalia/control/HtmlRedirect.java | 22 - .../java/nu/marginalia/control/Redirects.java | 32 ++ .../monitor/ProcessLivenessMonitorActor.java | 84 ---- .../control/{ => app}/model/ApiKeyModel.java | 2 +- .../model/BlacklistedDomainModel.java | 2 +- .../model/DomainComplaintCategory.java | 2 +- .../{ => app}/model/DomainComplaintModel.java | 2 +- .../control/{ => app}/svc/ApiKeyService.java | 62 ++- .../svc/ControlBlacklistService.java | 42 +- .../{ => app}/svc/DomainComplaintService.java | 60 ++- .../app/svc/RandomExplorationService.java | 113 +++++ .../{ => app}/svc/SearchToBanService.java | 15 +- .../control/{ => node}/model/ActorState.java | 2 +- .../{ => node}/model/ActorStateGraph.java | 14 +- .../model/FileStorageBaseWithStorage.java | 4 +- .../model/FileStorageFileModel.java | 2 +- .../model/FileStorageWithActions.java | 10 +- .../model/FileStorageWithRelatedEntries.java | 9 +- .../control/node/model/IndexNode.java | 4 + .../control/node/model/IndexNodeStatus.java | 4 + .../control/node/svc/ControlActorService.java | 103 ++++ .../node/svc/ControlFileStorageService.java | 67 +++ .../node/svc/ControlNodeActionsService.java | 107 ++++ .../control/node/svc/ControlNodeService.java | 465 ++++++++++++++++++ .../control/svc/ControlActionsService.java | 152 ------ .../control/svc/ControlActorService.java | 166 ------- .../svc/ControlFileStorageService.java | 199 -------- .../control/svc/RandomExplorationService.java | 60 --- .../{ => sys}/model/EventLogEntry.java | 2 +- .../model/EventLogServiceFilter.java | 2 +- .../{ => sys}/model/EventLogTypeFilter.java | 2 +- .../{ => sys}/model/MessageQueueEntry.java | 2 +- .../{ => sys}/model/ProcessHeartbeat.java | 25 +- .../{ => sys}/model/ServiceHeartbeat.java | 2 +- .../{ => sys}/model/TaskHeartbeat.java | 3 +- .../sys/svc/ControlSysActionsService.java | 99 ++++ .../{ => sys}/svc/EventLogService.java | 8 +- .../{ => sys}/svc/HeartbeatService.java | 78 ++- .../{ => sys}/svc/MessageQueueService.java | 34 +- .../main/resources/static/control/style.css | 123 ----- .../main/resources/static/control/tables.css | 21 + .../resources/templates/control/actions.hdb | 121 ----- .../resources/templates/control/actors.hdb | 21 - .../resources/templates/control/api-keys.hdb | 61 --- .../templates/control/app/api-keys.hdb | 65 +++ .../templates/control/app/blacklist.hdb | 66 +++ .../control/app/domain-complaints.hdb | 130 +++++ .../{ => app}/review-random-domains.hdb | 0 .../control/{ => app}/search-to-ban.hdb | 0 .../resources/templates/control/blacklist.hdb | 68 --- .../templates/control/domain-complaints.hdb | 111 ----- .../resources/templates/control/events.hdb | 20 - .../resources/templates/control/index.hdb | 17 +- .../templates/control/message-queue.hdb | 20 - .../templates/control/node/node-actions.hdb | 232 +++++++++ .../templates/control/node/node-actors.hdb | 43 ++ .../templates/control/node/node-config.hdb | 42 ++ .../control/node/node-new-specs-form.hdb | 85 ++++ .../templates/control/node/node-overview.hdb | 45 ++ .../control/node/node-storage-conf.hdb | 67 +++ .../control/node/node-storage-details.hdb | 115 +++++ .../control/node/node-storage-list.hdb | 149 ++++++ .../control/partials/actor-state-graph.hdb | 16 - .../control/partials/actors-table.hdb | 11 +- .../control/partials/events-table-summary.hdb | 4 +- .../control/partials/events-table.hdb | 4 +- .../control/partials/foot-includes.hdb | 9 + .../control/partials/head-includes.hdb | 3 + .../control/partials/message-queue-table.hdb | 4 +- .../templates/control/partials/nav.hdb | 48 +- .../control/partials/nodes-table.hdb | 21 + .../control/partials/processes-table.hdb | 8 +- .../control/partials/services-table.hdb | 2 +- .../partials/storage-details/files.hdb | 17 + .../partials/storage-details/related.hdb | 17 + .../control/partials/storage-table.hdb | 35 -- .../templates/control/service-by-id.hdb | 22 - .../templates/control/storage-backups.hdb | 27 - .../templates/control/storage-crawls.hdb | 28 -- .../templates/control/storage-details.hdb | 140 ------ .../templates/control/storage-overview.hdb | 52 -- .../templates/control/storage-processed.hdb | 26 - .../templates/control/storage-specs.hdb | 64 --- .../templates/control/sys/events.hdb | 14 + .../templates/control/sys/message-queue.hdb | 14 + .../control/{ => sys}/new-message.hdb | 19 +- .../templates/control/sys/service-by-id.hdb | 16 + .../templates/control/{ => sys}/services.hdb | 0 .../{ => sys}/update-message-state.hdb | 15 +- .../control/{ => sys}/view-message.hdb | 19 +- .../control/svc/ApiKeyServiceTest.java | 9 +- .../control/svc/HeartbeatServiceTest.java | 35 +- .../executor-service/build.gradle | 55 +++ .../nu/marginalia/executor/ExecutorMain.java | 33 ++ .../marginalia/executor/ExecutorModule.java | 13 + .../nu/marginalia/executor/ExecutorSvc.java | 101 ++++ .../executor/svc/BackupService.java | 24 + .../executor/svc/ProcessingService.java | 99 ++++ .../executor/svc/SideloadService.java | 32 ++ .../ExecutorSvcApiIntegrationTest.java | 202 ++++++++ .../java/nu/marginalia/index/IndexMain.java | 1 - .../java/nu/marginalia/index/IndexModule.java | 9 +- .../nu/marginalia/index/IndexService.java | 9 +- .../index/IndexServicesFactory.java | 15 +- .../marginalia/index/IndexTablesModule.java | 10 - ...IndexQueryServiceIntegrationSmokeTest.java | 57 ++- .../svc/IndexQueryServiceIntegrationTest.java | 43 +- ...ndexQueryServiceIntegrationTestModule.java | 25 +- .../nu/marginalia/index/util/TestUtil.java | 2 +- docker-compose.yml | 16 +- run/env/service.env | 2 +- settings.gradle | 5 + 209 files changed, 4878 insertions(+), 2672 deletions(-) create mode 100644 code/api/executor-api/build.gradle create mode 100644 code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java rename code/{services-core/control-service/src/main/java/nu/marginalia/control => api/executor-api/src/main/java/nu/marginalia/executor}/model/ActorRunState.java (94%) create mode 100644 code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java create mode 100644 code/api/executor-api/src/main/java/nu/marginalia/executor/model/crawl/RecrawlParameters.java create mode 100644 code/api/executor-api/src/main/java/nu/marginalia/executor/model/load/LoadParameters.java create mode 100644 code/common/config/src/main/java/nu/marginalia/IndexLocations.java create mode 100644 code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java create mode 100644 code/common/config/src/main/java/nu/marginalia/nodecfg/model/NodeConfiguration.java rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/FileStorageManifest.java (92%) rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/FileStorageService.java (79%) rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/model/FileStorage.java (88%) rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/model/FileStorageBase.java (71%) rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/model/FileStorageBaseId.java (74%) create mode 100644 code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/model/FileStorageId.java (89%) rename code/common/{db/src/main/java/nu/marginalia/db => config/src/main/java/nu/marginalia}/storage/model/FileStorageType.java (55%) create mode 100644 code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java rename code/common/{db/src/test/java/nu/marginalia/db => config/src/test/java/nu/marginalia}/storage/FileStorageServiceTest.java (79%) delete mode 100644 code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseType.java create mode 100644 code/common/db/src/main/resources/db/migration/V23_11_0_001__heartbeat_node.sql create mode 100644 code/common/db/src/main/resources/db/migration/V23_11_0_002__file_storage_state.sql create mode 100644 code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql create mode 100644 code/common/db/src/main/resources/db/migration/V23_11_0_004__file_storage_base_type.sql create mode 100644 code/common/db/src/main/resources/db/migration/V23_11_0_005__node_config.sql create mode 100644 code/features-control/actors/build.gradle rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/Actor.java (92%) create mode 100644 code/features-control/actors/src/main/java/nu/marginalia/actor/ActorApi.java rename code/{services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java => features-control/actors/src/main/java/nu/marginalia/actor/ActorControlService.java} (66%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/AbstractProcessSpawnerActor.java (96%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/ConverterMonitorActor.java (59%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/CrawlerMonitorActor.java (79%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/FileStorageMonitorActor.java (92%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/IndexConstructorMonitorActor.java (59%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/LoaderMonitorActor.java (71%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/monitor/MessageQueueMonitorActor.java (97%) create mode 100644 code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ProcessLivenessMonitorActor.java rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/ActorProcessWatcher.java (98%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/ConvertActor.java (96%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/ConvertAndLoadActor.java (94%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/CrawlActor.java (92%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/CrawlJobExtractorActor.java (87%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/ExportDataActor.java (97%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/RecrawlActor.java (81%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/RestoreBackupActor.java (73%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/TriggerAdjacencyCalculationActor.java (98%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/actor/task/TruncateLinkDatabase.java (96%) rename code/{services-core/control-service/src/main/java/nu/marginalia/control => features-control/actors/src/main/java/nu/marginalia}/svc/BackupService.java (58%) create mode 100644 code/features-control/process-execution/build.gradle rename code/{services-core/control-service => features-control/process-execution}/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java (83%) rename code/{services-core/control-service => features-control/process-execution}/src/main/java/nu/marginalia/control/process/ProcessService.java (98%) delete mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/HtmlRedirect.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/Redirects.java delete mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/model/ApiKeyModel.java (71%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/model/BlacklistedDomainModel.java (74%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/model/DomainComplaintCategory.java (94%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/model/DomainComplaintModel.java (92%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/svc/ApiKeyService.java (60%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/svc/ControlBlacklistService.java (63%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/svc/DomainComplaintService.java (58%) create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => app}/svc/SearchToBanService.java (78%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => node}/model/ActorState.java (93%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => node}/model/ActorStateGraph.java (63%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => node}/model/FileStorageBaseWithStorage.java (64%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => node}/model/FileStorageFileModel.java (78%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => node}/model/FileStorageWithActions.java (85%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => node}/model/FileStorageWithRelatedEntries.java (57%) create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNode.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNodeStatus.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlActorService.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java delete mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java delete mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java delete mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java delete mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/EventLogEntry.java (92%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/EventLogServiceFilter.java (73%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/EventLogTypeFilter.java (72%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/MessageQueueEntry.java (97%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/ProcessHeartbeat.java (57%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/ServiceHeartbeat.java (92%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/model/TaskHeartbeat.java (92%) create mode 100644 code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/svc/EventLogService.java (98%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/svc/HeartbeatService.java (60%) rename code/services-core/control-service/src/main/java/nu/marginalia/control/{ => sys}/svc/MessageQueueService.java (86%) delete mode 100644 code/services-core/control-service/src/main/resources/static/control/style.css create mode 100644 code/services-core/control-service/src/main/resources/static/control/tables.css delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/actors.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/app/api-keys.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/app/blacklist.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/app/domain-complaints.hdb rename code/services-core/control-service/src/main/resources/templates/control/{ => app}/review-random-domains.hdb (100%) rename code/services-core/control-service/src/main/resources/templates/control/{ => app}/search-to-ban.hdb (100%) delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/events.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/message-queue.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-new-specs-form.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-overview.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-storage-conf.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/partials/actor-state-graph.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/partials/foot-includes.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/partials/head-includes.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/partials/nodes-table.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/partials/storage-details/files.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/partials/storage-details/related.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/service-by-id.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/storage-backups.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/storage-crawls.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/storage-details.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/storage-overview.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/storage-processed.hdb delete mode 100644 code/services-core/control-service/src/main/resources/templates/control/storage-specs.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/sys/events.hdb create mode 100644 code/services-core/control-service/src/main/resources/templates/control/sys/message-queue.hdb rename code/services-core/control-service/src/main/resources/templates/control/{ => sys}/new-message.hdb (76%) create mode 100644 code/services-core/control-service/src/main/resources/templates/control/sys/service-by-id.hdb rename code/services-core/control-service/src/main/resources/templates/control/{ => sys}/services.hdb (100%) rename code/services-core/control-service/src/main/resources/templates/control/{ => sys}/update-message-state.hdb (84%) rename code/services-core/control-service/src/main/resources/templates/control/{ => sys}/view-message.hdb (83%) create mode 100644 code/services-core/executor-service/build.gradle create mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java create mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java create mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java create mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/BackupService.java create mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ProcessingService.java create mode 100644 code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java create mode 100644 code/services-core/executor-service/src/test/java/nu/marginalia/executor/ExecutorSvcApiIntegrationTest.java delete mode 100644 code/services-core/index-service/src/main/java/nu/marginalia/index/IndexTablesModule.java diff --git a/code/api/executor-api/build.gradle b/code/api/executor-api/build.gradle new file mode 100644 index 00000000..28defeb2 --- /dev/null +++ b/code/api/executor-api/build.gradle @@ -0,0 +1,34 @@ +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:common:model') + implementation project(':code:api:index-api') + implementation project(':code:common:config') + implementation project(':code:common:db') + implementation project(':code:libraries:message-queue') + implementation project(':code:common:service-discovery') + implementation project(':code:common:service-client') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.notnull + implementation libs.guice + implementation libs.rxjava + implementation libs.protobuf + implementation libs.gson + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + +} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java new file mode 100644 index 00000000..a1715f0e --- /dev/null +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -0,0 +1,108 @@ +package nu.marginalia.executor.client; + +import com.google.inject.Inject; +import nu.marginalia.WmsaHome; +import nu.marginalia.client.AbstractDynamicClient; +import nu.marginalia.client.Context; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.executor.model.ActorRunStates; +import nu.marginalia.executor.model.crawl.RecrawlParameters; +import nu.marginalia.executor.model.load.LoadParameters; +import nu.marginalia.model.gson.GsonFactory; +import nu.marginalia.service.descriptor.ServiceDescriptors; +import nu.marginalia.service.id.ServiceId; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +public class ExecutorClient extends AbstractDynamicClient { + @Inject + public ExecutorClient(ServiceDescriptors descriptors) { + super(descriptors.forId(ServiceId.Executor), WmsaHome.getHostsFile(), GsonFactory::get); + } + + public void startFsm(Context ctx, int node, String actorName) { + post(ctx, node, "/actor/"+actorName+"/start", "").blockingSubscribe(); + } + + public void stopFsm(Context ctx, int node, String actorName) { + post(ctx, node, "/actor/"+actorName+"/stop", "").blockingSubscribe(); + } + + public void triggerCrawl(Context ctx, int node, String fid) { + post(ctx, node, "/process/crawl/" + fid, "").blockingSubscribe(); + } + + public void triggerRecrawl(Context ctx, int node, RecrawlParameters parameters) { + post(ctx, node, "/process/recrawl", parameters).blockingSubscribe(); + } + + public void triggerConvert(Context ctx, int node, FileStorageId fid) { + post(ctx, node, "/process/convert/" + fid.id(), "").blockingSubscribe(); + } + @Deprecated + public void triggerConvert(Context ctx, int node, String fid) { + post(ctx, node, "/process/convert/" + fid, "").blockingSubscribe(); + } + + public void triggerProcessAndLoad(Context ctx, int node, String fid) { + post(ctx, node, "/process/convert-load/" + fid, "").blockingSubscribe(); + } + + @Deprecated + public void loadProcessedData(Context ctx, int node, String fid) { + loadProcessedData(ctx, node, new LoadParameters(List.of(new FileStorageId(Long.parseLong(fid))))); + } + + public void loadProcessedData(Context ctx, int node, LoadParameters ids) { + post(ctx, node, "/process/load", ids).blockingSubscribe(); + } + + public void calculateAdjacencies(Context ctx, int node) { + post(ctx, node, "/process/adjacency-calculation", "").blockingSubscribe(); + } + + public void exportData(Context ctx) { +// post(ctx, node, "/process/adjacency-calculation/", "").blockingSubscribe(); + // FIXME + } + + public void sideloadEncyclopedia(Context ctx, int node, Path sourcePath) { + post(ctx, node, + "/sideload/encyclopedia?path="+ URLEncoder.encode(sourcePath.toString(), StandardCharsets.UTF_8), + "").blockingSubscribe(); + + } + + public void sideloadDirtree(Context ctx, int node, Path sourcePath) { + post(ctx, node, + "/sideload/dirtree?path="+ URLEncoder.encode(sourcePath.toString(), StandardCharsets.UTF_8), + "").blockingSubscribe(); + } + + public void sideloadStackexchange(Context ctx, int node, Path sourcePath) { + post(ctx, node, + "/sideload/stackexchange?path="+URLEncoder.encode(sourcePath.toString(), StandardCharsets.UTF_8), + "").blockingSubscribe(); + } + + public void createCrawlSpecFromDb(Context context, int node, String description) { + post(context, node, "/process/crawl-spec/from-db?description="+URLEncoder.encode(description, StandardCharsets.UTF_8), "") + .blockingSubscribe(); + } + + public void createCrawlSpecFromDownload(Context context, int node, String description, String url) { + post(context, node, "/process/crawl-spec/from-download?description="+URLEncoder.encode(description, StandardCharsets.UTF_8)+"&url="+URLEncoder.encode(url, StandardCharsets.UTF_8), "") + .blockingSubscribe(); + } + + public void restoreBackup(Context context, int node, String fid) { + post(context, node, "/backup/" + fid + "/restore", "").blockingSubscribe(); + } + + public ActorRunStates getActorStates(Context context, int node) { + return get(context, node, "/actor", ActorRunStates.class).blockingFirst(); + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunState.java similarity index 94% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java rename to code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunState.java index 805038cb..a9d71b93 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunState.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.executor.model; public record ActorRunState(String name, String state, diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java new file mode 100644 index 00000000..86dd9520 --- /dev/null +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/ActorRunStates.java @@ -0,0 +1,5 @@ +package nu.marginalia.executor.model; + +import java.util.List; + +public record ActorRunStates(int node, List states) {} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/crawl/RecrawlParameters.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/crawl/RecrawlParameters.java new file mode 100644 index 00000000..e4fbefc3 --- /dev/null +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/crawl/RecrawlParameters.java @@ -0,0 +1,11 @@ +package nu.marginalia.executor.model.crawl; + +import nu.marginalia.storage.model.FileStorageId; + +import java.util.List; + +public record RecrawlParameters( + FileStorageId crawlDataId, + List crawlSpecIds +) { +} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/model/load/LoadParameters.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/load/LoadParameters.java new file mode 100644 index 00000000..1874406b --- /dev/null +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/model/load/LoadParameters.java @@ -0,0 +1,10 @@ +package nu.marginalia.executor.model.load; + +import nu.marginalia.storage.model.FileStorageId; + +import java.util.List; + +public record LoadParameters( + List ids +) { +} diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java index d9d136e8..c85d8899 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java +++ b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java @@ -2,6 +2,7 @@ package nu.marginalia.index.client; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.google.inject.name.Named; import io.prometheus.client.Summary; import io.reactivex.rxjava3.core.Observable; import nu.marginalia.WmsaHome; @@ -23,23 +24,21 @@ public class IndexClient extends AbstractDynamicClient { private static final Summary wmsa_search_index_api_time = Summary.build().name("wmsa_search_index_api_time").help("-").register(); - private final MqOutbox outbox; + MqOutbox outbox; @Inject public IndexClient(ServiceDescriptors descriptors, - MessageQueueFactory messageQueueFactory) + MessageQueueFactory messageQueueFactory, + @Named("wmsa-system-node") Integer nodeId) { super(descriptors.forId(ServiceId.Index), WmsaHome.getHostsFile(), GsonFactory::get); - String inboxName = ServiceId.Index.name + ":" + "0"; - String outboxName = System.getProperty("service-name", UUID.randomUUID().toString()); - - outbox = messageQueueFactory.createOutbox(inboxName, outboxName, UUID.randomUUID()); - + String inboxName = ServiceId.Index.name; + String outboxName = System.getProperty("service-name:"+nodeId, UUID.randomUUID().toString()); + outbox = messageQueueFactory.createOutbox(inboxName, nodeId, outboxName, nodeId, UUID.randomUUID()); setTimeout(30); } - public MqOutbox outbox() { return outbox; } diff --git a/code/api/process-mqapi/build.gradle b/code/api/process-mqapi/build.gradle index eace2289..b99fdf75 100644 --- a/code/api/process-mqapi/build.gradle +++ b/code/api/process-mqapi/build.gradle @@ -12,7 +12,7 @@ java { } dependencies { - implementation project(':code:common:db') + implementation project(':code:common:config') testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java index abacf8af..c5241914 100644 --- a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java +++ b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java @@ -1,7 +1,7 @@ package nu.marginalia.mqapi.converting; import lombok.AllArgsConstructor; -import nu.marginalia.db.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageId; @AllArgsConstructor public class ConvertRequest { diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java index 16cdc6f3..f8376d53 100644 --- a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java +++ b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java @@ -1,11 +1,13 @@ package nu.marginalia.mqapi.crawling; import lombok.AllArgsConstructor; -import nu.marginalia.db.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageId; + +import java.util.List; /** A request to start a crawl */ @AllArgsConstructor public class CrawlRequest { - public FileStorageId specStorage; + public List specStorage; public FileStorageId crawlStorage; } diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java index 10e22ed1..89410950 100644 --- a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java +++ b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java @@ -1,7 +1,7 @@ package nu.marginalia.mqapi.loading; import lombok.AllArgsConstructor; -import nu.marginalia.db.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageId; import java.util.List; diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java index 742c8200..ecb50102 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java @@ -29,19 +29,11 @@ public class QueryClient extends AbstractDynamicClient { private final Logger logger = LoggerFactory.getLogger(getClass()); - private final MqOutbox outbox; - @Inject public QueryClient(ServiceDescriptors descriptors, MessageQueueFactory messageQueueFactory) { super(descriptors.forId(ServiceId.Query), WmsaHome.getHostsFile(), GsonFactory::get); - - String inboxName = ServiceId.Query.name + ":" + "0"; - String outboxName = System.getProperty("service-name", UUID.randomUUID().toString()); - - outbox = messageQueueFactory.createOutbox(inboxName, outboxName, UUID.randomUUID()); - } /** Delegate an Index API style query directly to the index service */ @@ -57,8 +49,5 @@ public class QueryClient extends AbstractDynamicClient { () -> this.postGet(ctx, 0, "/search/", params, QueryResponse.class).blockingFirst() ); } - public MqOutbox outbox() { - return outbox; - } } diff --git a/code/common/config/build.gradle b/code/common/config/build.gradle index 60b2eb7e..d088d4aa 100644 --- a/code/common/config/build.gradle +++ b/code/common/config/build.gradle @@ -14,4 +14,22 @@ java { dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') + implementation project(':code:common:db') + implementation project(':code:common:model') + + implementation libs.bundles.slf4j + implementation libs.bundles.mariadb + implementation libs.mockito + implementation libs.guice + implementation libs.gson + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + + + + testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') + testImplementation 'org.testcontainers:mariadb:1.17.4' + testImplementation 'org.testcontainers:junit-jupiter:1.17.4' + } diff --git a/code/common/config/src/main/java/nu/marginalia/IndexLocations.java b/code/common/config/src/main/java/nu/marginalia/IndexLocations.java new file mode 100644 index 00000000..abdc9acf --- /dev/null +++ b/code/common/config/src/main/java/nu/marginalia/IndexLocations.java @@ -0,0 +1,67 @@ +package nu.marginalia; + +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBaseType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; + +/** The IndexLocations class is responsible for knowledge about the locations + * of various important system paths. The methods take a FileStorageService, + * as these paths are node-dependent. + */ +public class IndexLocations { + + private static final Logger logger = LoggerFactory.getLogger(IndexLocations.class); + /** Return the path to the current link database */ + public static Path getLinkdbLivePath(FileStorageService fileStorage) { + return getStorage(fileStorage, FileStorageBaseType.CURRENT, "ldbr"); + } + + /** Return the path to the next link database */ + public static Path getLinkdbWritePath(FileStorageService fileStorage) { + return getStorage(fileStorage, FileStorageBaseType.CURRENT, "ldbw"); + } + + /** Return the path to the current live index */ + public static Path getCurrentIndex(FileStorageService fileStorage) { + return getStorage(fileStorage, FileStorageBaseType.CURRENT, "ir"); + } + + /** Return the path to the designated index construction area */ + public static Path getIndexConstructionArea(FileStorageService fileStorage) { + return getStorage(fileStorage, FileStorageBaseType.CURRENT, "iw"); + } + + /** Return the path to the search sets */ + public static Path getSearchSetsPath(FileStorageService fileStorage) { + return getStorage(fileStorage, FileStorageBaseType.CURRENT, "ss"); + } + + private static Path getStorage(FileStorageService service, FileStorageBaseType baseType, String pathPart) { + try { + var base = service.getStorageBase(baseType); + if (base == null) { + throw new IllegalStateException("File storage base " + baseType + " is not configured!"); + } + + // Ensure the directory exists + Path ret = base.asPath().resolve(pathPart); + if (!Files.exists(ret)) { + logger.info("Creating system directory {}", ret); + + Files.createDirectories(ret); + } + + return ret; + } + catch (SQLException | IOException ex) { + throw new IllegalStateException("Error fetching storage " + baseType + " / " + pathPart, ex); + } + } + +} diff --git a/code/common/config/src/main/java/nu/marginalia/UserAgent.java b/code/common/config/src/main/java/nu/marginalia/UserAgent.java index 75c1aa7d..f3897592 100644 --- a/code/common/config/src/main/java/nu/marginalia/UserAgent.java +++ b/code/common/config/src/main/java/nu/marginalia/UserAgent.java @@ -1,5 +1,3 @@ package nu.marginalia; -public record UserAgent(String uaString) { - -} +public record UserAgent(String uaString) {} diff --git a/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java b/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java new file mode 100644 index 00000000..151868c0 --- /dev/null +++ b/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java @@ -0,0 +1,104 @@ +package nu.marginalia.nodecfg; + +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.nodecfg.model.NodeConfiguration; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class NodeConfigurationService { + + private final HikariDataSource dataSource; + + public NodeConfigurationService(HikariDataSource dataSource) { + this.dataSource = dataSource; + } + + public NodeConfiguration create(String description, boolean acceptQueries) throws SQLException { + try (var conn = dataSource.getConnection(); + var is = conn.prepareStatement(""" + INSERT INTO NODE_CONFIGURATION(DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?) + """); + var qs = conn.prepareStatement(""" + SELECT LAST_INSERT_ID() + """)) + { + is.setString(1, description); + is.setBoolean(2, acceptQueries); + + if (is.executeUpdate() <= 0) { + throw new IllegalStateException("Failed to insert configuration"); + } + + var rs = qs.executeQuery(); + + if (rs.next()) { + return get(rs.getInt(1)); + } + + throw new AssertionError("No LAST_INSERT_ID()"); + } + } + + public List getAll() throws SQLException { + try (var conn = dataSource.getConnection(); + var qs = conn.prepareStatement(""" + SELECT ID, DESCRIPTION, ACCEPT_QUERIES, DISABLED + FROM NODE_CONFIGURATION + """)) { + var rs = qs.executeQuery(); + List ret = new ArrayList<>(); + while (rs.next()) { + ret.add(new NodeConfiguration( + rs.getInt("ID"), + rs.getString("DESCRIPTION"), + rs.getBoolean("ACCEPT_QUERIES"), + rs.getBoolean("DISABLED") + )); + } + return ret; + } + } + + public NodeConfiguration get(int nodeId) throws SQLException { + try (var conn = dataSource.getConnection(); + var qs = conn.prepareStatement(""" + SELECT ID, DESCRIPTION, ACCEPT_QUERIES, DISABLED + FROM NODE_CONFIGURATION + WHERE ID=? + """)) { + qs.setInt(1, nodeId); + var rs = qs.executeQuery(); + if (rs.next()) { + return new NodeConfiguration( + rs.getInt("ID"), + rs.getString("DESCRIPTION"), + rs.getBoolean("ACCEPT_QUERIES"), + rs.getBoolean("DISABLED") + ); + } + } + + return null; + } + + public void save(NodeConfiguration config) throws SQLException { + try (var conn = dataSource.getConnection(); + var us = conn.prepareStatement(""" + UPDATE NODE_CONFIGURATION + SET DESCRIPTION=?, ACCEPT_QUERIES=?, DISABLED=? + WHERE ID=? + """)) + { + us.setString(1, config.description()); + us.setBoolean(2, config.acceptQueries()); + us.setBoolean(3, config.disabled()); + us.setInt(4, config.node()); + + if (us.executeUpdate() <= 0) + throw new IllegalStateException("Failed to update configuration"); + + } + } +} diff --git a/code/common/config/src/main/java/nu/marginalia/nodecfg/model/NodeConfiguration.java b/code/common/config/src/main/java/nu/marginalia/nodecfg/model/NodeConfiguration.java new file mode 100644 index 00000000..dc01ff39 --- /dev/null +++ b/code/common/config/src/main/java/nu/marginalia/nodecfg/model/NodeConfiguration.java @@ -0,0 +1,9 @@ +package nu.marginalia.nodecfg.model; + +public record NodeConfiguration(int node, + String description, + boolean acceptQueries, + boolean disabled + ) +{ +} diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageManifest.java similarity index 92% rename from code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java rename to code/common/config/src/main/java/nu/marginalia/storage/FileStorageManifest.java index f002a47d..ae275bc3 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageManifest.java @@ -1,9 +1,9 @@ -package nu.marginalia.db.storage; +package nu.marginalia.storage; import com.google.gson.Gson; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageType; import nu.marginalia.model.gson.GsonFactory; +import nu.marginalia.storage.model.FileStorage; +import nu.marginalia.storage.model.FileStorageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java similarity index 79% rename from code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java rename to code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java index 38a1573a..bec2a240 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java @@ -1,9 +1,9 @@ -package nu.marginalia.db.storage; +package nu.marginalia.storage; import com.google.inject.name.Named; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.db.storage.model.*; +import nu.marginalia.storage.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,7 +19,6 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; /** Manages file storage for processes and services */ @@ -34,7 +33,7 @@ public class FileStorageService { public Optional findFileStorageToDelete() { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT ID FROM FILE_STORAGE WHERE DO_PURGE LIMIT 1 + SELECT ID FROM FILE_STORAGE WHERE STATE='DELETE' LIMIT 1 """)) { var rs = stmt.executeQuery(); if (rs.next()) { @@ -46,6 +45,24 @@ public class FileStorageService { return Optional.empty(); } + public Set getConfiguredNodes() { + Set ret = new HashSet<>(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT DISTINCT(NODE) FROM FILE_STORAGE_BASE + """)) { + var rs = stmt.executeQuery(); + while (rs.next()) { + ret.add(rs.getInt(1)); + } + } catch (SQLException e) { + logger.warn("SQL error getting nodes", e); + } + + return ret; + } + @Inject public FileStorageService(HikariDataSource dataSource, @Named("wmsa-system-node") Integer node) { this.dataSource = dataSource; @@ -58,7 +75,7 @@ public class FileStorageService { continue; logger.info("FileStorage override present: {} -> {}", type, - FileStorage.createOverrideStorage(type, overrideProperty).asPath()); + FileStorage.createOverrideStorage(type, FileStorageBaseType.CURRENT, overrideProperty).asPath()); } } @@ -66,7 +83,7 @@ public class FileStorageService { public FileStorageBase getStorageBase(FileStorageBaseId type) throws SQLException { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT ID, NAME, PATH, TYPE, PERMIT_TEMP + SELECT ID, NAME, PATH, TYPE FROM FILE_STORAGE_BASE WHERE ID = ? """)) { stmt.setLong(1, type.id()); @@ -76,8 +93,7 @@ public class FileStorageService { new FileStorageBaseId(rs.getLong(1)), FileStorageBaseType.valueOf(rs.getString(4)), rs.getString(2), - rs.getString(3), - rs.getBoolean(5) + rs.getString(3) ); } } @@ -128,6 +144,7 @@ public class FileStorageService { } } } + public void relateFileStorages(FileStorageId source, FileStorageId target) { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" @@ -173,9 +190,13 @@ public class FileStorageService { /** @return the storage base with the given type, or null if it does not exist */ public FileStorageBase getStorageBase(FileStorageBaseType type) throws SQLException { + return getStorageBase(type, node); + } + + public FileStorageBase getStorageBase(FileStorageBaseType type, int node) throws SQLException { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT ID, NAME, PATH, TYPE, PERMIT_TEMP + SELECT ID, NAME, PATH, TYPE FROM FILE_STORAGE_BASE WHERE TYPE = ? AND NODE = ? """)) { stmt.setString(1, type.name()); @@ -186,16 +207,14 @@ public class FileStorageService { new FileStorageBaseId(rs.getLong(1)), FileStorageBaseType.valueOf(rs.getString(4)), rs.getString(2), - rs.getString(3), - rs.getBoolean(5) + rs.getString(3) ); } } } return null; } - - public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type, boolean permitTemp) throws SQLException, FileNotFoundException { + public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type) throws SQLException, FileNotFoundException { if (!Files.exists(path)) { throw new FileNotFoundException("Storage base path does not exist: " + path); @@ -203,14 +222,13 @@ public class FileStorageService { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - INSERT INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP, NODE) - VALUES (?, ?, ?, ?, ?) + INSERT INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, NODE) + VALUES (?, ?, ?, ?) """)) { stmt.setString(1, name); stmt.setString(2, path.toString()); stmt.setString(3, type.name()); - stmt.setBoolean(4, permitTemp); - stmt.setInt(5, node); + stmt.setInt(4, node); int update = stmt.executeUpdate(); if (update < 0) { @@ -250,10 +268,6 @@ public class FileStorageService { String prefix, String description) throws IOException, SQLException { - if (!base.permitTemp()) { - throw new IllegalArgumentException("Temporary storage not permitted in base " + base.name()); - } - Path newDir = allocateDirectory(base.asPath(), prefix); String relDir = base.asPath().relativize(newDir).normalize().toString(); @@ -299,7 +313,11 @@ public class FileStorageService { /** Allocate permanent storage in base */ - public FileStorage allocatePermanentStorage(FileStorageBase base, String relativePath, FileStorageType type, String description) throws IOException, SQLException { + public FileStorage allocatePermanentStorage(FileStorageBase base, + String relativePath, + FileStorageType type, + String description) throws IOException, SQLException + { Path newDir = base.asPath().resolve(relativePath); @@ -338,6 +356,7 @@ public class FileStorageService { type, LocalDateTime.now(), newDir.toString(), + "", description ); } @@ -359,12 +378,12 @@ public class FileStorageService { throw new IllegalStateException("FileStorageType " + type.name() + " was overridden, but location '" + override + "' does not exist!"); } - return FileStorage.createOverrideStorage(type, override); + return FileStorage.createOverrideStorage(type, FileStorageBaseType.CURRENT, override); } try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT PATH, DESCRIPTION, ID, BASE_ID, CREATE_DATE + SELECT PATH, STATE, DESCRIPTION, ID, BASE_ID, CREATE_DATE FROM FILE_STORAGE_VIEW WHERE TYPE = ? AND NODE = ? """)) { stmt.setString(1, type.name()); @@ -373,6 +392,7 @@ public class FileStorageService { long storageId; long baseId; String path; + String state; String description; LocalDateTime createDateTime; @@ -382,6 +402,7 @@ public class FileStorageService { storageId = rs.getLong("ID"); createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime(); path = rs.getString("PATH"); + state = rs.getString("STATE"); description = rs.getString("DESCRIPTION"); } else { @@ -396,18 +417,27 @@ public class FileStorageService { type, createDateTime, path, + state, description ); } } } + public List getStorage(List ids) throws SQLException { + List ret = new ArrayList<>(); + for (var id : ids) { + ret.add(getStorage(id)); + } + return ret; + } + /** @return the storage with the given id, or null if it does not exist */ public FileStorage getStorage(FileStorageId id) throws SQLException { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT PATH, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID + SELECT PATH, TYPE, STATE, DESCRIPTION, CREATE_DATE, ID, BASE_ID FROM FILE_STORAGE_VIEW WHERE ID = ? """)) { stmt.setLong(1, id.id()); @@ -415,6 +445,7 @@ public class FileStorageService { long storageId; long baseId; String path; + String state; String description; FileStorageType type; LocalDateTime createDateTime; @@ -425,6 +456,7 @@ public class FileStorageService { storageId = rs.getLong("ID"); type = FileStorageType.valueOf(rs.getString("TYPE")); path = rs.getString("PATH"); + state = rs.getString("STATE"); description = rs.getString("DESCRIPTION"); createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime(); } @@ -440,6 +472,7 @@ public class FileStorageService { type, createDateTime, path, + state, description ); } @@ -460,13 +493,14 @@ public class FileStorageService { List ret = new ArrayList<>(); try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT PATH, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID + SELECT PATH, STATE, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID FROM FILE_STORAGE_VIEW """)) { long storageId; long baseId; String path; + String state; String description; LocalDateTime createDateTime; FileStorageType type; @@ -476,6 +510,7 @@ public class FileStorageService { baseId = rs.getLong("BASE_ID"); storageId = rs.getLong("ID"); path = rs.getString("PATH"); + state = rs.getString("STATE"); type = FileStorageType.valueOf(rs.getString("TYPE")); description = rs.getString("DESCRIPTION"); createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime(); @@ -487,6 +522,7 @@ public class FileStorageService { type, createDateTime, path, + state, description )); } @@ -497,4 +533,62 @@ public class FileStorageService { return ret; } + + public void flagFileForDeletion(FileStorageId id) throws SQLException { + setFileStorageState(id, "DELETE"); + } + + public void enableFileStorage(FileStorageId id) throws SQLException { + setFileStorageState(id, "ACTIVE"); + } + public void disableFileStorage(FileStorageId id) throws SQLException { + setFileStorageState(id, ""); + } + + private void setFileStorageState(FileStorageId id, String state) throws SQLException { + try (var conn = dataSource.getConnection(); + var flagStmt = conn.prepareStatement("UPDATE FILE_STORAGE SET STATE = ? WHERE ID = ?")) { + flagStmt.setString(1, state); + flagStmt.setLong(2, id.id()); + flagStmt.executeUpdate(); + } + } + + public void disableFileStorageOfType(int nodeId, FileStorageType type) throws SQLException { + try (var conn = dataSource.getConnection(); + var flagStmt = conn.prepareStatement(""" + UPDATE FILE_STORAGE + INNER JOIN FILE_STORAGE_BASE ON BASE_ID=FILE_STORAGE_BASE.ID + SET FILE_STORAGE.STATE = '' + WHERE FILE_STORAGE.TYPE = ? + AND FILE_STORAGE_BASE.NODE=? + """)) { + flagStmt.setString(1, type.name()); + flagStmt.setInt(2, nodeId); + flagStmt.executeUpdate(); + } + } + + public List getActiveFileStorages(int nodeId, FileStorageType type) throws SQLException + { + + try (var conn = dataSource.getConnection(); + var queryStmt = conn.prepareStatement(""" + SELECT FILE_STORAGE.ID FROM FILE_STORAGE + INNER JOIN FILE_STORAGE_BASE ON BASE_ID=FILE_STORAGE_BASE.ID + WHERE FILE_STORAGE.TYPE = ? + AND STATE='ACTIVE' + AND FILE_STORAGE_BASE.NODE=? + """)) { + queryStmt.setString(1, type.name()); + queryStmt.setInt(2, nodeId); + var rs = queryStmt.executeQuery(); + List ids = new ArrayList<>(); + while (rs.next()) { + ids.add(new FileStorageId(rs.getInt(1))); + } + return ids; + } + } + } diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorage.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java similarity index 88% rename from code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorage.java rename to code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java index a7266da9..f0971349 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorage.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java @@ -1,4 +1,4 @@ -package nu.marginalia.db.storage.model; +package nu.marginalia.storage.model; import java.nio.file.Path; import java.time.LocalDateTime; @@ -19,19 +19,19 @@ public record FileStorage( FileStorageType type, LocalDateTime createDateTime, String path, + String state, String description) { /** It is sometimes desirable to be able to create an override that isn't * backed by the database. This constructor permits this. */ - public static FileStorage createOverrideStorage(FileStorageType type, String override) { + public static FileStorage createOverrideStorage(FileStorageType type, FileStorageBaseType baseType, String override) { var mockBase = new FileStorageBase( new FileStorageBaseId(-1), - FileStorageBaseType.SSD_INDEX, + baseType, "OVERRIDE:" + type.name(), - "INVALIDINVALIDINVALID", - false + "INVALIDINVALIDINVALID" ); return new FileStorage( @@ -40,6 +40,7 @@ public record FileStorage( type, LocalDateTime.now(), override, + "OVERRIDE", "OVERRIDE:" + type.name() ); } @@ -48,6 +49,9 @@ public record FileStorage( return Path.of(path); } + public boolean isActive() { + return "ACTIVE".equals(state); + } @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBase.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java similarity index 71% rename from code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBase.java rename to code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java index 1e8245ad..461d34d7 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBase.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java @@ -1,4 +1,4 @@ -package nu.marginalia.db.storage.model; +package nu.marginalia.storage.model; import java.nio.file.Path; @@ -9,15 +9,16 @@ import java.nio.file.Path; * @param type the type of the storage base * @param name the name of the storage base * @param path the path of the storage base - * @param permitTemp if true, the storage may be used for temporary files */ public record FileStorageBase(FileStorageBaseId id, FileStorageBaseType type, String name, - String path, - boolean permitTemp + String path ) { public Path asPath() { return Path.of(path); } + public boolean isValid() { + return id.id() >= 0; + } } diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseId.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseId.java similarity index 74% rename from code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseId.java rename to code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseId.java index 1c7ededd..3f3cf66f 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseId.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseId.java @@ -1,4 +1,4 @@ -package nu.marginalia.db.storage.model; +package nu.marginalia.storage.model; public record FileStorageBaseId(long id) { diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java new file mode 100644 index 00000000..d646c5e6 --- /dev/null +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java @@ -0,0 +1,12 @@ +package nu.marginalia.storage.model; + +public enum FileStorageBaseType { + CURRENT, + WORK, + STORAGE, + BACKUP; + + public String overrideName() { + return "FS_BASE_OVERRIDE:"+name(); + } +} diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageId.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageId.java similarity index 89% rename from code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageId.java rename to code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageId.java index a89ad9f8..f31c7ab8 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageId.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageId.java @@ -1,4 +1,4 @@ -package nu.marginalia.db.storage.model; +package nu.marginalia.storage.model; public record FileStorageId(long id) { public static FileStorageId parse(String str) { diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageType.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageType.java similarity index 55% rename from code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageType.java rename to code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageType.java index 33f30f5d..9363584c 100644 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageType.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageType.java @@ -1,17 +1,11 @@ -package nu.marginalia.db.storage.model; +package nu.marginalia.storage.model; public enum FileStorageType { CRAWL_SPEC, CRAWL_DATA, PROCESSED_DATA, - INDEX_STAGING, - LINKDB_STAGING, - LINKDB_LIVE, - INDEX_LIVE, BACKUP, - EXPORT, - SEARCH_SETS; - + EXPORT; public String overrideName() { return "FS_OVERRIDE:"+name(); } diff --git a/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java b/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java new file mode 100644 index 00000000..e5ef9517 --- /dev/null +++ b/code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java @@ -0,0 +1,70 @@ +package nu.marginalia.nodecfg; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.storage.FileStorageService; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +@Execution(ExecutionMode.SAME_THREAD) +@Tag("slow") +public class NodeConfigurationServiceTest { + @Container + static MariaDBContainer mariaDBContainer = new MariaDBContainer<>("mariadb") + .withDatabaseName("WMSA_prod") + .withUsername("wmsa") + .withPassword("wmsa") + .withInitScript("db/migration/V23_11_0_005__node_config.sql") + .withNetworkAliases("mariadb"); + + static HikariDataSource dataSource; + static NodeConfigurationService nodeConfigurationService; + + @BeforeAll + public static void setup() { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(mariaDBContainer.getJdbcUrl()); + config.setUsername("wmsa"); + config.setPassword("wmsa"); + + dataSource = new HikariDataSource(config); + + nodeConfigurationService = new NodeConfigurationService(dataSource); + } + + @Test + public void test() throws SQLException { + var a = nodeConfigurationService.create("Test", false); + var b = nodeConfigurationService.create("Foo", true); + + assertEquals(1, a.node()); + assertEquals("Test", a.description()); + assertFalse(a.acceptQueries()); + + assertEquals(2, b.node()); + assertEquals("Foo", b.description()); + assertTrue(b.acceptQueries()); + + var list = nodeConfigurationService.getAll(); + assertEquals(2, list.size()); + assertEquals(a, list.get(0)); + assertEquals(b, list.get(1)); + + } +} \ No newline at end of file diff --git a/code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java b/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java similarity index 79% rename from code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java rename to code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java index 466ed979..b9bb492f 100644 --- a/code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java +++ b/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java @@ -1,20 +1,19 @@ -package nu.marginalia.db.storage; +package nu.marginalia.storage; import com.google.common.collect.Lists; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageType; import org.junit.jupiter.api.*; import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.testcontainers.containers.MariaDBContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; @@ -23,11 +22,10 @@ import java.util.List; import java.util.Objects; import java.util.UUID; -import static org.junit.Assert.*; import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; @Testcontainers -@Execution(SAME_THREAD) +@Execution(ExecutionMode.SAME_THREAD) @Tag("slow") public class FileStorageServiceTest { @Container @@ -54,7 +52,11 @@ public class FileStorageServiceTest { // apply migrations - List migrations = List.of("db/migration/V23_11_0_000__file_storage_node.sql"); + List migrations = List.of( + "db/migration/V23_11_0_000__file_storage_node.sql", + "db/migration/V23_11_0_002__file_storage_state.sql", + "db/migration/V23_11_0_004__file_storage_base_type.sql" + ); for (String migration : migrations) { try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration), "Could not load migration script " + migration); @@ -135,38 +137,19 @@ public class FileStorageServiceTest { String name = "test-" + UUID.randomUUID(); var storage = new FileStorageService(dataSource, 0); - var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false); + var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.WORK); Assertions.assertEquals(name, base.name()); - Assertions.assertEquals(FileStorageBaseType.SLOW, base.type()); - Assertions.assertFalse(base.permitTemp()); + Assertions.assertEquals(FileStorageBaseType.WORK, base.type()); } + @Test - public void testAllocateTempInNonPermitted() throws SQLException, FileNotFoundException { + public void testAllocatePermanent() throws SQLException, IOException { String name = "test-" + UUID.randomUUID(); var storage = new FileStorageService(dataSource, 0); - var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false); - - try { - storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldFail"); - fail(); - } - catch (IllegalArgumentException ex) {} // ok - catch (Exception ex) { - ex.printStackTrace(); - fail(); - } - } - - @Test - public void testAllocatePermanentInNonPermitted() throws SQLException, IOException { - String name = "test-" + UUID.randomUUID(); - - var storage = new FileStorageService(dataSource, 0); - - var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false); + var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.WORK); var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed"); tempDirs.add(created.asPath()); @@ -176,12 +159,12 @@ public class FileStorageServiceTest { } @Test - public void testAllocateTempInPermitted() throws IOException, SQLException { + public void testAllocateTemp() throws IOException, SQLException { String name = "test-" + UUID.randomUUID(); var storage = new FileStorageService(dataSource, 0); - var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, true); + var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.WORK); var fileStorage = storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed"); System.out.println("Allocated " + fileStorage.asPath()); Assertions.assertTrue(Files.exists(fileStorage.asPath())); diff --git a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseType.java b/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseType.java deleted file mode 100644 index 08d67069..00000000 --- a/code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseType.java +++ /dev/null @@ -1,8 +0,0 @@ -package nu.marginalia.db.storage.model; - -public enum FileStorageBaseType { - SSD_INDEX, - SSD_WORK, - SLOW, - BACKUP -} diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_001__heartbeat_node.sql b/code/common/db/src/main/resources/db/migration/V23_11_0_001__heartbeat_node.sql new file mode 100644 index 00000000..9e17638d --- /dev/null +++ b/code/common/db/src/main/resources/db/migration/V23_11_0_001__heartbeat_node.sql @@ -0,0 +1,3 @@ +ALTER TABLE TASK_HEARTBEAT ADD COLUMN NODE INT NOT NULL DEFAULT -1; +ALTER TABLE PROCESS_HEARTBEAT ADD COLUMN NODE INT NOT NULL DEFAULT -1; +ALTER TABLE SERVICE_HEARTBEAT ADD COLUMN NODE INT NOT NULL DEFAULT -1; diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_002__file_storage_state.sql b/code/common/db/src/main/resources/db/migration/V23_11_0_002__file_storage_state.sql new file mode 100644 index 00000000..26e3f94a --- /dev/null +++ b/code/common/db/src/main/resources/db/migration/V23_11_0_002__file_storage_state.sql @@ -0,0 +1,17 @@ +ALTER TABLE FILE_STORAGE ADD COLUMN STATE VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE FILE_STORAGE DROP COLUMN DO_PURGE; + +DROP VIEW FILE_STORAGE_VIEW; + +CREATE VIEW FILE_STORAGE_VIEW +AS SELECT + CONCAT(BASE.PATH, '/', STORAGE.PATH) AS PATH, + STORAGE.TYPE AS TYPE, + STATE AS STATE, + NODE AS NODE, + DESCRIPTION AS DESCRIPTION, + CREATE_DATE AS CREATE_DATE, + STORAGE.ID AS ID, + BASE.ID AS BASE_ID +FROM FILE_STORAGE STORAGE +INNER JOIN FILE_STORAGE_BASE BASE ON STORAGE.BASE_ID=BASE.ID; diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql b/code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql new file mode 100644 index 00000000..9f7fa231 --- /dev/null +++ b/code/common/db/src/main/resources/db/migration/V23_11_0_003__node_configuration.sql @@ -0,0 +1,4 @@ +CREATE TABLE NODE_CONFIGURATION( + ID INT PRIMARY KEY, + DESCRIPTION VARCHAR(255) +); \ No newline at end of file diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_004__file_storage_base_type.sql b/code/common/db/src/main/resources/db/migration/V23_11_0_004__file_storage_base_type.sql new file mode 100644 index 00000000..7c597905 --- /dev/null +++ b/code/common/db/src/main/resources/db/migration/V23_11_0_004__file_storage_base_type.sql @@ -0,0 +1,10 @@ +ALTER TABLE FILE_STORAGE_BASE DROP COLUMN PERMIT_TEMP; +ALTER TABLE FILE_STORAGE_BASE ADD COLUMN TYPE_NEW VARCHAR(255) NOT NULL; + +UPDATE FILE_STORAGE_BASE SET TYPE_NEW = 'CURRENT' WHERE TYPE='SSD_INDEX'; +UPDATE FILE_STORAGE_BASE SET TYPE_NEW = 'WORK' WHERE TYPE='SSD_WORK'; +UPDATE FILE_STORAGE_BASE SET TYPE_NEW = 'STORAGE' WHERE TYPE='SLOW'; +UPDATE FILE_STORAGE_BASE SET TYPE_NEW = 'BACKUP' WHERE TYPE='BACKUP'; + +ALTER TABLE FILE_STORAGE_BASE DROP COLUMN TYPE; +ALTER TABLE FILE_STORAGE_BASE CHANGE COLUMN TYPE_NEW TYPE VARCHAR(255) NOT NULL; diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_005__node_config.sql b/code/common/db/src/main/resources/db/migration/V23_11_0_005__node_config.sql new file mode 100644 index 00000000..71364017 --- /dev/null +++ b/code/common/db/src/main/resources/db/migration/V23_11_0_005__node_config.sql @@ -0,0 +1,6 @@ +CREATE TABLE NODE_CONFIGURATION ( + ID INT PRIMARY KEY AUTO_INCREMENT, + DESCRIPTION VARCHAR(255), + ACCEPT_QUERIES BOOLEAN, + DISABLED BOOLEAN DEFAULT FALSE +); \ No newline at end of file diff --git a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbReader.java b/code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbReader.java index 625c3214..027b2371 100644 --- a/code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbReader.java +++ b/code/common/linkdb/src/main/java/nu/marginalia/linkdb/LinkdbReader.java @@ -24,7 +24,7 @@ import java.util.List; @Singleton public class LinkdbReader { - private Path dbFile; + private final Path dbFile; private volatile Connection connection; private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -34,13 +34,7 @@ public class LinkdbReader { this.dbFile = dbFile; if (Files.exists(dbFile)) { - try { - connection = createConnection(); - } - catch (SQLException ex) { - connection = null; - logger.error("Failed to load linkdb file", ex); - } + connection = createConnection(); } else { logger.warn("No linkdb file {}", dbFile); @@ -48,15 +42,28 @@ public class LinkdbReader { } private Connection createConnection() throws SQLException { - String connStr = "jdbc:sqlite:" + dbFile.toString(); - return DriverManager.getConnection(connStr); + try { + String connStr = "jdbc:sqlite:" + dbFile.toString(); + return DriverManager.getConnection(connStr); + } + catch (SQLException ex) { + logger.error("Failed to connect to link database " + dbFile, ex); + return null; + } } public void switchInput(Path newDbFile) throws IOException, SQLException { + if (!Files.isRegularFile(newDbFile)) { + logger.error("Source is not a file, refusing switch-over {}", newDbFile); + return; + } + if (connection != null) { connection.close(); } + logger.info("Moving {} to {}", newDbFile, dbFile); + Files.move(newDbFile, dbFile, StandardCopyOption.REPLACE_EXISTING); connection = createConnection(); diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java b/code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java index 41c963cf..9c0eeddf 100644 --- a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java +++ b/code/common/process/src/main/java/nu/marginalia/process/control/ProcessAdHocTaskHeartbeatImpl.java @@ -19,6 +19,7 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo private final Logger logger = LoggerFactory.getLogger(ProcessAdHocTaskHeartbeatImpl.class); private final String taskName; private final String taskBase; + private final int node; private final String instanceUUID; private final HikariDataSource dataSource; @@ -37,6 +38,7 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo { this.taskName = configuration.processName() + "." + taskName + ":" + configuration.node(); this.taskBase = configuration.processName() + "." + taskName; + this.node = configuration.node(); this.dataSource = dataSource; this.instanceUUID = UUID.randomUUID().toString(); @@ -110,8 +112,8 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo try (var connection = dataSource.getConnection()) { try (var stmt = connection.prepareStatement( """ - INSERT INTO TASK_HEARTBEAT (TASK_NAME, TASK_BASE, INSTANCE, SERVICE_INSTANCE, HEARTBEAT_TIME, STATUS) - VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') + INSERT INTO TASK_HEARTBEAT (TASK_NAME, TASK_BASE, NODE, INSTANCE, SERVICE_INSTANCE, HEARTBEAT_TIME, STATUS) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') ON DUPLICATE KEY UPDATE INSTANCE = ?, SERVICE_INSTANCE = ?, @@ -122,10 +124,11 @@ public class ProcessAdHocTaskHeartbeatImpl implements AutoCloseable, ProcessAdHo { stmt.setString(1, taskName); stmt.setString(2, taskBase); - stmt.setString(3, instanceUUID); - stmt.setString(4, serviceInstanceUUID); - stmt.setString(5, instanceUUID); - stmt.setString(6, serviceInstanceUUID); + stmt.setInt(3, node); + stmt.setString(4, instanceUUID); + stmt.setString(5, serviceInstanceUUID); + stmt.setString(6, instanceUUID); + stmt.setString(7, serviceInstanceUUID); stmt.executeUpdate(); } } diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java b/code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java index 05fc7ae5..6f85d890 100644 --- a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java +++ b/code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeatImpl.java @@ -18,6 +18,7 @@ public class ProcessHeartbeatImpl implements ProcessHeartbeat { private final Logger logger = LoggerFactory.getLogger(ProcessHeartbeatImpl.class); private final String processName; private final String processBase; + private final int node; private final String instanceUUID; @org.jetbrains.annotations.NotNull private final ProcessConfiguration configuration; @@ -37,6 +38,7 @@ public class ProcessHeartbeatImpl implements ProcessHeartbeat { { this.processName = configuration.processName() + ":" + configuration.node(); this.processBase = configuration.processName(); + this.node = configuration.node(); this.configuration = configuration; this.dataSource = dataSource; @@ -115,8 +117,8 @@ public class ProcessHeartbeatImpl implements ProcessHeartbeat { try (var connection = dataSource.getConnection()) { try (var stmt = connection.prepareStatement( """ - INSERT INTO PROCESS_HEARTBEAT (PROCESS_NAME, PROCESS_BASE, INSTANCE, HEARTBEAT_TIME, STATUS) - VALUES (?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') + INSERT INTO PROCESS_HEARTBEAT (PROCESS_NAME, PROCESS_BASE, NODE, INSTANCE, HEARTBEAT_TIME, STATUS) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') ON DUPLICATE KEY UPDATE INSTANCE = ?, HEARTBEAT_TIME = CURRENT_TIMESTAMP(6), @@ -126,8 +128,9 @@ public class ProcessHeartbeatImpl implements ProcessHeartbeat { { stmt.setString(1, processName); stmt.setString(2, processBase); - stmt.setString(3, instanceUUID); + stmt.setInt(3, node); stmt.setString(4, instanceUUID); + stmt.setString(5, instanceUUID); stmt.executeUpdate(); } } diff --git a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java b/code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java index eb848e57..f8d68b7e 100644 --- a/code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java +++ b/code/common/process/src/main/java/nu/marginalia/process/control/ProcessTaskHeartbeatImpl.java @@ -19,6 +19,8 @@ public class ProcessTaskHeartbeatImpl> implements AutoCloseabl private final Logger logger = LoggerFactory.getLogger(ProcessTaskHeartbeatImpl.class); private final String taskName; private final String taskBase; + private final int node; + private final String instanceUUID; private final HikariDataSource dataSource; @@ -39,6 +41,7 @@ public class ProcessTaskHeartbeatImpl> implements AutoCloseabl { this.taskName = configuration.processName() + "." + taskName + ":" + configuration.node(); this.taskBase = configuration.processName() + "." + taskName; + this.node = configuration.node(); this.dataSource = dataSource; this.instanceUUID = UUID.randomUUID().toString(); @@ -115,8 +118,8 @@ public class ProcessTaskHeartbeatImpl> implements AutoCloseabl try (var connection = dataSource.getConnection()) { try (var stmt = connection.prepareStatement( """ - INSERT INTO TASK_HEARTBEAT (TASK_NAME, TASK_BASE, INSTANCE, SERVICE_INSTANCE, HEARTBEAT_TIME, STATUS) - VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') + INSERT INTO TASK_HEARTBEAT (TASK_NAME, TASK_BASE, NODE, INSTANCE, SERVICE_INSTANCE, HEARTBEAT_TIME, STATUS) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') ON DUPLICATE KEY UPDATE INSTANCE = ?, SERVICE_INSTANCE = ?, @@ -127,10 +130,11 @@ public class ProcessTaskHeartbeatImpl> implements AutoCloseabl { stmt.setString(1, taskName); stmt.setString(2, taskBase); - stmt.setString(3, instanceUUID); - stmt.setString(4, serviceInstanceUUID); - stmt.setString(5, instanceUUID); - stmt.setString(6, serviceInstanceUUID); + stmt.setInt(3, node); + stmt.setString(4, instanceUUID); + stmt.setString(5, serviceInstanceUUID); + stmt.setString(6, instanceUUID); + stmt.setString(7, serviceInstanceUUID); stmt.executeUpdate(); } } diff --git a/code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java b/code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java index a77768be..f4244112 100644 --- a/code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java +++ b/code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java @@ -14,12 +14,11 @@ import java.util.concurrent.TimeUnit; @Singleton public class ServiceMonitors { private final HikariDataSource dataSource; - private final Logger logger = LoggerFactory.getLogger(getClass()); + private static final Logger logger = LoggerFactory.getLogger(ServiceMonitors.class); - private final Set runningServices = new HashSet<>(); + private final Set runningServices = new HashSet<>(); private final Set callbacks = new HashSet<>(); - private final int heartbeatInterval = Integer.getInteger("mcp.heartbeat.interval", 5); private volatile boolean running; @@ -80,14 +79,14 @@ public class ServiceMonitors { private boolean updateRunningServices() { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT SERVICE_BASE, TIMESTAMPDIFF(SECOND, HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) + SELECT SERVICE_NAME, TIMESTAMPDIFF(SECOND, HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) FROM SERVICE_HEARTBEAT WHERE ALIVE=1 """)) { try (var rs = stmt.executeQuery()) { - Set newRunningServices = new HashSet<>(10); + Set newRunningServices = new HashSet<>(10); while (rs.next()) { - String svc = rs.getString(1); + ServiceNode svc = ServiceNode.parse(rs.getString(1)); int dtime = rs.getInt(2); if (dtime < 2.5 * heartbeatInterval) { newRunningServices.add(svc); @@ -113,21 +112,37 @@ public class ServiceMonitors { return false; } - public boolean isServiceUp(ServiceId serviceId) { + public boolean isServiceUp(ServiceId serviceId, int node) { synchronized (runningServices) { - return runningServices.contains(serviceId.name); + return runningServices.contains(new ServiceNode(serviceId.name, node)); } } - public List getRunningServices() { - List ret = new ArrayList<>(ServiceId.values().length); + public List getRunningServices() { + List ret = new ArrayList<>(ServiceId.values().length); synchronized (runningServices) { - for (var runningService : runningServices) { - ret.add(ServiceId.byName(runningService)); - } + ret.addAll(runningServices); } return ret; } + + public record ServiceNode(String service, int node) { + public static ServiceNode parse(String serviceName) { + + if (serviceName.contains(":")) { + String[] parts = serviceName.split(":", 2); + try { + return new ServiceNode(parts[0], Integer.parseInt(parts[1])); + } + catch (NumberFormatException ex) { + logger.warn("Failed to parse serviceName '" + serviceName + "'", ex); + //fallthrough + } + } + + return new ServiceNode(serviceName, -1); + } + } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java index 9fbdf9fd..943b392a 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java @@ -12,6 +12,7 @@ public class SearchServiceDescriptors { new ServiceDescriptor(ServiceId.Index, 5021), new ServiceDescriptor(ServiceId.Query, 5022), new ServiceDescriptor(ServiceId.Search, 5023), + new ServiceDescriptor(ServiceId.Executor, 5024), new ServiceDescriptor(ServiceId.Assistant, 5025), new ServiceDescriptor(ServiceId.Dating, 5070), new ServiceDescriptor(ServiceId.Explorer, 5071), diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java index 89f3fe22..287c4e83 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/descriptor/ServiceDescriptor.java @@ -12,7 +12,11 @@ public class ServiceDescriptor { this.name = id.name; this.port = port; } - + public ServiceDescriptor(ServiceId id, String host, int port) { + this.id = id; + this.name = host; + this.port = port; + } public String toString() { return name; } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java index d24441b5..5d1bfd0c 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java @@ -7,6 +7,7 @@ public enum ServiceId { Search("search-service"), Index("index-service"), Query("query-service"), + Executor("executor-service"), Control("control-service"), diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java index 63746567..800d6712 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java +++ b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java @@ -18,6 +18,7 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat { private final Logger logger = LoggerFactory.getLogger(ServiceHeartbeatImpl.class); private final String serviceName; private final String serviceBase; + private final int node; private final String instanceUUID; private final ServiceConfiguration configuration; private final ServiceEventLog eventLog; @@ -36,6 +37,7 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat { { this.serviceName = configuration.serviceName() + ":" + configuration.node(); this.serviceBase = configuration.serviceName(); + this.node = configuration.node(); this.configuration = configuration; this.eventLog = eventLog; this.dataSource = dataSource; @@ -105,8 +107,8 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat { try (var connection = dataSource.getConnection()) { try (var stmt = connection.prepareStatement( """ - INSERT INTO SERVICE_HEARTBEAT (SERVICE_NAME, SERVICE_BASE, INSTANCE, HEARTBEAT_TIME, ALIVE) - VALUES (?, ?, ?, CURRENT_TIMESTAMP(6), 1) + INSERT INTO SERVICE_HEARTBEAT (SERVICE_NAME, SERVICE_BASE, NODE, INSTANCE, HEARTBEAT_TIME, ALIVE) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP(6), 1) ON DUPLICATE KEY UPDATE INSTANCE = ?, HEARTBEAT_TIME = CURRENT_TIMESTAMP(6), @@ -116,8 +118,9 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat { { stmt.setString(1, serviceName); stmt.setString(2, serviceBase); - stmt.setString(3, instanceUUID); + stmt.setInt(3, node); stmt.setString(4, instanceUUID); + stmt.setString(5, instanceUUID); stmt.executeUpdate(); } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java index c16bcc8c..b52e2201 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java +++ b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceTaskHeartbeatImpl.java @@ -19,6 +19,7 @@ public class ServiceTaskHeartbeatImpl> implements ServiceTaskH private final Logger logger = LoggerFactory.getLogger(ServiceTaskHeartbeatImpl.class); private final String taskName; private final String taskBase; + private final int node; private final String instanceUUID; private final HikariDataSource dataSource; @@ -27,6 +28,7 @@ public class ServiceTaskHeartbeatImpl> implements ServiceTaskH private final int heartbeatInterval = Integer.getInteger("mcp.heartbeat.interval", 1); private final String serviceInstanceUUID; private final int stepCount; + private final ServiceEventLog eventLog; private volatile boolean running = false; @@ -42,6 +44,7 @@ public class ServiceTaskHeartbeatImpl> implements ServiceTaskH this.eventLog = eventLog; this.taskName = configuration.serviceName() + "." + taskName + ":" + configuration.node(); this.taskBase = configuration.serviceName() + "." + taskName; + this.node = configuration.node(); this.dataSource = dataSource; this.instanceUUID = UUID.randomUUID().toString(); @@ -118,8 +121,8 @@ public class ServiceTaskHeartbeatImpl> implements ServiceTaskH try (var connection = dataSource.getConnection()) { try (var stmt = connection.prepareStatement( """ - INSERT INTO TASK_HEARTBEAT (TASK_NAME, TASK_BASE, INSTANCE, SERVICE_INSTANCE, HEARTBEAT_TIME, STATUS) - VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') + INSERT INTO TASK_HEARTBEAT (TASK_NAME, TASK_BASE, NODE, INSTANCE, SERVICE_INSTANCE, HEARTBEAT_TIME, STATUS) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING') ON DUPLICATE KEY UPDATE INSTANCE = ?, SERVICE_INSTANCE = ?, @@ -130,10 +133,11 @@ public class ServiceTaskHeartbeatImpl> implements ServiceTaskH { stmt.setString(1, taskName); stmt.setString(2, taskBase); - stmt.setString(3, instanceUUID); - stmt.setString(4, serviceInstanceUUID); - stmt.setString(5, instanceUUID); - stmt.setString(6, serviceInstanceUUID); + stmt.setInt(3, node); + stmt.setString(4, instanceUUID); + stmt.setString(5, serviceInstanceUUID); + stmt.setString(6, instanceUUID); + stmt.setString(7, serviceInstanceUUID); stmt.executeUpdate(); } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java index 4185aad6..3e202d78 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java @@ -47,11 +47,11 @@ public class Service { this.initialization = params.initialization; var config = params.configuration; - String inboxName = config.serviceName() + ":" + config.node(); + String inboxName = config.serviceName(); logger.info("Inbox name: {}", inboxName); var mqInboxFactory = params.messageQueueInboxFactory; - messageQueueInbox = mqInboxFactory.createAsynchronousInbox(inboxName, config.instanceUuid()); + messageQueueInbox = mqInboxFactory.createAsynchronousInbox(inboxName, config.node(), config.instanceUuid()); messageQueueInbox.subscribe(new ServiceMqSubscription(this)); serviceName = System.getProperty("service-name"); diff --git a/code/features-control/actors/build.gradle b/code/features-control/actors/build.gradle new file mode 100644 index 00000000..da12c712 --- /dev/null +++ b/code/features-control/actors/build.gradle @@ -0,0 +1,41 @@ + +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + + implementation project(':code:libraries:message-queue') + implementation project(':code:common:service') + implementation project(':code:common:process') + implementation project(':code:common:model') + implementation project(':code:common:service-client') + implementation project(':code:common:db') + implementation project(':code:common:config') + implementation project(':code:api:process-mqapi') + implementation project(':code:api:index-api') + implementation project(':code:features-control:process-execution') + implementation project(':code:features-index:index-journal') + implementation project(':code:process-models:crawl-spec') + + implementation libs.bundles.slf4j + implementation libs.guice + implementation libs.notnull + implementation libs.spark + implementation libs.jsoup + implementation libs.zstd + implementation libs.bundles.mariadb + implementation libs.commons.io + implementation libs.bundles.gson + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/Actor.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/Actor.java index 9eb625ce..f5655307 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/Actor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor; +package nu.marginalia.actor; public enum Actor { CRAWL, diff --git a/code/features-control/actors/src/main/java/nu/marginalia/actor/ActorApi.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/ActorApi.java new file mode 100644 index 00000000..fb73903b --- /dev/null +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/ActorApi.java @@ -0,0 +1,55 @@ +package nu.marginalia.actor; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; +import spark.Spark; + +@Singleton +public class ActorApi { + private final ActorControlService actors; + private final Logger logger = LoggerFactory.getLogger(getClass()); + @Inject + public ActorApi(ActorControlService actors) { + this.actors = actors; + } + + public Object startActorFromState(Request request, Response response) throws Exception { + Actor actor = translateActor(request.params("id")); + String state = request.params("state"); + + actors.startFromJSON(actor, state, request.body()); + + return ""; + } + + public Object startActor(Request request, Response response) throws Exception { + Actor actor = translateActor(request.params("id")); + + actors.startJSON(actor, request.body()); + + return ""; + } + + public Object stopActor(Request request, Response response) { + Actor actor = translateActor(request.params("id")); + + actors.stop(actor); + + return "OK"; + } + + public Actor translateActor(String name) { + try { + return Actor.valueOf(name.toUpperCase()); + } + catch (IllegalArgumentException ex) { + logger.error("Unknown actor {}", name); + Spark.halt(400, "Unknown actor name provided"); + return null; + } + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/ActorControlService.java similarity index 66% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/ActorControlService.java index 3aea2bf9..3c4d7f5a 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/ActorControlService.java @@ -1,18 +1,15 @@ -package nu.marginalia.control.actor; +package nu.marginalia.actor; import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; -import nu.marginalia.control.actor.task.*; -import nu.marginalia.control.actor.monitor.*; -import nu.marginalia.control.actor.monitor.ConverterMonitorActor; -import nu.marginalia.control.actor.monitor.LoaderMonitorActor; -import nu.marginalia.model.gson.GsonFactory; -import nu.marginalia.mq.MessageQueueFactory; -import nu.marginalia.actor.ActorStateMachine; +import nu.marginalia.actor.monitor.*; import nu.marginalia.actor.prototype.AbstractActorPrototype; import nu.marginalia.actor.state.ActorStateInstance; +import nu.marginalia.actor.task.*; +import nu.marginalia.model.gson.GsonFactory; +import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.BaseServiceParams; @@ -21,39 +18,40 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; -/** This class is responsible for starting and stopping the various actors in the controller service */ +/** This class is responsible for starting and stopping the various actors in the responsible service */ @Singleton -public class ControlActors { +public class ActorControlService { private final ServiceEventLog eventLog; private final Gson gson; private final MessageQueueFactory messageQueueFactory; public Map stateMachines = new HashMap<>(); public Map actorDefinitions = new HashMap<>(); - + private final int node; @Inject - public ControlActors(MessageQueueFactory messageQueueFactory, - GsonFactory gsonFactory, - BaseServiceParams baseServiceParams, - ConvertActor convertActor, - ConvertAndLoadActor convertAndLoadActor, - CrawlActor crawlActor, - RecrawlActor recrawlActor, - RestoreBackupActor restoreBackupActor, - ConverterMonitorActor converterMonitorFSM, - CrawlerMonitorActor crawlerMonitorActor, - LoaderMonitorActor loaderMonitor, - MessageQueueMonitorActor messageQueueMonitor, - ProcessLivenessMonitorActor processMonitorFSM, - FileStorageMonitorActor fileStorageMonitorActor, - IndexConstructorMonitorActor indexConstructorMonitorActor, - TriggerAdjacencyCalculationActor triggerAdjacencyCalculationActor, - CrawlJobExtractorActor crawlJobExtractorActor, - ExportDataActor exportDataActor, - TruncateLinkDatabase truncateLinkDatabase + public ActorControlService(MessageQueueFactory messageQueueFactory, + GsonFactory gsonFactory, + BaseServiceParams baseServiceParams, + ConvertActor convertActor, + ConvertAndLoadActor convertAndLoadActor, + CrawlActor crawlActor, + RecrawlActor recrawlActor, + RestoreBackupActor restoreBackupActor, + ConverterMonitorActor converterMonitorFSM, + CrawlerMonitorActor crawlerMonitorActor, + LoaderMonitorActor loaderMonitor, + MessageQueueMonitorActor messageQueueMonitor, + ProcessLivenessMonitorActor processMonitorFSM, + FileStorageMonitorActor fileStorageMonitorActor, + IndexConstructorMonitorActor indexConstructorMonitorActor, + TriggerAdjacencyCalculationActor triggerAdjacencyCalculationActor, + CrawlJobExtractorActor crawlJobExtractorActor, + ExportDataActor exportDataActor, + TruncateLinkDatabase truncateLinkDatabase ) { this.messageQueueFactory = messageQueueFactory; this.eventLog = baseServiceParams.eventLog; this.gson = gsonFactory.get(); + this.node = baseServiceParams.configuration.node(); register(Actor.CRAWL, crawlActor); register(Actor.RECRAWL, recrawlActor); @@ -76,7 +74,7 @@ public class ControlActors { } private void register(Actor process, AbstractActorPrototype graph) { - var sm = new ActorStateMachine(messageQueueFactory, process.id(), UUID.randomUUID(), graph); + var sm = new ActorStateMachine(messageQueueFactory, process.id(), node, UUID.randomUUID(), graph); sm.listen((function, param) -> logStateChange(process, function)); stateMachines.put(process, sm); @@ -105,12 +103,22 @@ public class ControlActors { stateMachines.get(process).initFrom(state, gson.toJson(arg)); } + public void startFromJSON(Actor process, String state, String json) throws Exception { + eventLog.logEvent("FSM-START", process.id()); + + stateMachines.get(process).initFrom(state, json); + } + public void start(Actor process, Object arg) throws Exception { eventLog.logEvent("FSM-START", process.id()); stateMachines.get(process).init(gson.toJson(arg)); } + public void startJSON(Actor process, String json) throws Exception { + eventLog.logEvent("FSM-START", process.id()); + stateMachines.get(process).init(json); + } @SneakyThrows public void stop(Actor process) { eventLog.logEvent("FSM-STOP", process.id()); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java similarity index 96% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java index d95c9475..4b8cfffa 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java @@ -1,15 +1,16 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.actor.ActorStateFactory; +import nu.marginalia.actor.prototype.AbstractActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.actor.state.ActorTerminalState; import nu.marginalia.control.process.ProcessService; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.persistence.MqPersistence; -import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorResumeBehavior; -import nu.marginalia.actor.state.ActorTerminalState; +import nu.marginalia.service.module.ServiceConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,7 @@ public class AbstractProcessSpawnerActor extends AbstractActorPrototype { private final String inboxName; private final ProcessService.ProcessId processId; private final ExecutorService executorService = Executors.newSingleThreadExecutor(); + private final int node; public String describe() { return "Spawns a(n) " + processId + " process and monitors its inbox for messages"; @@ -46,14 +48,16 @@ public class AbstractProcessSpawnerActor extends AbstractActorPrototype { @Inject public AbstractProcessSpawnerActor(ActorStateFactory stateFactory, + ServiceConfiguration configuration, MqPersistence persistence, ProcessService processService, String inboxName, ProcessService.ProcessId processId) { super(stateFactory); + this.node = configuration.node(); this.persistence = persistence; this.processService = processService; - this.inboxName = inboxName; + this.inboxName = inboxName + ":" + node; this.processId = processId; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ConverterMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ConverterMonitorActor.java similarity index 59% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ConverterMonitorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ConverterMonitorActor.java index aebb4a38..f39d21d7 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ConverterMonitorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ConverterMonitorActor.java @@ -1,11 +1,12 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.actor.ActorStateFactory; import nu.marginalia.control.process.ProcessService; -import nu.marginalia.mqapi.ProcessInboxNames; import nu.marginalia.mq.persistence.MqPersistence; +import nu.marginalia.mqapi.ProcessInboxNames; +import nu.marginalia.service.module.ServiceConfiguration; @Singleton public class ConverterMonitorActor extends AbstractProcessSpawnerActor { @@ -13,9 +14,15 @@ public class ConverterMonitorActor extends AbstractProcessSpawnerActor { @Inject public ConverterMonitorActor(ActorStateFactory stateFactory, + ServiceConfiguration configuration, MqPersistence persistence, ProcessService processService) { - super(stateFactory, persistence, processService, ProcessInboxNames.CONVERTER_INBOX, ProcessService.ProcessId.CONVERTER); + super(stateFactory, + configuration, + persistence, + processService, + ProcessInboxNames.CONVERTER_INBOX, + ProcessService.ProcessId.CONVERTER); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/CrawlerMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/CrawlerMonitorActor.java similarity index 79% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/CrawlerMonitorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/CrawlerMonitorActor.java index 631f29da..c431a25c 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/CrawlerMonitorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/CrawlerMonitorActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -6,15 +6,18 @@ import nu.marginalia.actor.ActorStateFactory; import nu.marginalia.control.process.ProcessService; import nu.marginalia.mq.persistence.MqPersistence; import nu.marginalia.mqapi.ProcessInboxNames; +import nu.marginalia.service.module.ServiceConfiguration; @Singleton public class CrawlerMonitorActor extends AbstractProcessSpawnerActor { @Inject public CrawlerMonitorActor(ActorStateFactory stateFactory, + ServiceConfiguration configuration, MqPersistence persistence, ProcessService processService) { super(stateFactory, + configuration, persistence, processService, ProcessInboxNames.CRAWLER_INBOX, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java index f69ab3de..f2f80d6b 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java @@ -1,15 +1,15 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageId; import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorage; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageId; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +74,7 @@ public class FileStorageMonitorActor extends AbstractActorPrototype { transition(REMOVE_STALE, missing.get().id()); } - fileStorageService.synchronizeStorageManifests(fileStorageService.getStorageBase(FileStorageBaseType.SLOW)); + fileStorageService.synchronizeStorageManifests(fileStorageService.getStorageBase(FileStorageBaseType.WORK)); TimeUnit.SECONDS.sleep(10); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/IndexConstructorMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/IndexConstructorMonitorActor.java similarity index 59% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/IndexConstructorMonitorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/IndexConstructorMonitorActor.java index abc44d6b..a62a4741 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/IndexConstructorMonitorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/IndexConstructorMonitorActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -6,6 +6,7 @@ import nu.marginalia.actor.ActorStateFactory; import nu.marginalia.control.process.ProcessService; import nu.marginalia.mq.persistence.MqPersistence; import nu.marginalia.mqapi.ProcessInboxNames; +import nu.marginalia.service.module.ServiceConfiguration; @Singleton public class IndexConstructorMonitorActor extends AbstractProcessSpawnerActor { @@ -13,9 +14,15 @@ public class IndexConstructorMonitorActor extends AbstractProcessSpawnerActor { @Inject public IndexConstructorMonitorActor(ActorStateFactory stateFactory, + ServiceConfiguration configuration, MqPersistence persistence, ProcessService processService) { - super(stateFactory, persistence, processService, ProcessInboxNames.INDEX_CONSTRUCTOR_INBOX, ProcessService.ProcessId.INDEX_CONSTRUCTOR); + super(stateFactory, + configuration, + persistence, + processService, + ProcessInboxNames.INDEX_CONSTRUCTOR_INBOX, + ProcessService.ProcessId.INDEX_CONSTRUCTOR); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/LoaderMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/LoaderMonitorActor.java similarity index 71% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/LoaderMonitorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/LoaderMonitorActor.java index 281b021b..2c34e6dd 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/LoaderMonitorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/LoaderMonitorActor.java @@ -1,11 +1,12 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.actor.ActorStateFactory; import nu.marginalia.control.process.ProcessService; -import nu.marginalia.mqapi.ProcessInboxNames; import nu.marginalia.mq.persistence.MqPersistence; +import nu.marginalia.mqapi.ProcessInboxNames; +import nu.marginalia.service.module.ServiceConfiguration; @Singleton public class LoaderMonitorActor extends AbstractProcessSpawnerActor { @@ -13,10 +14,13 @@ public class LoaderMonitorActor extends AbstractProcessSpawnerActor { @Inject public LoaderMonitorActor(ActorStateFactory stateFactory, + ServiceConfiguration configuration, MqPersistence persistence, ProcessService processService) { - super(stateFactory, persistence, processService, + super(stateFactory, + configuration, + persistence, processService, ProcessInboxNames.LOADER_INBOX, ProcessService.ProcessId.LOADER); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/MessageQueueMonitorActor.java similarity index 97% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/MessageQueueMonitorActor.java index 8b7f3354..b10b279c 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/MessageQueueMonitorActor.java @@ -1,12 +1,12 @@ -package nu.marginalia.control.actor.monitor; +package nu.marginalia.actor.monitor; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.mq.persistence.MqPersistence; import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.mq.persistence.MqPersistence; import java.util.concurrent.TimeUnit; diff --git a/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ProcessLivenessMonitorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ProcessLivenessMonitorActor.java new file mode 100644 index 00000000..85fe463b --- /dev/null +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/ProcessLivenessMonitorActor.java @@ -0,0 +1,217 @@ +package nu.marginalia.actor.monitor; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.actor.ActorStateFactory; +import nu.marginalia.actor.prototype.AbstractActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.control.process.ProcessService; +import nu.marginalia.service.control.ServiceEventLog; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Singleton +public class ProcessLivenessMonitorActor extends AbstractActorPrototype { + + // STATES + + private static final String INITIAL = "INITIAL"; + private static final String MONITOR = "MONITOR"; + private static final String END = "END"; + private final ServiceEventLog eventLogService; + private final ProcessService processService; + private final HikariDataSource dataSource; + + + @Inject + public ProcessLivenessMonitorActor(ActorStateFactory stateFactory, + ServiceEventLog eventLogService, + ProcessService processService, + HikariDataSource dataSource) { + super(stateFactory); + this.eventLogService = eventLogService; + this.processService = processService; + this.dataSource = dataSource; + } + + @Override + public String describe() { + return "Periodically check to ensure that the control service's view of running processes is agreement with the process heartbeats table."; + } + + @ActorState(name = INITIAL, next = MONITOR) + public void init() { + } + + @ActorState(name = MONITOR, next = MONITOR, resume = ActorResumeBehavior.RETRY, description = """ + Periodically check to ensure that the control service's view of + running processes is agreement with the process heartbeats table. + + If the process is not running, mark the process as stopped in the table. + """) + public void monitor() throws Exception { + + for (;;) { + for (var heartbeat : getProcessHeartbeats()) { + if (!heartbeat.isRunning()) { + continue; + } + + var processId = heartbeat.getProcessId(); + if (null == processId) + continue; + + if (processService.isRunning(processId) && heartbeat.lastSeenMillis() < 10_000) { + continue; + } + + flagProcessAsStopped(heartbeat); + } + + for (var heartbeat : getTaskHeartbeats()) { + if (heartbeat.lastSeenMillis() < 10_000) { + continue; + } + + removeTaskHeartbeat(heartbeat); + } + + + TimeUnit.SECONDS.sleep(60); + } + } + + + private List getProcessHeartbeats() { + List heartbeats = new ArrayList<>(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT PROCESS_NAME, PROCESS_BASE, INSTANCE, STATUS, PROGRESS, + TIMESTAMPDIFF(MICROSECOND, HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF + FROM PROCESS_HEARTBEAT + """)) { + + var rs = stmt.executeQuery(); + while (rs.next()) { + int progress = rs.getInt("PROGRESS"); + heartbeats.add(new ProcessHeartbeat( + rs.getString("PROCESS_NAME"), + rs.getString("PROCESS_BASE"), + rs.getString("INSTANCE"), + rs.getLong("TSDIFF") / 1000., + progress < 0 ? null : progress, + rs.getString("STATUS") + )); + } + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + + return heartbeats; + } + + private void flagProcessAsStopped(ProcessHeartbeat processHeartbeat) { + eventLogService.logEvent("PROCESS-MISSING", "Marking stale process heartbeat " + + processHeartbeat.processId() + " / " + processHeartbeat.uuidFull() + " as stopped"); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + UPDATE PROCESS_HEARTBEAT + SET STATUS = 'STOPPED' + WHERE INSTANCE = ? + """)) { + + stmt.setString(1, processHeartbeat.uuidFull()); + stmt.executeUpdate(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + + private List getTaskHeartbeats() { + List heartbeats = new ArrayList<>(); + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT TASK_NAME, TASK_BASE, INSTANCE, SERVICE_INSTANCE, STATUS, STAGE_NAME, PROGRESS, TIMESTAMPDIFF(MICROSECOND, TASK_HEARTBEAT.HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF + FROM TASK_HEARTBEAT + """)) { + var rs = stmt.executeQuery(); + while (rs.next()) { + int progress = rs.getInt("PROGRESS"); + heartbeats.add(new TaskHeartbeat( + rs.getString("TASK_NAME"), + rs.getString("TASK_BASE"), + rs.getString("INSTANCE"), + rs.getString("SERVICE_INSTANCE"), + rs.getLong("TSDIFF") / 1000., + progress < 0 ? null : progress, + rs.getString("STAGE_NAME"), + rs.getString("STATUS") + )); + } + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + return heartbeats; + } + + private void removeTaskHeartbeat(TaskHeartbeat heartbeat) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + DELETE FROM TASK_HEARTBEAT + WHERE INSTANCE = ? + """)) { + + stmt.setString(1, heartbeat.instanceUuidFull()); + stmt.executeUpdate(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + private record ProcessHeartbeat( + String processId, + String processBase, + String uuidFull, + double lastSeenMillis, + Integer progress, + String status + ) { + public boolean isRunning() { + return "RUNNING".equals(status); + } + public ProcessService.ProcessId getProcessId() { + return switch (processBase) { + case "converter" -> ProcessService.ProcessId.CONVERTER; + case "crawler" -> ProcessService.ProcessId.CRAWLER; + case "loader" -> ProcessService.ProcessId.LOADER; + case "website-adjacencies-calculator" -> ProcessService.ProcessId.ADJACENCIES_CALCULATOR; + case "index-constructor" -> ProcessService.ProcessId.INDEX_CONSTRUCTOR; + default -> null; + }; + } + } + + private record TaskHeartbeat( + String taskName, + String taskBase, + String instanceUuidFull, + String serviceUuuidFull, + double lastSeenMillis, + Integer progress, + String stage, + String status + ) { } + +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ActorProcessWatcher.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java similarity index 98% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ActorProcessWatcher.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java index b8fc8261..83835e00 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ActorProcessWatcher.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ActorProcessWatcher.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ConvertActor.java similarity index 96% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/ConvertActor.java index 299e952a..d9867e86 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ConvertActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.gson.Gson; import com.google.inject.Inject; @@ -6,20 +6,20 @@ import com.google.inject.Singleton; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.With; +import nu.marginalia.actor.ActorStateFactory; +import nu.marginalia.actor.prototype.AbstractActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; import nu.marginalia.control.process.ProcessOutboxes; import nu.marginalia.control.process.ProcessService; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageType; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mqapi.converting.ConvertAction; import nu.marginalia.mqapi.converting.ConvertRequest; -import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorResumeBehavior; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,7 +92,7 @@ public class ConvertActor extends AbstractActorPrototype { // Create processed data area var toProcess = storageService.getStorage(sourceStorageId); - var base = storageService.getStorageBase(FileStorageBaseType.SLOW); + var base = storageService.getStorageBase(FileStorageBaseType.WORK); var processedArea = storageService.allocateTemporaryStorage(base, FileStorageType.PROCESSED_DATA, "processed-data", "Processed Data; " + toProcess.description()); @@ -125,7 +125,7 @@ public class ConvertActor extends AbstractActorPrototype { String fileName = sourcePath.toFile().getName(); - var base = storageService.getStorageBase(FileStorageBaseType.SLOW); + var base = storageService.getStorageBase(FileStorageBaseType.WORK); var processedArea = storageService.allocateTemporaryStorage(base, FileStorageType.PROCESSED_DATA, "processed-data", "Processed Encylopedia Data; " + fileName); @@ -157,7 +157,7 @@ public class ConvertActor extends AbstractActorPrototype { String fileName = sourcePath.toFile().getName(); - var base = storageService.getStorageBase(FileStorageBaseType.SLOW); + var base = storageService.getStorageBase(FileStorageBaseType.WORK); var processedArea = storageService.allocateTemporaryStorage(base, FileStorageType.PROCESSED_DATA, "processed-data", "Processed Dirtree Data; " + fileName); @@ -188,7 +188,7 @@ public class ConvertActor extends AbstractActorPrototype { String fileName = sourcePath.toFile().getName(); - var base = storageService.getStorageBase(FileStorageBaseType.SLOW); + var base = storageService.getStorageBase(FileStorageBaseType.WORK); var processedArea = storageService.allocateTemporaryStorage(base, FileStorageType.PROCESSED_DATA, "processed-data", "Processed Stackexchange Data; " + fileName); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java similarity index 94% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java index 2525656c..e13cb22e 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.gson.Gson; import com.google.inject.Inject; @@ -7,25 +7,25 @@ import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.With; import nu.marginalia.actor.ActorStateFactory; +import nu.marginalia.actor.prototype.AbstractActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; import nu.marginalia.control.process.ProcessOutboxes; import nu.marginalia.control.process.ProcessService; -import nu.marginalia.control.svc.BackupService; +import nu.marginalia.svc.BackupService; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageType; import nu.marginalia.index.client.IndexClient; import nu.marginalia.index.client.IndexMqEndpoints; +import nu.marginalia.mq.MqMessageState; +import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mqapi.converting.ConvertAction; import nu.marginalia.mqapi.converting.ConvertRequest; import nu.marginalia.mqapi.index.CreateIndexRequest; import nu.marginalia.mqapi.index.IndexName; import nu.marginalia.mqapi.loading.LoadRequest; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; -import nu.marginalia.mq.MqMessageState; -import nu.marginalia.mq.outbox.MqOutbox; -import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorResumeBehavior; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +64,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { @AllArgsConstructor @With @NoArgsConstructor public static class Message { public FileStorageId crawlStorageId = null; - public FileStorageId processedStorageId = null; + public List processedStorageId = null; public long converterMsgId = 0L; public long loaderMsgId = 0L; }; @@ -126,7 +126,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { var toProcess = storageService.getStorage(message.crawlStorageId); - var base = storageService.getStorageBase(FileStorageBaseType.SLOW); + var base = storageService.getStorageBase(FileStorageBaseType.WORK); var processedArea = storageService.allocateTemporaryStorage(base, FileStorageType.PROCESSED_DATA, "processed-data", "Processed Data; " + toProcess.description()); @@ -140,7 +140,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { long id = mqConverterOutbox.sendAsync(ConvertRequest.class.getSimpleName(), gson.toJson(request)); return message - .withProcessedStorageId(processedArea.id()) + .withProcessedStorageId(List.of(processedArea.id())) .withConverterMsgId(id); } @@ -171,7 +171,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { """) public Message load(Message message) throws Exception { if (message.loaderMsgId <= 0) { - var request = new LoadRequest(List.of(message.processedStorageId)); + var request = new LoadRequest(message.processedStorageId); long id = mqLoaderOutbox.sendAsync(LoadRequest.class.getSimpleName(), gson.toJson(request)); transition(LOAD, message.withLoaderMsgId(id)); @@ -192,7 +192,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { Create a backup snapshot of the new data """) public void createBackup(Message message) throws SQLException, IOException { - backupService.createBackupFromStaging(List.of(message.processedStorageId)); + backupService.createBackupFromStaging(message.processedStorageId); } @ActorState( diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/CrawlActor.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/CrawlActor.java index 77d2a86c..04b75f84 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/CrawlActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.gson.Gson; import com.google.inject.Inject; @@ -7,21 +7,23 @@ import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.With; import nu.marginalia.actor.ActorStateFactory; +import nu.marginalia.actor.prototype.AbstractActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; import nu.marginalia.control.process.ProcessOutboxes; import nu.marginalia.control.process.ProcessService; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageType; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mqapi.crawling.CrawlRequest; -import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorResumeBehavior; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + @Singleton public class CrawlActor extends AbstractActorPrototype { @@ -96,7 +98,7 @@ public class CrawlActor extends AbstractActorPrototype { var toCrawl = storageService.getStorage(message.crawlSpecId); - var base = storageService.getStorageBase(FileStorageBaseType.SLOW); + var base = storageService.getStorageBase(FileStorageBaseType.WORK); var dataArea = storageService.allocateTemporaryStorage( base, FileStorageType.CRAWL_DATA, @@ -106,7 +108,7 @@ public class CrawlActor extends AbstractActorPrototype { storageService.relateFileStorages(toCrawl.id(), dataArea.id()); // Pre-send convert request - var request = new CrawlRequest(message.crawlSpecId, dataArea.id()); + var request = new CrawlRequest(List.of(message.crawlSpecId), dataArea.id()); long id = mqCrawlerOutbox.sendAsync(CrawlRequest.class.getSimpleName(), gson.toJson(request)); return message diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java similarity index 87% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java index 8fe19d5a..ebe01f04 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/CrawlJobExtractorActor.java @@ -1,19 +1,17 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.control.svc.ControlFileStorageService; -import nu.marginalia.crawlspec.CrawlSpecFileNames; -import nu.marginalia.crawlspec.CrawlSpecGenerator; -import nu.marginalia.db.DbDomainStatsExportMultitool; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageType; import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.crawlspec.CrawlSpecFileNames; +import nu.marginalia.db.DbDomainStatsExportMultitool; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,18 +32,15 @@ public class CrawlJobExtractorActor extends AbstractActorPrototype { public static final String CREATE_FROM_LINK = "CREATE_FROM_LINK"; public static final String END = "END"; private final FileStorageService fileStorageService; - private final ControlFileStorageService controlFileStorageService; private final HikariDataSource dataSource; @Inject public CrawlJobExtractorActor(ActorStateFactory stateFactory, FileStorageService fileStorageService, - ControlFileStorageService controlFileStorageService, HikariDataSource dataSource ) { super(stateFactory); this.fileStorageService = fileStorageService; - this.controlFileStorageService = controlFileStorageService; this.dataSource = dataSource; } @@ -70,7 +65,7 @@ public class CrawlJobExtractorActor extends AbstractActorPrototype { error("This actor requires a CrawlJobExtractorArgumentsWithURL argument"); } - var base = fileStorageService.getStorageBase(FileStorageBaseType.SLOW); + var base = fileStorageService.getStorageBase(FileStorageBaseType.WORK); var storage = fileStorageService.allocateTemporaryStorage(base, FileStorageType.CRAWL_SPEC, "crawl-spec", arg.description()); Path urlsTxt = storage.asPath().resolve("urls.txt"); @@ -81,7 +76,7 @@ public class CrawlJobExtractorActor extends AbstractActorPrototype { is.transferTo(os); } catch (Exception ex) { - controlFileStorageService.flagFileForDeletion(storage.id()); + fileStorageService.flagFileForDeletion(storage.id()); error("Error downloading " + arg.url()); } @@ -107,7 +102,7 @@ public class CrawlJobExtractorActor extends AbstractActorPrototype { error("This actor requires a CrawlJobExtractorArguments argument"); } - var base = fileStorageService.getStorageBase(FileStorageBaseType.SLOW); + var base = fileStorageService.getStorageBase(FileStorageBaseType.WORK); var storage = fileStorageService.allocateTemporaryStorage(base, FileStorageType.CRAWL_SPEC, "crawl-spec", arg.description()); final Path path = CrawlSpecFileNames.resolve(storage); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ExportDataActor.java similarity index 97% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/ExportDataActor.java index 5e2a3cdd..88448824 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/ExportDataActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -7,12 +7,12 @@ import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.With; import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/RecrawlActor.java similarity index 81% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/RecrawlActor.java index 2351e1aa..b065f01f 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/RecrawlActor.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.gson.Gson; import com.google.inject.Inject; @@ -7,21 +7,22 @@ import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.With; import nu.marginalia.actor.ActorStateFactory; +import nu.marginalia.actor.prototype.AbstractActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; import nu.marginalia.control.process.ProcessOutboxes; import nu.marginalia.control.process.ProcessService; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorage; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageType; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mqapi.crawling.CrawlRequest; -import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorResumeBehavior; import java.nio.file.Files; import java.sql.SQLException; +import java.util.List; import java.util.Optional; @Singleton @@ -41,7 +42,7 @@ public class RecrawlActor extends AbstractActorPrototype { @AllArgsConstructor @With @NoArgsConstructor public static class RecrawlMessage { - public FileStorageId crawlSpecId = null; + public List crawlSpecId = null; public FileStorageId crawlStorageId = null; public long crawlerMsgId = 0L; }; @@ -50,10 +51,8 @@ public class RecrawlActor extends AbstractActorPrototype { public String describe() { return "Run the crawler with the given crawl spec using previous crawl data for a reference"; } - public static RecrawlMessage recrawlFromCrawlData(FileStorageId crawlData) { - return new RecrawlMessage(null, crawlData, 0L); - } - public static RecrawlMessage recrawlFromCrawlDataAndCralSpec(FileStorageId crawlData, FileStorageId crawlSpec) { + + public static RecrawlMessage recrawlFromCrawlDataAndCralSpec(FileStorageId crawlData, List crawlSpec) { return new RecrawlMessage(crawlSpec, crawlData, 0L); } @@ -83,24 +82,22 @@ public class RecrawlActor extends AbstractActorPrototype { } var crawlStorage = storageService.getStorage(recrawlMessage.crawlStorageId); - FileStorage specStorage; - if (recrawlMessage.crawlSpecId != null) { - specStorage = storageService.getStorage(recrawlMessage.crawlSpecId); - } - else { - specStorage = getSpec(crawlStorage).orElse(null); + for (var specs : recrawlMessage.crawlSpecId) { + FileStorage specStorage = storageService.getStorage(specs); + + if (specStorage == null) error("Bad storage id"); + if (specStorage.type() != FileStorageType.CRAWL_SPEC) error("Bad storage type " + specStorage.type()); } - if (specStorage == null) error("Bad storage id"); - if (specStorage.type() != FileStorageType.CRAWL_SPEC) error("Bad storage type " + specStorage.type()); + if (crawlStorage == null) error("Bad storage id"); - if (crawlStorage.type() != FileStorageType.CRAWL_DATA) error("Bad storage type " + specStorage.type()); + if (crawlStorage.type() != FileStorageType.CRAWL_DATA) error("Bad storage type " + crawlStorage.type()); Files.deleteIfExists(crawlStorage.asPath().resolve("crawler.log")); return recrawlMessage - .withCrawlSpecId(specStorage.id()); + .withCrawlSpecId(recrawlMessage.crawlSpecId); } private Optional getSpec(FileStorage crawlStorage) throws SQLException { @@ -119,6 +116,7 @@ public class RecrawlActor extends AbstractActorPrototype { ) public RecrawlMessage crawl(RecrawlMessage recrawlMessage) throws Exception { // Pre-send crawl request + var request = new CrawlRequest(recrawlMessage.crawlSpecId, recrawlMessage.crawlStorageId); long id = mqCrawlerOutbox.sendAsync(CrawlRequest.class.getSimpleName(), gson.toJson(request)); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RestoreBackupActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java similarity index 73% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RestoreBackupActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java index 96629208..87dc0f78 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RestoreBackupActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/RestoreBackupActor.java @@ -1,13 +1,14 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.inject.Inject; import nu.marginalia.actor.ActorStateFactory; import nu.marginalia.actor.prototype.AbstractActorPrototype; import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.actor.state.ActorState; -import nu.marginalia.control.actor.Actor; -import nu.marginalia.control.svc.BackupService; -import nu.marginalia.db.storage.model.FileStorageId; +import nu.marginalia.actor.Actor; +import nu.marginalia.service.module.ServiceConfiguration; +import nu.marginalia.svc.BackupService; +import nu.marginalia.storage.model.FileStorageId; import nu.marginalia.mq.persistence.MqPersistence; @@ -18,6 +19,7 @@ public class RestoreBackupActor extends AbstractActorPrototype { public static final String END = "END"; private final BackupService backupService; + private final int node; private final MqPersistence mqPersistence; @Override @@ -27,11 +29,13 @@ public class RestoreBackupActor extends AbstractActorPrototype { @Inject public RestoreBackupActor(ActorStateFactory stateFactory, MqPersistence mqPersistence, - BackupService backupService + BackupService backupService, + ServiceConfiguration configuration ) { super(stateFactory); this.mqPersistence = mqPersistence; this.backupService = backupService; + this.node = configuration.node(); } @ActorState(name=RESTORE, next = END, resume = ActorResumeBehavior.ERROR) @@ -39,7 +43,7 @@ public class RestoreBackupActor extends AbstractActorPrototype { backupService.restoreBackup(id); mqPersistence.sendNewMessage( - Actor.CONVERT_AND_LOAD.id(), + Actor.CONVERT_AND_LOAD.id() + ":" + node, null, null, ConvertAndLoadActor.REPARTITION, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java similarity index 98% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java index 0082c024..14a3d500 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/TriggerAdjacencyCalculationActor.java @@ -1,12 +1,12 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.control.process.ProcessService; import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.control.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/TruncateLinkDatabase.java similarity index 96% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java rename to code/features-control/actors/src/main/java/nu/marginalia/actor/task/TruncateLinkDatabase.java index 70dd06a3..9eff9fce 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/actor/task/TruncateLinkDatabase.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.actor.task; +package nu.marginalia.actor.task; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -7,10 +7,10 @@ import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.With; import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.db.storage.model.FileStorageId; import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; +import nu.marginalia.actor.state.ActorState; +import nu.marginalia.storage.model.FileStorageId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/BackupService.java b/code/features-control/actors/src/main/java/nu/marginalia/svc/BackupService.java similarity index 58% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/BackupService.java rename to code/features-control/actors/src/main/java/nu/marginalia/svc/BackupService.java index 68e87dd4..b84d2bec 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/BackupService.java +++ b/code/features-control/actors/src/main/java/nu/marginalia/svc/BackupService.java @@ -1,18 +1,19 @@ -package nu.marginalia.control.svc; +package nu.marginalia.svc; import com.github.luben.zstd.ZstdInputStream; import com.github.luben.zstd.ZstdOutputStream; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageBaseType; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.IndexLocations; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBaseType; +import nu.marginalia.storage.model.FileStorageId; +import nu.marginalia.storage.model.FileStorageType; import nu.marginallia.index.journal.IndexJournalFileNames; import org.apache.commons.io.IOUtils; import com.google.inject.Inject; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.sql.SQLException; import java.time.LocalDateTime; import java.util.List; @@ -34,37 +35,39 @@ public class BackupService { String desc = "Pre-load backup snapshot " + LocalDateTime.now(); - var backupStorage = storageService.allocateTemporaryStorage(backupBase, FileStorageType.BACKUP, "snapshot", desc); + var backupStorage = storageService.allocateTemporaryStorage(backupBase, + FileStorageType.BACKUP, "snapshot", desc); for (var associatedId : associatedIds) { storageService.relateFileStorages(associatedId, backupStorage.id()); } - var indexStagingStorage = storageService.getStorageByType(FileStorageType.INDEX_STAGING); - var linkdbStagingStorage = storageService.getStorageByType(FileStorageType.LINKDB_STAGING); - backupFileCompressed("links.db", linkdbStagingStorage, backupStorage); + var indexStagingStorage = IndexLocations.getIndexConstructionArea(storageService); + var linkdbStagingStorage = IndexLocations.getLinkdbWritePath(storageService); + + backupFileCompressed("links.db", linkdbStagingStorage, backupStorage.asPath()); // This file format is already compressed - backupJournal(indexStagingStorage, backupStorage); + backupJournal(indexStagingStorage, backupStorage.asPath()); } /** Read back a backup into _STAGING */ public void restoreBackup(FileStorageId backupId) throws SQLException, IOException { - var backupStorage = storageService.getStorage(backupId); + var backupStorage = storageService.getStorage(backupId).asPath(); - var indexStagingStorage = storageService.getStorageByType(FileStorageType.INDEX_STAGING); - var linkdbStagingStorage = storageService.getStorageByType(FileStorageType.LINKDB_STAGING); + var indexStagingStorage = IndexLocations.getIndexConstructionArea(storageService); + var linkdbStagingStorage = IndexLocations.getLinkdbWritePath(storageService); restoreBackupCompressed("links.db", linkdbStagingStorage, backupStorage); restoreJournal(indexStagingStorage, backupStorage); } - private void backupJournal(FileStorage inputStorage, FileStorage backupStorage) throws IOException + private void backupJournal(Path inputStorage, Path backupStorage) throws IOException { - for (var source : IndexJournalFileNames.findJournalFiles(inputStorage.asPath())) { - var dest = backupStorage.asPath().resolve(source.toFile().getName()); + for (var source : IndexJournalFileNames.findJournalFiles(inputStorage)) { + var dest = backupStorage.resolve(source.toFile().getName()); try (var is = Files.newInputStream(source); var os = Files.newOutputStream(dest) @@ -75,15 +78,15 @@ public class BackupService { } - private void restoreJournal(FileStorage destStorage, FileStorage backupStorage) throws IOException { + private void restoreJournal(Path destStorage, Path backupStorage) throws IOException { // Remove any old journal files first to avoid them getting loaded - for (var garbage : IndexJournalFileNames.findJournalFiles(destStorage.asPath())) { + for (var garbage : IndexJournalFileNames.findJournalFiles(destStorage)) { Files.delete(garbage); } - for (var source : IndexJournalFileNames.findJournalFiles(backupStorage.asPath())) { - var dest = destStorage.asPath().resolve(source.toFile().getName()); + for (var source : IndexJournalFileNames.findJournalFiles(backupStorage)) { + var dest = destStorage.resolve(source.toFile().getName()); try (var is = Files.newInputStream(source); var os = Files.newOutputStream(dest) @@ -94,18 +97,18 @@ public class BackupService { } - private void backupFileCompressed(String fileName, FileStorage inputStorage, FileStorage backupStorage) throws IOException + private void backupFileCompressed(String fileName, Path inputStorage, Path backupStorage) throws IOException { - try (var is = Files.newInputStream(inputStorage.asPath().resolve(fileName)); - var os = new ZstdOutputStream(Files.newOutputStream(backupStorage.asPath().resolve(fileName))) + try (var is = Files.newInputStream(inputStorage.resolve(fileName)); + var os = new ZstdOutputStream(Files.newOutputStream(backupStorage.resolve(fileName))) ) { IOUtils.copyLarge(is, os); } } - private void restoreBackupCompressed(String fileName, FileStorage destStorage, FileStorage backupStorage) throws IOException + private void restoreBackupCompressed(String fileName, Path destStorage, Path backupStorage) throws IOException { - try (var is = new ZstdInputStream(Files.newInputStream(backupStorage.asPath().resolve(fileName))); - var os = Files.newOutputStream(destStorage.asPath().resolve(fileName)) + try (var is = new ZstdInputStream(Files.newInputStream(backupStorage.resolve(fileName))); + var os = Files.newOutputStream(destStorage.resolve(fileName)) ) { IOUtils.copyLarge(is, os); } diff --git a/code/features-control/process-execution/build.gradle b/code/features-control/process-execution/build.gradle new file mode 100644 index 00000000..b2ab9536 --- /dev/null +++ b/code/features-control/process-execution/build.gradle @@ -0,0 +1,28 @@ + +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + + implementation project(':code:libraries:message-queue') + implementation project(':code:common:service') + implementation project(':code:common:process') + implementation project(':code:api:process-mqapi') + + implementation libs.bundles.slf4j + implementation libs.guice + implementation libs.notnull + implementation libs.jsoup + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java b/code/features-control/process-execution/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java similarity index 83% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java rename to code/features-control/process-execution/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java index cb45b6f5..9f8d835d 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java +++ b/code/features-control/process-execution/src/main/java/nu/marginalia/control/process/ProcessOutboxes.java @@ -2,9 +2,9 @@ package nu.marginalia.control.process; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.mqapi.ProcessInboxNames; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.mq.persistence.MqPersistence; +import nu.marginalia.mqapi.ProcessInboxNames; import nu.marginalia.service.server.BaseServiceParams; @Singleton @@ -18,22 +18,30 @@ public class ProcessOutboxes { public ProcessOutboxes(BaseServiceParams params, MqPersistence persistence) { converterOutbox = new MqOutbox(persistence, ProcessInboxNames.CONVERTER_INBOX, + params.configuration.node(), params.configuration.serviceName(), + params.configuration.node(), params.configuration.instanceUuid() ); loaderOutbox = new MqOutbox(persistence, ProcessInboxNames.LOADER_INBOX, + params.configuration.node(), params.configuration.serviceName(), + params.configuration.node(), params.configuration.instanceUuid() ); crawlerOutbox = new MqOutbox(persistence, ProcessInboxNames.CRAWLER_INBOX, + params.configuration.node(), params.configuration.serviceName(), + params.configuration.node(), params.configuration.instanceUuid() ); indexConstructorOutbox = new MqOutbox(persistence, ProcessInboxNames.INDEX_CONSTRUCTOR_INBOX, + params.configuration.node(), params.configuration.serviceName(), + params.configuration.node(), params.configuration.instanceUuid() ); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/process/ProcessService.java b/code/features-control/process-execution/src/main/java/nu/marginalia/control/process/ProcessService.java similarity index 98% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/process/ProcessService.java rename to code/features-control/process-execution/src/main/java/nu/marginalia/control/process/ProcessService.java index e8361868..0dbe147e 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/process/ProcessService.java +++ b/code/features-control/process-execution/src/main/java/nu/marginalia/control/process/ProcessService.java @@ -1,5 +1,7 @@ package nu.marginalia.control.process; +import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.name.Named; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.BaseServiceParams; @@ -8,14 +10,14 @@ import org.slf4j.LoggerFactory; import org.slf4j.Marker; import org.slf4j.MarkerFactory; -import com.google.inject.Inject; -import com.google.inject.Singleton; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; @Singleton diff --git a/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderPagingImpl.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderPagingImpl.java index 37db0b70..d86931a1 100644 --- a/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderPagingImpl.java +++ b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderPagingImpl.java @@ -2,6 +2,8 @@ package nu.marginalia.index.journal.reader; import nu.marginalia.index.journal.reader.pointer.IndexJournalPointer; import nu.marginallia.index.journal.IndexJournalFileNames; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; @@ -10,10 +12,16 @@ import java.util.List; public class IndexJournalReaderPagingImpl implements IndexJournalReader { + private static final Logger logger = LoggerFactory.getLogger(IndexJournalReaderPagingImpl.class); private final List readers; public IndexJournalReaderPagingImpl(Path baseDir) throws IOException { var inputFiles = IndexJournalFileNames.findJournalFiles(baseDir); + if (inputFiles.isEmpty()) + logger.warn("Creating paging index journal file in {}, found no inputs!", baseDir); + else + logger.info("Creating paging index journal reader for {} inputs", inputFiles.size()); + this.readers = new ArrayList<>(inputFiles.size()); for (var inputFile : inputFiles) { diff --git a/code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java b/code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java index dc1646b4..5bd4baf6 100644 --- a/code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java +++ b/code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java @@ -49,13 +49,14 @@ public class SimpleBlockingThreadPool { public void shutDownNow() { this.shutDown = true; + tasks.clear(); for (Thread worker : workers) { worker.interrupt(); } } private void worker() { - while (!shutDown) { + while (!tasks.isEmpty() || !shutDown) { try { Task task = tasks.poll(1, TimeUnit.SECONDS); if (task == null) { @@ -89,6 +90,14 @@ public class SimpleBlockingThreadPool { final long start = System.currentTimeMillis(); final long deadline = start + timeUnit.toMillis(i); + // Drain the queue + while (!tasks.isEmpty()) { + long timeRemaining = deadline - System.currentTimeMillis(); + if (timeRemaining <= 0) + return false; + } + + // Wait for termination for (var thread : workers) { if (!thread.isAlive()) continue; diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java b/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java index 88e57028..fa34f893 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/actor/ActorStateMachine.java @@ -44,14 +44,15 @@ public class ActorStateMachine { private final boolean isDirectlyInitializable; public ActorStateMachine(MessageQueueFactory messageQueueFactory, - String queueName, + String fsmName, + int node, UUID instanceUUID, ActorPrototype statePrototype) { - this.queueName = queueName; + this.queueName = fsmName; - smInbox = messageQueueFactory.createSynchronousInbox(queueName, instanceUUID); - smOutbox = messageQueueFactory.createOutbox(queueName, queueName+"//out", instanceUUID); + smInbox = messageQueueFactory.createSynchronousInbox(queueName, node, instanceUUID); + smOutbox = messageQueueFactory.createOutbox(queueName, node, queueName+"//out", node, instanceUUID); smInbox.subscribe(new StateEventSubscription()); diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java index e682de17..672556ea 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java @@ -20,25 +20,25 @@ public class MessageQueueFactory { this.persistence = persistence; } - public MqSingleShotInbox createSingleShotInbox(String inboxName, UUID instanceUUID) + public MqSingleShotInbox createSingleShotInbox(String inboxName, int node, UUID instanceUUID) { - return new MqSingleShotInbox(persistence, inboxName, instanceUUID); + return new MqSingleShotInbox(persistence, inboxName + ":" + node, instanceUUID); } - public MqAsynchronousInbox createAsynchronousInbox(String inboxName, UUID instanceUUID) + public MqAsynchronousInbox createAsynchronousInbox(String inboxName, int node, UUID instanceUUID) { - return new MqAsynchronousInbox(persistence, inboxName, instanceUUID); + return new MqAsynchronousInbox(persistence, inboxName + ":" + node, instanceUUID); } - public MqSynchronousInbox createSynchronousInbox(String inboxName, UUID instanceUUID) + public MqSynchronousInbox createSynchronousInbox(String inboxName, int node, UUID instanceUUID) { - return new MqSynchronousInbox(persistence, inboxName, instanceUUID); + return new MqSynchronousInbox(persistence, inboxName + ":" + node, instanceUUID); } - public MqOutbox createOutbox(String inboxName, String outboxName, UUID instanceUUID) + public MqOutbox createOutbox(String inboxName, int inboxNode, String outboxName, int outboxNode, UUID instanceUUID) { - return new MqOutbox(persistence, inboxName, outboxName, instanceUUID); + return new MqOutbox(persistence, inboxName, inboxNode, outboxName, outboxNode, instanceUUID); } } diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java index 61e73bec..f658e1a1 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java @@ -30,12 +30,14 @@ public class MqOutbox { public MqOutbox(MqPersistence persistence, String inboxName, + int inboxNode, String outboxName, + int outboxNode, UUID instanceUUID) { this.persistence = persistence; - this.inboxName = inboxName; - this.replyInboxName = outboxName + "//" + inboxName; + this.inboxName = inboxName + ":" + inboxNode; + this.replyInboxName = String.format("%s:%d//%s:%d", outboxName, outboxNode, inboxName, inboxNode); this.instanceUUID = instanceUUID.toString(); pollThread = new Thread(this::poll, "mq-outbox-poll-thread:" + inboxName); diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineErrorTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineErrorTest.java index 6a39d0ec..3f4feebe 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineErrorTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineErrorTest.java @@ -88,14 +88,14 @@ public class ActorStateMachineErrorTest { @Test public void smResumeResumableFromNew() throws Exception { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ErrorHurdles(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new ErrorHurdles(stateFactory)); sm.init(); sm.join(2, TimeUnit.SECONDS); sm.stop(); - List states = MqTestUtil.getMessages(dataSource, inboxId) + List states = MqTestUtil.getMessages(dataSource, inboxId, 0) .stream() .peek(System.out::println) .map(MqMessageRow::function) diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineNullTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineNullTest.java index d1e1f0c6..9f24858a 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineNullTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineNullTest.java @@ -86,7 +86,7 @@ public class ActorStateMachineNullTest { var graph = new TestPrototypeActor(stateFactory); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), graph); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), graph); sm.registerStates(graph); sm.init(); @@ -94,7 +94,7 @@ public class ActorStateMachineNullTest { sm.join(2, TimeUnit.SECONDS); sm.stop(); - MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println); + MqTestUtil.getMessages(dataSource, inboxId, 0).forEach(System.out::println); } diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineResumeTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineResumeTest.java index 69381c51..d91ad882 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineResumeTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineResumeTest.java @@ -87,14 +87,14 @@ public class ActorStateMachineResumeTest { public void smResumeResumableFromNew() throws Exception { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); + sendMessage(inboxId, 0, "RESUMABLE"); - persistence.sendNewMessage(inboxId, null, -1L, "RESUMABLE", "", null); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); sm.join(2, TimeUnit.SECONDS); sm.stop(); - List states = MqTestUtil.getMessages(dataSource, inboxId) + List states = MqTestUtil.getMessages(dataSource, inboxId, 0) .stream() .peek(System.out::println) .map(MqMessageRow::function) @@ -103,19 +103,23 @@ public class ActorStateMachineResumeTest { assertEquals(List.of("RESUMABLE", "NON-RESUMABLE", "OK", "END"), states); } + private long sendMessage(String inboxId, int node, String function) throws Exception { + return persistence.sendNewMessage(inboxId+":"+node, null, -1L, function, "", null); + } + @Test public void smResumeFromAck() throws Exception { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); - long id = persistence.sendNewMessage(inboxId, null, -1L, "RESUMABLE", "", null); + long id = sendMessage(inboxId, 0, "RESUMABLE"); persistence.updateMessageState(id, MqMessageState.ACK); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); sm.join(4, TimeUnit.SECONDS); sm.stop(); - List states = MqTestUtil.getMessages(dataSource, inboxId) + List states = MqTestUtil.getMessages(dataSource, inboxId, 0) .stream() .peek(System.out::println) .map(MqMessageRow::function) @@ -129,15 +133,14 @@ public class ActorStateMachineResumeTest { public void smResumeNonResumableFromNew() throws Exception { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); + sendMessage(inboxId, 0, "NON-RESUMABLE"); - persistence.sendNewMessage(inboxId, null, -1L, "NON-RESUMABLE", "", null); - - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); sm.join(2, TimeUnit.SECONDS); sm.stop(); - List states = MqTestUtil.getMessages(dataSource, inboxId) + List states = MqTestUtil.getMessages(dataSource, inboxId, 0) .stream() .peek(System.out::println) .map(MqMessageRow::function) @@ -151,15 +154,15 @@ public class ActorStateMachineResumeTest { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); - long id = persistence.sendNewMessage(inboxId, null, null, "NON-RESUMABLE", "", null); + long id = sendMessage(inboxId, 0, "NON-RESUMABLE"); persistence.updateMessageState(id, MqMessageState.ACK); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); sm.join(2, TimeUnit.SECONDS); sm.stop(); - List states = MqTestUtil.getMessages(dataSource, inboxId) + List states = MqTestUtil.getMessages(dataSource, inboxId, 0) .stream() .peek(System.out::println) .map(MqMessageRow::function) @@ -172,13 +175,12 @@ public class ActorStateMachineResumeTest { public void smResumeEmptyQueue() throws Exception { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); - - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new ResumeTrialsPrototypeActor(stateFactory)); sm.join(2, TimeUnit.SECONDS); sm.stop(); - List states = MqTestUtil.getMessages(dataSource, inboxId) + List states = MqTestUtil.getMessages(dataSource, inboxId, 0) .stream() .peek(System.out::println) .map(MqMessageRow::function) diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineTest.java index ac9147a9..02cfcff7 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorStateMachineTest.java @@ -93,7 +93,7 @@ public class ActorStateMachineTest { var graph = new TestPrototypeActor(stateFactory); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), graph); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), graph); sm.registerStates(graph); sm.init(); @@ -101,14 +101,14 @@ public class ActorStateMachineTest { sm.join(2, TimeUnit.SECONDS); sm.stop(); - MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println); + MqTestUtil.getMessages(dataSource, inboxId, 0).forEach(System.out::println); } @Test public void testStartStopStartStop() throws Exception { var stateFactory = new ActorStateFactory(new GsonBuilder().create()); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new TestPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new TestPrototypeActor(stateFactory)); sm.init(); @@ -117,11 +117,11 @@ public class ActorStateMachineTest { System.out.println("-------------------- "); - var sm2 = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new TestPrototypeActor(stateFactory)); + var sm2 = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new TestPrototypeActor(stateFactory)); sm2.join(2, TimeUnit.SECONDS); sm2.stop(); - MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println); + MqTestUtil.getMessages(dataSource, inboxId, 0).forEach(System.out::println); } @Test @@ -134,14 +134,14 @@ public class ActorStateMachineTest { persistence.sendNewMessage(inboxId, null, null, "INITIAL", "", null); persistence.sendNewMessage(inboxId, null, null, "INITIAL", "", null); - var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new TestPrototypeActor(stateFactory)); + var sm = new ActorStateMachine(messageQueueFactory, inboxId, 0, UUID.randomUUID(), new TestPrototypeActor(stateFactory)); Thread.sleep(50); sm.join(2, TimeUnit.SECONDS); sm.stop(); - MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println); + MqTestUtil.getMessages(dataSource, inboxId, 0).forEach(System.out::println); } } diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java b/code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java index b3ba62cf..b55f9e09 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java @@ -8,7 +8,7 @@ import java.util.ArrayList; import java.util.List; public class MqTestUtil { - public static List getMessages(HikariDataSource dataSource, String inbox) { + public static List getMessages(HikariDataSource dataSource, String inbox, int node) { List messages = new ArrayList<>(); try (var conn = dataSource.getConnection(); @@ -24,7 +24,7 @@ public class MqTestUtil { WHERE RECIPIENT_INBOX = ? """)) { - stmt.setString(1, inbox); + stmt.setString(1, inbox+":"+node); var rsp = stmt.executeQuery(); while (rsp.next()) { messages.add(new MqMessageRow( diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java index ea2105bd..52f44659 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java @@ -54,7 +54,7 @@ public class MqOutboxTest { @Test public void testOpenClose() throws InterruptedException { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); outbox.stop(); } @@ -67,7 +67,7 @@ public class MqOutboxTest { @Test public void testOutboxTimeout() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); long id = outbox.sendAsync("test", "Hello World"); try { outbox.waitResponse(id, 100, TimeUnit.MILLISECONDS); @@ -84,11 +84,11 @@ public class MqOutboxTest { @Test public void testSingleShotInbox() throws Exception { // Send a message to the inbox - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); long id = outbox.sendAsync("test", "Hello World"); // Create a single-shot inbox - var inbox = new MqSingleShotInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var inbox = new MqSingleShotInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); // Wait for the message to arrive var message = inbox.waitForMessage(1, TimeUnit.SECONDS); @@ -110,12 +110,12 @@ public class MqOutboxTest { @Test public void testSend() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); Executors.newSingleThreadExecutor().submit(() -> outbox.send("test", "Hello World")); TimeUnit.MILLISECONDS.sleep(100); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(1, messages.size()); System.out.println(messages.get(0)); @@ -125,9 +125,9 @@ public class MqOutboxTest { @Test public void testSendAndRespondAsyncInbox() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); - var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); inbox.subscribe(justRespond("Alright then")); inbox.start(); @@ -136,7 +136,7 @@ public class MqOutboxTest { assertEquals(MqMessageState.OK, rsp.state()); assertEquals("Alright then", rsp.payload()); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(1, messages.size()); assertEquals(MqMessageState.OK, messages.get(0).state()); @@ -146,9 +146,9 @@ public class MqOutboxTest { @Test public void testSendAndRespondSyncInbox() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); - var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); inbox.subscribe(justRespond("Alright then")); inbox.start(); @@ -157,7 +157,7 @@ public class MqOutboxTest { assertEquals(MqMessageState.OK, rsp.state()); assertEquals("Alright then", rsp.payload()); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(1, messages.size()); assertEquals(MqMessageState.OK, messages.get(0).state()); @@ -167,9 +167,9 @@ public class MqOutboxTest { @Test public void testSendMultipleAsyncInbox() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); - var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); inbox.subscribe(echo()); inbox.start(); @@ -189,7 +189,7 @@ public class MqOutboxTest { assertEquals(MqMessageState.OK, rsp4.state()); assertEquals("four", rsp4.payload()); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(4, messages.size()); for (var message : messages) { assertEquals(MqMessageState.OK, message.state()); @@ -201,9 +201,9 @@ public class MqOutboxTest { @Test public void testSendMultipleSyncInbox() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); - var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); inbox.subscribe(echo()); inbox.start(); @@ -223,7 +223,7 @@ public class MqOutboxTest { assertEquals(MqMessageState.OK, rsp4.state()); assertEquals("four", rsp4.payload()); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(4, messages.size()); for (var message : messages) { assertEquals(MqMessageState.OK, message.state()); @@ -235,8 +235,8 @@ public class MqOutboxTest { @Test public void testSendAndRespondWithErrorHandlerAsyncInbox() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); - var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); + var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); inbox.start(); @@ -244,7 +244,7 @@ public class MqOutboxTest { assertEquals(MqMessageState.ERR, rsp.state()); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(1, messages.size()); assertEquals(MqMessageState.ERR, messages.get(0).state()); @@ -254,8 +254,8 @@ public class MqOutboxTest { @Test public void testSendAndRespondWithErrorHandlerSyncInbox() throws Exception { - var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID()); - var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID()); + var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, 0, inboxId+"/reply", 0, UUID.randomUUID()); + var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId+":0", UUID.randomUUID()); inbox.start(); @@ -263,7 +263,7 @@ public class MqOutboxTest { assertEquals(MqMessageState.ERR, rsp.state()); - var messages = MqTestUtil.getMessages(dataSource, inboxId); + var messages = MqTestUtil.getMessages(dataSource, inboxId, 0); assertEquals(1, messages.size()); assertEquals(MqMessageState.ERR, messages.get(0).state()); diff --git a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java b/code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java index bab700c0..6697a0c4 100644 --- a/code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java +++ b/code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java @@ -54,13 +54,18 @@ public class MqPersistenceTest { dataSource.close(); } + public long sendMessage(String recipient, String sender, String function, String payload, Duration ttl) throws Exception { + return persistence.sendNewMessage(recipient+":0", sender != null ? (sender+":0") : null, null, function, payload, ttl); + } + @Test public void testReaper() throws Exception { - long id = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(2)); + sendMessage(recipientId, senderId, "function", "payload", Duration.ofSeconds(2)); + persistence.reapDeadMessages(); - var messages = MqTestUtil.getMessages(dataSource, recipientId); + var messages = MqTestUtil.getMessages(dataSource, recipientId, 0); assertEquals(1, messages.size()); assertEquals(MqMessageState.NEW, messages.get(0).state()); System.out.println(messages); @@ -69,7 +74,7 @@ public class MqPersistenceTest { persistence.reapDeadMessages(); - messages = MqTestUtil.getMessages(dataSource, recipientId); + messages = MqTestUtil.getMessages(dataSource, recipientId, 0); assertEquals(1, messages.size()); assertEquals(MqMessageState.DEAD, messages.get(0).state()); } @@ -77,9 +82,9 @@ public class MqPersistenceTest { @Test public void sendWithReplyAddress() throws Exception { - long id = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(30)); + long id = sendMessage(recipientId, senderId, "function", "payload", Duration.ofSeconds(30)); - var messages = MqTestUtil.getMessages(dataSource, recipientId); + var messages = MqTestUtil.getMessages(dataSource, recipientId, 0); assertEquals(1, messages.size()); var message = messages.get(0); @@ -95,9 +100,9 @@ public class MqPersistenceTest { @Test public void sendNoReplyAddress() throws Exception { - long id = persistence.sendNewMessage(recipientId, null, null, "function", "payload", Duration.ofSeconds(30)); + long id = sendMessage(recipientId, null, "function", "payload", Duration.ofSeconds(30)); - var messages = MqTestUtil.getMessages(dataSource, recipientId); + var messages = MqTestUtil.getMessages(dataSource, recipientId, 0); assertEquals(1, messages.size()); var message = messages.get(0); @@ -114,11 +119,13 @@ public class MqPersistenceTest { @Test public void updateState() throws Exception { - long id = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(30)); + + long id = sendMessage(recipientId, senderId, "function", "payload", Duration.ofSeconds(30)); + persistence.updateMessageState(id, MqMessageState.OK); System.out.println(id); - var messages = MqTestUtil.getMessages(dataSource, recipientId); + var messages = MqTestUtil.getMessages(dataSource, recipientId, 0); assertEquals(1, messages.size()); var message = messages.get(0); @@ -131,10 +138,10 @@ public class MqPersistenceTest { @Test public void testReply() throws Exception { - long request = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(30)); + long request = sendMessage(recipientId, senderId, "function", "payload", Duration.ofSeconds(30)); long response = persistence.sendResponse(request, MqMessageState.OK, "response"); - var sentMessages = MqTestUtil.getMessages(dataSource, recipientId); + var sentMessages = MqTestUtil.getMessages(dataSource, recipientId, 0); System.out.println(sentMessages); assertEquals(1, sentMessages.size()); @@ -143,7 +150,7 @@ public class MqPersistenceTest { assertEquals(MqMessageState.OK, requestMessage.state()); - var replies = MqTestUtil.getMessages(dataSource, senderId); + var replies = MqTestUtil.getMessages(dataSource, senderId, 0); System.out.println(replies); assertEquals(1, replies.size()); @@ -159,9 +166,9 @@ public class MqPersistenceTest { String instanceId = "BATMAN"; long tick = 1234L; - long id = persistence.sendNewMessage(recipientId, null, null, "function", "payload", Duration.ofSeconds(30)); + long id = sendMessage(recipientId, null, "function", "payload", Duration.ofSeconds(30)); - var messagesPollFirstTime = persistence.pollInbox(recipientId, instanceId , tick, 10); + var messagesPollFirstTime = persistence.pollInbox(recipientId+":0", instanceId , tick, 10); /** CHECK POLL RESULT */ assertEquals(1, messagesPollFirstTime.size()); @@ -171,7 +178,7 @@ public class MqPersistenceTest { assertEquals("payload", firstPollMessage.payload()); /** CHECK DB TABLE */ - var messages = MqTestUtil.getMessages(dataSource, recipientId); + var messages = MqTestUtil.getMessages(dataSource, recipientId, 0); assertEquals(1, messages.size()); var message = messages.get(0); @@ -184,7 +191,7 @@ public class MqPersistenceTest { assertEquals(tick, message.ownerTick()); /** VERIFY SECOND POLL IS EMPTY */ - var messagePollSecondTime = persistence.pollInbox(recipientId, instanceId , 1, 10); + var messagePollSecondTime = persistence.pollInbox(recipientId+":0", instanceId , 1, 10); assertEquals(0, messagePollSecondTime.size()); } } diff --git a/code/process-models/crawl-spec/build.gradle b/code/process-models/crawl-spec/build.gradle index 148aebc1..a0045a22 100644 --- a/code/process-models/crawl-spec/build.gradle +++ b/code/process-models/crawl-spec/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation libs.bundles.slf4j implementation project(':third-party:parquet-floor') + implementation project(':code:common:config') implementation project(':code:common:db') implementation project(':code:common:linkdb') diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java b/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java index a359eaa1..ec715b97 100644 --- a/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java +++ b/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecFileNames.java @@ -1,9 +1,11 @@ package nu.marginalia.crawlspec; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.model.FileStorage; +import nu.marginalia.storage.model.FileStorageType; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; public class CrawlSpecFileNames { public static Path resolve(Path base) { @@ -17,4 +19,16 @@ public class CrawlSpecFileNames { return resolve(storage.asPath()); } + + public static List resolve(List storageList) { + List ret = new ArrayList<>(); + for (var storage : storageList) { + if (storage.type() != FileStorageType.CRAWL_SPEC) + throw new IllegalArgumentException("Provided file storage is of unexpected type " + + storage.type() + ", expected CRAWL_SPEC"); + ret.add(resolve(storage)); + } + + return ret; + } } diff --git a/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java b/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java index d67ebfda..17c4012b 100644 --- a/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java +++ b/code/process-models/crawl-spec/src/main/java/nu/marginalia/crawlspec/CrawlSpecGenerator.java @@ -84,8 +84,12 @@ public class CrawlSpecGenerator { static DomainSource fromFile(Path file) { return () -> { var lines = Files.readAllLines(file); - lines.replaceAll(s -> s.trim().toLowerCase()); - lines.removeIf(line -> line.isBlank() || line.startsWith("#")); + lines.replaceAll(s -> + s.split("#", 2)[0] + .trim() + .toLowerCase() + ); + lines.removeIf(String::isBlank); return lines; }; } diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java index 3d75d3b3..6b9201af 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java @@ -4,13 +4,14 @@ import com.google.gson.Gson; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; +import nu.marginalia.ProcessConfiguration; import nu.marginalia.ProcessConfigurationModule; import nu.marginalia.converting.model.ProcessedDomain; import nu.marginalia.converting.sideload.SideloadSource; import nu.marginalia.converting.sideload.SideloadSourceFactory; import nu.marginalia.converting.writer.ConverterBatchWriter; import nu.marginalia.converting.writer.ConverterWriter; -import nu.marginalia.db.storage.FileStorageService; +import nu.marginalia.storage.FileStorageService; import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.mq.MqMessage; import nu.marginalia.mq.inbox.MqInboxResponse; @@ -46,6 +47,8 @@ public class ConverterMain { private final FileStorageService fileStorageService; private final SideloadSourceFactory sideloadSourceFactory; + private final int node; + public static void main(String... args) throws Exception { Injector injector = Guice.createInjector( new ConverterModule(), @@ -73,7 +76,8 @@ public class ConverterMain { ProcessHeartbeatImpl heartbeat, MessageQueueFactory messageQueueFactory, FileStorageService fileStorageService, - SideloadSourceFactory sideloadSourceFactory + SideloadSourceFactory sideloadSourceFactory, + ProcessConfiguration processConfiguration ) { this.processor = processor; @@ -82,6 +86,7 @@ public class ConverterMain { this.messageQueueFactory = messageQueueFactory; this.fileStorageService = fileStorageService; this.sideloadSourceFactory = sideloadSourceFactory; + this.node = processConfiguration.node(); heartbeat.start(); } @@ -214,7 +219,7 @@ public class ConverterMain { private ConvertRequest fetchInstructions() throws Exception { - var inbox = messageQueueFactory.createSingleShotInbox(CONVERTER_INBOX, UUID.randomUUID()); + var inbox = messageQueueFactory.createSingleShotInbox(CONVERTER_INBOX, node, UUID.randomUUID()); var msgOpt = getMessage(inbox, nu.marginalia.mqapi.converting.ConvertRequest.class.getSimpleName()); var msg = msgOpt.orElseThrow(() -> new RuntimeException("No message received")); diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java index 4f88120f..cd6f5bd1 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java @@ -4,6 +4,7 @@ import com.google.gson.Gson; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; +import nu.marginalia.ProcessConfiguration; import nu.marginalia.ProcessConfigurationModule; import nu.marginalia.UserAgent; import nu.marginalia.WmsaHome; @@ -11,7 +12,7 @@ import nu.marginalia.crawl.retreival.CrawlDataReference; import nu.marginalia.crawl.retreival.fetcher.HttpFetcherImpl; import nu.marginalia.crawling.io.CrawledDomainReader; import nu.marginalia.crawlspec.CrawlSpecFileNames; -import nu.marginalia.db.storage.FileStorageService; +import nu.marginalia.storage.FileStorageService; import nu.marginalia.io.crawlspec.CrawlSpecRecordParquetFileReader; import nu.marginalia.model.crawlspec.CrawlSpecRecord; import nu.marginalia.mq.MessageQueueFactory; @@ -43,8 +44,6 @@ import static nu.marginalia.mqapi.ProcessInboxNames.CRAWLER_INBOX; public class CrawlerMain { private final Logger logger = LoggerFactory.getLogger(getClass()); - private Path crawlDataDir; - private final ProcessHeartbeatImpl heartbeat; private final ConnectionPool connectionPool = new ConnectionPool(5, 10, TimeUnit.SECONDS); @@ -55,6 +54,7 @@ public class CrawlerMain { private final MessageQueueFactory messageQueueFactory; private final FileStorageService fileStorageService; private final Gson gson; + private final int node; private final SimpleBlockingThreadPool pool; private final Map processingIds = new ConcurrentHashMap<>(); @@ -71,12 +71,14 @@ public class CrawlerMain { ProcessHeartbeatImpl heartbeat, MessageQueueFactory messageQueueFactory, FileStorageService fileStorageService, + ProcessConfiguration processConfiguration, Gson gson) { this.heartbeat = heartbeat; this.userAgent = userAgent; this.messageQueueFactory = messageQueueFactory; this.fileStorageService = fileStorageService; this.gson = gson; + this.node = processConfiguration.node(); // maybe need to set -Xss for JVM to deal with this? pool = new SimpleBlockingThreadPool("CrawlerPool", CrawlLimiter.maxPoolSize, 1); @@ -121,24 +123,30 @@ public class CrawlerMain { System.exit(0); } - public void run(Path crawlSpec, Path outputDir) throws InterruptedException, IOException { + public void run(List crawlSpec, Path outputDir) throws InterruptedException, IOException { heartbeat.start(); try (WorkLog workLog = new WorkLog(outputDir.resolve("crawler.log"))) { // First a validation run to ensure the file is all good to parse logger.info("Validating JSON"); - totalTasks = CrawlSpecRecordParquetFileReader.count(crawlSpec); + int taskCount = 0; + for (var specs : crawlSpec) { + taskCount += CrawlSpecRecordParquetFileReader.count(specs); + } + totalTasks = taskCount; - logger.info("Let's go"); + logger.info("Queued {} crawl tasks, let's go", taskCount); - try (var specStream = CrawlSpecRecordParquetFileReader.stream(crawlSpec)) { - specStream - .takeWhile((e) -> abortMonitor.isAlive()) - .filter(e -> workLog.isJobFinished(e.domain)) - .filter(e -> processingIds.put(e.domain, "") == null) - .map(e -> new CrawlTask(e, workLog)) - .forEach(pool::submitQuietly); + for (var specs : crawlSpec) { + try (var specStream = CrawlSpecRecordParquetFileReader.stream(specs)) { + specStream + .takeWhile((e) -> abortMonitor.isAlive()) + .filter(e -> !workLog.isJobFinished(e.domain)) + .filter(e -> processingIds.put(e.domain, "") == null) + .map(e -> new CrawlTask(e, outputDir, workLog)) + .forEach(pool::submitQuietly); + } } logger.info("Shutting down the pool, waiting for tasks to complete..."); @@ -160,10 +168,14 @@ public class CrawlerMain { private final String domain; private final String id; + private final Path outputDir; private final WorkLog workLog; - CrawlTask(CrawlSpecRecord specification, WorkLog workLog) { + CrawlTask(CrawlSpecRecord specification, + Path outputDir, + WorkLog workLog) { this.specification = specification; + this.outputDir = outputDir; this.workLog = workLog; this.domain = specification.domain; @@ -177,7 +189,7 @@ public class CrawlerMain { HttpFetcher fetcher = new HttpFetcherImpl(userAgent.uaString(), dispatcher, connectionPool); - try (CrawledDomainWriter writer = new CrawledDomainWriter(crawlDataDir, domain, id); + try (CrawledDomainWriter writer = new CrawledDomainWriter(outputDir, domain, id); CrawlDataReference reference = getReference()) { Thread.currentThread().setName("crawling:" + specification.domain); @@ -202,7 +214,7 @@ public class CrawlerMain { private CrawlDataReference getReference() { try { - var dataStream = reader.createDataStream(crawlDataDir, domain, id); + var dataStream = reader.createDataStream(outputDir, domain, id); return new CrawlDataReference(dataStream); } catch (IOException e) { logger.debug("Failed to read previous crawl data for {}", specification.domain); @@ -215,12 +227,12 @@ public class CrawlerMain { private static class CrawlRequest { - private final Path crawlSpec; + private final List crawlSpec; private final Path outputDir; private final MqMessage message; private final MqSingleShotInbox inbox; - CrawlRequest(Path crawlSpec, Path outputDir, MqMessage message, MqSingleShotInbox inbox) { + CrawlRequest(List crawlSpec, Path outputDir, MqMessage message, MqSingleShotInbox inbox) { this.message = message; this.inbox = inbox; this.crawlSpec = crawlSpec; @@ -239,7 +251,7 @@ public class CrawlerMain { private CrawlRequest fetchInstructions() throws Exception { - var inbox = messageQueueFactory.createSingleShotInbox(CRAWLER_INBOX, UUID.randomUUID()); + var inbox = messageQueueFactory.createSingleShotInbox(CRAWLER_INBOX, node, UUID.randomUUID()); logger.info("Waiting for instructions"); var msgOpt = getMessage(inbox, nu.marginalia.mqapi.crawling.CrawlRequest.class.getSimpleName()); diff --git a/code/processes/index-constructor-process/build.gradle b/code/processes/index-constructor-process/build.gradle index 5d8268e5..d3b81107 100644 --- a/code/processes/index-constructor-process/build.gradle +++ b/code/processes/index-constructor-process/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation project(':code:common:process') implementation project(':code:common:service') implementation project(':code:common:db') + implementation project(':code:common:config') implementation project(':code:common:model') implementation project(':code:libraries:message-queue') @@ -31,6 +32,8 @@ dependencies { implementation project(':code:features-index:index-journal') implementation project(':code:features-index:domain-ranking') + implementation project(':code:services-core:index-service') + implementation libs.bundles.slf4j implementation libs.guice implementation libs.bundles.mariadb diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java index bcf4ec1c..263b54ba 100644 --- a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java +++ b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java @@ -3,10 +3,10 @@ package nu.marginalia.index; import com.google.gson.Gson; import com.google.inject.Guice; import com.google.inject.Inject; +import nu.marginalia.IndexLocations; +import nu.marginalia.ProcessConfiguration; import nu.marginalia.ProcessConfigurationModule; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.construction.ReverseIndexConstructor; import nu.marginalia.index.forward.ForwardIndexConverter; import nu.marginalia.index.forward.ForwardIndexFileNames; @@ -43,6 +43,8 @@ public class IndexConstructorMain { private final ProcessHeartbeatImpl heartbeat; private final MessageQueueFactory messageQueueFactory; private final DomainRankings domainRankings; + private final int node; + private static final Logger logger = LoggerFactory.getLogger(IndexConstructorMain.class); private final Gson gson = GsonFactory.get(); public static void main(String[] args) throws Exception { @@ -74,12 +76,14 @@ public class IndexConstructorMain { public IndexConstructorMain(FileStorageService fileStorageService, ProcessHeartbeatImpl heartbeat, MessageQueueFactory messageQueueFactory, + ProcessConfiguration processConfiguration, DomainRankings domainRankings) { this.fileStorageService = fileStorageService; this.heartbeat = heartbeat; this.messageQueueFactory = messageQueueFactory; this.domainRankings = domainRankings; + this.node = processConfiguration.node(); } private void run(CreateIndexInstructions instructions) throws SQLException, IOException { @@ -96,33 +100,27 @@ public class IndexConstructorMain { private void createFullReverseIndex() throws SQLException, IOException { - FileStorage indexLive = fileStorageService.getStorageByType(FileStorageType.INDEX_LIVE); - FileStorage indexStaging = fileStorageService.getStorageByType(FileStorageType.INDEX_STAGING); + Path outputFileDocs = ReverseIndexFullFileNames.resolve(IndexLocations.getCurrentIndex(fileStorageService), ReverseIndexFullFileNames.FileIdentifier.DOCS, ReverseIndexFullFileNames.FileVersion.NEXT); + Path outputFileWords = ReverseIndexFullFileNames.resolve(IndexLocations.getCurrentIndex(fileStorageService), ReverseIndexFullFileNames.FileIdentifier.WORDS, ReverseIndexFullFileNames.FileVersion.NEXT); + Path workDir = IndexLocations.getIndexConstructionArea(fileStorageService); + Path tmpDir = workDir.resolve("tmp"); - Path outputFileDocs = ReverseIndexFullFileNames.resolve(indexLive.asPath(), ReverseIndexFullFileNames.FileIdentifier.DOCS, ReverseIndexFullFileNames.FileVersion.NEXT); - Path outputFileWords = ReverseIndexFullFileNames.resolve(indexLive.asPath(), ReverseIndexFullFileNames.FileIdentifier.WORDS, ReverseIndexFullFileNames.FileVersion.NEXT); - - Path tmpDir = indexStaging.asPath().resolve("tmp"); if (!Files.isDirectory(tmpDir)) Files.createDirectories(tmpDir); new ReverseIndexConstructor(outputFileDocs, outputFileWords, IndexJournalReader::singleFile, this::addRankToIdEncoding, tmpDir) - .createReverseIndex(heartbeat, indexStaging.asPath()); + .createReverseIndex(heartbeat, workDir); } private void createPrioReverseIndex() throws SQLException, IOException { - FileStorage indexLive = fileStorageService.getStorageByType(FileStorageType.INDEX_LIVE); - FileStorage indexStaging = fileStorageService.getStorageByType(FileStorageType.INDEX_STAGING); - - Path outputFileDocs = ReverseIndexPrioFileNames.resolve(indexLive.asPath(), ReverseIndexPrioFileNames.FileIdentifier.DOCS, ReverseIndexPrioFileNames.FileVersion.NEXT); - Path outputFileWords = ReverseIndexPrioFileNames.resolve(indexLive.asPath(), ReverseIndexPrioFileNames.FileIdentifier.WORDS, ReverseIndexPrioFileNames.FileVersion.NEXT); - - Path tmpDir = indexStaging.asPath().resolve("tmp"); - if (!Files.isDirectory(tmpDir)) Files.createDirectories(tmpDir); + Path outputFileDocs = ReverseIndexFullFileNames.resolve(IndexLocations.getCurrentIndex(fileStorageService), ReverseIndexFullFileNames.FileIdentifier.DOCS, ReverseIndexFullFileNames.FileVersion.NEXT); + Path outputFileWords = ReverseIndexFullFileNames.resolve(IndexLocations.getCurrentIndex(fileStorageService), ReverseIndexFullFileNames.FileIdentifier.WORDS, ReverseIndexFullFileNames.FileVersion.NEXT); + Path workDir = IndexLocations.getIndexConstructionArea(fileStorageService); + Path tmpDir = workDir.resolve("tmp"); // The priority index only includes words that have bits indicating they are // important to the document. This filter will act on the encoded {@see WordMetadata} @@ -131,7 +129,7 @@ public class IndexConstructorMain { new ReverseIndexConstructor(outputFileDocs, outputFileWords, (path) -> IndexJournalReader.singleFile(path).filtering(wordMetaFilter), this::addRankToIdEncoding, tmpDir) - .createReverseIndex(heartbeat, indexStaging.asPath()); + .createReverseIndex(heartbeat, workDir); } private static LongPredicate getPriorityIndexWordMetaFilter() { @@ -149,16 +147,14 @@ public class IndexConstructorMain { return r -> WordMetadata.hasAnyFlags(r, highPriorityFlags); } - private void createForwardIndex() throws SQLException, IOException { + private void createForwardIndex() throws IOException { - FileStorage indexLive = fileStorageService.getStorageByType(FileStorageType.INDEX_LIVE); - FileStorage indexStaging = fileStorageService.getStorageByType(FileStorageType.INDEX_STAGING); - - Path outputFileDocsId = ForwardIndexFileNames.resolve(indexLive.asPath(), ForwardIndexFileNames.FileIdentifier.DOC_ID, ForwardIndexFileNames.FileVersion.NEXT); - Path outputFileDocsData = ForwardIndexFileNames.resolve(indexLive.asPath(), ForwardIndexFileNames.FileIdentifier.DOC_DATA, ForwardIndexFileNames.FileVersion.NEXT); + Path workDir = IndexLocations.getIndexConstructionArea(fileStorageService); + Path outputFileDocsId = ForwardIndexFileNames.resolve(IndexLocations.getCurrentIndex(fileStorageService), ForwardIndexFileNames.FileIdentifier.DOC_ID, ForwardIndexFileNames.FileVersion.NEXT); + Path outputFileDocsData = ForwardIndexFileNames.resolve(IndexLocations.getCurrentIndex(fileStorageService), ForwardIndexFileNames.FileIdentifier.DOC_DATA, ForwardIndexFileNames.FileVersion.NEXT); ForwardIndexConverter converter = new ForwardIndexConverter(heartbeat, - IndexJournalReader.paging(indexStaging.asPath()), + IndexJournalReader.paging(workDir), outputFileDocsId, outputFileDocsData, domainRankings @@ -198,7 +194,7 @@ public class IndexConstructorMain { private CreateIndexInstructions fetchInstructions() throws Exception { - var inbox = messageQueueFactory.createSingleShotInbox(INDEX_CONSTRUCTOR_INBOX, UUID.randomUUID()); + var inbox = messageQueueFactory.createSingleShotInbox(INDEX_CONSTRUCTOR_INBOX, node, UUID.randomUUID()); logger.info("Waiting for instructions"); var msgOpt = getMessage(inbox, CreateIndexRequest.class.getSimpleName()); diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderIndexJournalWriter.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderIndexJournalWriter.java index c3c9d6f9..2f2ebe48 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderIndexJournalWriter.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderIndexJournalWriter.java @@ -3,8 +3,8 @@ package nu.marginalia.loading; import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.IndexLocations; +import nu.marginalia.storage.FileStorageService; import nu.marginalia.hash.MurmurHash3_128; import nu.marginalia.index.journal.model.IndexJournalEntryData; import nu.marginalia.index.journal.model.IndexJournalEntryHeader; @@ -34,14 +34,14 @@ public class LoaderIndexJournalWriter { @Inject public LoaderIndexJournalWriter(FileStorageService fileStorageService) throws IOException, SQLException { - var indexArea = fileStorageService.getStorageByType(FileStorageType.INDEX_STAGING); + var indexArea = IndexLocations.getIndexConstructionArea(fileStorageService); - var existingIndexFiles = IndexJournalFileNames.findJournalFiles(indexArea.asPath()); + var existingIndexFiles = IndexJournalFileNames.findJournalFiles(indexArea); for (var existingFile : existingIndexFiles) { Files.delete(existingFile); } - indexWriter = new IndexJournalWriterPagingImpl(indexArea.asPath()); + indexWriter = new IndexJournalWriterPagingImpl(indexArea); } public void putWords(long combinedId, diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java index df8653e4..6ca8e316 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java @@ -6,8 +6,9 @@ import com.google.inject.Inject; import com.google.inject.Injector; import lombok.Getter; import lombok.SneakyThrows; +import nu.marginalia.ProcessConfiguration; import nu.marginalia.ProcessConfigurationModule; -import nu.marginalia.db.storage.FileStorageService; +import nu.marginalia.storage.FileStorageService; import nu.marginalia.linkdb.LinkdbWriter; import nu.marginalia.loading.documents.DocumentLoaderService; import nu.marginalia.loading.documents.KeywordLoaderService; @@ -16,11 +17,10 @@ import nu.marginalia.loading.domains.DomainLoaderService; import nu.marginalia.loading.links.DomainLinksLoaderService; import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.mq.MqMessage; +import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.inbox.MqInboxResponse; import nu.marginalia.mq.inbox.MqSingleShotInbox; import nu.marginalia.process.control.ProcessHeartbeatImpl; -import nu.marginalia.worklog.BatchingWorkLogInspector; -import plan.CrawlPlan; import nu.marginalia.service.module.DatabaseModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +49,7 @@ public class LoaderMain { private final DomainLinksLoaderService linksService; private final KeywordLoaderService keywordLoaderService; private final DocumentLoaderService documentLoaderService; + private final int node; private final Gson gson; public static void main(String... args) throws Exception { @@ -81,9 +82,10 @@ public class LoaderMain { DomainLinksLoaderService linksService, KeywordLoaderService keywordLoaderService, DocumentLoaderService documentLoaderService, + ProcessConfiguration processConfiguration, Gson gson ) { - + this.node = processConfiguration.node(); this.heartbeat = heartbeat; this.messageQueueFactory = messageQueueFactory; this.fileStorageService = fileStorageService; @@ -157,7 +159,7 @@ public class LoaderMain { private LoadRequest fetchInstructions() throws Exception { - var inbox = messageQueueFactory.createSingleShotInbox(LOADER_INBOX, UUID.randomUUID()); + var inbox = messageQueueFactory.createSingleShotInbox(LOADER_INBOX, node, UUID.randomUUID()); var msgOpt = getMessage(inbox, nu.marginalia.mqapi.loading.LoadRequest.class.getSimpleName()); if (msgOpt.isEmpty()) @@ -168,14 +170,20 @@ public class LoaderMain { throw new RuntimeException("Unexpected message in inbox: " + msg); } - var request = gson.fromJson(msg.payload(), nu.marginalia.mqapi.loading.LoadRequest.class); + try { + var request = gson.fromJson(msg.payload(), nu.marginalia.mqapi.loading.LoadRequest.class); - List inputSources = new ArrayList<>(); - for (var storageId : request.inputProcessDataStorageIds) { - inputSources.add(fileStorageService.getStorage(storageId).asPath()); + List inputSources = new ArrayList<>(); + for (var storageId : request.inputProcessDataStorageIds) { + inputSources.add(fileStorageService.getStorage(storageId).asPath()); + } + + return new LoadRequest(new LoaderInputData(inputSources), msg, inbox); + } + catch (Exception ex) { + inbox.sendResponse(msg, new MqInboxResponse("FAILED", MqMessageState.ERR)); + throw ex; } - - return new LoadRequest(new LoaderInputData(inputSources), msg, inbox); } private Optional getMessage(MqSingleShotInbox inbox, String expectedFunction) throws SQLException, InterruptedException { diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java index 0700b73a..954b5b60 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderModule.java @@ -7,10 +7,9 @@ import com.google.inject.Provides; import com.google.inject.Singleton; import com.google.inject.name.Names; import nu.marginalia.LanguageModels; -import nu.marginalia.ProcessConfiguration; import nu.marginalia.WmsaHome; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.IndexLocations; +import nu.marginalia.storage.FileStorageService; import nu.marginalia.linkdb.LinkdbStatusWriter; import nu.marginalia.linkdb.LinkdbWriter; import nu.marginalia.model.gson.GsonFactory; @@ -21,7 +20,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; -import java.util.UUID; public class LoaderModule extends AbstractModule { @@ -38,8 +36,8 @@ public class LoaderModule extends AbstractModule { @Inject @Provides @Singleton private LinkdbWriter createLinkdbWriter(FileStorageService service) throws SQLException, IOException { - var storage = service.getStorageByType(FileStorageType.LINKDB_STAGING); - Path dbPath = storage.asPath().resolve("links.db"); + + Path dbPath = IndexLocations.getLinkdbWritePath(service).resolve("links.db"); if (Files.exists(dbPath)) { Files.delete(dbPath); @@ -49,8 +47,7 @@ public class LoaderModule extends AbstractModule { @Inject @Provides @Singleton private LinkdbStatusWriter createLinkdbStatusWriter(FileStorageService service) throws SQLException, IOException { - var storage = service.getStorageByType(FileStorageType.LINKDB_STAGING); - Path dbPath = storage.asPath().resolve("urlstatus.db"); + Path dbPath = IndexLocations.getLinkdbWritePath(service).resolve("urlstatus.db"); if (Files.exists(dbPath)) { Files.delete(dbPath); diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java b/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java index 472e692d..14f188ee 100644 --- a/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java +++ b/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java @@ -1,8 +1,8 @@ package nu.marginalia.loading.loader; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.FileStorage; -import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageBase; +import nu.marginalia.storage.model.FileStorageBaseType; import nu.marginalia.index.journal.reader.IndexJournalReaderSingleFile; import nu.marginalia.keyword.model.DocumentKeywords; import nu.marginalia.loading.LoaderIndexJournalWriter; @@ -31,18 +31,19 @@ class LoaderIndexJournalWriterTest { public void setUp() throws IOException, SQLException { tempDir = Files.createTempDirectory(getClass().getSimpleName()); FileStorageService storageService = Mockito.mock(FileStorageService.class); - Mockito.when(storageService.getStorageByType(FileStorageType.INDEX_STAGING)). - thenReturn(new FileStorage(null, null, null, null, tempDir.toString(), - "test")); + + Mockito.when(storageService.getStorageBase(FileStorageBaseType.CURRENT)).thenReturn(new FileStorageBase(null, null, null, tempDir.toString())); + writer = new LoaderIndexJournalWriter(storageService); } @AfterEach public void tearDown() throws Exception { writer.close(); - List junk = Files.list(tempDir).toList(); + List junk = Files.list(tempDir.resolve("iw")).toList(); for (var item : junk) Files.delete(item); + Files.delete(tempDir.resolve("iw")); Files.delete(tempDir); } @@ -60,7 +61,7 @@ class LoaderIndexJournalWriterTest { writer.close(); - List journalFiles =IndexJournalFileNames.findJournalFiles(tempDir); + List journalFiles = IndexJournalFileNames.findJournalFiles(tempDir.resolve("iw")); assertEquals(1, journalFiles.size()); var reader = new IndexJournalReaderSingleFile(journalFiles.get(0)); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java index af3735e1..3069ac2a 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java @@ -5,11 +5,9 @@ import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; import nu.marginalia.client.Context; -import nu.marginalia.db.storage.FileStorageService; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.search.svc.SearchFrontPageService; import nu.marginalia.search.svc.*; -import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index c1e18501..6eb878b9 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -34,10 +34,12 @@ dependencies { implementation project(':code:common:service-client') implementation project(':code:api:index-api') implementation project(':code:api:query-api') + implementation project(':code:api:executor-api') implementation project(':code:api:process-mqapi') implementation project(':code:features-search:screenshots') implementation project(':code:features-index:index-journal') implementation project(':code:features-index:index-query') + implementation project(':code:process-models:crawl-spec') implementation libs.bundles.slf4j diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java index 2bf111aa..02f6b142 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlProcessModule.java @@ -8,8 +8,5 @@ import java.nio.file.Path; public class ControlProcessModule extends AbstractModule { @Override - protected void configure() { - String dist = System.getProperty("distPath", System.getProperty("WMSA_HOME", "/var/lib/wmsa") + "/dist/current"); - bind(Path.class).annotatedWith(Names.named("distPath")).toInstance(Path.of(dist)); - } + protected void configure() {} } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index dcd7496b..3623a873 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -2,19 +2,20 @@ package nu.marginalia.control; import com.google.gson.Gson; import com.google.inject.Inject; -import gnu.trove.list.array.TIntArrayList; import nu.marginalia.client.ServiceMonitors; -import nu.marginalia.control.actor.Actor; -import nu.marginalia.control.model.*; -import nu.marginalia.control.svc.*; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.db.storage.model.FileStorageType; -import nu.marginalia.model.EdgeDomain; +import nu.marginalia.control.app.svc.*; +import nu.marginalia.control.node.svc.ControlNodeActionsService; +import nu.marginalia.control.node.svc.ControlActorService; +import nu.marginalia.control.node.svc.ControlFileStorageService; +import nu.marginalia.control.node.svc.ControlNodeService; +import nu.marginalia.control.sys.svc.ControlSysActionsService; +import nu.marginalia.control.sys.svc.EventLogService; +import nu.marginalia.control.sys.svc.HeartbeatService; +import nu.marginalia.control.sys.svc.MessageQueueService; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.renderer.RendererFactory; import nu.marginalia.screenshot.ScreenshotService; import nu.marginalia.service.server.*; -import org.eclipse.jetty.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -22,9 +23,7 @@ import spark.Response; import spark.Spark; import java.io.IOException; -import java.sql.SQLException; import java.util.*; -import java.util.stream.Collectors; public class ControlService extends Service { @@ -34,15 +33,10 @@ public class ControlService extends Service { private final ServiceMonitors monitors; private final HeartbeatService heartbeatService; private final EventLogService eventLogService; - private final ApiKeyService apiKeyService; - private final DomainComplaintService domainComplaintService; - private final ControlBlacklistService blacklistService; - private final SearchToBanService searchToBanService; - private final RandomExplorationService randomExplorationService; + private final ControlNodeService controlNodeService; private final ControlActorService controlActorService; private final StaticResources staticResources; private final MessageQueueService messageQueueService; - private final ControlFileStorageService controlFileStorageService; @Inject @@ -58,54 +52,48 @@ public class ControlService extends Service { ApiKeyService apiKeyService, DomainComplaintService domainComplaintService, ControlBlacklistService blacklistService, - ControlActionsService controlActionsService, + ControlNodeActionsService nodeActionsService, + ControlSysActionsService sysActionsService, ScreenshotService screenshotService, SearchToBanService searchToBanService, - RandomExplorationService randomExplorationService + RandomExplorationService randomExplorationService, + ControlNodeService controlNodeService ) throws IOException { super(params); this.monitors = monitors; this.heartbeatService = heartbeatService; this.eventLogService = eventLogService; - this.apiKeyService = apiKeyService; - this.domainComplaintService = domainComplaintService; - this.blacklistService = blacklistService; - this.searchToBanService = searchToBanService; - this.randomExplorationService = randomExplorationService; + this.controlNodeService = controlNodeService; + + // sys + messageQueueService.register(); + sysActionsService.register(); + + // node + controlFileStorageService.register(); + controlActorService.register(); + nodeActionsService.register(); + controlNodeService.register(); + + // app + blacklistService.register(); + searchToBanService.register(); + apiKeyService.register(); + domainComplaintService.register(); + randomExplorationService.register(); var indexRenderer = rendererFactory.renderer("control/index"); - var eventsRenderer = rendererFactory.renderer("control/events"); - var servicesRenderer = rendererFactory.renderer("control/services"); - var serviceByIdRenderer = rendererFactory.renderer("control/service-by-id"); - var actorsRenderer = rendererFactory.renderer("control/actors"); - var actorDetailsRenderer = rendererFactory.renderer("control/actor-details"); - var storageRenderer = rendererFactory.renderer("control/storage-overview"); - var storageSpecsRenderer = rendererFactory.renderer("control/storage-specs"); - var storageCrawlsRenderer = rendererFactory.renderer("control/storage-crawls"); - var storageBackupsRenderer = rendererFactory.renderer("control/storage-backups"); - var storageProcessedRenderer = rendererFactory.renderer("control/storage-processed"); - var reviewRandomDomainsRenderer = rendererFactory.renderer("control/review-random-domains"); - - var apiKeysRenderer = rendererFactory.renderer("control/api-keys"); - var domainComplaintsRenderer = rendererFactory.renderer("control/domain-complaints"); - - var messageQueueRenderer = rendererFactory.renderer("control/message-queue"); - - var storageDetailsRenderer = rendererFactory.renderer("control/storage-details"); - var updateMessageStateRenderer = rendererFactory.renderer("control/update-message-state"); - var newMessageRenderer = rendererFactory.renderer("control/new-message"); - var viewMessageRenderer = rendererFactory.renderer("control/view-message"); + var eventsRenderer = rendererFactory.renderer("control/sys/events"); + var servicesRenderer = rendererFactory.renderer("control/sys/services"); + var serviceByIdRenderer = rendererFactory.renderer("control/sys/service-by-id"); var actionsViewRenderer = rendererFactory.renderer("control/actions"); - var blacklistRenderer = rendererFactory.renderer("control/blacklist"); - var searchToBanRenderer = rendererFactory.renderer("control/search-to-ban"); this.controlActorService = controlActorService; this.staticResources = staticResources; this.messageQueueService = messageQueueService; - this.controlFileStorageService = controlFileStorageService; Spark.get("/public/heartbeats", (req, res) -> { res.type("application/json"); @@ -114,224 +102,30 @@ public class ControlService extends Service { Spark.get("/public/", this::overviewModel, indexRenderer::render); - Spark.get("/public/actions", (rq,rsp) -> new Object() , actionsViewRenderer::render); + Spark.get("/public/actions", (req,rs) -> new Object() , actionsViewRenderer::render); Spark.get("/public/events", eventLogService::eventsListModel , eventsRenderer::render); Spark.get("/public/services", this::servicesModel, servicesRenderer::render); Spark.get("/public/services/:id", this::serviceModel, serviceByIdRenderer::render); - Spark.get("/public/actors", this::processesModel, actorsRenderer::render); - Spark.get("/public/actors/:fsm", this::actorDetailsModel, actorDetailsRenderer::render); - - final HtmlRedirect redirectToServices = new HtmlRedirect("/services"); - final HtmlRedirect redirectToActors = new HtmlRedirect("/actors"); - final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys"); - final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage"); - final HtmlRedirect redirectToBlacklist = new HtmlRedirect("/blacklist"); - final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints"); - final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue"); // Needed to be able to show website screenshots Spark.get("/public/screenshot/:id", screenshotService::serveScreenshotRequest); - // FSMs - - Spark.post("/public/fsms/:fsm/start", controlActorService::startFsm, redirectToActors); - Spark.post("/public/fsms/:fsm/stop", controlActorService::stopFsm, redirectToActors); - - // Message Queue - - Spark.get("/public/message-queue", messageQueueService::listMessageQueueModel, messageQueueRenderer::render); - Spark.post("/public/message-queue/", messageQueueService::createMessage, redirectToMessageQueue); - Spark.get("/public/message-queue/new", messageQueueService::newMessageModel, newMessageRenderer::render); - Spark.get("/public/message-queue/:id", messageQueueService::viewMessageModel, viewMessageRenderer::render); - Spark.get("/public/message-queue/:id/reply", messageQueueService::replyMessageModel, newMessageRenderer::render); - Spark.get("/public/message-queue/:id/edit", messageQueueService::viewMessageForEditStateModel, updateMessageStateRenderer::render); - Spark.post("/public/message-queue/:id/edit", messageQueueService::editMessageState, redirectToMessageQueue); - - // Storage - Spark.get("/public/storage", this::storageModel, storageRenderer::render); - Spark.get("/public/storage/specs", this::storageModelSpecs, storageSpecsRenderer::render); - Spark.get("/public/storage/crawls", this::storageModelCrawls, storageCrawlsRenderer::render); - Spark.get("/public/storage/backups", this::storageModelBackups, storageBackupsRenderer::render); - Spark.get("/public/storage/processed", this::storageModelProcessed, storageProcessedRenderer::render); - Spark.get("/public/storage/:id", this::storageDetailsModel, storageDetailsRenderer::render); - Spark.get("/public/storage/:id/file", controlFileStorageService::downloadFileFromStorage); - - // Storage Actions - - Spark.post("/public/storage/:fid/crawl", controlActorService::triggerCrawling, redirectToActors); - Spark.post("/public/storage/:fid/recrawl", controlActorService::triggerRecrawling, redirectToActors); - Spark.post("/public/storage/:fid/process", controlActorService::triggerProcessing, redirectToActors); - Spark.post("/public/storage/:fid/process-and-load", controlActorService::triggerProcessingWithLoad, redirectToActors); - Spark.post("/public/storage/:fid/load", controlActorService::loadProcessedData, redirectToActors); - Spark.post("/public/storage/:fid/restore-backup", controlActorService::restoreBackup, redirectToActors); - - Spark.post("/public/storage/specs", controlActorService::createCrawlSpecification, redirectToStorage); - Spark.post("/public/storage/:fid/delete", controlFileStorageService::flagFileForDeletionRequest, redirectToStorage); - - // Blacklist - - Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render); - Spark.post("/public/blacklist", this::updateBlacklist, redirectToBlacklist); - - Spark.get("/public/search-to-ban", searchToBanService::handle, searchToBanRenderer::render); - Spark.post("/public/search-to-ban", searchToBanService::handle, searchToBanRenderer::render); - - // API Keys - - Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render); - Spark.post("/public/api-keys", this::createApiKey, redirectToApiKeys); - Spark.delete("/public/api-keys/:key", this::deleteApiKey, redirectToApiKeys); - // HTML forms don't support the DELETE verb :-( - Spark.post("/public/api-keys/:key/delete", this::deleteApiKey, redirectToApiKeys); - - Spark.get("/public/complaints", this::complaintsModel, domainComplaintsRenderer::render); - Spark.post("/public/complaints/:domain", this::reviewComplaint, redirectToComplaints); - - // Actions - - Spark.post("/public/actions/calculate-adjacencies", controlActionsService::calculateAdjacencies, redirectToActors); - Spark.post("/public/actions/reload-blogs-list", controlActionsService::reloadBlogsList, redirectToActors); - Spark.post("/public/actions/repartition-index", controlActionsService::triggerRepartition, redirectToActors); - Spark.post("/public/actions/trigger-data-exports", controlActionsService::triggerDataExports, redirectToActors); - Spark.post("/public/actions/flush-api-caches", controlActionsService::flushApiCaches, redirectToActors); - Spark.post("/public/actions/truncate-links-database", controlActionsService::truncateLinkDatabase, redirectToActors); - Spark.post("/public/actions/sideload-encyclopedia", controlActionsService::sideloadEncyclopedia, redirectToActors); - Spark.post("/public/actions/sideload-dirtree", controlActionsService::sideloadDirtree, redirectToActors); - Spark.post("/public/actions/sideload-stackexchange", controlActionsService::sideloadStackexchange, redirectToActors); - - // Review Random Domains - Spark.get("/public/review-random-domains", this::reviewRandomDomainsModel, reviewRandomDomainsRenderer::render); - - Spark.post("/public/review-random-domains", this::reviewRandomDomainsAction); - - Spark.get("/public/:resource", this::serveStatic); monitors.subscribe(this::logMonitorStateChange); } - private Object reviewRandomDomainsModel(Request request, Response response) throws SQLException { - String afterVal = Objects.requireNonNullElse(request.queryParams("after"), "0"); - int after = Integer.parseInt(afterVal); - var domains = randomExplorationService.getDomains(after, 25); - int nextAfter = domains.stream().mapToInt(RandomExplorationService.RandomDomainResult::id).max().orElse(Integer.MAX_VALUE); - - return Map.of("domains", domains, - "after", nextAfter); - - } - - private Object reviewRandomDomainsAction(Request request, Response response) throws SQLException { - TIntArrayList idList = new TIntArrayList(); - - request.queryParams().forEach(key -> { - if (key.startsWith("domain-")) { - String value = request.queryParams(key); - if ("on".equalsIgnoreCase(value)) { - int id = Integer.parseInt(key.substring(7)); - idList.add(id); - } - } - }); - - randomExplorationService.removeRandomDomains(idList.toArray()); - - String after = request.queryParams("after"); - - return """ - - - """.formatted(after); - } - - - private Object blacklistModel(Request request, Response response) { - return Map.of("blacklist", blacklistService.lastNAdditions(100)); - } - - private Object updateBlacklist(Request request, Response response) { - var domain = new EdgeDomain(request.queryParams("domain")); - if ("add".equals(request.queryParams("act"))) { - var comment = Objects.requireNonNullElse(request.queryParams("comment"), ""); - blacklistService.addToBlacklist(domain, comment); - } else if ("del".equals(request.queryParams("act"))) { - blacklistService.removeFromBlacklist(domain); - } - - return ""; - } - private Object overviewModel(Request request, Response response) { return Map.of("processes", heartbeatService.getProcessHeartbeats(), + "nodes", controlNodeService.getNodeStatusList(), "jobs", heartbeatService.getTaskHeartbeats(), - "actors", controlActorService.getActorStates(), "services", heartbeatService.getServiceHeartbeats(), "events", eventLogService.getLastEntries(Long.MAX_VALUE, 20) ); } - private Object complaintsModel(Request request, Response response) { - Map> complaintsByReviewed = - domainComplaintService.getComplaints().stream().collect(Collectors.partitioningBy(DomainComplaintModel::reviewed)); - - var reviewed = complaintsByReviewed.get(true); - var unreviewed = complaintsByReviewed.get(false); - - reviewed.sort(Comparator.comparing(DomainComplaintModel::reviewDate).reversed()); - unreviewed.sort(Comparator.comparing(DomainComplaintModel::fileDate).reversed()); - - return Map.of("complaintsNew", unreviewed, "complaintsReviewed", reviewed); - } - - private Object reviewComplaint(Request request, Response response) { - var domain = new EdgeDomain(request.params("domain")); - String action = request.queryParams("action"); - - logger.info("Reviewing complaint for domain {} with action {}", domain, action); - - switch (action) { - case "noop" -> domainComplaintService.reviewNoAction(domain); - case "appeal" -> domainComplaintService.approveAppealBlacklisting(domain); - case "blacklist" -> domainComplaintService.blacklistDomain(domain); - default -> throw new UnsupportedOperationException(); - } - - return ""; - } - - private Object createApiKey(Request request, Response response) { - String license = request.queryParams("license"); - String name = request.queryParams("name"); - String email = request.queryParams("email"); - int rate = Integer.parseInt(request.queryParams("rate")); - - if (StringUtil.isBlank(license) || - StringUtil.isBlank(name) || - StringUtil.isBlank(email) || - rate <= 0) - { - response.status(400); - return ""; - } - - apiKeyService.addApiKey(license, name, email, rate); - - return ""; - } - - private Object deleteApiKey(Request request, Response response) { - String licenseKey = request.params("key"); - apiKeyService.deleteApiKey(licenseKey); - return ""; - } - - private Object apiKeysModel(Request request, Response response) { - return Map.of("apikeys", apiKeyService.getApiKeys()); - } - - @Override public void logRequest(Request request) { if ("GET".equals(request.requestMethod())) @@ -358,25 +152,6 @@ public class ControlService extends Service { "events", eventLogService.getLastEntriesForService(serviceName, Long.MAX_VALUE, 20)); } - private Object storageModel(Request request, Response response) { - return Map.of("storage", controlFileStorageService.getStorageList()); - } - - private Object storageDetailsModel(Request request, Response response) throws SQLException { - return Map.of("storage", controlFileStorageService.getFileStorageWithRelatedEntries(FileStorageId.parse(request.params("id")))); - } - private Object storageModelSpecs(Request request, Response response) { - return Map.of("storage", controlFileStorageService.getStorageList(FileStorageType.CRAWL_SPEC)); - } - private Object storageModelCrawls(Request request, Response response) { - return Map.of("storage", controlFileStorageService.getStorageList(FileStorageType.CRAWL_DATA)); - } - private Object storageModelBackups(Request request, Response response) { - return Map.of("storage", controlFileStorageService.getStorageList(FileStorageType.BACKUP)); - } - private Object storageModelProcessed(Request request, Response response) { - return Map.of("storage", controlFileStorageService.getStorageList(FileStorageType.PROCESSED_DATA)); - } private Object servicesModel(Request request, Response response) { return Map.of("services", heartbeatService.getServiceHeartbeats(), "events", eventLogService.getLastEntries(Long.MAX_VALUE, 20)); @@ -388,18 +163,20 @@ public class ControlService extends Service { return Map.of("processes", processes, "jobs", jobs, - "actors", controlActorService.getActorStates(), + "actors", controlActorService.getActorStates(request), "messages", messageQueueService.getLastEntries(20)); } - private Object actorDetailsModel(Request request, Response response) { - final Actor actor = Actor.valueOf(request.params("fsm").toUpperCase()); - final String inbox = actor.id(); - return Map.of( - "actor", actor, - "state-graph", controlActorService.getActorStateGraph(actor), - "messages", messageQueueService.getLastEntriesForInbox(inbox, 20)); - } +// private Object actorDetailsModel(Request request, Response response) { +// final Actor actor = Actor.valueOf(request.params("fsm").toUpperCase()); +// final String inbox = actor.id(); +// +// return Map.of( +// "actor", actor, +// "state-graph", controlActorService.getActorStateGraph(actor), +// "messages", messageQueueService.getLastEntriesForInbox(inbox, 20)); +// } + private Object serveStatic(Request request, Response response) { String resource = request.params("resource"); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/HtmlRedirect.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/HtmlRedirect.java deleted file mode 100644 index ff1e2368..00000000 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/HtmlRedirect.java +++ /dev/null @@ -1,22 +0,0 @@ -package nu.marginalia.control; - -import spark.ResponseTransformer; - -public class HtmlRedirect implements ResponseTransformer { - private final String html; - - /** Because Spark doesn't have a redirect method that works with relative URLs - * (without explicitly providing the external address),we use HTML and let the - * browser resolve the relative redirect instead */ - public HtmlRedirect(String destination) { - this.html = """ - - - """.formatted(destination); - } - - @Override - public String render(Object any) throws Exception { - return html; - } -} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/Redirects.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/Redirects.java new file mode 100644 index 00000000..864edc1a --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/Redirects.java @@ -0,0 +1,32 @@ +package nu.marginalia.control; + +import spark.ResponseTransformer; + +public class Redirects { + public static final HtmlRedirect redirectToServices = new HtmlRedirect("/services"); + public static final HtmlRedirect redirectToActors = new HtmlRedirect("/actors"); + public static final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys"); + public static final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage"); + public static final HtmlRedirect redirectToBlacklist = new HtmlRedirect("/blacklist"); + public static final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints"); + public static final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue"); + + public static class HtmlRedirect implements ResponseTransformer { + private final String html; + + /** Because Spark doesn't have a redirect method that works with relative URLs + * (without explicitly providing the external address),we use HTML and let the + * browser resolve the relative redirect instead */ + public HtmlRedirect(String destination) { + this.html = """ + + + """.formatted(destination); + } + + @Override + public String render(Object any) throws Exception { + return html; + } + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java deleted file mode 100644 index ca3009cc..00000000 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java +++ /dev/null @@ -1,84 +0,0 @@ -package nu.marginalia.control.actor.monitor; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.actor.ActorStateFactory; -import nu.marginalia.control.model.ServiceHeartbeat; -import nu.marginalia.control.svc.HeartbeatService; -import nu.marginalia.control.process.ProcessService; -import nu.marginalia.actor.prototype.AbstractActorPrototype; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorResumeBehavior; - -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -@Singleton -public class ProcessLivenessMonitorActor extends AbstractActorPrototype { - - // STATES - - private static final String INITIAL = "INITIAL"; - private static final String MONITOR = "MONITOR"; - private static final String END = "END"; - private final ProcessService processService; - private final HeartbeatService heartbeatService; - - - @Inject - public ProcessLivenessMonitorActor(ActorStateFactory stateFactory, - ProcessService processService, - HeartbeatService heartbeatService) { - super(stateFactory); - this.processService = processService; - this.heartbeatService = heartbeatService; - } - - @Override - public String describe() { - return "Periodically check to ensure that the control service's view of running processes is agreement with the process heartbeats table."; - } - - @ActorState(name = INITIAL, next = MONITOR) - public void init() { - } - - @ActorState(name = MONITOR, next = MONITOR, resume = ActorResumeBehavior.RETRY, description = """ - Periodically check to ensure that the control service's view of - running processes is agreement with the process heartbeats table. - - If the process is not running, mark the process as stopped in the table. - """) - public void monitor() throws Exception { - - for (;;) { - for (var heartbeat : heartbeatService.getProcessHeartbeats()) { - if (!heartbeat.isRunning()) { - continue; - } - - var processId = heartbeat.getProcessId(); - if (null == processId) - continue; - - if (processService.isRunning(processId) && heartbeat.lastSeenMillis() < 10_000) { - continue; - } - - heartbeatService.flagProcessAsStopped(heartbeat); - } - - for (var heartbeat : heartbeatService.getTaskHeartbeats()) { - if (heartbeat.lastSeenMillis() < 10_000) { - continue; - } - - heartbeatService.removeTaskHeartbeat(heartbeat); - } - - - TimeUnit.SECONDS.sleep(60); - } - } - -} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ApiKeyModel.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/ApiKeyModel.java similarity index 71% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/ApiKeyModel.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/ApiKeyModel.java index 15eda2ba..7e2168de 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ApiKeyModel.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/ApiKeyModel.java @@ -1,2 +1,2 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.app.model; public record ApiKeyModel(String licenseKey, String license, String name, String email, int rate) {} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java similarity index 74% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java index e7db4805..ae409cd6 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/BlacklistedDomainModel.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.app.model; import nu.marginalia.model.EdgeDomain; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintCategory.java similarity index 94% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintCategory.java index d1743ba9..c575f845 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintCategory.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.app.model; public enum DomainComplaintCategory { SPAM("spam"), diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintModel.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintModel.java index 603b6fc8..db3fc400 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/model/DomainComplaintModel.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.app.model; public record DomainComplaintModel(String domain, DomainComplaintCategory category, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ApiKeyService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java similarity index 60% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ApiKeyService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java index 505d2220..b719084d 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ApiKeyService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java @@ -1,23 +1,79 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.app.svc; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.control.model.ApiKeyModel; +import nu.marginalia.control.Redirects; +import nu.marginalia.control.app.model.ApiKeyModel; +import nu.marginalia.renderer.RendererFactory; +import org.eclipse.jetty.util.StringUtil; +import spark.Request; +import spark.Response; +import spark.Spark; +import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; public class ApiKeyService { private final HikariDataSource dataSource; + private final RendererFactory rendererFactory; @Inject - public ApiKeyService(HikariDataSource dataSource) { + public ApiKeyService(HikariDataSource dataSource, + RendererFactory rendererFactory + ) { this.dataSource = dataSource; + this.rendererFactory = rendererFactory; } + public void register() throws IOException { + + var apiKeysRenderer = rendererFactory.renderer("control/app/api-keys"); + + Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render); + Spark.post("/public/api-keys", this::createApiKey, Redirects.redirectToApiKeys); + Spark.delete("/public/api-keys/:key", this::deleteApiKey, Redirects.redirectToApiKeys); + // HTML forms don't support the DELETE verb :-( + Spark.post("/public/api-keys/:key/delete", this::deleteApiKey, Redirects.redirectToApiKeys); + + } + + private Object createApiKey(Request request, Response response) { + String license = request.queryParams("license"); + String name = request.queryParams("name"); + String email = request.queryParams("email"); + int rate = Integer.parseInt(request.queryParams("rate")); + + if (StringUtil.isBlank(license) || + StringUtil.isBlank(name) || + StringUtil.isBlank(email) || + rate <= 0) + { + response.status(400); + return ""; + } + + addApiKey(license, name, email, rate); + + return ""; + } + + private Object deleteApiKey(Request request, Response response) { + String licenseKey = request.params("key"); + deleteApiKey(licenseKey); + return ""; + } + + private Object apiKeysModel(Request request, Response response) { + return Map.of("apikeys", getApiKeys()); + } + + + public List getApiKeys() { try (var conn = dataSource.getConnection()) { try (var stmt = conn.prepareStatement(""" diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java similarity index 63% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java index 5d692f52..9181f325 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java @@ -1,24 +1,59 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.app.svc; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.control.model.BlacklistedDomainModel; +import nu.marginalia.control.Redirects; +import nu.marginalia.control.app.model.BlacklistedDomainModel; import nu.marginalia.model.EdgeDomain; +import nu.marginalia.renderer.RendererFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; +import spark.Spark; +import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; public class ControlBlacklistService { private final HikariDataSource dataSource; + private final RendererFactory rendererFactory; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject - public ControlBlacklistService(HikariDataSource dataSource) { + public ControlBlacklistService(HikariDataSource dataSource, + RendererFactory rendererFactory) { this.dataSource = dataSource; + this.rendererFactory = rendererFactory; + } + + + public void register() throws IOException { + var blacklistRenderer = rendererFactory.renderer("control/app/blacklist"); + + Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render); + Spark.post("/public/blacklist", this::updateBlacklist, Redirects.redirectToBlacklist); + } + + private Object blacklistModel(Request request, Response response) { + return Map.of("blacklist", lastNAdditions(100)); + } + + private Object updateBlacklist(Request request, Response response) { + var domain = new EdgeDomain(request.queryParams("domain")); + if ("add".equals(request.queryParams("act"))) { + var comment = Objects.requireNonNullElse(request.queryParams("comment"), ""); + addToBlacklist(domain, comment); + } else if ("del".equals(request.queryParams("act"))) { + removeFromBlacklist(domain); + } + + return ""; } public void addToBlacklist(EdgeDomain domain, String comment) { @@ -83,4 +118,5 @@ public class ControlBlacklistService { return ret; } + } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java similarity index 58% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java index 758d0313..b3e12176 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java @@ -1,16 +1,23 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.app.svc; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.control.model.DomainComplaintCategory; -import nu.marginalia.control.model.DomainComplaintModel; +import nu.marginalia.control.Redirects; +import nu.marginalia.control.app.model.DomainComplaintCategory; +import nu.marginalia.control.app.model.DomainComplaintModel; import nu.marginalia.model.EdgeDomain; +import nu.marginalia.renderer.RendererFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; +import spark.Spark; +import java.io.IOException; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; +import java.util.stream.Collectors; /** Service for handling domain complaints. This code has an user-facing correspondent in @@ -18,16 +25,57 @@ import java.util.Optional; */ public class DomainComplaintService { private final HikariDataSource dataSource; + private final RendererFactory rendererFactory; private final ControlBlacklistService blacklistService; + private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject public DomainComplaintService(HikariDataSource dataSource, + RendererFactory rendererFactory, ControlBlacklistService blacklistService ) { this.dataSource = dataSource; + this.rendererFactory = rendererFactory; this.blacklistService = blacklistService; } + public void register() throws IOException { + var domainComplaintsRenderer = rendererFactory.renderer("control/app/domain-complaints"); + + Spark.get("/public/complaints", this::complaintsModel, domainComplaintsRenderer::render); + Spark.post("/public/complaints/:domain", this::reviewComplaint, Redirects.redirectToComplaints); + } + + private Object complaintsModel(Request request, Response response) { + Map> complaintsByReviewed = + getComplaints().stream().collect(Collectors.partitioningBy(DomainComplaintModel::reviewed)); + + var reviewed = complaintsByReviewed.get(true); + var unreviewed = complaintsByReviewed.get(false); + + reviewed.sort(Comparator.comparing(DomainComplaintModel::reviewDate).reversed()); + unreviewed.sort(Comparator.comparing(DomainComplaintModel::fileDate).reversed()); + + return Map.of("complaintsNew", unreviewed, "complaintsReviewed", reviewed); + } + + private Object reviewComplaint(Request request, Response response) { + var domain = new EdgeDomain(request.params("domain")); + String action = request.queryParams("action"); + + logger.info("Reviewing complaint for domain {} with action {}", domain, action); + + switch (action) { + case "noop" -> reviewNoAction(domain); + case "appeal" -> approveAppealBlacklisting(domain); + case "blacklist" -> blacklistDomain(domain); + default -> throw new UnsupportedOperationException(); + } + + return ""; + } + + public List getComplaints() { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java new file mode 100644 index 00000000..6d2d0dad --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java @@ -0,0 +1,113 @@ +package nu.marginalia.control.app.svc; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import gnu.trove.list.array.TIntArrayList; +import nu.marginalia.renderer.RendererFactory; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class RandomExplorationService { + + private final HikariDataSource dataSource; + private final RendererFactory rendererFactory; + + @Inject + public RandomExplorationService(HikariDataSource dataSource, + RendererFactory rendererFactory + ) { + this.dataSource = dataSource; + this.rendererFactory = rendererFactory; + } + + public void register() throws IOException { + var reviewRandomDomainsRenderer = rendererFactory.renderer("control/app/review-random-domains"); + + Spark.get("/public/review-random-domains", this::reviewRandomDomainsModel, reviewRandomDomainsRenderer::render); + Spark.post("/public/review-random-domains", this::reviewRandomDomainsAction); + } + + private Object reviewRandomDomainsModel(Request request, Response response) throws SQLException { + String afterVal = Objects.requireNonNullElse(request.queryParams("after"), "0"); + int after = Integer.parseInt(afterVal); + var domains = getDomains(after, 25); + int nextAfter = domains.stream().mapToInt(RandomExplorationService.RandomDomainResult::id).max().orElse(Integer.MAX_VALUE); + + return Map.of("domains", domains, + "after", nextAfter); + + } + + private Object reviewRandomDomainsAction(Request request, Response response) throws SQLException { + TIntArrayList idList = new TIntArrayList(); + + request.queryParams().forEach(key -> { + if (key.startsWith("domain-")) { + String value = request.queryParams(key); + if ("on".equalsIgnoreCase(value)) { + int id = Integer.parseInt(key.substring(7)); + idList.add(id); + } + } + }); + + removeRandomDomains(idList.toArray()); + + String after = request.queryParams("after"); + + return """ + + + """.formatted(after); + } + + public void removeRandomDomains(int[] ids) throws SQLException { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + DELETE FROM EC_RANDOM_DOMAINS + WHERE DOMAIN_ID = ? + AND DOMAIN_SET = 0 + """)) + { + for (var id : ids) { + stmt.setInt(1, id); + stmt.addBatch(); + } + stmt.executeBatch(); + if (!conn.getAutoCommit()) { + conn.commit(); + } + } + } + + public List getDomains(int afterId, int numResults) throws SQLException { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT DOMAIN_ID, DOMAIN_NAME FROM EC_RANDOM_DOMAINS + INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID + WHERE DOMAIN_ID >= ? + LIMIT ? + """)) + { + List ret = new ArrayList<>(numResults); + stmt.setInt(1, afterId); + stmt.setInt(2, numResults); + var rs = stmt.executeQuery(); + while (rs.next()) { + ret.add(new RandomDomainResult(rs.getInt(1), rs.getString(2))); + } + return ret; + } + } + + + public record RandomDomainResult(int id, String domainName) {} +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/SearchToBanService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java similarity index 78% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/SearchToBanService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java index c4fb4e2c..95afc9b3 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/SearchToBanService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.app.svc; import com.google.inject.Inject; import nu.marginalia.client.Context; @@ -7,28 +7,41 @@ import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.EdgeUrl; import nu.marginalia.query.client.QueryClient; import nu.marginalia.query.model.QueryParams; +import nu.marginalia.renderer.RendererFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; +import spark.Spark; +import java.io.IOException; import java.util.Map; import java.util.Objects; public class SearchToBanService { private final ControlBlacklistService blacklistService; + private final RendererFactory rendererFactory; private final QueryClient queryClient; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject public SearchToBanService(ControlBlacklistService blacklistService, + RendererFactory rendererFactory, QueryClient queryClient) { this.blacklistService = blacklistService; + this.rendererFactory = rendererFactory; this.queryClient = queryClient; } + public void register() throws IOException { + var searchToBanRenderer = rendererFactory.renderer("control/app/search-to-ban"); + + Spark.get("/public/search-to-ban", this::handle, searchToBanRenderer::render); + Spark.post("/public/search-to-ban", this::handle, searchToBanRenderer::render); + } + public Object handle(Request request, Response response) { if (Objects.equals(request.requestMethod(), "POST")) { executeBlacklisting(request); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorState.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/ActorState.java similarity index 93% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorState.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/ActorState.java index 7d4681ee..b04908f4 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorState.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/ActorState.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.node.model; import java.util.Arrays; import java.util.List; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/ActorStateGraph.java similarity index 63% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/ActorStateGraph.java index 757bdd9a..99a67512 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/ActorStateGraph.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.node.model; import nu.marginalia.actor.prototype.AbstractActorPrototype; import nu.marginalia.actor.state.ActorState; @@ -6,13 +6,13 @@ import nu.marginalia.actor.state.ActorStateInstance; import java.util.*; -public record ActorStateGraph(String description, List states) { +public record ActorStateGraph(String description, List states) { public ActorStateGraph(AbstractActorPrototype graph, ActorStateInstance currentState) { this(graph.describe(), getStateList(graph, currentState)); } - private static List getStateList( + private static List getStateList( AbstractActorPrototype graph, ActorStateInstance currentState) { @@ -20,7 +20,7 @@ public record ActorStateGraph(String description, List seenStates = new HashSet<>(declaredStates.size()); LinkedList edge = new LinkedList<>(); - List statesList = new ArrayList<>(declaredStates.size()); + List statesList = new ArrayList<>(declaredStates.size()); edge.add(declaredStates.get("INITIAL")); @@ -29,7 +29,7 @@ public record ActorStateGraph(String description, List related, List files ) { + public FileStorageType type() { + return self().storage().type(); + } } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNode.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNode.java new file mode 100644 index 00000000..6748a2ce --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNode.java @@ -0,0 +1,4 @@ +package nu.marginalia.control.node.model; + +public record IndexNode(int id) { +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNodeStatus.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNodeStatus.java new file mode 100644 index 00000000..b413808f --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/IndexNodeStatus.java @@ -0,0 +1,4 @@ +package nu.marginalia.control.node.model; + +public record IndexNodeStatus(IndexNode node, boolean indexServiceOnline, boolean executorServiceOnline) { +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlActorService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlActorService.java new file mode 100644 index 00000000..8d3df93a --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlActorService.java @@ -0,0 +1,103 @@ +package nu.marginalia.control.node.svc; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.client.Context; +import nu.marginalia.control.Redirects; +import nu.marginalia.executor.client.ExecutorClient; +import nu.marginalia.executor.model.ActorRunState; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.util.List; + +@Singleton +public class ControlActorService { + + private final ExecutorClient executorClient; + + @Inject + public ControlActorService(ExecutorClient executorClient) { + this.executorClient = executorClient; + } + + public void register() { + Spark.post("/public/nodes/:node/storage/:fid/crawl", this::triggerCrawling, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/storage/:fid/process", this::triggerProcessing, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/storage/:fid/process-and-load", this::triggerProcessingWithLoad, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/storage/:fid/load", this::loadProcessedData, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/storage/:fid/restore-backup", this::restoreBackup, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/storage/specs", this::createCrawlSpecification, Redirects.redirectToStorage); + + Spark.post("/public/nodes/:node/fsms/:fsm/start", this::startFsm, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/fsms/:fsm/stop", this::stopFsm, Redirects.redirectToActors); + + } + + public Object startFsm(Request req, Response rsp) throws Exception { + executorClient.startFsm(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fsm").toUpperCase()); + + return ""; + } + + public Object stopFsm(Request req, Response rsp) throws Exception { + executorClient.stopFsm(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fsm").toUpperCase()); + + return ""; + } + + public Object triggerCrawling(Request req, Response response) throws Exception { + executorClient.triggerCrawl(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fid")); + + return ""; + } + + public Object triggerProcessing(Request req, Response response) throws Exception { + executorClient.triggerConvert(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fid")); + + return ""; + } + + public Object triggerProcessingWithLoad(Request req, Response response) throws Exception { + executorClient.triggerProcessAndLoad(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fid")); + + return ""; + } + + public Object loadProcessedData(Request req, Response response) throws Exception { + executorClient.loadProcessedData(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fid")); + + return ""; + } + + public List getActorStates(Request req) { + return executorClient.getActorStates(Context.fromRequest(req), Integer.parseInt(req.params("node"))).states(); + } + + public Object createCrawlSpecification(Request request, Response response) throws Exception { + final String description = request.queryParams("description"); + final String url = request.queryParams("url"); + final String source = request.queryParams("source"); + + if ("db".equals(source)) { + executorClient.createCrawlSpecFromDb(Context.fromRequest(request), 0, description); + } + else if ("download".equals(source)) { + executorClient.createCrawlSpecFromDownload(Context.fromRequest(request), 0, description, url); + } + else { + throw new IllegalArgumentException("Unknown source: " + source); + } + + return ""; + } + + public Object restoreBackup(Request req, Response response) throws Exception { + executorClient.restoreBackup(Context.fromRequest(req), Integer.parseInt(req.params("node")), req.params("fid")); + + return ""; + } + + +} \ No newline at end of file diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java new file mode 100644 index 00000000..4bb38978 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlFileStorageService.java @@ -0,0 +1,67 @@ +package nu.marginalia.control.node.svc; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.control.Redirects; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; + +@Singleton +public class ControlFileStorageService { + private final FileStorageService fileStorageService; + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Inject + public ControlFileStorageService( FileStorageService fileStorageService) + { + this.fileStorageService = fileStorageService; + } + + public void register() throws IOException { + Spark.get("/public/storage/:id/file", this::downloadFileFromStorage); + Spark.post("/public/storage/:fid/delete", this::flagFileForDeletionRequest, Redirects.redirectToStorage); + + } + + public Object flagFileForDeletionRequest(Request request, Response response) throws SQLException { + FileStorageId fid = new FileStorageId(Long.parseLong(request.params(":fid"))); + fileStorageService.flagFileForDeletion(fid); + return ""; + } + + public Object downloadFileFromStorage(Request request, Response response) throws SQLException { + var fileStorageId = FileStorageId.parse(request.params("id")); + String filename = request.queryParams("name"); + + Path root = fileStorageService.getStorage(fileStorageId).asPath(); + Path filePath = root.resolve(filename).normalize(); + + if (!filePath.startsWith(root)) { + response.status(403); + return ""; + } + + if (filePath.endsWith(".txt") || filePath.endsWith(".log")) response.type("text/plain"); + else response.type("application/octet-stream"); + + try (var is = Files.newInputStream(filePath)) { + is.transferTo(response.raw().getOutputStream()); + } + catch (IOException ex) { + logger.error("Failed to download file", ex); + throw new RuntimeException(ex); + } + + return ""; + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java new file mode 100644 index 00000000..460626a0 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java @@ -0,0 +1,107 @@ +package nu.marginalia.control.node.svc; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.client.Context; +import nu.marginalia.control.Redirects; +import nu.marginalia.db.DomainTypes; +import nu.marginalia.executor.client.ExecutorClient; +import nu.marginalia.index.client.IndexClient; +import nu.marginalia.index.client.IndexMqEndpoints; +import nu.marginalia.mq.MessageQueueFactory; +import nu.marginalia.mq.outbox.MqOutbox; +import nu.marginalia.mq.persistence.MqPersistence; +import nu.marginalia.service.control.ServiceEventLog; +import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.module.ServiceConfiguration; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +@Singleton +public class ControlNodeActionsService { + + private final IndexClient indexClient; + private final ServiceEventLog eventLog; + private final ExecutorClient executorClient; + + @Inject + public ControlNodeActionsService(ExecutorClient executorClient, + IndexClient indexClient, + ServiceEventLog eventLog) + { + this.executorClient = executorClient; + + this.indexClient = indexClient; + this.eventLog = eventLog; + + } + + public void register() { + Spark.post("/public/nodes/:node/actions/repartition-index", this::triggerRepartition, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/actions/sideload-encyclopedia", this::sideloadEncyclopedia, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/actions/sideload-dirtree", this::sideloadDirtree, Redirects.redirectToActors); + Spark.post("/public/nodes/:node/actions/sideload-stackexchange", this::sideloadStackexchange, Redirects.redirectToActors); + } + + public Object sideloadEncyclopedia(Request request, Response response) throws Exception { + + Path sourcePath = Path.of(request.queryParams("source")); + if (!Files.exists(sourcePath)) { + Spark.halt(404); + return "No such file " + sourcePath; + } + + final int nodeId = Integer.parseInt(request.params("node")); + + eventLog.logEvent("USER-ACTION", "SIDELOAD ENCYCLOPEDIA " + nodeId); + + executorClient.sideloadEncyclopedia(Context.fromRequest(request), nodeId, sourcePath); + + return ""; + } + + public Object sideloadDirtree(Request request, Response response) throws Exception { + + Path sourcePath = Path.of(request.queryParams("source")); + if (!Files.exists(sourcePath)) { + Spark.halt(404); + return "No such file " + sourcePath; + } + + final int nodeId = Integer.parseInt(request.params("node")); + + eventLog.logEvent("USER-ACTION", "SIDELOAD DIRTREE " + nodeId); + + executorClient.sideloadDirtree(Context.fromRequest(request), nodeId, sourcePath); + + return ""; + } + + public Object sideloadStackexchange(Request request, Response response) throws Exception { + + Path sourcePath = Path.of(request.queryParams("source")); + if (!Files.exists(sourcePath)) { + Spark.halt(404); + return "No such file " + sourcePath; + } + + final int nodeId = Integer.parseInt(request.params("node")); + + eventLog.logEvent("USER-ACTION", "SIDELOAD STACKEXCHANGE " + nodeId); + + executorClient.sideloadStackexchange(Context.fromRequest(request), nodeId, sourcePath); + return ""; + } + + public Object triggerRepartition(Request request, Response response) throws Exception { + indexClient.outbox().sendAsync(IndexMqEndpoints.INDEX_REPARTITION, ""); + return ""; + } + + +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java new file mode 100644 index 00000000..af5be047 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java @@ -0,0 +1,465 @@ +package nu.marginalia.control.node.svc; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import lombok.SneakyThrows; +import nu.marginalia.client.Context; +import nu.marginalia.client.ServiceMonitors; +import nu.marginalia.control.Redirects; +import nu.marginalia.control.node.model.*; +import nu.marginalia.control.sys.model.EventLogEntry; +import nu.marginalia.control.sys.svc.EventLogService; +import nu.marginalia.control.sys.svc.HeartbeatService; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.executor.client.ExecutorClient; +import nu.marginalia.executor.model.crawl.RecrawlParameters; +import nu.marginalia.executor.model.load.LoadParameters; +import nu.marginalia.renderer.RendererFactory; +import nu.marginalia.service.id.ServiceId; +import nu.marginalia.storage.model.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.*; + +public class ControlNodeService { + private final FileStorageService fileStorageService; + private final RendererFactory rendererFactory; + private final EventLogService eventLogService; + private final HeartbeatService heartbeatService; + private final ExecutorClient executorClient; + private final HikariDataSource dataSource; + private final ServiceMonitors monitors; + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Inject + public ControlNodeService( + FileStorageService fileStorageService, + RendererFactory rendererFactory, + EventLogService eventLogService, + HeartbeatService heartbeatService, + ExecutorClient executorClient, + HikariDataSource dataSource, + ServiceMonitors monitors) + { + this.fileStorageService = fileStorageService; + this.rendererFactory = rendererFactory; + this.eventLogService = eventLogService; + this.heartbeatService = heartbeatService; + this.executorClient = executorClient; + this.dataSource = dataSource; + this.monitors = monitors; + } + + public void register() throws IOException { + var overviewRenderer = rendererFactory.renderer("control/node/node-overview"); + var actionsRenderer = rendererFactory.renderer("control/node/node-actions"); + var actorsRenderer = rendererFactory.renderer("control/node/node-actors"); + var storageConfRenderer = rendererFactory.renderer("control/node/node-storage-conf"); + var storageListRenderer = rendererFactory.renderer("control/node/node-storage-list"); + var storageDetailsRenderer = rendererFactory.renderer("control/node/node-storage-details"); + var configRenderer = rendererFactory.renderer("control/node/node-config"); + + var newSpecsFormRenderer = rendererFactory.renderer("control/node/node-new-specs-form"); + + Spark.get("/public/nodes/:id", this::nodeOverviewModel, overviewRenderer::render); + Spark.get("/public/nodes/:id/", this::nodeOverviewModel, overviewRenderer::render); + Spark.get("/public/nodes/:id/actors", this::nodeActorsModel, actorsRenderer::render); + Spark.get("/public/nodes/:id/actions", this::nodeActionsModel, actionsRenderer::render); + Spark.get("/public/nodes/:id/storage/", this::nodeStorageConfModel, storageConfRenderer::render); + Spark.get("/public/nodes/:id/storage/conf", this::nodeStorageConfModel, storageConfRenderer::render); + Spark.get("/public/nodes/:id/storage/details", this::nodeStorageDetailsModel, storageDetailsRenderer::render); + + Spark.get("/public/nodes/:id/storage/new-specs", this::newSpecsModel, newSpecsFormRenderer::render); + Spark.post("/public/nodes/:id/storage/new-specs", this::createNewSpecsAction); + + Spark.get("/public/nodes/:id/storage/:view", this::nodeStorageListModel, storageListRenderer::render); + Spark.get("/public/nodes/:id/configuration", this::nodeConfigModel, configRenderer::render); + + Spark.post("/public/nodes/:id/storage/recrawl-auto", this::triggerAutoRecrawl); + Spark.post("/public/nodes/:id/storage/process-auto", this::triggerAutoProcess); + Spark.post("/public/nodes/:id/storage/load-selected", this::triggerLoadSelected); + Spark.post("/public/nodes/:id/storage/crawl/:fid", this::triggerCrawl); + Spark.post("/public/nodes/:id/storage/backup-restore/:fid", this::triggerRestoreBackup); + + Spark.post("/public/nodes/:id/storage/:fid/delete", this::deleteFileStorage); + Spark.post("/public/nodes/:id/storage/:fid/enable", this::enableFileStorage); + Spark.post("/public/nodes/:id/storage/:fid/disable", this::disableFileStorage); + + } + + private Object triggerCrawl(Request request, Response response) throws Exception { + int nodeId = Integer.parseInt(request.params("id")); + + executorClient.triggerCrawl(Context.fromRequest(request), nodeId, request.params("fid")); + + return redirectToOverview(request); + } + + private Object triggerRestoreBackup(Request request, Response response) throws Exception { + int nodeId = Integer.parseInt(request.params("id")); + + executorClient.restoreBackup(Context.fromRequest(request), nodeId, request.params("fid")); + + return redirectToOverview(request); + } + + @SneakyThrows + public String redirectToOverview(Request request) { + return new Redirects.HtmlRedirect("/nodes/"+request.params("id")).render(null); + } + + private Object createNewSpecsAction(Request request, Response response) { + final String description = request.queryParams("description"); + final String url = request.queryParams("url"); + final String source = request.queryParams("source"); + + if ("db".equals(source)) { + executorClient.createCrawlSpecFromDb(Context.fromRequest(request), 0, description); + } + else if ("download".equals(source)) { + executorClient.createCrawlSpecFromDownload(Context.fromRequest(request), 0, description, url); + } + else { + throw new IllegalArgumentException("Unknown source: " + source); + } + + return redirectToOverview(request); + } + + private Object newSpecsModel(Request request, Response response) { + int nodeId = Integer.parseInt(request.params("id")); + + return Map.of( + "node", new IndexNode(nodeId), + "view", Map.of("specs", true) + ); + } + + private Object triggerAutoRecrawl(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + + var toCrawl = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.CRAWL_DATA); + if (toCrawl.size() != 1) + throw new IllegalStateException(); + + var specs = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.CRAWL_SPEC); + + executorClient.triggerRecrawl(Context.fromRequest(request), + nodeId, + new RecrawlParameters(toCrawl.get(0), specs)); + + return redirectToOverview(request); + } + + private Object triggerAutoProcess(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + + var toConvert = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.CRAWL_DATA); + if (toConvert.size() != 1) + throw new IllegalStateException(); + + executorClient.triggerConvert(Context.fromRequest(request), + nodeId, + toConvert.get(0)); + + return redirectToOverview(request); + } + + private Object triggerLoadSelected(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + + var toLoad = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.PROCESSED_DATA); + + executorClient.loadProcessedData(Context.fromRequest(request), + nodeId, + new LoadParameters(toLoad) + ); + + return redirectToOverview(request); + } + + private Object deleteFileStorage(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + int fileId = Integer.parseInt(request.params("fid")); + + fileStorageService.flagFileForDeletion(new FileStorageId(fileId)); + + return redirectToOverview(request); + } + + private Object enableFileStorage(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + FileStorageId fileId = new FileStorageId(Integer.parseInt(request.params("fid"))); + + var storage = fileStorageService.getStorage(fileId); + if (storage.type() == FileStorageType.CRAWL_DATA) { + fileStorageService.disableFileStorageOfType(nodeId, storage.type()); + } + + fileStorageService.enableFileStorage(fileId); + + return ""; + } + + private Object disableFileStorage(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + int fileId = Integer.parseInt(request.params("fid")); + + fileStorageService.disableFileStorage(new FileStorageId(fileId)); + + return ""; + } + + private Object nodeActorsModel(Request request, Response response) { + int nodeId = Integer.parseInt(request.params("id")); + + return Map.of( + "node", new IndexNode(nodeId), + "actors", executorClient.getActorStates(Context.fromRequest(request), nodeId).states() + ); + } + + private Object nodeActionsModel(Request request, Response response) { + int nodeId = Integer.parseInt(request.params("id")); + + return Map.of( + "node", new IndexNode(nodeId) + ); + } + + private Object nodeStorageConfModel(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + + return Map.of( + "view", Map.of("conf", true), + "node", new IndexNode(nodeId), + "storagebase", getStorageBaseList(nodeId) + ); + } + + + private Object nodeStorageListModel(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + String view = request.params("view"); + + FileStorageType type = switch(view) { + case "backup" -> FileStorageType.BACKUP; + case "crawl" -> FileStorageType.CRAWL_DATA; + case "processed" -> FileStorageType.PROCESSED_DATA; + case "specs" -> FileStorageType.CRAWL_SPEC; + default -> throw new IllegalArgumentException(view); + }; + + return Map.of( + "view", Map.of(view, true), + "node", new IndexNode(nodeId), + "storage", makeFileStorageBaseWithStorage(getFileStorageIds(type, nodeId)) + ); + } + + private Object nodeStorageDetailsModel(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + + var storage = getFileStorageWithRelatedEntries(FileStorageId.parse(request.queryParams("fid"))); + String view = switch(storage.type()) { + case BACKUP -> "backup"; + case CRAWL_DATA -> "crawl"; + case CRAWL_SPEC -> "specs"; + case PROCESSED_DATA -> "processed"; + default -> throw new IllegalStateException(storage.type().toString()); + }; + + return Map.of( + "view", Map.of(view, true), + "node", new IndexNode(nodeId), + "storage", storage); + } + + private Object nodeConfigModel(Request request, Response response) { + int nodeId = Integer.parseInt(request.params("id")); + return Map.of( + "node", new IndexNode(nodeId) + ); + } + + private Object nodeOverviewModel(Request request, Response response) throws SQLException { + int nodeId = Integer.parseInt(request.params("id")); + return Map.of( + "node", new IndexNode(nodeId), + "status", getStatus(new IndexNode(nodeId)), + "events", getEvents(nodeId), + "processes", heartbeatService.getProcessHeartbeatsForNode(nodeId), + "jobs", heartbeatService.getTaskHeartbeatsForNode(nodeId) + ); + } + + private Object getStorageBaseList(int nodeId) throws SQLException { + List bases = new ArrayList<>(); + + for (var type : FileStorageBaseType.values()) { + var base = fileStorageService.getStorageBase(type, nodeId); + bases.add(Objects.requireNonNullElseGet(base, + () -> new FileStorageBase(new FileStorageBaseId(-1), type, "MISSING", "MISSING")) + ); + } + + return bases; + } + + private List getEvents(int nodeId) { + List services = List.of(ServiceId.Index.name+":"+nodeId, ServiceId.Executor.name+":"+nodeId); + List events = new ArrayList<>(20); + for (var service :services) { + events.addAll(eventLogService.getLastEntriesForService(service, Long.MAX_VALUE, 10)); + } + events.sort(Comparator.comparing(EventLogEntry::id).reversed()); + return events; + } + + public List getConfiguredNodes() { + return fileStorageService + .getConfiguredNodes() + .stream() + .sorted() + .map(IndexNode::new) + .toList(); + } + + public List getNodeStatusList() { + return fileStorageService + .getConfiguredNodes() + .stream() + .sorted() + .map(IndexNode::new) + .map(this::getStatus) + .toList(); + } + + IndexNodeStatus getStatus(IndexNode node) { + return new IndexNodeStatus(node, + monitors.isServiceUp(ServiceId.Index, node.id()), + monitors.isServiceUp(ServiceId.Executor, node.id()) + ); + } + + private List getFileStorageIds(FileStorageType type, int node) throws SQLException { + List storageIds = new ArrayList<>(); + + try (var conn = dataSource.getConnection(); + var storageByIdStmt = conn.prepareStatement(""" + SELECT FILE_STORAGE.ID + FROM FILE_STORAGE + INNER JOIN FILE_STORAGE_BASE + ON BASE_ID=FILE_STORAGE_BASE.ID + WHERE FILE_STORAGE.TYPE = ? + AND NODE = ? + """)) + { + storageByIdStmt.setString(1, type.name()); + storageByIdStmt.setInt(2, node); + var rs = storageByIdStmt.executeQuery(); + while (rs.next()) { + storageIds.add(new FileStorageId(rs.getLong("ID"))); + } + } + + return storageIds; + } + + private List makeFileStorageBaseWithStorage(List storageIds) throws SQLException { + + Map fileStorageBaseByBaseId = new HashMap<>(); + Map> fileStoragByBaseId = new HashMap<>(); + + for (var id : storageIds) { + var storage = fileStorageService.getStorage(id); + fileStorageBaseByBaseId.computeIfAbsent(storage.base().id(), k -> storage.base()); + fileStoragByBaseId.computeIfAbsent(storage.base().id(), k -> new ArrayList<>()).add(new FileStorageWithActions(storage)); + } + + List result = new ArrayList<>(); + for (var baseId : fileStorageBaseByBaseId.keySet()) { + result.add(new FileStorageBaseWithStorage(fileStorageBaseByBaseId.get(baseId), + fileStoragByBaseId.get(baseId) + + )); + } + + return result; + } + + + public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(FileStorageId id) throws SQLException { + var storage = fileStorageService.getStorage(id); + var related = getRelatedEntries(id); + + List files = new ArrayList<>(); + + try (var filesStream = Files.list(storage.asPath())) { + filesStream + .filter(Files::isRegularFile) + .map(this::createFileModel) + .sorted(Comparator.comparing(FileStorageFileModel::filename)) + .forEach(files::add); + } + catch (IOException ex) { + logger.error("Failed to list files in storage", ex); + } + + return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related, files); + } + + private FileStorageFileModel createFileModel(Path p) { + try { + String mTime = Files.getLastModifiedTime(p).toInstant().toString(); + String size; + if (Files.isDirectory(p)) { + size = "-"; + } + else { + long sizeBytes = Files.size(p); + + if (sizeBytes < 1024) size = sizeBytes + " B"; + else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB"; + else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB"; + else size = sizeBytes / (1024 * 1024 * 1024) + " GB"; + } + + return new FileStorageFileModel(p.toFile().getName(), mTime, size); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + private List getRelatedEntries(FileStorageId id) { + List ret = new ArrayList<>(); + try (var conn = dataSource.getConnection(); + var relatedIds = conn.prepareStatement(""" + (SELECT SOURCE_ID AS ID FROM FILE_STORAGE_RELATION WHERE TARGET_ID = ?) + UNION + (SELECT TARGET_ID AS ID FROM FILE_STORAGE_RELATION WHERE SOURCE_ID = ?) + """)) + { + + relatedIds.setLong(1, id.id()); + relatedIds.setLong(2, id.id()); + var rs = relatedIds.executeQuery(); + while (rs.next()) { + ret.add(fileStorageService.getStorage(new FileStorageId(rs.getLong("ID")))); + } + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + return ret; + } + +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java deleted file mode 100644 index 5b4bfd5c..00000000 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java +++ /dev/null @@ -1,152 +0,0 @@ -package nu.marginalia.control.svc; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.control.actor.ControlActors; -import nu.marginalia.control.actor.Actor; -import nu.marginalia.control.actor.task.ConvertActor; -import nu.marginalia.db.DomainTypes; -import nu.marginalia.index.client.IndexClient; -import nu.marginalia.index.client.IndexMqEndpoints; -import nu.marginalia.mq.MessageQueueFactory; -import nu.marginalia.mq.outbox.MqOutbox; -import nu.marginalia.service.control.ServiceEventLog; -import nu.marginalia.service.id.ServiceId; -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.UUID; - -@Singleton -public class ControlActionsService { - - private final ControlActors actors; - private final IndexClient indexClient; - private final MqOutbox apiOutbox; - private final ServiceEventLog eventLog; - private final DomainTypes domainTypes; - - @Inject - public ControlActionsService(ControlActors actors, - IndexClient indexClient, - MessageQueueFactory mqFactory, - ServiceEventLog eventLog, - DomainTypes domainTypes) { - - this.actors = actors; - this.indexClient = indexClient; - this.apiOutbox = createApiOutbox(mqFactory); - this.eventLog = eventLog; - - this.domainTypes = domainTypes; - } - - /** This is a hack to get around the fact that the API service is not a core service - * and lacks a proper internal API - */ - private MqOutbox createApiOutbox(MessageQueueFactory mqFactory) { - String inboxName = ServiceId.Api.name + ":" + "0"; - String outboxName = System.getProperty("service-name", UUID.randomUUID().toString()); - return mqFactory.createOutbox(inboxName, outboxName, UUID.randomUUID()); - } - - public Object calculateAdjacencies(Request request, Response response) throws Exception { - eventLog.logEvent("USER-ACTION", "CALCULATE-ADJACENCIES"); - - actors.start(Actor.ADJACENCY_CALCULATION); - - return ""; - } - - public Object triggerDataExports(Request request, Response response) throws Exception { - eventLog.logEvent("USER-ACTION", "EXPORT-DATA"); - actors.start(Actor.EXPORT_DATA); - - return ""; - } - - public Object reloadBlogsList(Request request, Response response) throws Exception { - eventLog.logEvent("USER-ACTION", "RELOAD-BLOGS-LIST"); - - domainTypes.reloadDomainsList(DomainTypes.Type.BLOG); - - return ""; - } - - public Object flushApiCaches(Request request, Response response) throws Exception { - eventLog.logEvent("USER-ACTION", "FLUSH-API-CACHES"); - apiOutbox.sendNotice("FLUSH_CACHES", ""); - - return ""; - } - - public Object truncateLinkDatabase(Request request, Response response) throws Exception { - - String footgunLicense = request.queryParams("footgun-license"); - - if (!"YES".equals(footgunLicense)) { - Spark.halt(403); - return "You must agree to the footgun license to truncate the link database"; - } - - eventLog.logEvent("USER-ACTION", "FLUSH-LINK-DATABASE"); - - actors.start(Actor.TRUNCATE_LINK_DATABASE); - - return ""; - } - - public Object sideloadEncyclopedia(Request request, Response response) throws Exception { - - Path sourcePath = Path.of(request.queryParams("source")); - if (!Files.exists(sourcePath)) { - Spark.halt(404); - return "No such file " + sourcePath; - } - - eventLog.logEvent("USER-ACTION", "SIDELOAD ENCYCLOPEDIA"); - - actors.startFrom(Actor.CONVERT, ConvertActor.CONVERT_ENCYCLOPEDIA, sourcePath.toString()); - - return ""; - } - - public Object sideloadDirtree(Request request, Response response) throws Exception { - - Path sourcePath = Path.of(request.queryParams("source")); - if (!Files.exists(sourcePath)) { - Spark.halt(404); - return "No such file " + sourcePath; - } - - eventLog.logEvent("USER-ACTION", "SIDELOAD DIRTREE"); - - actors.startFrom(Actor.CONVERT, ConvertActor.CONVERT_DIRTREE, sourcePath.toString()); - - return ""; - } - - public Object sideloadStackexchange(Request request, Response response) throws Exception { - - Path sourcePath = Path.of(request.queryParams("source")); - if (!Files.exists(sourcePath)) { - Spark.halt(404); - return "No such file " + sourcePath; - } - - eventLog.logEvent("USER-ACTION", "SIDELOAD STACKEXCHANGE"); - - actors.startFrom(Actor.CONVERT, ConvertActor.CONVERT_STACKEXCHANGE, sourcePath.toString()); - - return ""; - } - - public Object triggerRepartition(Request request, Response response) throws Exception { - indexClient.outbox().sendAsync(IndexMqEndpoints.INDEX_REPARTITION, ""); - - return null; - } -} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java deleted file mode 100644 index fbb2b818..00000000 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java +++ /dev/null @@ -1,166 +0,0 @@ -package nu.marginalia.control.svc; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.control.actor.ControlActors; -import nu.marginalia.control.actor.task.*; -import nu.marginalia.control.actor.Actor; -import nu.marginalia.control.model.ActorRunState; -import nu.marginalia.control.model.ActorStateGraph; -import nu.marginalia.db.storage.model.FileStorageId; -import nu.marginalia.actor.state.ActorState; -import nu.marginalia.actor.state.ActorStateInstance; -import spark.Request; -import spark.Response; - -import java.util.Comparator; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -@Singleton -public class ControlActorService { - private final ControlActors controlActors; - - @Inject - public ControlActorService(ControlActors controlActors) { - this.controlActors = controlActors; - } - - public Object getActorStateGraph(Actor actor) { - var currentState = controlActors.getActorStates().get(actor); - - return new ActorStateGraph(controlActors.getActorDefinition(actor), currentState); - } - - public Object startFsm(Request req, Response rsp) throws Exception { - controlActors.start( - Actor.valueOf(req.params("fsm").toUpperCase()) - ); - return ""; - } - - public Object stopFsm(Request req, Response rsp) throws Exception { - controlActors.stop( - Actor.valueOf(req.params("fsm").toUpperCase()) - ); - return ""; - } - - public Object triggerCrawling(Request request, Response response) throws Exception { - controlActors.start( - Actor.CRAWL, - FileStorageId.parse(request.params("fid")) - ); - return ""; - } - - public Object triggerRecrawling(Request request, Response response) throws Exception { - controlActors.start( - Actor.RECRAWL, - RecrawlActor.recrawlFromCrawlData( - FileStorageId.parse(request.params("fid")) - ) - ); - return ""; - } - - public Object triggerProcessing(Request request, Response response) throws Exception { - controlActors.startFrom( - Actor.CONVERT, - ConvertActor.CONVERT, - FileStorageId.parse(request.params("fid")) - ); - - return ""; - } - - public Object triggerProcessingWithLoad(Request request, Response response) throws Exception { - controlActors.start( - Actor.CONVERT_AND_LOAD, - FileStorageId.parse(request.params("fid")) - ); - return ""; - } - - public Object loadProcessedData(Request request, Response response) throws Exception { - var fid = FileStorageId.parse(request.params("fid")); - - // Start the FSM from the intermediate state that triggers the load - controlActors.startFrom( - Actor.CONVERT_AND_LOAD, - ConvertAndLoadActor.LOAD, - new ConvertAndLoadActor.Message(null, fid, 0L, 0L) - ); - - return ""; - } - - private final ConcurrentHashMap actorStateDescriptions = new ConcurrentHashMap<>(); - - public Object getActorStates() { - return controlActors.getActorStates().entrySet().stream().map(e -> { - - final var stateGraph = controlActors.getActorDefinition(e.getKey()); - - final ActorStateInstance state = e.getValue(); - final String actorDescription = stateGraph.describe(); - - final String machineName = e.getKey().name(); - final String stateName = state.name(); - - final String stateDescription = actorStateDescriptions.computeIfAbsent( - (machineName + "." + stateName), - k -> Optional.ofNullable(stateGraph.declaredStates().get(stateName)) - .map(ActorState::description) - .orElse("Description missing for " + stateName) - ); - - - - final boolean terminal = state.isFinal(); - final boolean canStart = controlActors.isDirectlyInitializable(e.getKey()) && terminal; - - return new ActorRunState(machineName, - stateName, - actorDescription, - stateDescription, - terminal, - canStart); - }) - .filter(s -> !s.terminal() || s.canStart()) - .sorted(Comparator.comparing(ActorRunState::name)) - .toList(); - } - - public Object createCrawlSpecification(Request request, Response response) throws Exception { - final String description = request.queryParams("description"); - final String url = request.queryParams("url"); - final String source = request.queryParams("source"); - - if ("db".equals(source)) { - controlActors.startFrom(Actor.CRAWL_JOB_EXTRACTOR, - CrawlJobExtractorActor.CREATE_FROM_DB, - new CrawlJobExtractorActor.CrawlJobExtractorArguments(description) - ); - } - else if ("download".equals(source)) { - controlActors.startFrom(Actor.CRAWL_JOB_EXTRACTOR, - CrawlJobExtractorActor.CREATE_FROM_LINK, - new CrawlJobExtractorActor.CrawlJobExtractorArgumentsWithURL(description, url) - ); - } - else { - throw new IllegalArgumentException("Unknown source: " + source); - } - - return ""; - } - - public Object restoreBackup(Request request, Response response) throws Exception { - var fid = FileStorageId.parse(request.params("fid")); - controlActors.startFrom(Actor.RESTORE_BACKUP, RestoreBackupActor.RESTORE, fid); - return ""; - } - - -} \ No newline at end of file diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java deleted file mode 100644 index 65be9614..00000000 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java +++ /dev/null @@ -1,199 +0,0 @@ -package nu.marginalia.control.svc; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.zaxxer.hikari.HikariDataSource; -import lombok.SneakyThrows; -import nu.marginalia.control.model.*; -import nu.marginalia.db.storage.FileStorageService; -import nu.marginalia.db.storage.model.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.*; - -@Singleton -public class ControlFileStorageService { - private final HikariDataSource dataSource; - private final FileStorageService fileStorageService; - private Logger logger = LoggerFactory.getLogger(getClass()); - - @Inject - public ControlFileStorageService(HikariDataSource dataSource, FileStorageService fileStorageService) { - this.dataSource = dataSource; - this.fileStorageService = fileStorageService; - } - - public Object flagFileForDeletionRequest(Request request, Response response) throws SQLException { - FileStorageId fid = new FileStorageId(Long.parseLong(request.params(":fid"))); - flagFileForDeletion(fid); - return ""; - } - - public void flagFileForDeletion(FileStorageId id) throws SQLException { - try (var conn = dataSource.getConnection(); - var flagStmt = conn.prepareStatement("UPDATE FILE_STORAGE SET DO_PURGE = TRUE WHERE ID = ?")) { - flagStmt.setLong(1, id.id()); - flagStmt.executeUpdate(); - } - } - - @SneakyThrows - public List getStorageList() { - var storageIds = getFileStorageIds(); - return makeFileStorageBaseWithStorage(storageIds); - } - - @SneakyThrows - public List getStorageList(FileStorageType type) { - var storageIds = getFileStorageIds(type); - return makeFileStorageBaseWithStorage(storageIds); - } - - private List getFileStorageIds() throws SQLException { - List storageIds = new ArrayList<>(); - - try (var conn = dataSource.getConnection(); - var storageByIdStmt = conn.prepareStatement("SELECT ID FROM FILE_STORAGE")) { - var rs = storageByIdStmt.executeQuery(); - while (rs.next()) { - storageIds.add(new FileStorageId(rs.getLong("ID"))); - } - } - - return storageIds; - } - - private List getFileStorageIds(FileStorageType type) throws SQLException { - List storageIds = new ArrayList<>(); - - try (var conn = dataSource.getConnection(); - var storageByIdStmt = conn.prepareStatement("SELECT ID FROM FILE_STORAGE WHERE TYPE = ?")) { - storageByIdStmt.setString(1, type.name()); - var rs = storageByIdStmt.executeQuery(); - while (rs.next()) { - storageIds.add(new FileStorageId(rs.getLong("ID"))); - } - } - - return storageIds; - } - - private List makeFileStorageBaseWithStorage(List storageIds) throws SQLException { - - Map fileStorageBaseByBaseId = new HashMap<>(); - Map> fileStoragByBaseId = new HashMap<>(); - - for (var id : storageIds) { - var storage = fileStorageService.getStorage(id); - fileStorageBaseByBaseId.computeIfAbsent(storage.base().id(), k -> storage.base()); - fileStoragByBaseId.computeIfAbsent(storage.base().id(), k -> new ArrayList<>()).add(new FileStorageWithActions(storage)); - } - - List result = new ArrayList<>(); - for (var baseId : fileStorageBaseByBaseId.keySet()) { - result.add(new FileStorageBaseWithStorage(fileStorageBaseByBaseId.get(baseId), - fileStoragByBaseId.get(baseId) - - )); - } - - return result; - } - - public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(FileStorageId id) throws SQLException { - var storage = fileStorageService.getStorage(id); - var related = getRelatedEntries(id); - - List files = new ArrayList<>(); - - try (var filesStream = Files.list(storage.asPath())) { - filesStream - .filter(Files::isRegularFile) - .map(this::createFileModel) - .sorted(Comparator.comparing(FileStorageFileModel::filename)) - .forEach(files::add); - } - catch (IOException ex) { - logger.error("Failed to list files in storage", ex); - } - - return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related, files); - } - - private FileStorageFileModel createFileModel(Path p) { - try { - String mTime = Files.getLastModifiedTime(p).toInstant().toString(); - String size; - if (Files.isDirectory(p)) { - size = "-"; - } - else { - long sizeBytes = Files.size(p); - - if (sizeBytes < 1024) size = sizeBytes + " B"; - else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB"; - else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB"; - else size = sizeBytes / (1024 * 1024 * 1024) + " GB"; - } - - return new FileStorageFileModel(p.toFile().getName(), mTime, size); - } - catch (IOException ex) { - throw new RuntimeException(ex); - } - } - private List getRelatedEntries(FileStorageId id) { - List ret = new ArrayList<>(); - try (var conn = dataSource.getConnection(); - var relatedIds = conn.prepareStatement(""" - (SELECT SOURCE_ID AS ID FROM FILE_STORAGE_RELATION WHERE TARGET_ID = ?) - UNION - (SELECT TARGET_ID AS ID FROM FILE_STORAGE_RELATION WHERE SOURCE_ID = ?) - """)) - { - - relatedIds.setLong(1, id.id()); - relatedIds.setLong(2, id.id()); - var rs = relatedIds.executeQuery(); - while (rs.next()) { - ret.add(fileStorageService.getStorage(new FileStorageId(rs.getLong("ID")))); - } - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - return ret; - } - - public Object downloadFileFromStorage(Request request, Response response) throws SQLException { - var fileStorageId = FileStorageId.parse(request.params("id")); - String filename = request.queryParams("name"); - - Path root = fileStorageService.getStorage(fileStorageId).asPath(); - Path filePath = root.resolve(filename).normalize(); - - if (!filePath.startsWith(root)) { - response.status(403); - return ""; - } - - if (filePath.endsWith(".txt") || filePath.endsWith(".log")) response.type("text/plain"); - else response.type("application/octet-stream"); - - try (var is = Files.newInputStream(filePath)) { - is.transferTo(response.raw().getOutputStream()); - } - catch (IOException ex) { - logger.error("Failed to download file", ex); - throw new RuntimeException(ex); - } - - return ""; - } -} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java deleted file mode 100644 index 7a414761..00000000 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java +++ /dev/null @@ -1,60 +0,0 @@ -package nu.marginalia.control.svc; - -import com.google.inject.Inject; -import com.zaxxer.hikari.HikariDataSource; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -public class RandomExplorationService { - - private final HikariDataSource dataSource; - - @Inject - public RandomExplorationService(HikariDataSource dataSource) { - this.dataSource = dataSource; - } - - public void removeRandomDomains(int[] ids) throws SQLException { - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(""" - DELETE FROM EC_RANDOM_DOMAINS - WHERE DOMAIN_ID = ? - AND DOMAIN_SET = 0 - """)) - { - for (var id : ids) { - stmt.setInt(1, id); - stmt.addBatch(); - } - stmt.executeBatch(); - if (!conn.getAutoCommit()) { - conn.commit(); - } - } - } - - public List getDomains(int afterId, int numResults) throws SQLException { - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(""" - SELECT DOMAIN_ID, DOMAIN_NAME FROM EC_RANDOM_DOMAINS - INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID - WHERE DOMAIN_ID >= ? - LIMIT ? - """)) - { - List ret = new ArrayList<>(numResults); - stmt.setInt(1, afterId); - stmt.setInt(2, numResults); - var rs = stmt.executeQuery(); - while (rs.next()) { - ret.add(new RandomDomainResult(rs.getInt(1), rs.getString(2))); - } - return ret; - } - } - - - public record RandomDomainResult(int id, String domainName) {} -} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogEntry.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogEntry.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java index aa064bf1..b16d01f0 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogEntry.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogEntry.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.sys.model; public record EventLogEntry( long id, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogServiceFilter.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java similarity index 73% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogServiceFilter.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java index 4bed3dd3..0359d102 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogServiceFilter.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogServiceFilter.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.sys.model; public record EventLogServiceFilter( String name, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogTypeFilter.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java similarity index 72% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogTypeFilter.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java index eb64b9df..cb7c6a1f 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogTypeFilter.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/EventLogTypeFilter.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.sys.model; public record EventLogTypeFilter( String name, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/MessageQueueEntry.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java similarity index 97% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/MessageQueueEntry.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java index c90bda76..dd0146bd 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/MessageQueueEntry.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/MessageQueueEntry.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.sys.model; public record MessageQueueEntry ( long id, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ProcessHeartbeat.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java similarity index 57% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/ProcessHeartbeat.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java index ae3f4fae..0f789aee 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ProcessHeartbeat.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ProcessHeartbeat.java @@ -1,10 +1,9 @@ -package nu.marginalia.control.model; - -import nu.marginalia.control.process.ProcessService; +package nu.marginalia.control.sys.model; public record ProcessHeartbeat( String processId, String processBase, + int node, String uuidFull, double lastSeenMillis, Integer progress, @@ -37,24 +36,8 @@ public record ProcessHeartbeat( return ""; } - public ProcessService.ProcessId getProcessId() { - return switch (processBase) { - case "converter" -> ProcessService.ProcessId.CONVERTER; - case "crawler" -> ProcessService.ProcessId.CRAWLER; - case "loader" -> ProcessService.ProcessId.LOADER; - case "website-adjacencies-calculator" -> ProcessService.ProcessId.ADJACENCIES_CALCULATOR; - case "index-constructor" -> ProcessService.ProcessId.INDEX_CONSTRUCTOR; - default -> null; - }; + public String displayName() { + return processId; } - public String displayName() { - var pid = getProcessId(); - if (pid != null) { - return pid.name(); - } - else { - return processBase; - } - } } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ServiceHeartbeat.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/ServiceHeartbeat.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java index f43d9058..f1d6c705 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ServiceHeartbeat.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/ServiceHeartbeat.java @@ -1,4 +1,4 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.sys.model; public record ServiceHeartbeat( String serviceId, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/TaskHeartbeat.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java similarity index 92% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/model/TaskHeartbeat.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java index 65070cea..467ae493 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/TaskHeartbeat.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/model/TaskHeartbeat.java @@ -1,9 +1,10 @@ -package nu.marginalia.control.model; +package nu.marginalia.control.sys.model; public record TaskHeartbeat( String taskName, String taskBase, + int node, String instanceUuidFull, String serviceUuuidFull, double lastSeenMillis, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java new file mode 100644 index 00000000..1779d24d --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java @@ -0,0 +1,99 @@ +package nu.marginalia.control.sys.svc; + +import com.google.inject.Inject; +import nu.marginalia.client.Context; +import nu.marginalia.control.Redirects; +import nu.marginalia.db.DomainTypes; +import nu.marginalia.executor.client.ExecutorClient; +import nu.marginalia.mq.MessageQueueFactory; +import nu.marginalia.mq.outbox.MqOutbox; +import nu.marginalia.service.control.ServiceEventLog; +import nu.marginalia.service.id.ServiceId; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.util.UUID; + +public class ControlSysActionsService { + private final MqOutbox apiOutbox; + private final DomainTypes domainTypes; + private final ServiceEventLog eventLog; + private final ExecutorClient executorClient; + + @Inject + public ControlSysActionsService(MessageQueueFactory mqFactory, DomainTypes domainTypes, ServiceEventLog eventLog, ExecutorClient executorClient) { + this.apiOutbox = createApiOutbox(mqFactory); + this.eventLog = eventLog; + this.domainTypes = domainTypes; + this.executorClient = executorClient; + } + + /** This is a hack to get around the fact that the API service is not a core service + * and lacks a proper internal API + */ + private MqOutbox createApiOutbox(MessageQueueFactory mqFactory) { + String inboxName = ServiceId.Api.name + ":" + "0"; + String outboxName = System.getProperty("service-name", UUID.randomUUID().toString()); + return mqFactory.createOutbox(inboxName, 0, outboxName, 0, UUID.randomUUID()); + } + + public void register() { + Spark.post("/public/actions/flush-api-caches", this::flushApiCaches, Redirects.redirectToActors); + Spark.post("/public/actions/reload-blogs-list", this::reloadBlogsList, Redirects.redirectToActors); + Spark.post("/public/actions/calculate-adjacencies", this::calculateAdjacencies, Redirects.redirectToActors); + Spark.post("/public/actions/truncate-links-database", this::truncateLinkDatabase, Redirects.redirectToActors); + Spark.post("/public/actions/trigger-data-exports", this::triggerDataExports, Redirects.redirectToActors); + } + + public Object triggerDataExports(Request request, Response response) throws Exception { + eventLog.logEvent("USER-ACTION", "EXPORT-DATA"); + + executorClient.exportData(Context.fromRequest(request)); + + return ""; + } + + public Object truncateLinkDatabase(Request request, Response response) throws Exception { + + String footgunLicense = request.queryParams("footgun-license"); + + if (!"YES".equals(footgunLicense)) { + Spark.halt(403); + return "You must agree to the footgun license to truncate the link database"; + } + + eventLog.logEvent("USER-ACTION", "FLUSH-LINK-DATABASE"); + + // FIXME: +// actors.start(Actor.TRUNCATE_LINK_DATABASE); + + return ""; + } + + public Object reloadBlogsList(Request request, Response response) throws Exception { + eventLog.logEvent("USER-ACTION", "RELOAD-BLOGS-LIST"); + + domainTypes.reloadDomainsList(DomainTypes.Type.BLOG); + + return ""; + } + + public Object flushApiCaches(Request request, Response response) throws Exception { + eventLog.logEvent("USER-ACTION", "FLUSH-API-CACHES"); + apiOutbox.sendNotice("FLUSH_CACHES", ""); + + return ""; + } + + public Object calculateAdjacencies(Request request, Response response) throws Exception { + eventLog.logEvent("USER-ACTION", "CALCULATE-ADJACENCIES"); + + // This is technically not a partitioned operation, but we execute it at node zero + // and let the effects be global :-) + + executorClient.calculateAdjacencies(Context.fromRequest(request), 0); + + return ""; + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/EventLogService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java similarity index 98% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/EventLogService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java index ddbc1974..c4b42bf3 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/EventLogService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/EventLogService.java @@ -1,11 +1,11 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.sys.svc; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.control.model.EventLogEntry; -import nu.marginalia.control.model.EventLogServiceFilter; -import nu.marginalia.control.model.EventLogTypeFilter; +import nu.marginalia.control.sys.model.EventLogEntry; +import nu.marginalia.control.sys.model.EventLogServiceFilter; +import nu.marginalia.control.sys.model.EventLogTypeFilter; import org.apache.logging.log4j.util.Strings; import spark.Request; import spark.Response; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/HeartbeatService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java similarity index 60% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/HeartbeatService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java index 73cc2b23..d1db3752 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/HeartbeatService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/HeartbeatService.java @@ -1,11 +1,11 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.sys.svc; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.control.model.ProcessHeartbeat; -import nu.marginalia.control.model.ServiceHeartbeat; -import nu.marginalia.control.model.TaskHeartbeat; +import nu.marginalia.control.sys.model.ProcessHeartbeat; +import nu.marginalia.control.sys.model.ServiceHeartbeat; +import nu.marginalia.control.sys.model.TaskHeartbeat; import nu.marginalia.service.control.ServiceEventLog; import java.sql.SQLException; @@ -56,7 +56,7 @@ public class HeartbeatService { List heartbeats = new ArrayList<>(); try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT TASK_NAME, TASK_BASE, INSTANCE, SERVICE_INSTANCE, STATUS, STAGE_NAME, PROGRESS, TIMESTAMPDIFF(MICROSECOND, TASK_HEARTBEAT.HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF + SELECT TASK_NAME, TASK_BASE, NODE, INSTANCE, SERVICE_INSTANCE, STATUS, STAGE_NAME, PROGRESS, TIMESTAMPDIFF(MICROSECOND, TASK_HEARTBEAT.HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF FROM TASK_HEARTBEAT """)) { var rs = stmt.executeQuery(); @@ -65,6 +65,7 @@ public class HeartbeatService { heartbeats.add(new TaskHeartbeat( rs.getString("TASK_NAME"), rs.getString("TASK_BASE"), + rs.getInt("NODE"), rs.getString("INSTANCE"), rs.getString("SERVICE_INSTANCE"), rs.getLong("TSDIFF") / 1000., @@ -80,6 +81,36 @@ public class HeartbeatService { return heartbeats; } + public List getTaskHeartbeatsForNode(int node) { + List heartbeats = new ArrayList<>(); + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT TASK_NAME, TASK_BASE, NODE, INSTANCE, SERVICE_INSTANCE, STATUS, STAGE_NAME, PROGRESS, TIMESTAMPDIFF(MICROSECOND, TASK_HEARTBEAT.HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF + FROM TASK_HEARTBEAT + WHERE NODE=? + """)) { + stmt.setInt(1, node); + var rs = stmt.executeQuery(); + while (rs.next()) { + int progress = rs.getInt("PROGRESS"); + heartbeats.add(new TaskHeartbeat( + rs.getString("TASK_NAME"), + rs.getString("TASK_BASE"), + rs.getInt("NODE"), + rs.getString("INSTANCE"), + rs.getString("SERVICE_INSTANCE"), + rs.getLong("TSDIFF") / 1000., + progress < 0 ? null : progress, + rs.getString("STAGE_NAME"), + rs.getString("STATUS") + )); + } + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + return heartbeats; + } public void removeTaskHeartbeat(TaskHeartbeat heartbeat) { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" @@ -100,7 +131,7 @@ public class HeartbeatService { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT PROCESS_NAME, PROCESS_BASE, INSTANCE, STATUS, PROGRESS, + SELECT PROCESS_NAME, PROCESS_BASE, NODE, INSTANCE, STATUS, PROGRESS, TIMESTAMPDIFF(MICROSECOND, HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF FROM PROCESS_HEARTBEAT """)) { @@ -111,6 +142,41 @@ public class HeartbeatService { heartbeats.add(new ProcessHeartbeat( rs.getString("PROCESS_NAME"), rs.getString("PROCESS_BASE"), + rs.getInt("NODE"), + rs.getString("INSTANCE"), + rs.getLong("TSDIFF") / 1000., + progress < 0 ? null : progress, + rs.getString("STATUS") + )); + } + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + + return heartbeats; + } + + public List getProcessHeartbeatsForNode(int node) { + List heartbeats = new ArrayList<>(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT PROCESS_NAME, PROCESS_BASE, NODE, INSTANCE, STATUS, PROGRESS, + TIMESTAMPDIFF(MICROSECOND, HEARTBEAT_TIME, CURRENT_TIMESTAMP(6)) AS TSDIFF + FROM PROCESS_HEARTBEAT + WHERE NODE=? + """)) { + + stmt.setInt(1, node); + + var rs = stmt.executeQuery(); + while (rs.next()) { + int progress = rs.getInt("PROGRESS"); + heartbeats.add(new ProcessHeartbeat( + rs.getString("PROCESS_NAME"), + rs.getString("PROCESS_BASE"), + rs.getInt("NODE"), rs.getString("INSTANCE"), rs.getLong("TSDIFF") / 1000., progress < 0 ? null : progress, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/MessageQueueService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/MessageQueueService.java similarity index 86% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/svc/MessageQueueService.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/MessageQueueService.java index b045cd48..75e97e2b 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/MessageQueueService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/MessageQueueService.java @@ -1,14 +1,18 @@ -package nu.marginalia.control.svc; +package nu.marginalia.control.sys.svc; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.control.model.MessageQueueEntry; +import nu.marginalia.control.Redirects; +import nu.marginalia.control.sys.model.MessageQueueEntry; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.persistence.MqPersistence; +import nu.marginalia.renderer.RendererFactory; import spark.Request; import spark.Response; +import spark.Spark; +import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; @@ -20,12 +24,34 @@ import java.util.Optional; public class MessageQueueService { private final HikariDataSource dataSource; + private final RendererFactory rendererFactory; private final MqPersistence persistence; @Inject - public MessageQueueService(HikariDataSource dataSource, MqPersistence persistence) { + public MessageQueueService(HikariDataSource dataSource, + RendererFactory rendererFactory, + MqPersistence persistence) { this.dataSource = dataSource; + this.rendererFactory = rendererFactory; this.persistence = persistence; + + + } + + public void register() throws IOException { + var messageQueueRenderer = rendererFactory.renderer("control/sys/message-queue"); + var updateMessageStateRenderer = rendererFactory.renderer("control/sys/update-message-state"); + var newMessageRenderer = rendererFactory.renderer("control/sys/new-message"); + var viewMessageRenderer = rendererFactory.renderer("control/sys/view-message"); + + Spark.get("/public/message-queue", this::listMessageQueueModel, messageQueueRenderer::render); + Spark.post("/public/message-queue/", this::createMessage, Redirects.redirectToMessageQueue); + Spark.get("/public/message-queue/new", this::newMessageModel, newMessageRenderer::render); + Spark.get("/public/message-queue/:id", this::viewMessageModel, viewMessageRenderer::render); + Spark.get("/public/message-queue/:id/reply", this::replyMessageModel, newMessageRenderer::render); + Spark.get("/public/message-queue/:id/edit", this::viewMessageForEditStateModel, updateMessageStateRenderer::render); + Spark.post("/public/message-queue/:id/edit", this::editMessageState, Redirects.redirectToMessageQueue); + } @@ -118,7 +144,6 @@ public class MessageQueueService { persistence.updateMessageState(id, state); return ""; } - public List getLastEntries(int n) { try (var conn = dataSource.getConnection(); var query = conn.prepareStatement(""" @@ -140,6 +165,7 @@ public class MessageQueueService { throw new RuntimeException(ex); } } + public MessageQueueEntry getMessage(long id) { try (var conn = dataSource.getConnection(); var query = conn.prepareStatement(""" diff --git a/code/services-core/control-service/src/main/resources/static/control/style.css b/code/services-core/control-service/src/main/resources/static/control/style.css deleted file mode 100644 index 762aa8e5..00000000 --- a/code/services-core/control-service/src/main/resources/static/control/style.css +++ /dev/null @@ -1,123 +0,0 @@ -body { - background-color: #f8f8ee; - font-family: sans-serif; - line-height: 1.6; - - display: grid; - grid-template-columns: 20ch auto; - grid-gap: 1em; - grid-template-areas: - "left right"; -} - - - -section nav.tabs > a { - color: #000; - text-decoration: none; - background-color: #ccc; - padding: 0.5ch; - border-radius: .5ch; -} -section nav.tabs a.selected { - background-color: #eee; -} - -.toggle-switch-off { - border-left: 5px solid #f00; - width: 8ch; -} -.toggle-switch-on { - border-right: 5px solid #080; - width: 8ch; -} -.toggle-switch-active { - border-left: 5px solid #00f; - border-right: 5px solid #00f; - width: 8ch; -} -#services .missing { - color: #800; -} -.uuidPip { - margin-left: 0.25ch; - border-radius: 2ch; - border: 1px solid #ccc; -} -h1 { - font-family: serif; -} -table { - font-family: monospace; -} -th { text-align: left; } -td,th { padding-right: 1ch; border: 1px solid #ccc; } - -tr:nth-of-type(2n) { - background-color: #eee; -} - - -table.table-rh-2 tr:nth-of-type(4n+1) { background-color: #eee; } -table.table-rh-2 tr:nth-of-type(4n+2) { background-color: #eee; } -table.table-rh-2 tr:nth-of-type(4n+3) { background-color: unset; } -table.table-rh-2 tr:nth-of-type(4n) { background-color: unset; } - -table.table-rh-2 tr:nth-of-type(4n) td, -table.table-rh-2 tr:nth-of-type(4n) th { border-bottom: 1px solid #888; } -table.table-rh-2 tr:nth-of-type(4n+2) td, -table.table-rh-2 tr:nth-of-type(4n+2) th { border-bottom: 1px solid #888; } - -table.table-rh-3 tr:nth-of-type(6n+1) { background-color: #eee; } -table.table-rh-3 tr:nth-of-type(6n+2) { background-color: #eee; } -table.table-rh-3 tr:nth-of-type(6n+3) { background-color: #eee; } -table.table-rh-3 tr:nth-of-type(6n+4) { background-color: unset; } -table.table-rh-3 tr:nth-of-type(6n+5) { background-color: unset; } -table.table-rh-3 tr:nth-of-type(6n) { background-color: unset; } - -table.table-rh-3 tr:nth-of-type(6n) td, -table.table-rh-3 tr:nth-of-type(6n) th { border-bottom: 1px solid #888; } -table.table-rh-3 tr:nth-of-type(6n+3) td, -table.table-rh-3 tr:nth-of-type(6n+3) th { border-bottom: 1px solid #888; } - -body > nav { - grid-area: left; -} -nav ul { - list-style-type: none; - padding: 0; -} -nav ul li { - line-height: 2; -} -nav ul li a { - text-decoration: none; - padding: 0.5ch; - display: block; - color: #000; - background-color: #ccc; - border: 2px #ccc solid; -} -nav ul li a:focus { - text-decoration: underline; - border: 2px #eee inset; -} - -nav ul li a:hover { - border: 2px #eee outset; -} - -nav ul li a.current { - color: #000; - background-color: #fff; -} - -body > section { - grid-area: right; -} - -#state-graph .current-state td:first-of-type { - border-right: 1em solid #000; - font-weight: bold; - border-color: #000; -} \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/static/control/tables.css b/code/services-core/control-service/src/main/resources/static/control/tables.css new file mode 100644 index 00000000..68e87e23 --- /dev/null +++ b/code/services-core/control-service/src/main/resources/static/control/tables.css @@ -0,0 +1,21 @@ +table.table-rh-2 tr:nth-of-type(4n+1) { background-color: #eee; } +table.table-rh-2 tr:nth-of-type(4n+2) { background-color: #eee; } +table.table-rh-2 tr:nth-of-type(4n+3) { background-color: unset; } +table.table-rh-2 tr:nth-of-type(4n) { background-color: unset; } + +table.table-rh-2 tr:nth-of-type(4n) td, +table.table-rh-2 tr:nth-of-type(4n) th { border-bottom: 1px solid #888; } +table.table-rh-2 tr:nth-of-type(4n+2) td, +table.table-rh-2 tr:nth-of-type(4n+2) th { border-bottom: 1px solid #888; } + +table.table-rh-3 tr:nth-of-type(6n+1) { background-color: #eee; } +table.table-rh-3 tr:nth-of-type(6n+2) { background-color: #eee; } +table.table-rh-3 tr:nth-of-type(6n+3) { background-color: #eee; } +table.table-rh-3 tr:nth-of-type(6n+4) { background-color: unset; } +table.table-rh-3 tr:nth-of-type(6n+5) { background-color: unset; } +table.table-rh-3 tr:nth-of-type(6n) { background-color: unset; } + +table.table-rh-3 tr:nth-of-type(6n) td, +table.table-rh-3 tr:nth-of-type(6n) th { border-bottom: 1px solid #888; } +table.table-rh-3 tr:nth-of-type(6n+3) td, +table.table-rh-3 tr:nth-of-type(6n+3) th { border-bottom: 1px solid #888; } \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/actions.hdb b/code/services-core/control-service/src/main/resources/templates/control/actions.hdb index b372b304..af216ae5 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/actions.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/actions.hdb @@ -9,128 +9,7 @@ {{> control/partials/nav}}

Actions

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ActionTrigger
Trigger Adjacency Calculation

- This will trigger a recalculation of website similarities, which affects - the rankings calculations. -

-
- -
-
Sideload Encyclopedia

- This will load pre-digested encyclopedia data - from a encyclopedia.marginalia.nu-style database. -

-
-
- -

- -
-
Sideload Dirtree

- This will load HTML from a directory structure as specified - by a YAML file. -

-
-
- -

- - -
-
Sideload Stackexchange

- Will load a set of pre-converted stackexchange .db files. -

-
-
- -

- - -
-
- Reload Blogs List -

This will reload the list of blogs from its source.

- 		-
- -
-
Repartition Index

- This will recalculate the rankings and search sets for the index. -

-
- -
-
Flush api-service Caches

- This will instruct the api-service to flush its caches, - getting rid of any stale data. This will be necessary after - changes to the API licenses directly through the database. -

-
- -
-
Trigger Data Exports

- This exports the data from the database into a set of CSV files -

-
- -
-
- WARNING -- Destructive Actions Below This Line -
Truncate Links Database.

- This will drop all known URLs and domain links.
- This action is not reversible. -

-
-
- -

- -
-
\ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/actors.hdb b/code/services-core/control-service/src/main/resources/templates/control/actors.hdb deleted file mode 100644 index 0e6a5672..00000000 --- a/code/services-core/control-service/src/main/resources/templates/control/actors.hdb +++ /dev/null @@ -1,21 +0,0 @@ - - - - Control Service - - - - - {{> control/partials/nav}} -
- {{> control/partials/processes-table}} - {{> control/partials/actors-table}} -
- - - - \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb b/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb deleted file mode 100644 index e58b6b8a..00000000 --- a/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb +++ /dev/null @@ -1,61 +0,0 @@ - - - - Control Service - - - - -{{> control/partials/nav}} -
- -

API Keys

- - - - - - - - - - - - {{#each apikeys}} - - - - - - - - - - - {{/each}} -
Key 
LicenseNameContactRate
{{licenseKey}} -
- -
-
{{license}}{{name}}{{email}}{{rate}}
-

Add New

-
-
-
-
-
-
-
-
-

- -
-
- - - - \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/api-keys.hdb b/code/services-core/control-service/src/main/resources/templates/control/app/api-keys.hdb new file mode 100644 index 00000000..92f834c2 --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/app/api-keys.hdb @@ -0,0 +1,65 @@ + + + + Control Service + {{> control/partials/head-includes }} + + +{{> control/partials/nav}} +
+

API Keys

+ + + + + + + + + + + + {{#each apikeys}} + + + + + + + + + + + {{/each}} +
Key 
LicenseNameContactRate
{{licenseKey}} +
+ +
+
{{license}}{{name}}{{email}}{{rate}}
+ +
+

Add New

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+ +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/blacklist.hdb b/code/services-core/control-service/src/main/resources/templates/control/app/blacklist.hdb new file mode 100644 index 00000000..d3bf3087 --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/app/blacklist.hdb @@ -0,0 +1,66 @@ + + + + Control Service + {{> control/partials/head-includes }} + + +{{> control/partials/nav}} +
+

Blacklist

+ +

+ The blacklist is a list of sanctioned domains that will not be + crawled, indexed, or returned from the search results. +

+ +
+
+
+

Add Sanction

+
+
+ + +
+
+ + +
+ +
+
+
+

Remove Sanction

+
+
+ + +
+ +
+
+
+
+ + +
+

Recent Additions

+ + + + + + {{#each blacklist}} + + + + + {{/each}} +
DomainComment
{{domain}}{{comment}}
+
+
+ + +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/app/domain-complaints.hdb b/code/services-core/control-service/src/main/resources/templates/control/app/domain-complaints.hdb new file mode 100644 index 00000000..038a65a3 --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/app/domain-complaints.hdb @@ -0,0 +1,130 @@ + + + + Control Service + {{> control/partials/head-includes }} + + +{{> control/partials/nav}} +
+

Domain Complaints

+ {{#unless complaintsNew}} +

No new complaints!

+ {{/unless}} + {{#if complaintsNew}} + + + + + + + + + + + + + + + {{#each complaintsNew}} + + + + + + + + + + + + + + {{/each}} +
DateCategory
DomainSample
Description
{{fileDate}}{{category}} +
+ + + +
+ + + + + + + + + + + + + + + + + + + + +
{{domain}}{{sample}}
{{description}}
+ {{/if}} + + {{#if complaintsReviewed}} +

Review Log

+ + + + + + + + + + + + + + + {{#each complaintsReviewed}} + + + + + + + + + + + + + {{/each}} +
Review DateCategoryAction
DomainSample
Description
{{fileDate}}{{category}} + {{decision}} +
{{domain}}{{sample}}
{{description}}
+ {{/if}} +
+ + +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/review-random-domains.hdb b/code/services-core/control-service/src/main/resources/templates/control/app/review-random-domains.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/review-random-domains.hdb rename to code/services-core/control-service/src/main/resources/templates/control/app/review-random-domains.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/search-to-ban.hdb b/code/services-core/control-service/src/main/resources/templates/control/app/search-to-ban.hdb similarity index 100% rename from code/services-core/control-service/src/main/resources/templates/control/search-to-ban.hdb rename to code/services-core/control-service/src/main/resources/templates/control/app/search-to-ban.hdb diff --git a/code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb b/code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb deleted file mode 100644 index 5622659c..00000000 --- a/code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb +++ /dev/null @@ -1,68 +0,0 @@ - - - - Control Service - - - - -{{> control/partials/nav}} -
-

Blacklist

- -

- The blacklist is a list of sanctioned domains that will not be - crawled, indexed, or returned from the search results. -

- - - - - - - - - - - - - -
DescriptionAction
Add To Blacklist

- This will add the given domain to the blacklist. -

-
-  
-   -
-
- -
-
Remove from blacklist

- Remove the specified domain from the blacklist. This will ensure that - the domain is not blacklisted, in doing so it may remove the root domain - from the blacklist as well. -

-
-   -
-
- -
-
- -

Recent Additions

- - - - - - {{#each blacklist}} - - - - - {{/each}} -
DomainComment
{{domain}}{{comment}}
-
- - \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb b/code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb deleted file mode 100644 index ac1f6c88..00000000 --- a/code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb +++ /dev/null @@ -1,111 +0,0 @@ - - - - Control Service - - - - -{{> control/partials/nav}} -
- -

Domain Complaints

- {{#unless complaintsNew}} -

No new complaints!

- {{/unless}} - {{#if complaintsNew}} - - - - - - - - - - - - - - - {{#each complaintsNew}} - - - - - - - - - - - - - - {{/each}} -
DateCategory
DomainSample
Description
{{fileDate}}{{category}} -
- - - -
-
{{domain}}{{sample}}
{{description}}
- {{/if}} - - {{#if complaintsReviewed}} -

Review Log

- - - - - - - - - - - - - - - {{#each complaintsReviewed}} - - - - - - - - - - - - - {{/each}} -
Review DateCategoryAction
DomainSample
Description
{{fileDate}}{{category}} - {{decision}} -
{{domain}}{{sample}}
{{description}}
- {{/if}} -
- - - - \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/events.hdb b/code/services-core/control-service/src/main/resources/templates/control/events.hdb deleted file mode 100644 index 6af83bf5..00000000 --- a/code/services-core/control-service/src/main/resources/templates/control/events.hdb +++ /dev/null @@ -1,20 +0,0 @@ - - - - Control Service - - - - - {{> control/partials/nav}} -
- {{> control/partials/events-table}} -
- - - - \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/index.hdb b/code/services-core/control-service/src/main/resources/templates/control/index.hdb index 95642fa6..81358481 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/index.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/index.hdb @@ -4,19 +4,22 @@ Control Service + {{> control/partials/head-includes }} {{> control/partials/nav}} -
- {{> control/partials/services-table }} +
+

Overview

+ {{> control/partials/nodes-table }} {{> control/partials/processes-table}} + {{> control/partials/services-table }} {{> control/partials/events-table-summary }} -
+ - +{{> control/partials/foot-includes }} diff --git a/code/services-core/control-service/src/main/resources/templates/control/message-queue.hdb b/code/services-core/control-service/src/main/resources/templates/control/message-queue.hdb deleted file mode 100644 index cc5b5da9..00000000 --- a/code/services-core/control-service/src/main/resources/templates/control/message-queue.hdb +++ /dev/null @@ -1,20 +0,0 @@ - - - - Control Service - - - - - {{> control/partials/nav}} -
- {{> control/partials/message-queue-table }} -
- - - - \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb new file mode 100644 index 00000000..807078f7 --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb @@ -0,0 +1,232 @@ + + +{{> control/partials/head-includes }} +Control Service: Node {{node.id}} + +{{> control/partials/nav}} + +
+

Index Node {{node.id}}

+ + + +
+
+

+ +

+
+ + This will load pre-digested encyclopedia data from an encyclopedia.marginalia.nu-style database. + +
+
+ + +
+
+ +
+
+ +
+
+
+
+
+
+ +
+

+ +

+
+ This will load a set of pre-converted stackexchange .db files + +
+
+ + +
+
+ +
+
+ +
+
+
+
+
+
+ +
+

+ +

+
+ This will load HTML from a directory structure as specified by a YAML file. + +
+
+ + +
+
+ +
+
+ +
+
+
+
+
+
+ +
+

+ +

+
+

This will recalculate the rankings and search sets for the index

+ +
+ +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb new file mode 100644 index 00000000..2074bfef --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb @@ -0,0 +1,43 @@ + + +{{> control/partials/head-includes }} +Control Service: Node {{node.id}} + +{{> control/partials/nav}} + +
+

Index Node {{node.id}}

+ + + +
+ {{> control/partials/actors-table }} +
+ +
+ + +{{> control/partials/foot-includes }} + + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb new file mode 100644 index 00000000..cee517fb --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb @@ -0,0 +1,42 @@ + + +{{> control/partials/head-includes }} +Control Service: Node {{node.id}} + +{{> control/partials/nav}} + +