161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(control/exec) File listings in control GUI

Browse Source
This commit is contained in:
Viktor Lofgren 2023-10-15 19:15:44 +02:00
parent 16e0738731
commit 9a38a455c9
5 changed files with 83 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java
View File

@ -6,6 +6,7 @@ import nu.marginalia.client.AbstractDynamicClient;
import nu.marginalia.client.Context;
import nu.marginalia.client.route.RouteProvider;
import nu.marginalia.client.route.ServiceRoute;
import nu.marginalia.executor.storage.FileStorageContent;
import nu.marginalia.service.descriptor.ServiceDescriptor;
import nu.marginalia.storage.model.FileStorageId;
import nu.marginalia.executor.model.ActorRunStates;
@ -69,7 +70,7 @@ public class ExecutorClient extends AbstractDynamicClient {
public void exportData(Context ctx) {
// post(ctx, node, "/process/adjacency-calculation/", "").blockingSubscribe();
// FIXME
// FIXME this shouldn't be done in the executor
}
public void sideloadEncyclopedia(Context ctx, int node, Path sourcePath) {
@ -108,4 +109,9 @@ public class ExecutorClient extends AbstractDynamicClient {
public ActorRunStates getActorStates(Context context, int node) {
return get(context, node, "/actor", ActorRunStates.class).blockingFirst();
}
public FileStorageContent listFileStorage(Context context, int node, FileStorageId fileId) {
return get(context, node, "/storage/"+fileId.id(), FileStorageContent.class).blockingFirst();
}
}

7
code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java Normal file
View File

@ -0,0 +1,7 @@
package nu.marginalia.executor.storage;
import java.util.List;
public record FileStorageContent(List<FileStorageFile> files)
{
}

5
code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java Normal file
View File

@ -0,0 +1,5 @@
package nu.marginalia.executor.storage;
public record FileStorageFile(String name, long size, String modTime) {
}

43
code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java
View File

@ -303,8 +303,8 @@ public class ControlNodeService {
private Object nodeStorageDetailsModel(Request request, Response response) throws SQLException {
int nodeId = Integer.parseInt(request.params("id"));
var storage = getFileStorageWithRelatedEntries(Context.fromRequest(request), nodeId, FileStorageId.parse(request.queryParams("fid")));
var storage = getFileStorageWithRelatedEntries(FileStorageId.parse(request.queryParams("fid")));
String view = switch(storage.type()) {
case BACKUP -> "backup";
case CRAWL_DATA -> "crawl";
@ -460,48 +460,37 @@ public class ControlNodeService {
}
public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(FileStorageId id) throws SQLException {
var storage = fileStorageService.getStorage(id);
var related = getRelatedEntries(id);
public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(
Context context,
int node,
FileStorageId fileId
) throws SQLException {
var storage = fileStorageService.getStorage(fileId);
var related = getRelatedEntries(fileId);
List<FileStorageFileModel> files = new ArrayList<>();
try (var filesStream = Files.list(storage.asPath())) {
filesStream
.filter(Files::isRegularFile)
.map(this::createFileModel)
.sorted(Comparator.comparing(FileStorageFileModel::filename))
.forEach(files::add);
}
catch (IOException ex) {
logger.error("Failed to list files in storage", ex);
for (var execFile : executorClient.listFileStorage(context, node, fileId).files()) {
files.add(new FileStorageFileModel(
execFile.name(),
execFile.modTime(),
sizeString(execFile.size())
));
}
return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related, files);
}
private FileStorageFileModel createFileModel(Path p) {
try {
String mTime = Files.getLastModifiedTime(p).toInstant().toString();
private String sizeString(long sizeBytes) {
String size;
if (Files.isDirectory(p)) {
size = "-";
}
else {
long sizeBytes = Files.size(p);
if (sizeBytes < 1024) size = sizeBytes + " B";
else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB";
else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB";
else size = sizeBytes / (1024 * 1024 * 1024) + " GB";
return size;
}
return new FileStorageFileModel(p.toFile().getName(), mTime, size);
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
}
private List<FileStorage> getRelatedEntries(FileStorageId id) {
List<FileStorage> ret = new ArrayList<>();
try (var conn = dataSource.getConnection();

42
code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java
View File

@ -2,22 +2,34 @@ package nu.marginalia.executor;
import com.google.gson.Gson;
import com.google.inject.Inject;
import lombok.SneakyThrows;
import nu.marginalia.actor.ActorApi;
import nu.marginalia.actor.ActorControlService;
import nu.marginalia.actor.state.ActorState;
import nu.marginalia.actor.state.ActorStateInstance;
import nu.marginalia.executor.model.ActorRunState;
import nu.marginalia.executor.model.ActorRunStates;
import nu.marginalia.executor.storage.FileStorageContent;
import nu.marginalia.executor.storage.FileStorageFile;
import nu.marginalia.executor.svc.BackupService;
import nu.marginalia.executor.svc.ProcessingService;
import nu.marginalia.executor.svc.SideloadService;
import nu.marginalia.service.server.BaseServiceParams;
import nu.marginalia.service.server.Service;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorageId;
import spark.Request;
import spark.Response;
import spark.Spark;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.SQLException;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
@ -25,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class ExecutorSvc extends Service {
private final BaseServiceParams params;
private final ActorControlService actorControlService;
private final FileStorageService fileStorageService;
@Inject
public ExecutorSvc(BaseServiceParams params,
@ -32,11 +45,13 @@ public class ExecutorSvc extends Service {
ProcessingService processingService,
SideloadService sideloadService,
BackupService backupService,
FileStorageService fileStorageService,
Gson gson,
ActorApi actorApi) {
super(params);
this.params = params;
this.actorControlService = actorControlService;
this.fileStorageService = fileStorageService;
Spark.post("/actor/:id/start", actorApi::startActor);
Spark.post("/actor/:id/start/:state", actorApi::startActorFromState);
@ -57,9 +72,36 @@ public class ExecutorSvc extends Service {
Spark.post("/sideload/encyclopedia", sideloadService::sideloadEncyclopedia);
Spark.post("/backup/:fid/restore", backupService::restore);
Spark.get("/storage/:fid", this::listFiles, gson::toJson);
}
private FileStorageContent listFiles(Request request, Response response) throws SQLException, IOException {
FileStorageId fileStorageId = FileStorageId.parse(request.params("fid"));
var storage = fileStorageService.getStorage(fileStorageId);
List<FileStorageFile> files;
try (var fs = Files.list(storage.asPath())) {
files = fs.filter(Files::isRegularFile)
.map(this::createFileModel)
.sorted(Comparator.comparing(FileStorageFile::name))
.toList();
}
return new FileStorageContent(files);
}
@SneakyThrows
private FileStorageFile createFileModel(Path path) {
return new FileStorageFile(
path.toFile().getName(),
Files.size(path),
Files.getLastModifiedTime(path).toInstant().toString()
);
}
private final ConcurrentHashMap<String, String> actorStateDescriptions = new ConcurrentHashMap<>();