161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(file-storage) Delete unused code

Browse Source
This commit is contained in:
Viktor Lofgren 2023-10-15 22:25:08 +02:00
parent c245f7ce3a
commit 0b19b28a64
6 changed files with 61 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java
View File

@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.PosixFilePermissions;
@ -30,46 +29,6 @@ public class FileStorageService {
private static final DateTimeFormatter dirNameDatePattern = DateTimeFormatter.ofPattern("__uu-MM-dd'T'HH_mm_ss.SSS"); // filesystem safe ISO8601
public Optional<FileStorage> findFileStorageToDelete() {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT FILE_STORAGE.ID FROM FILE_STORAGE
INNER JOIN FILE_STORAGE_BASE ON BASE_ID=FILE_STORAGE_BASE.ID
WHERE STATE='DELETE'
AND NODE = ?
LIMIT 1
""")) {
stmt.setInt(1, node);
var rs = stmt.executeQuery();
if (rs.next()) {
return Optional.of(getStorage(new FileStorageId(rs.getLong(1))));
}
} catch (SQLException e) {
return Optional.empty();
}
return Optional.empty();
}
public Set<Integer> getConfiguredNodes() {
Set<Integer> ret = new HashSet<>();
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT DISTINCT(NODE) FROM FILE_STORAGE_BASE
""")) {
var rs = stmt.executeQuery();
while (rs.next()) {
ret.add(rs.getInt(1));
}
} catch (SQLException e) {
logger.warn("SQL error getting nodes", e);
}
return ret;
}
@Inject
public FileStorageService(HikariDataSource dataSource, @Named("wmsa-system-node") Integer node) {
this.dataSource = dataSource;
@ -87,13 +46,13 @@ public class FileStorageService {
}
/** @return the storage base with the given id, or null if it does not exist */
public FileStorageBase getStorageBase(FileStorageBaseId type) throws SQLException {
public FileStorageBase getStorageBase(FileStorageBaseId id) throws SQLException {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT ID, NAME, PATH, TYPE
FROM FILE_STORAGE_BASE WHERE ID = ?
""")) {
stmt.setLong(1, type.id());
stmt.setLong(1, id.id());
try (var rs = stmt.executeQuery()) {
if (rs.next()) {
return new FileStorageBase(
@ -187,21 +146,6 @@ public class FileStorageService {
}
}
public List<FileStorage> getTargetFromStorage(FileStorage storage) throws SQLException {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT TARGET_ID FROM FILE_STORAGE_RELATION WHERE SOURCE_ID = ?
""")) {
stmt.setLong(1, storage.id().id());
var rs = stmt.executeQuery();
List<FileStorage> ret = new ArrayList<>();
while (rs.next()) {
ret.add(getStorage(new FileStorageId(rs.getLong(1))));
}
return ret;
}
}
/** @return the storage base with the given type, or null if it does not exist */
public FileStorageBase getStorageBase(FileStorageBaseType type) throws SQLException {
return getStorageBase(type, node);
@ -228,11 +172,12 @@ public class FileStorageService {
}
return null;
}
public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type) throws SQLException, FileNotFoundException {
public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type) throws SQLException {
return createStorageBase(name, path, node, type);
}
public FileStorageBase createStorageBase(String name, Path path, int node, FileStorageBaseType type) throws SQLException, FileNotFoundException {
public FileStorageBase createStorageBase(String name, Path path, int node, FileStorageBaseType type) throws SQLException {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
@ -252,6 +197,7 @@ public class FileStorageService {
return getStorageBase(type);
}
@SneakyThrows
private Path allocateDirectory(Path basePath, String prefix) throws IOException {
LocalDateTime now = LocalDateTime.now();
@ -275,12 +221,15 @@ public class FileStorageService {
return maybePath;
}
/** Allocate a temporary storage of the given type if temporary allocation is permitted */
/** Allocate a temporary storage of the given type */
public FileStorage allocateTemporaryStorage(FileStorageBase base,
FileStorageType type,
String prefix,
String description) throws IOException, SQLException
{
if (!base.type().permitsStorageType(type))
throw new RuntimeException("Attempting to allocate storage of type " + type + " in base of type " + base.type());
Path newDir = allocateDirectory(base.asPath(), prefix);
String relDir = base.asPath().relativize(newDir).normalize().toString();
@ -325,60 +274,6 @@ public class FileStorageService {
}
/** Allocate permanent storage in base */
public FileStorage allocatePermanentStorage(FileStorageBase base,
String relativePath,
FileStorageType type,
String description) throws IOException, SQLException
{
Path newDir = base.asPath().resolve(relativePath);
if (Files.exists(newDir)) {
throw new IllegalArgumentException("Storage already exists: " + newDir);
}
Files.createDirectory(newDir, PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x")));
try (var conn = dataSource.getConnection();
var update = conn.prepareStatement("""
INSERT INTO FILE_STORAGE(PATH, TYPE, DESCRIPTION, BASE_ID)
VALUES (?, ?, ?, ?)
""");
var query = conn.prepareStatement("""
SELECT ID
FROM FILE_STORAGE WHERE PATH = ? AND BASE_ID = ?
""")
) {
update.setString(1, relativePath);
update.setString(2, type.name());
update.setString(3, description);
update.setLong(4, base.id().id());
if (update.executeUpdate() < 1)
throw new SQLException("Failed to insert storage");
query.setString(1, relativePath);
query.setLong(2, base.id().id());
var rs = query.executeQuery();
if (rs.next()) {
return new FileStorage(
new FileStorageId(rs.getLong("ID")),
base,
type,
LocalDateTime.now(),
newDir.toString(),
"",
description
);
}
}
throw new SQLException("Failed to insert storage");
}
public FileStorage getStorageByType(FileStorageType type) throws SQLException {
String override = System.getProperty(type.overrideName());
@ -492,7 +387,7 @@ public class FileStorageService {
}
}
public void removeFileStorage(FileStorageId id) throws SQLException {
public void deregisterFileStorage(FileStorageId id) throws SQLException {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
DELETE FROM FILE_STORAGE WHERE ID = ?

1
code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java
View File

@ -21,4 +21,5 @@ public record FileStorageBase(FileStorageBaseId id,
public boolean isValid() {
return id.id() >= 0;
}
}

10
code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java
View File

@ -1,12 +1,18 @@
package nu.marginalia.storage.model;
import java.util.EnumSet;
public enum FileStorageBaseType {
CURRENT,
WORK,
STORAGE,
BACKUP;
public String overrideName() {
return "FS_BASE_OVERRIDE:"+name();
public boolean permitsStorageType(FileStorageType type) {
return switch (this) {
case BACKUP -> FileStorageType.BACKUP.equals(type);
case STORAGE -> EnumSet.of(FileStorageType.CRAWL_DATA, FileStorageType.PROCESSED_DATA, FileStorageType.CRAWL_SPEC).contains(type);
default -> false;
};
}
}

19
code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java
View File

@ -133,7 +133,7 @@ public class FileStorageServiceTest {
Assertions.assertEquals(Path.of("/tmp"), fileStorageService.getStorageByType(FileStorageType.BACKUP).asPath());
}
@Test
public void testCreateBase() throws SQLException, FileNotFoundException {
public void testCreateBase() throws SQLException {
String name = "test-" + UUID.randomUUID();
var storage = new FileStorageService(dataSource, 0);
@ -143,28 +143,13 @@ public class FileStorageServiceTest {
Assertions.assertEquals(FileStorageBaseType.WORK, base.type());
}
@Test
public void testAllocatePermanent() throws SQLException, IOException {
String name = "test-" + UUID.randomUUID();
var storage = new FileStorageService(dataSource, 0);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.WORK);
var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed");
tempDirs.add(created.asPath());
var actual = storage.getStorage(created.id());
Assertions.assertEquals(created, actual);
}
@Test
public void testAllocateTemp() throws IOException, SQLException {
String name = "test-" + UUID.randomUUID();
var storage = new FileStorageService(dataSource, 0);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.WORK);
var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.STORAGE);
var fileStorage = storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed");
System.out.println("Allocated " + fileStorage.asPath());
Assertions.assertTrue(Files.exists(fileStorage.asPath()));

42
code/features-control/actors/src/main/java/nu/marginalia/actor/monitor/FileStorageMonitorActor.java
View File

@ -2,10 +2,12 @@ package nu.marginalia.actor.monitor;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.actor.ActorStateFactory;
import nu.marginalia.actor.prototype.AbstractActorPrototype;
import nu.marginalia.actor.state.ActorResumeBehavior;
import nu.marginalia.actor.state.ActorState;
import nu.marginalia.service.module.ServiceConfiguration;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorage;
import nu.marginalia.storage.model.FileStorageBaseType;
@ -32,7 +34,10 @@ public class FileStorageMonitorActor extends AbstractActorPrototype {
private static final String PURGE = "PURGE";
private static final String REMOVE_STALE = "REMOVE-STALE";
private static final String END = "END";
private final HikariDataSource dataSource;
private final FileStorageService fileStorageService;
private final int node;
@Override
public String describe() {
@ -42,9 +47,13 @@ public class FileStorageMonitorActor extends AbstractActorPrototype {
@Inject
public FileStorageMonitorActor(ActorStateFactory stateFactory,
HikariDataSource dataSource,
ServiceConfiguration serviceConfiguration,
FileStorageService fileStorageService) {
super(stateFactory);
this.dataSource = dataSource;
this.fileStorageService = fileStorageService;
this.node = serviceConfiguration.node();
}
@ActorState(name = INITIAL, next = MONITOR)
@ -62,7 +71,7 @@ public class FileStorageMonitorActor extends AbstractActorPrototype {
public void monitor() throws Exception {
for (;;) {
Optional<FileStorage> toDeleteOpt = fileStorageService.findFileStorageToDelete();
Optional<FileStorage> toDeleteOpt = findFileStorageToDelete();
if (toDeleteOpt.isPresent()) {
transition(PURGE, toDeleteOpt.get().id());
@ -96,7 +105,7 @@ public class FileStorageMonitorActor extends AbstractActorPrototype {
FileUtils.deleteDirectory(path.toFile());
}
fileStorageService.removeFileStorage(storage.id());
fileStorageService.deregisterFileStorage(storage.id());
}
@ActorState(
@ -108,6 +117,33 @@ public class FileStorageMonitorActor extends AbstractActorPrototype {
"""
)
public void removeStale(FileStorageId id) throws SQLException {
fileStorageService.removeFileStorage(id);
fileStorageService.deregisterFileStorage(id);
}
public Optional<FileStorage> findFileStorageToDelete() {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT FILE_STORAGE.ID FROM FILE_STORAGE
INNER JOIN FILE_STORAGE_BASE ON BASE_ID=FILE_STORAGE_BASE.ID
WHERE STATE='DELETE'
AND NODE = ?
LIMIT 1
""")) {
stmt.setInt(1, node);
var rs = stmt.executeQuery();
if (rs.next()) {
var id = new FileStorageId(rs.getLong("ID"));
return Optional.of(fileStorageService.getStorage(id));
}
} catch (SQLException e) {
logger.warn("SQL error", e);
}
return Optional.empty();
}
}

1
code/features-control/actors/src/main/java/nu/marginalia/actor/task/ExportDataActor.java
View File

@ -69,7 +69,6 @@ public class ExportDataActor extends AbstractActorPrototype {
Find EXPORT storage area, then transition to EXPORT-BLACKLIST.
""")
public Message init(Integer i) throws Exception {
var storage = storageService.getStorageByType(FileStorageType.EXPORT);
if (storage == null) error("Bad storage id");