161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(control) Wean the ExportDataActor off EC_DOMAIN_LINK

Browse Source 
The EC_DOMAIN_LINK table is deprecated and slated for removal, use QueryClient.getAllDomainLinks() instead.

The ExportDataActor now uses the QueryClient appropriately.  The CSV format was also changed to quote the values, to prevent e.g. Excel from interpreting the comma as a decimal separator when previewing the file.

Finally the form for triggering an export was overhauled.
This commit is contained in:
Viktor Lofgren 2024-01-12 17:09:11 +01:00
parent 98c0972619
commit 83776a8dce
2 changed files with 22 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-data.hdb
View File

@ -1,6 +1,9 @@
<h1 class="my-3">Export Data</h1>
This will export database data: Domains, blacklist and domain links.
<div class="my-3 p-3 border bg-light">
This will export database data: Domains, blacklist and domain links. The exported data will be saved as
a new <a href="/nodes/{{node.id}}/storage/exports">exports storage object</a>.
</div>
<form method="post" action="actions/export-data" onsubmit="return confirm('Confirm export')">
<div class="my-3 py-3">

30
code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java
View File

@ -6,6 +6,7 @@ import com.google.inject.Singleton;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.actor.prototype.RecordActorPrototype;
import nu.marginalia.actor.state.ActorStep;
import nu.marginalia.query.client.QueryClient;
import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorageBaseType;
import nu.marginalia.storage.model.FileStorageId;
@ -32,6 +33,7 @@ public class ExportDataActor extends RecordActorPrototype {
private final FileStorageService storageService;
private final HikariDataSource dataSource;
private final Logger logger = LoggerFactory.getLogger(getClass());
private final QueryClient queryClient;
public record Export() implements ActorStep {}
public record ExportBlacklist(FileStorageId fid) implements ActorStep {}
@ -114,19 +116,22 @@ public class ExportDataActor extends RecordActorPrototype {
var tmpFile = Files.createTempFile(storage.asPath(), "export", ".csv.gz",
PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))));
var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("SELECT SOURCE_DOMAIN_ID, DEST_DOMAIN_ID FROM EC_DOMAIN_LINK");
)
var allLinks = queryClient.getAllDomainLinks();
try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))))
{
stmt.setFetchSize(1000);
var rs = stmt.executeQuery();
while (rs.next()) {
bw.write(rs.getString("SOURCE_DOMAIN_ID"));
bw.write(",");
bw.write(rs.getString("DEST_DOMAIN_ID"));
bw.write("\n");
var iter = allLinks.iterator();
while (iter.advance()) {
bw.write('"');
bw.write(Integer.toString(iter.source()));
bw.write('"');
bw.write(',');
bw.write('"');
bw.write(Integer.toString(iter.dest()));
bw.write('"');
bw.write('\n');
}
Files.move(tmpFile, storage.asPath().resolve(linkGraphFilename), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
}
catch (Exception ex) {
@ -151,11 +156,12 @@ public class ExportDataActor extends RecordActorPrototype {
@Inject
public ExportDataActor(Gson gson,
FileStorageService storageService,
HikariDataSource dataSource)
HikariDataSource dataSource, QueryClient queryClient)
{
super(gson);
this.storageService = storageService;
this.dataSource = dataSource;
this.queryClient = queryClient;
}
}