161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(control) Filterable event log view

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-12 14:43:11 +02:00
parent 0961f627b1
commit 998f239ed9
7 changed files with 281 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java
View File

@ -4,7 +4,7 @@ import com.google.gson.Gson;
import com.google.inject.Inject;
import nu.marginalia.client.ServiceMonitors;
import nu.marginalia.control.actor.Actor;
import nu.marginalia.control.model.DomainComplaintModel;
import nu.marginalia.control.model.*;
import nu.marginalia.control.svc.*;
import nu.marginalia.db.storage.model.FileStorageId;
import nu.marginalia.db.storage.model.FileStorageType;
@ -21,10 +21,7 @@ import spark.Spark;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
public class ControlService extends Service {
@ -69,6 +66,7 @@ public class ControlService extends Service {
this.blacklistService = blacklistService;
var indexRenderer = rendererFactory.renderer("control/index");
var eventsRenderer = rendererFactory.renderer("control/events");
var servicesRenderer = rendererFactory.renderer("control/services");
var serviceByIdRenderer = rendererFactory.renderer("control/service-by-id");
var actorsRenderer = rendererFactory.renderer("control/actors");
@ -105,6 +103,7 @@ public class ControlService extends Service {
Spark.get("/public/", this::overviewModel, indexRenderer::render);
Spark.get("/public/actions", (rq,rsp) -> new Object() , actionsViewRenderer::render);
Spark.get("/public/events", eventLogService::eventsListModel , eventsRenderer::render);
Spark.get("/public/services", this::servicesModel, servicesRenderer::render);
Spark.get("/public/services/:id", this::serviceModel, serviceByIdRenderer::render);
Spark.get("/public/actors", this::processesModel, actorsRenderer::render);
@ -182,6 +181,7 @@ public class ControlService extends Service {
monitors.subscribe(this::logMonitorStateChange);
}
private Object blacklistModel(Request request, Response response) {
return Map.of("blacklist", blacklistService.lastNAdditions(100));
}
@ -204,7 +204,7 @@ public class ControlService extends Service {
"jobs", heartbeatService.getTaskHeartbeats(),
"actors", controlActorService.getActorStates(),
"services", heartbeatService.getServiceHeartbeats(),
"events", eventLogService.getLastEntries(20)
"events", eventLogService.getLastEntries(Long.MAX_VALUE, 20)
);
}
@ -292,7 +292,7 @@ public class ControlService extends Service {
return Map.of(
"id", serviceName,
"messages", messageQueueService.getEntriesForInbox(serviceName, Long.MAX_VALUE, 20),
"events", eventLogService.getLastEntriesForService(serviceName, 20));
"events", eventLogService.getLastEntriesForService(serviceName, Long.MAX_VALUE, 20));
}
private Object storageModel(Request request, Response response) {
@ -313,7 +313,7 @@ public class ControlService extends Service {
}
private Object servicesModel(Request request, Response response) {
return Map.of("services", heartbeatService.getServiceHeartbeats(),
"events", eventLogService.getLastEntries(20));
"events", eventLogService.getLastEntries(Long.MAX_VALUE, 20));
}
private Object processesModel(Request request, Response response) {

2
code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogEntry.java
View File

@ -1,9 +1,11 @@
package nu.marginalia.control.model;
public record EventLogEntry(
long id,
String serviceName,
String instanceFull,
String eventTime,
String eventDateTime,
String eventType,
String eventMessage)
{

9
code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogServiceFilter.java Normal file
View File

@ -0,0 +1,9 @@
package nu.marginalia.control.model;
public record EventLogServiceFilter(
String name,
String value,
boolean current
)
{
}

9
code/services-core/control-service/src/main/java/nu/marginalia/control/model/EventLogTypeFilter.java Normal file
View File

@ -0,0 +1,9 @@
package nu.marginalia.control.model;
public record EventLogTypeFilter(
String name,
String value,
boolean current
)
{
}

237
code/services-core/control-service/src/main/java/nu/marginalia/control/svc/EventLogService.java
View File

@ -4,10 +4,17 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.model.EventLogEntry;
import nu.marginalia.control.model.EventLogServiceFilter;
import nu.marginalia.control.model.EventLogTypeFilter;
import org.apache.logging.log4j.util.Strings;
import spark.Request;
import spark.Response;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Singleton
public class EventLogService {
@ -19,52 +26,163 @@ public class EventLogService {
this.dataSource = dataSource;
}
public List<EventLogEntry> getLastEntries(int n) {
try (var conn = dataSource.getConnection();
var query = conn.prepareStatement("""
SELECT SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
FROM SERVICE_EVENTLOG ORDER BY ID DESC LIMIT ?
""")) {
public Object eventsListModel(Request request, Response response) {
query.setInt(1, n);
List<EventLogEntry> entries = new ArrayList<>(n);
var rs = query.executeQuery();
while (rs.next()) {
entries.add(new EventLogEntry(
rs.getString("SERVICE_NAME"),
rs.getString("INSTANCE"),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toLocalTime().toString(),
rs.getString("EVENT_TYPE"),
rs.getString("EVENT_MESSAGE")
));
}
return entries;
String serviceParam = request.queryParams("service");
String typeParam = request.queryParams("type");
String afterParam = request.queryParams("after");
if (Strings.isBlank(serviceParam)) serviceParam = null;
if (Strings.isBlank(typeParam)) typeParam = null;
if (Strings.isBlank(afterParam)) afterParam = null;
long afterId = Optional.ofNullable(afterParam).map(Long::parseLong).orElse(Long.MAX_VALUE);
List<EventLogTypeFilter> typeFilterList = new ArrayList<>();
List<String> typenames = getTypeNames();
typeFilterList.add(new EventLogTypeFilter("Show All", "", typeParam == null));
for (String typename : typenames) {
typeFilterList.add(new EventLogTypeFilter(typename, typename,
typename.equalsIgnoreCase(typeParam)));
}
catch (SQLException ex) {
throw new RuntimeException(ex);
List<EventLogServiceFilter> serviceFilterList = new ArrayList<>();
List<String> serviceNames = getServiceNames();
serviceFilterList.add(new EventLogServiceFilter("Show All", "", serviceParam == null));
for (String serviceName : serviceNames) {
serviceFilterList.add(new EventLogServiceFilter(serviceName, serviceName,
serviceName.equalsIgnoreCase(serviceParam)));
}
List<EventLogEntry> entries;
String elFilter = "filter=none";
if (serviceParam != null && typeParam != null) {
elFilter = "service=" + serviceParam + "&type=" + typeParam;
entries = getLastEntriesForTypeAndService(typeParam, serviceParam, afterId, 20);
}
else if (serviceParam != null) {
elFilter = "service=" + serviceParam;
entries = getLastEntriesForService(serviceParam, afterId, 20);
}
else if (typeParam != null) {
elFilter = "type=" + typeParam;
entries = getLastEntriesForType(typeParam, afterId, 20);
}
else {
entries = getLastEntries(afterId, 20);
}
Object next;
if (entries.size() == 20)
next = entries.stream().mapToLong(EventLogEntry::id).min().getAsLong();
else
next = "";
Object prev = afterParam == null ? "" : afterParam;
return Map.of(
"events", entries,
"types", typeFilterList,
"services", serviceFilterList,
"next", next,
"prev", prev,
"elFilter", elFilter);
}
public List<EventLogEntry> getLastEntriesForService(String serviceName, int n) {
public List<EventLogEntry> getLastEntries(long afterId, int n) {
try (var conn = dataSource.getConnection();
var query = conn.prepareStatement("""
SELECT SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
SELECT ID, SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
FROM SERVICE_EVENTLOG
WHERE SERVICE_NAME = ?
WHERE ID < ?
ORDER BY ID DESC
LIMIT ?
""")) {
query.setString(1, serviceName);
query.setLong(1, afterId);
query.setInt(2, n);
List<EventLogEntry> entries = new ArrayList<>(n);
var rs = query.executeQuery();
while (rs.next()) {
entries.add(new EventLogEntry(
rs.getLong("ID"),
rs.getString("SERVICE_NAME"),
rs.getString("INSTANCE"),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toLocalTime().toString(),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toString(),
rs.getString("EVENT_TYPE"),
rs.getString("EVENT_MESSAGE")
));
}
return entries;
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
public List<EventLogEntry> getLastEntriesForService(String serviceName, long afterId, int n) {
try (var conn = dataSource.getConnection();
var query = conn.prepareStatement("""
SELECT ID, SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
FROM SERVICE_EVENTLOG
WHERE SERVICE_NAME = ?
AND ID < ?
ORDER BY ID DESC
LIMIT ?
""")) {
query.setString(1, serviceName);
query.setLong(2, afterId);
query.setInt(3, n);
List<EventLogEntry> entries = new ArrayList<>(n);
var rs = query.executeQuery();
while (rs.next()) {
entries.add(new EventLogEntry(
rs.getLong("ID"),
rs.getString("SERVICE_NAME"),
rs.getString("INSTANCE"),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toLocalTime().toString(),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toString(),
rs.getString("EVENT_TYPE"),
rs.getString("EVENT_MESSAGE")
));
}
return entries;
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
public List<EventLogEntry> getLastEntriesForTypeAndService(String typeName, String serviceName, long afterId, int n) {
try (var conn = dataSource.getConnection();
var query = conn.prepareStatement("""
SELECT ID, SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
FROM SERVICE_EVENTLOG
WHERE SERVICE_NAME = ? AND EVENT_TYPE=?
AND ID < ?
ORDER BY ID DESC
LIMIT ?
""")) {
query.setString(1, serviceName);
query.setString(2, typeName);
query.setLong(3, afterId);
query.setInt(4, n);
List<EventLogEntry> entries = new ArrayList<>(n);
var rs = query.executeQuery();
while (rs.next()) {
entries.add(new EventLogEntry(
rs.getLong("ID"),
rs.getString("SERVICE_NAME"),
rs.getString("INSTANCE"),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toLocalTime().toString(),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toString(),
rs.getString("EVENT_TYPE"),
rs.getString("EVENT_MESSAGE")
));
@ -77,10 +195,45 @@ public class EventLogService {
}
public List<EventLogEntry> getLastEntriesForType(String eventType, long afterId, int n) {
try (var conn = dataSource.getConnection();
var query = conn.prepareStatement("""
SELECT ID, SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
FROM SERVICE_EVENTLOG
WHERE EVENT_TYPE = ?
AND ID < ?
ORDER BY ID DESC
LIMIT ?
""")) {
query.setString(1, eventType);
query.setLong(2, afterId);
query.setInt(3, n);
List<EventLogEntry> entries = new ArrayList<>(n);
var rs = query.executeQuery();
while (rs.next()) {
entries.add(new EventLogEntry(
rs.getLong("ID"),
rs.getString("SERVICE_NAME"),
rs.getString("INSTANCE"),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toLocalTime().toString(),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toString(),
rs.getString("EVENT_TYPE"),
rs.getString("EVENT_MESSAGE")
));
}
return entries;
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
public List<EventLogEntry> getLastEntriesForInstance(String instance, int n) {
try (var conn = dataSource.getConnection();
var query = conn.prepareStatement("""
SELECT SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
SELECT ID, SERVICE_NAME, INSTANCE, EVENT_TIME, EVENT_TYPE, EVENT_MESSAGE
FROM SERVICE_EVENTLOG
WHERE INSTANCE = ?
ORDER BY ID DESC
@ -94,9 +247,11 @@ public class EventLogService {
var rs = query.executeQuery();
while (rs.next()) {
entries.add(new EventLogEntry(
rs.getLong("ID"),
rs.getString("SERVICE_NAME"),
rs.getString("INSTANCE"),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toLocalTime().toString(),
rs.getTimestamp("EVENT_TIME").toLocalDateTime().toString(),
rs.getString("EVENT_TYPE"),
rs.getString("EVENT_MESSAGE")
));
@ -107,4 +262,34 @@ public class EventLogService {
throw new RuntimeException(ex);
}
}
public List<String> getTypeNames() {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("SELECT DISTINCT(EVENT_TYPE) FROM SERVICE_EVENTLOG")) {
List<String> types = new ArrayList<>();
var rs = stmt.executeQuery();
while (rs.next()) {
types.add(rs.getString(1));
}
return types;
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
public List<String> getServiceNames() {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("SELECT DISTINCT(SERVICE_NAME) FROM SERVICE_EVENTLOG")) {
List<String> types = new ArrayList<>();
var rs = stmt.executeQuery();
while (rs.next()) {
types.add(rs.getString(1));
}
return types;
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
}

20
code/services-core/control-service/src/main/resources/templates/control/events.hdb Normal file
View File

@ -0,0 +1,20 @@
<!DOCTYPE html>
<html>
<head>
<title>Control Service</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="/style.css" />
</head>
<body>
{{> control/partials/nav}}
<section>
{{> control/partials/events-table}}
</section>
</body>
<script src="/refresh.js"></script>
<script>
window.setInterval(() => {
refresh(["events"]);
}, 2000);
</script>
</html>

23
code/services-core/control-service/src/main/resources/templates/control/partials/events-table.hdb
View File

@ -8,6 +8,19 @@
<th>Type</th>
<th>Message</th>
</tr>
<tr>
<td colspan="6" style="padding: 0.5ch">
<form method="GET" action="/events">
<select name="service" id="service">
{{#each services}}<option value="{{value}}" {{#if current}}selected{{/if}} >{{name}}</option>{{/each}}
</select>
<select name="type" id="type">
{{#each types}}<option value="{{value}}" {{#if current}}selected{{/if}} >{{name}}</option>{{/each}}
</select>
<input type="submit" value="Filter Results">
</form>
</td>
</tr>
{{#each events}}
<tr>
<td>{{serviceName}}</td>
@ -15,9 +28,17 @@
<span style="background-color: {{instanceColor}}" class="uuidPip">&nbsp;</span><span style="background-color: {{instanceColor2}}" class="uuidPip">&nbsp;</span>
{{instance}}
</td>
<td>{{eventTime}}</td>
<td title="{{eventDateTime}}">{{eventTime}}</td>
<td>{{eventType}}</td>
<td>{{eventMessage}}</td>
</tr>
{{/each}}
<tfoot>
<tr>
<td colspan="6" style="padding: 0.5ch">
{{#if prev}}<a href="/events?after={{prev}}&{{elFilter}}">Prev</a>{{/if}}
{{#if next}}<a href="/events?after={{next}}&{{elFilter}}" style="float:right">Next</a>{{/if}}
</td>
</tr>
</tfoot>
</table>