161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(control) Update GUI for crawl specs. They are now less important than they were before.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-10-19 14:20:58 +02:00
parent 23f0c79fba
commit 809b3ee023
5 changed files with 25 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java
View File

@ -140,15 +140,9 @@ public class ControlNodeService {
private Object createNewSpecsAction(Request request, Response response) {
final String description = request.queryParams("description");
final String url = request.queryParams("url");
final String source = request.queryParams("source");
int nodeId = Integer.parseInt(request.params("id"));
if ("download".equals(source)) {
executorClient.createCrawlSpecFromDownload(Context.fromRequest(request), nodeId, description, url);
}
else {
throw new IllegalArgumentException("Unknown source: " + source);
}
executorClient.createCrawlSpecFromDownload(Context.fromRequest(request), nodeId, description, url);
return redirectToOverview(request);
}

26
code/services-core/control-service/src/main/resources/templates/control/node/node-new-specs-form.hdb
View File

@ -38,9 +38,6 @@
<li class="nav-item">
<a href="conf" {{#unless view.conf}}class="nav-link"{{/unless}} {{#if view.conf}}aria-current="page" class="active nav-link"{{/if}}>Base Config</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
<li class="nav-item">
<a href="crawl" {{#unless view.crawl}}class="nav-link"{{/unless}} {{#if view.crawl}}aria-current="page" class="active nav-link"{{/if}}>Crawl Data</a>
</li>
@ -50,32 +47,25 @@
<li class="nav-item">
<a href="backup" {{#unless view.backup}}class="nav-link"{{/unless}} {{#if view.backup}}aria-current="page" class="active nav-link"{{/if}}>Backups</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
</nav>
<div class="col-4">
<h2>Create New Specification</h2>
<p>To create a new specification fill out the form below. </p>
<form method="post" action="new-specs">
<div class="form">
<label class="form-label" for="description">Description</label>
<input class="form-control" type="text" name="description" id="description" maxlength="255">
<div class="form-text" id="basic-addon4">(This is how you'll be able to find the
specification later so give it a good and descriptive name)</div>
<div class="form-text" id="basic-addon4">This is how you'll be able to find the
specification later so give it a good and descriptive name</div>
<h3 class="my-3">Source</h3>
<div class="form-check">
<input class="form-check-input" type="radio" name="source" value="db" id="db" checked onclick="document.getElementById('spec-url-options').style.display = 'none';">
<label for="db" class="form-check-label">Use links in database</label>
</div>
<div class="form-check">
<input class="form-check-input" type="radio" name="source" value="download" id="download" onclick="document.getElementById('spec-url-options').style.display = 'block';">
<label for="download" class="form-check-label">Download a list of domains from a URL</label>
</div>
<div id="spec-url-options" class="py-3" style="display: none">
<label class="form-label" for="url">URL to list of domains</label>
<div id="spec-url-options" class="py-3">
<label class="form-label" for="url">URL to list of domain names</label>
<input class="form-control" type="text" name="url" id="url" />
</div>

6
code/services-core/control-service/src/main/resources/templates/control/node/node-storage-conf.hdb
View File

@ -40,9 +40,6 @@
<li class="nav-item">
<a href="conf" {{#unless view.conf}}class="nav-link"{{/unless}} {{#if view.conf}}aria-current="page" class="active nav-link"{{/if}}>Base Config</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
<li class="nav-item">
<a href="crawl" {{#unless view.crawl}}class="nav-link"{{/unless}} {{#if view.crawl}}aria-current="page" class="active nav-link"{{/if}}>Crawl Data</a>
</li>
@ -52,6 +49,9 @@
<li class="nav-item">
<a href="backup" {{#unless view.backup}}class="nav-link"{{/unless}} {{#if view.backup}}aria-current="page" class="active nav-link"{{/if}}>Backups</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
</nav>
<h1>Storage Base Configuration</h1>

6
code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb
View File

@ -38,9 +38,6 @@
<li class="nav-item">
<a href="conf" {{#unless view.conf}}class="nav-link"{{/unless}} {{#if view.conf}}aria-current="page" class="active nav-link"{{/if}}>Base Config</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
<li class="nav-item">
<a href="crawl" {{#unless view.crawl}}class="nav-link"{{/unless}} {{#if view.crawl}}aria-current="page" class="active nav-link"{{/if}}>Crawl Data</a>
</li>
@ -50,6 +47,9 @@
<li class="nav-item">
<a href="backup" {{#unless view.backup}}class="nav-link"{{/unless}} {{#if view.backup}}aria-current="page" class="active nav-link"{{/if}}>Backups</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
</nav>
<h1>Storage Details</h1>

19
code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb
View File

@ -39,9 +39,6 @@
<li class="nav-item">
<a href="conf" {{#unless view.conf}}class="nav-link"{{/unless}} {{#if view.conf}}aria-current="page" class="active nav-link"{{/if}}>Base Config</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
<li class="nav-item">
<a href="crawl" {{#unless view.crawl}}class="nav-link"{{/unless}} {{#if view.crawl}}aria-current="page" class="active nav-link"{{/if}}>Crawl Data</a>
</li>
@ -51,6 +48,9 @@
<li class="nav-item">
<a href="backup" {{#unless view.backup}}class="nav-link"{{/unless}} {{#if view.backup}}aria-current="page" class="active nav-link"{{/if}}>Backups</a>
</li>
<li class="nav-item">
<a href="specs" {{#unless view.specs}}class="nav-link"{{/unless}} {{#if view.specs}}aria-current="page" class="active nav-link"{{/if}}>Specs</a>
</li>
</nav>
@ -149,22 +149,23 @@
{{#if view.specs}}
<div class="m-3 p-3 border bg-light">
<h3 class="my-3">About Specs</h3>
<p>Crawl specifications are set of domains to be crawled. A specification is necessary for the crawler
to understand which domains to visit.</p>
<p>Setting a spec as active means it will be used in <em>recrawls</em>. A fresh crawl can be triggered
from within the details view of a specification.</p>
<p>Crawl specifications are set of domains to be crawled. Normally the crawler uses the domain database
to find domains to crawl, but sometimes, when bootstrapping a system or during testing, it's useful
to be able to specify which domains to crawl with more control.
</p>
<p>The <em>ACTIVE</em> status does nothing for crawl specs.</p>
</div>
{{/if}}
{{#if view.crawl}}
<div class="m-3 p-3 border bg-light">
<h3 class="my-3">About Crawls</h3>
<p>Crawl data is the downloaded and compressed raw HTML of a website.</p>
<p>To start a new crawl, first a <em>crawl specification</em> must be created.</p>
<p>A new crawl may be bootstrappped from a Crawl Set</p>
<p>Unlike other data sets, only a single crawl set may be <em>ACTIVE</em> at any given time.</p>
<h4 class="my-3">Recrawl</h4>
<p>A set of existing crawl data can be <em>recrawled</em> and amended with new domains
and a fresh look at the existing domains. This is an operation that uses the currently <em>ACTIVE</em>
crawl data and crawl set(s).
crawl data.
</p>
<h4 class="my-3">Processing</h4>
<p>