161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Put news into a database table instead of keeping them hardcoded.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-03-19 11:46:13 +01:00
parent 3402b31c30
commit bdd2b4a43e
8 changed files with 157 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/common/model/src/main/resources/sql/edge-crawler-cache.sql
View File

@ -295,3 +295,11 @@ CREATE TABLE WMSA_PROCESS(
TIMEOUT INT NOT NULL DEFAULT 60
);
---
CREATE TABLE SEARCH_NEWS_FEED(
ID INT PRIMARY KEY AUTO_INCREMENT,
TITLE VARCHAR(255),
LINK VARCHAR(255),
LIST_DATE DATE
) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_bin;

48
code/services-core/search-service/src/main/java/nu/marginalia/search/command/IndexCommand.java
View File

@ -2,28 +2,62 @@ package nu.marginalia.search.command;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.browse.model.BrowseResultSet;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.renderer.MustacheRenderer;
import nu.marginalia.renderer.RendererFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Request;
import spark.Response;
import java.io.IOException;
import java.util.Collections;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
@Singleton
public class IndexCommand {
private final MustacheRenderer<BrowseResultSet> template;
@Inject
public IndexCommand(RendererFactory rendererFactory) throws IOException {
private final MustacheRenderer<IndexModel> template;
private final HikariDataSource dataSource;
template = rendererFactory.renderer("search/index");
private final Logger logger = LoggerFactory.getLogger(getClass());
@Inject
public IndexCommand(RendererFactory rendererFactory, HikariDataSource dataSource) throws IOException {
this.template = rendererFactory.renderer("search/index/index");
this.dataSource = dataSource;
}
public String render(Request request, Response response) {
response.header("Cache-control", "public,max-age=3600");
return template.render(new BrowseResultSet(Collections.emptyList()));
return template.render(new IndexModel(getNewsItems()));
}
private List<NewsItem> getNewsItems() {
List<NewsItem> items = new ArrayList<>();
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT TITLE, LINK, LIST_DATE FROM SEARCH_NEWS_FEED
""")) {
var rep = stmt.executeQuery();
while (rep.next()) {
items.add(new NewsItem(rep.getString(1), rep.getString(2), rep.getDate(3).toLocalDate()));
}
}
catch (SQLException ex) {
logger.warn("Failed to fetch news items", ex);
}
return items;
}
private record IndexModel(List<NewsItem> news) { }
private record NewsItem(String title, String url, LocalDate date) {}
}

124
code/services-core/search-service/src/main/resources/templates/search/index.hdb
View File

@ -1,124 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Marginalia Search</title>
<link rel="stylesheet" href="/style-new.css" />
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Marginalia">
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta property="og:description" content="search.marginalia.nu is a small independent do-it-yourself search engine for surprising but content-rich websites that never ask you to accept cookies or subscribe to newsletters. The goal is to bring you the sort of grass fed, free range HTML your grandma used to write. " />
<meta property="og:locale" content="en_US" />
<meta property="og:site_name" content="search.marginalia.nu" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://search.marginalia.nu/" />
</head>
<body>
<!-- Hi there, fellow human being :-) -->
{{>search/parts/search-header}}
<article>
{{>search/parts/search-form}}
<section class="card rightbox">
<h2>Publicity, Discussion and Events</h2>
<div class="info">
<dl>
<dt><a href="https://memex.marginalia.nu/log/64-hundred-million.gmi" rel="nofollow">Marginalia's Index Reaches 100,000,000 Documents</a> &#127882;</dt>
<dd>2022-10-21</dd>
<dt><a href="https://www.hs.fi/visio/art-2000009139237.html" rel="nofollow">Google ei en&auml;&auml; tide&auml;</a></dt>
<dd>Helsing Sanomat &#127467;&#127470; 2022-10-19</dd>
<dt><a href="https://www.deutschlandfunkkultur.de/google-suche-100.html" rel="nofollow">Kritik an Googles Suche - Platzhirsch auf dem Nebenschauplatz</a></dt>
<dd>Deutschlandfunk Kultur &#x1f1e9;&#x1f1ea;, 2022-08-18</dd>
<dt><a href="https://news.ycombinator.com/item?id=31536626" rel="nofollow">Marginalia Goes Open Source</a></dt>
<dd>Hacker News, 2022-05-28</dd>
<dt><a href="https://www.youtube.com/watch?v=rTSEr0cRJY8" rel="nofollow">You Should Check Out the Indie Web</a> &#x1F39E;&#xFE0F;</dt>
<dd>YouTube, You've Got Kat, 2022-03-15 </dd>
<dt> <a href="https://www.newyorker.com/culture/infinite-scroll/what-google-search-isnt-showing-you" rel="nofollow">What Google Search Isn't Showing You</a> </dt>
<dd>The New Yorker &#127913;, 2022-03-10</dd>
<dt> <a href="https://www.metafilter.com/194653/Marginalia-Search-Serendipity-Engineering" rel="nofollow">Marginalia Search - Serendipity Engineering</a> </dt>
<dd>MetaFilter, 2022-03-09</dd>
<dt> &#127874; <a href="https://memex.marginalia.nu/log/49-marginalia-1-year.gmi">First anniversary</a>! &#127882; </dt>
<dd> 2022-02-26 </dd>
<dt> <a href="https://onezero.medium.com/a-search-engine-designed-to-surprise-you-b81944ed5c06" rel="nofollow">A Search Engine Designed To Surprise You</a> </dt>
<dd>Clive Thompson OneZero, 2021-09-16</dd>
<dt> <a href="https://news.ycombinator.com/item?id=28550764" rel="nofollow"> A search engine that favors text-heavy sites and punishes modern web design</a> </dt>
<dd> Hacker News, 2021-09-16 </dd>
<dt>Development begins </dt>
<dd>2021-02-26</dd>
</dl>
</div>
</section>
<div class="cards big">
<section class="card">
<h2>About</h2>
<div class="info">
<p>This is an independent DIY search engine that focuses on non-commercial content, and attempts to
show you sites you perhaps weren't aware of in favor of the sort of sites you probably already knew
existed. </p>
<p>
The software for this search engine is all custom-built, and all crawling and indexing is
done in-house. The project is open source. Feel free to poke about in the <a
href="https://git.marginalia.nu/marginalia/marginalia.nu">source code</a> or contribute
to the development!
</p>
<p>
The entire search engine is hosted off a single PC in Sweden, albeit with pretty solid specs.
</p>
<p>Consider <a href="https://memex.marginalia.nu/projects/edge/supporting.gmi">supporting the
project</a>!</p>
</div>
<div class="utils">
<a href="https://memex.marginalia.nu/projects/edge/about.gmi">Read More</a>
</div>
</section>
<section class="card">
<h2>Tips</h2>
<div class="info">
<p>
This search engine isn't particularly well equipped to answering queries
posed like questions, instead try to imagine some text that might appear
in the website you are looking for, and search for that.</p>
<p>
Where this search engine really shines is finding small, old and obscure websites about some
given topic, perhaps
<a href="/search?query=commander+keen&profile=yolo&js=default">old video games</a>,
<a href="/search?query=voynich+&profile=yolo&js=default">a mystery</a>,
<a href="/search?query=augustine+confessions&profile=yolo&js=default">theology</a>,
<a href="/search?query=Hermes+Trismegistus&profile=yolo&js=default">the occult</a>,
<a href="/search?query=knitting&profile=yolo&js=default">knitting</a>,
<a href="/search?query=scc+graph+algorithm&profile=yolo&js=default">computer science</a>,
or <a href="/search?query=salvador+dali&profile=yolo&js=default">art</a>.
</p>
</div>
<div class="utils">
<a href="https://memex.marginalia.nu/projects/edge/search-tips.gmi">Additional Tips</a>
</div>
</section>
<section class="card">
<h2>Updates</h2>
<div class="info">
<p>☛ A recipe filter has been added to the algorithm selector.</p>
<p>☛ The <a href="https://search.marginalia.nu/explore/random">Random Mode</a> has been overhauled, and is
quite entertaining. I encourage you to give it a spin. </p>
<p>☛ A simple <a href="https://api.marginalia.nu/">public API</a> is now available.</p>
</div>
<div class="utils">
<a href="https://memex.marginalia.nu/projects/edge/changelog.gmi">Change Log</a>
</div>
</section>
</div>
</article>
{{>search/parts/search-footer}}
</body>

22
code/services-core/search-service/src/main/resources/templates/search/index/index-about.hdb Normal file
View File

@ -0,0 +1,22 @@
<section class="card">
<h2>About</h2>
<div class="info">
<p>This is an independent DIY search engine that focuses on non-commercial content, and attempts to
show you sites you perhaps weren't aware of in favor of the sort of sites you probably already knew
existed. </p>
<p>
The software for this search engine is all custom-built, and all crawling and indexing is
done in-house. The project is open source. Feel free to poke about in the <a
href="https://git.marginalia.nu/marginalia/marginalia.nu">source code</a> or contribute
to the development!
</p>
<p>
The entire search engine is hosted off a single PC in Sweden, albeit with pretty solid specs.
</p>
<p>Consider <a href="https://memex.marginalia.nu/projects/edge/supporting.gmi">supporting the
project</a>!</p>
</div>
<div class="utils">
<a href="https://memex.marginalia.nu/projects/edge/about.gmi">Read More</a>
</div>
</section>

14
code/services-core/search-service/src/main/resources/templates/search/index/index-news.hdb Normal file
View File

@ -0,0 +1,14 @@
{{#if news}}
<section class="card rightbox">
<h2>Publicity, Discussion and Events</h2>
<div class="info">
<dl>
{{#each news}}
<dt><a href="{{link}}" rel="nofollow">{{title}}</a></dt>
<dd>{{date}}</dd>
{{/each}}
</dl>
</div>
</section>
{{/if}}

23
code/services-core/search-service/src/main/resources/templates/search/index/index-tips.hdb Normal file
View File

@ -0,0 +1,23 @@
<section class="card">
<h2>Tips</h2>
<div class="info">
<p>
This search engine isn't particularly well equipped to answering queries
posed like questions, instead try to imagine some text that might appear
in the website you are looking for, and search for that.</p>
<p>
Where this search engine really shines is finding small, old and obscure websites about some
given topic, perhaps
<a href="/search?query=commander+keen&profile=yolo&js=default">old video games</a>,
<a href="/search?query=voynich+&profile=yolo&js=default">a mystery</a>,
<a href="/search?query=augustine+confessions&profile=yolo&js=default">theology</a>,
<a href="/search?query=Hermes+Trismegistus&profile=yolo&js=default">the occult</a>,
<a href="/search?query=knitting&profile=yolo&js=default">knitting</a>,
<a href="/search?query=scc+graph+algorithm&profile=yolo&js=default">computer science</a>,
or <a href="/search?query=salvador+dali&profile=yolo&js=default">art</a>.
</p>
</div>
<div class="utils">
</div>
</section>

12
code/services-core/search-service/src/main/resources/templates/search/index/index-updates.hdb Normal file
View File

@ -0,0 +1,12 @@
<section class="card">
<h2>Updates</h2>
<div class="info">
<p>☛ A recipe filter has been added to the algorithm selector.</p>
<p>☛ The <a href="https://search.marginalia.nu/explore/random">Random Mode</a> has been overhauled, and is
quite entertaining. I encourage you to give it a spin. </p>
<p>☛ A simple <a href="https://api.marginalia.nu/">public API</a> is now available.</p>
</div>
<div class="utils">
<a href="https://memex.marginalia.nu/projects/edge/changelog.gmi">Change Log</a>
</div>
</section>

37
code/services-core/search-service/src/main/resources/templates/search/index/index.hdb Normal file
View File

@ -0,0 +1,37 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Marginalia Search</title>
<link rel="stylesheet" href="/style-new.css" />
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Marginalia">
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta property="og:description" content="search.marginalia.nu is a small independent do-it-yourself search engine for surprising but content-rich websites that never ask you to accept cookies or subscribe to newsletters. The goal is to bring you the sort of grass fed, free range HTML your grandma used to write. " />
<meta property="og:locale" content="en_US" />
<meta property="og:site_name" content="search.marginalia.nu" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://search.marginalia.nu/" />
</head>
<body>
<!-- Hi there, fellow human being :-) -->
{{>search/parts/search-header}}
<article>
{{>search/parts/search-form}}
{{>search/index/index-news}}
<div class="cards big">
{{>search/index/index-about}}
{{>search/index/index-tips}}
{{>search/index/index-updates}}
</div>
</article>
{{>search/parts/search-footer}}
</body>