161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(minor) Remove dead code.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-12-15 21:49:35 +01:00
parent 2001d0f707
commit 54ed3b86ba
1 changed files with 0 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View File

@ -312,17 +312,6 @@ public class CrawlerRetreiver implements AutoCloseable {
|| proto.equalsIgnoreCase("https"); || proto.equalsIgnoreCase("https");
} }
// FIXME this does not belong in the crawler
private Optional<EdgeUrl> findCanonicalUrl(EdgeUrl baseUrl, Document parsed) {
baseUrl = baseUrl.domain.toRootUrl();
for (var link : parsed.select("link[rel=canonical]")) {
return linkParser.parseLink(baseUrl, link);
}
return Optional.empty();
}
private String findIp(String domain) { private String findIp(String domain) {
try { try {
return InetAddress.getByName(domain).getHostAddress(); return InetAddress.getByName(domain).getHostAddress();