161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

(minor) Remove dead code.

Browse source
This commit is contained in:
Viktor Lofgren 2023-12-15 21:49:35 +01:00
parent 2001d0f707
commit 54ed3b86ba
1 changed files with 0 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View file

@ -312,17 +312,6 @@ public class CrawlerRetreiver implements AutoCloseable {
|| proto.equalsIgnoreCase("https");
}
// FIXME this does not belong in the crawler
private Optional<EdgeUrl> findCanonicalUrl(EdgeUrl baseUrl, Document parsed) {
baseUrl = baseUrl.domain.toRootUrl();
for (var link : parsed.select("link[rel=canonical]")) {
return linkParser.parseLink(baseUrl, link);
}
return Optional.empty();
}
private String findIp(String domain) {
try {
return InetAddress.getByName(domain).getHostAddress();