161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove link filtering for mediawiki, it's too strict and not every site uses the /wiki/-pattern.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-07-01 19:32:25 +02:00
parent eda615de0f
commit 24dce8c03b
1 changed files with 0 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/LinkFilterSelector.java
View File

@ -22,9 +22,6 @@ public class LinkFilterSelector {
if (isLemmy(head)) {
return url -> url.path.startsWith("/post/") || url.path.startsWith("/c/");
}
if (isMediawiki(head)) {
return url -> url.path.startsWith("/wiki/") && !url.path.contains(":");
}
if (isDiscourse(head)) {
return url -> url.path.startsWith("/t/") || url.path.contains("/latest");
}