f655ec5a5c
In this commit, GeoIP-related classes are refactored and relocated to a common library as they are shared across multiple services. The crawler is refactored to enable the GeoIpBlocklist to use the new GeoIpDictionary as the base of its decisions. The converter is modified ot query this data to add a geoip:-keyword to documents to permit limiting a search to the country of the hosting server. The commit also adds due BY-SA attribution in the search engine footer for the source of the IP geolocation data.
101 lines
3.1 KiB
Groovy
101 lines
3.1 KiB
Groovy
plugins {
|
|
id 'java'
|
|
|
|
id 'application'
|
|
id 'jvm-test-suite'
|
|
}
|
|
|
|
java {
|
|
toolchain {
|
|
languageVersion.set(JavaLanguageVersion.of(21))
|
|
}
|
|
}
|
|
|
|
application {
|
|
mainClass = 'nu.marginalia.converting.ConverterMain'
|
|
applicationName = 'converter-process'
|
|
}
|
|
|
|
tasks.distZip.enabled = false
|
|
|
|
dependencies {
|
|
|
|
implementation project(':third-party:monkey-patch-gson')
|
|
|
|
implementation project(':code:common:process')
|
|
|
|
implementation project(':third-party:porterstemmer')
|
|
implementation project(':third-party:count-min-sketch')
|
|
|
|
implementation project(':code:api:index-api')
|
|
implementation project(':code:api:process-mqapi')
|
|
|
|
implementation project(':code:common:model')
|
|
implementation project(':code:common:db')
|
|
implementation project(':code:common:service')
|
|
implementation project(':code:common:config')
|
|
implementation project(':code:libraries:message-queue')
|
|
implementation project(':code:libraries:blocking-thread-pool')
|
|
implementation project(':code:common:service-discovery')
|
|
implementation project(':code:common:service-client')
|
|
|
|
implementation project(':code:libraries:guarded-regex')
|
|
implementation project(':code:libraries:easy-lsh')
|
|
implementation project(':code:libraries:geo-ip')
|
|
implementation project(':code:libraries:big-string')
|
|
implementation project(':code:libraries:language-processing')
|
|
|
|
implementation project(':code:process-models:processed-data')
|
|
implementation project(':code:process-models:work-log')
|
|
implementation project(':code:process-models:crawling-model')
|
|
|
|
implementation project(':code:features-convert:adblock')
|
|
implementation project(':code:features-convert:anchor-keywords')
|
|
implementation project(':code:features-convert:topic-detection')
|
|
implementation project(':code:features-convert:pubdate')
|
|
implementation project(':code:features-convert:keyword-extraction')
|
|
implementation project(':code:features-convert:summary-extraction')
|
|
implementation project(':code:features-convert:stackexchange-xml')
|
|
|
|
implementation project(':code:features-crawl:crawl-blocklist')
|
|
implementation project(':code:features-crawl:link-parser')
|
|
|
|
testImplementation project(':code:libraries:term-frequency-dict')
|
|
testImplementation project(':code:process-models:crawl-spec')
|
|
|
|
implementation libs.bundles.slf4j
|
|
|
|
implementation libs.notnull
|
|
|
|
implementation libs.jsoup
|
|
|
|
implementation libs.guice
|
|
implementation libs.guava
|
|
implementation libs.bundles.gson
|
|
|
|
implementation libs.zstd
|
|
|
|
implementation libs.bundles.mariadb
|
|
implementation libs.bundles.nlp
|
|
|
|
implementation libs.trove
|
|
implementation libs.fastutil
|
|
|
|
implementation libs.snakeyaml
|
|
|
|
implementation libs.crawlercommons
|
|
|
|
implementation libs.commons.lang3
|
|
implementation libs.commons.compress
|
|
implementation libs.sqlite
|
|
|
|
testImplementation libs.bundles.slf4j.test
|
|
testImplementation libs.bundles.junit
|
|
testImplementation libs.mockito
|
|
|
|
implementation 'org.tukaani:xz:1.8'
|
|
|
|
testImplementation project(':code:processes:test-data')
|
|
testImplementation project(':code:processes:crawling-process')
|
|
}
|