064265b0b9
This functionality needs to be accessed by the WarcSideloader, which is in the converter. The resultant microlibrary is tiny, but I think in this case it's justifiable.
68 lines
2.0 KiB
Groovy
68 lines
2.0 KiB
Groovy
plugins {
|
|
id 'java'
|
|
|
|
id 'application'
|
|
id 'jvm-test-suite'
|
|
}
|
|
|
|
java {
|
|
toolchain {
|
|
languageVersion.set(JavaLanguageVersion.of(21))
|
|
}
|
|
}
|
|
|
|
application {
|
|
mainClass = 'nu.marginalia.crawl.CrawlerMain'
|
|
applicationName = 'crawler-process'
|
|
}
|
|
|
|
tasks.distZip.enabled = false
|
|
|
|
dependencies {
|
|
implementation project(':code:common:process')
|
|
|
|
implementation project(':code:common:db')
|
|
implementation project(':code:common:model')
|
|
implementation project(':code:common:config')
|
|
implementation project(':code:common:service')
|
|
implementation project(':code:libraries:big-string')
|
|
implementation project(':code:libraries:blocking-thread-pool')
|
|
implementation project(':code:api:index-api')
|
|
implementation project(':code:api:process-mqapi')
|
|
implementation project(':code:common:service-discovery')
|
|
implementation project(':code:common:service-client')
|
|
implementation project(':code:libraries:message-queue')
|
|
implementation project(':code:libraries:language-processing')
|
|
implementation project(':code:libraries:easy-lsh')
|
|
implementation project(':code:process-models:crawling-model')
|
|
implementation project(':code:process-models:crawl-spec')
|
|
|
|
|
|
implementation project(':code:features-convert:anchor-keywords')
|
|
implementation project(':code:features-crawl:crawl-blocklist')
|
|
implementation project(':code:features-crawl:link-parser')
|
|
implementation project(':code:features-crawl:content-type')
|
|
|
|
implementation libs.bundles.slf4j
|
|
|
|
implementation libs.notnull
|
|
implementation libs.guice
|
|
implementation libs.gson
|
|
implementation libs.zstd
|
|
implementation libs.crawlercommons
|
|
implementation libs.okhttp3
|
|
implementation libs.jsoup
|
|
implementation libs.opencsv
|
|
implementation libs.rxjava
|
|
implementation libs.fastutil
|
|
|
|
implementation libs.bundles.mariadb
|
|
|
|
testImplementation libs.bundles.slf4j.test
|
|
testImplementation libs.bundles.junit
|
|
testImplementation libs.mockito
|
|
|
|
testImplementation project(':code:processes:test-data')
|
|
}
|
|
|