3113b5a551
There really is no fantastic place to put this logic, but we need to remove entries with an X-Robots-Tags header where that header indicates it doesn't want to be crawled by Marginalia.
47 lines
1.3 KiB
Groovy
47 lines
1.3 KiB
Groovy
plugins {
|
|
id 'java'
|
|
|
|
|
|
id 'jvm-test-suite'
|
|
}
|
|
|
|
|
|
java {
|
|
toolchain {
|
|
languageVersion.set(JavaLanguageVersion.of(21))
|
|
}
|
|
}
|
|
|
|
dependencies {
|
|
implementation project(':code:common:model')
|
|
implementation project(':code:common:db')
|
|
implementation project(':code:common:config')
|
|
implementation project(':code:common:process')
|
|
implementation project(':code:libraries:big-string')
|
|
implementation project(':code:api:index-api')
|
|
implementation project(':code:common:service-discovery')
|
|
implementation project(':code:common:service-client')
|
|
implementation project(':code:features-crawl:content-type')
|
|
implementation project(':code:libraries:language-processing')
|
|
implementation project(':third-party:parquet-floor')
|
|
implementation project(':third-party:commons-codec')
|
|
|
|
implementation libs.bundles.slf4j
|
|
|
|
implementation libs.notnull
|
|
implementation libs.bundles.parquet
|
|
|
|
implementation libs.jwarc
|
|
implementation libs.gson
|
|
implementation libs.commons.io
|
|
implementation libs.commons.lang3
|
|
implementation libs.okhttp3
|
|
implementation libs.jsoup
|
|
implementation libs.snakeyaml
|
|
implementation libs.zstd
|
|
|
|
testImplementation libs.bundles.slf4j.test
|
|
testImplementation libs.bundles.junit
|
|
testImplementation libs.mockito
|
|
}
|