3101b74580
This is a system-wide change. The index used to have a lexicon, mapping words to wordIds using a large in-memory hash table. This made index-construction easier, but it also added a fairly significant RAM penalty to both the index service and the loader. The new design moves to 64 bit word identifiers calculated using the murmur hash of the keyword, and an index construction based on merging smaller indices. It also became necessary half-way through to upgrade guice as its error reporting wasn't *quite* compatible with JDK20.
73 lines
2.2 KiB
Groovy
73 lines
2.2 KiB
Groovy
plugins {
|
|
id 'java'
|
|
id "io.freefair.lombok" version "8.2.2"
|
|
id 'application'
|
|
id 'jvm-test-suite'
|
|
}
|
|
java {
|
|
toolchain {
|
|
languageVersion.set(JavaLanguageVersion.of(20))
|
|
}
|
|
}
|
|
|
|
application {
|
|
mainClass = 'nu.marginalia.loading.LoaderMain'
|
|
applicationName = 'loader-process'
|
|
}
|
|
|
|
tasks.distZip.enabled = false
|
|
|
|
dependencies {
|
|
implementation project(':code:common:process')
|
|
implementation project(':code:api:process-mqapi')
|
|
implementation project(':code:api:index-api')
|
|
implementation project(':code:common:model')
|
|
implementation project(':code:common:db')
|
|
implementation project(':code:common:config')
|
|
implementation project(':code:common:service')
|
|
implementation project(':code:common:service-discovery')
|
|
implementation project(':code:common:service-client')
|
|
implementation project(':code:common:linkdb')
|
|
implementation project(':code:features-index:index-journal')
|
|
implementation project(':code:libraries:message-queue')
|
|
implementation project(':code:libraries:language-processing')
|
|
implementation project(':third-party:commons-codec')
|
|
testImplementation project(':code:services-core:search-service')
|
|
|
|
implementation project(':code:process-models:crawling-model')
|
|
implementation project(':code:process-models:converting-model')
|
|
|
|
implementation project(':code:features-convert:keyword-extraction')
|
|
|
|
|
|
implementation libs.lombok
|
|
annotationProcessor libs.lombok
|
|
implementation libs.bundles.slf4j
|
|
|
|
implementation libs.guice
|
|
implementation libs.gson
|
|
implementation libs.commons.lang3
|
|
implementation libs.zstd
|
|
implementation libs.trove
|
|
implementation libs.bundles.mariadb
|
|
|
|
testImplementation libs.bundles.slf4j.test
|
|
testImplementation libs.bundles.junit
|
|
testImplementation libs.mockito
|
|
|
|
testImplementation libs.bundles.selenium
|
|
testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
|
|
testImplementation 'org.testcontainers:mariadb:1.17.4'
|
|
testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
|
|
}
|
|
|
|
test {
|
|
useJUnitPlatform()
|
|
}
|
|
|
|
task fastTests(type: Test) {
|
|
useJUnitPlatform {
|
|
excludeTags "slow"
|
|
}
|
|
}
|