c41e68aaab
This commit also refactors the executor a bit, and introduces a new converter-feature called data-extractors for this class of jobs.
229 lines
10 KiB
Groovy
229 lines
10 KiB
Groovy
rootProject.name = 'marginalia.nu'
|
|
|
|
include 'code:services-core:index-service'
|
|
include 'code:services-core:assistant-service'
|
|
include 'code:services-core:control-service'
|
|
include 'code:services-core:query-service'
|
|
include 'code:services-core:executor-service'
|
|
|
|
include 'code:services-application:search-service'
|
|
include 'code:services-application:api-service'
|
|
include 'code:services-application:dating-service'
|
|
include 'code:services-application:explorer-service'
|
|
|
|
include 'code:libraries:array'
|
|
include 'code:libraries:geo-ip'
|
|
include 'code:libraries:btree'
|
|
include 'code:libraries:easy-lsh'
|
|
include 'code:libraries:guarded-regex'
|
|
include 'code:libraries:big-string'
|
|
include 'code:libraries:random-write-funnel'
|
|
include 'code:libraries:next-prime'
|
|
include 'code:libraries:blocking-thread-pool'
|
|
include 'code:libraries:braille-block-punch-cards'
|
|
include 'code:libraries:language-processing'
|
|
include 'code:libraries:term-frequency-dict'
|
|
include 'code:libraries:test-helpers'
|
|
|
|
include 'code:libraries:message-queue'
|
|
|
|
include 'code:features-search:screenshots'
|
|
include 'code:features-search:random-websites'
|
|
include 'code:features-search:feedlot-client'
|
|
include 'code:features-qs:query-parser'
|
|
include 'code:features-index:result-ranking'
|
|
|
|
include 'code:features-convert:adblock'
|
|
include 'code:features-convert:anchor-keywords'
|
|
include 'code:features-convert:data-extractors'
|
|
include 'code:features-convert:stackexchange-xml'
|
|
include 'code:features-convert:pubdate'
|
|
include 'code:features-convert:summary-extraction'
|
|
include 'code:features-convert:keyword-extraction'
|
|
include 'code:features-convert:topic-detection'
|
|
|
|
include 'code:features-crawl:crawl-blocklist'
|
|
include 'code:features-crawl:link-parser'
|
|
include 'code:features-crawl:content-type'
|
|
|
|
include 'code:features-index:index-journal'
|
|
include 'code:features-index:index-query'
|
|
include 'code:features-index:index-forward'
|
|
include 'code:features-index:index-reverse'
|
|
include 'code:features-index:domain-ranking'
|
|
|
|
include 'code:api:actor-api'
|
|
include 'code:api:query-api'
|
|
include 'code:api:index-api'
|
|
include 'code:api:assistant-api'
|
|
include 'code:api:process-mqapi'
|
|
include 'code:api:executor-api'
|
|
|
|
include 'code:common:service-discovery'
|
|
include 'code:common:service-client'
|
|
include 'code:common:db'
|
|
include 'code:common:linkdb'
|
|
include 'code:common:service'
|
|
include 'code:common:config'
|
|
include 'code:common:model'
|
|
include 'code:common:renderer'
|
|
include 'code:common:process'
|
|
|
|
include 'code:processes:converting-process'
|
|
include 'code:processes:crawling-process'
|
|
include 'code:processes:loading-process'
|
|
include 'code:processes:index-constructor-process'
|
|
include 'code:processes:test-data'
|
|
include 'code:processes:website-adjacencies-calculator'
|
|
|
|
include 'code:process-models:crawling-model'
|
|
include 'code:process-models:work-log'
|
|
include 'code:process-models:crawl-spec'
|
|
include 'code:process-models:processed-data'
|
|
|
|
include 'code:tools:term-frequency-extractor'
|
|
include 'code:tools:experiment-runner'
|
|
include 'code:tools:screenshot-capture-tool'
|
|
include 'code:tools:load-test'
|
|
include 'code:tools:stackexchange-converter'
|
|
include 'code:tools:crawl-data-unfcker'
|
|
|
|
include 'third-party:porterstemmer'
|
|
include 'third-party:xz'
|
|
include 'third-party:symspell'
|
|
include 'third-party:rdrpostagger'
|
|
include 'third-party:uppend'
|
|
include 'third-party:openzim'
|
|
include 'third-party:count-min-sketch'
|
|
include 'third-party:monkey-patch-opennlp'
|
|
include 'third-party:monkey-patch-gson'
|
|
include 'third-party:commons-codec'
|
|
include 'third-party:parquet-floor'
|
|
|
|
|
|
dependencyResolutionManagement {
|
|
|
|
repositories {
|
|
mavenLocal()
|
|
maven { url "https://artifactory.cronapp.io/public-release/" }
|
|
maven { url "https://repo1.maven.org/maven2/" }
|
|
maven { url "https://www2.ph.ed.ac.uk/maven2/" }
|
|
maven { url "https://jitpack.io/" }
|
|
exclusiveContent {
|
|
forRepository {
|
|
maven {
|
|
url = uri("https://jitpack.io")
|
|
}
|
|
}
|
|
filter {
|
|
// Only use JitPack for the `gson-record-type-adapter-factory` library
|
|
includeModule("com.github.Marcono1234", "gson-record-type-adapter-factory")
|
|
}
|
|
}
|
|
}
|
|
|
|
versionCatalogs {
|
|
libs {
|
|
library('lombok', 'org.projectlombok', 'lombok').version('1.18.30')
|
|
library('mariadb-client', 'org.mariadb.jdbc', 'mariadb-java-client').version('3.0.6')
|
|
library('hikaricp', 'com.zaxxer:HikariCP:5.0.1')
|
|
|
|
library('spark', 'com.sparkjava', 'spark-core').version('2.9.4')
|
|
library('guice', 'com.google.inject', 'guice').version('7.0.0')
|
|
library('guava', 'com.google.guava', 'guava').version('32.0.1-jre')
|
|
library('protobuf', 'com.google.protobuf', 'protobuf-java').version('3.16.3')
|
|
library('grpc-protobuf', 'io.grpc', 'grpc-protobuf').version('1.49.2')
|
|
library('grpc-stub', 'io.grpc', 'grpc-stub').version('1.49.2')
|
|
library('grpc-netty', 'io.grpc', 'grpc-netty-shaded').version('1.49.2')
|
|
library('rxjava', 'io.reactivex.rxjava3', 'rxjava').version('3.1.6')
|
|
|
|
library('prometheus', 'io.prometheus', 'simpleclient').version('0.16.0')
|
|
library('prometheus-servlet', 'io.prometheus', 'simpleclient_servlet').version('0.16.0')
|
|
library('prometheus-server', 'io.prometheus', 'simpleclient_httpserver').version('0.16.0')
|
|
library('prometheus-hotspot', 'io.prometheus', 'simpleclient_hotspot').version('0.16.0')
|
|
|
|
library('slf4j.api', 'org.slf4j', 'slf4j-api').version('1.7.36')
|
|
library('slf4j.jdk14', 'org.slf4j', 'slf4j-jdk14').version('2.0.3')
|
|
|
|
library('log4j.api', 'org.apache.logging.log4j', 'log4j-api').version('2.17.2')
|
|
library('log4j.core', 'org.apache.logging.log4j', 'log4j-core').version('2.17.2')
|
|
library('log4j.slf4j', 'org.apache.logging.log4j', 'log4j-slf4j-impl').version('2.17.2')
|
|
|
|
library('notnull','org.jetbrains','annotations').version('24.0.0')
|
|
|
|
library('trove', 'net.sf.trove4j', 'trove4j').version('3.0.3')
|
|
library('fastutil', 'it.unimi.dsi', 'fastutil').version('8.5.8')
|
|
|
|
library('hll', 'net.agkn', 'hll').version('1.6.0')
|
|
library('duckdb', 'org.duckdb', 'duckdb_jdbc').version('0.9.1')
|
|
library('okhttp3','com.squareup.okhttp3','okhttp').version('4.11.0')
|
|
|
|
library('jwarc', 'org.netpreserve', 'jwarc').version('0.28.5')
|
|
|
|
library('httpcomponents.core','org.apache.httpcomponents','httpcore').version('4.4.15')
|
|
library('httpcomponents.client','org.apache.httpcomponents','httpclient').version('4.5.13')
|
|
library('commons.net', 'commons-net','commons-net').version('3.9.0')
|
|
library('commons.lang3', 'org.apache.commons','commons-lang3').version('3.12.0')
|
|
library('commons.compress','org.apache.commons','commons-compress').version('1.21')
|
|
library('commons.io','commons-io','commons-io').version('2.11.0')
|
|
library('commons.codec', 'commons-codec', 'commons-codec').version('1.16.0')
|
|
|
|
library('ffi','com.github.jnr','jnr-ffi').version('2.2.12')
|
|
library('databind','com.fasterxml.jackson.core','jackson-databind').version('2.13.5')
|
|
|
|
library('crawlercommons', 'com.github.crawler-commons', 'crawler-commons').version('1.3')
|
|
|
|
library('stanford.corenlp','edu.stanford.nlp','stanford-corenlp').version('4.4.0')
|
|
library('opennlp','org.apache.opennlp','opennlp-tools').version('1.9.4')
|
|
library('fasttext','com.github.vinhkhuc','jfasttext').version('0.5')
|
|
library('roaringbitmap','org.roaringbitmap','RoaringBitmap').version('0.9.32')
|
|
library('opencsv','com.opencsv','opencsv').version('5.6')
|
|
library('bucket4j','com.github.vladimir-bukhtoyarov','bucket4j-core').version('7.5.0')
|
|
|
|
library('gson','com.google.code.gson','gson').version('2.10.1')
|
|
library('gson-type-adapter','com.github.Marcono1234','gson-record-type-adapter-factory').version('0.2.0')
|
|
|
|
library('zstd','com.github.luben','zstd-jni').version('1.5.2-2')
|
|
library('lz4','org.lz4','lz4-java').version('1.8.0')
|
|
|
|
library('flyway.core','org.flywaydb','flyway-core').version('10.4.1')
|
|
library('flyway.mysql','org.flywaydb','flyway-mysql').version('10.4.1')
|
|
|
|
library('jsoup','org.jsoup','jsoup').version('1.15.3')
|
|
library('snakeyaml','org.yaml','snakeyaml').version('1.33') // Known CVE, but this library only parses internal config files so it's fine
|
|
|
|
library('junit.jupiter','org.junit.jupiter','junit-jupiter-api').version('5.8.2')
|
|
library('junit.jupiter.engine','org.junit.jupiter','junit-jupiter-engine').version('')
|
|
library('mockito','org.mockito','mockito-junit-jupiter').version('4.5.1')
|
|
|
|
library('selenium.chrome','org.seleniumhq.selenium','selenium-chrome-driver').version('4.8.3')
|
|
library('selenium.java','org.seleniumhq.selenium','selenium-java').version('4.8.3')
|
|
|
|
library('handlebars','com.github.jknack','handlebars').version('4.3.1')
|
|
library('handlebars.markdown','com.github.jknack','handlebars-markdown').version('4.2.1')
|
|
|
|
library('sqlite','org.xerial','sqlite-jdbc').version('3.41.2.2')
|
|
library('javax.annotation','javax.annotation','javax.annotation-api').version('1.3.2')
|
|
library('parquet-column', 'org.apache.parquet','parquet-column').version('1.13.1')
|
|
library('parquet-hadoop', 'org.apache.parquet','parquet-hadoop').version('1.13.1')
|
|
|
|
bundle('slf4j', ['slf4j.api', 'log4j.api', 'log4j.core', 'log4j.slf4j'])
|
|
bundle('slf4j.test', ['slf4j.jdk14'])
|
|
bundle('prometheus', ['prometheus', 'prometheus-servlet', 'prometheus-server', 'prometheus-hotspot'])
|
|
bundle('mariadb', ['mariadb-client', 'hikaricp'])
|
|
bundle('nlp', ['stanford.corenlp', 'opennlp', 'fasttext'])
|
|
bundle('selenium', ['selenium.chrome', 'selenium.java'])
|
|
bundle('handlebars', ['handlebars', 'handlebars.markdown'])
|
|
bundle('grpc', ['protobuf', 'grpc-stub', 'grpc-protobuf', 'grpc-netty'])
|
|
bundle('gson', ['gson', 'gson-type-adapter'])
|
|
bundle('httpcomponents', ['httpcomponents.core', 'httpcomponents.client'])
|
|
bundle('parquet', ['parquet-column', 'parquet-hadoop'])
|
|
bundle('junit', ['junit.jupiter', 'junit.jupiter.engine'])
|
|
bundle('flyway', ['flyway.core', 'flyway.mysql'])
|
|
}
|
|
|
|
|
|
}
|
|
|
|
}
|