plugins { id 'java' id "io.freefair.lombok" version "5.3.3.3" id "me.champeau.jmh" version "0.6.6" id "de.undercouch.download" version "5.1.0" id 'jvm-test-suite' } repositories { mavenLocal() maven { url "https://artifactory.cronapp.io/public-release/" } maven { url "https://repo1.maven.org/maven2/" } maven { url "https://www2.ph.ed.ac.uk/maven2/" } maven { url "https://jitpack.io/" } exclusiveContent { forRepository { maven { url = uri("https://jitpack.io") } } filter { // Only use JitPack for the `gson-record-type-adapter-factory` library includeModule("com.github.Marcono1234", "gson-record-type-adapter-factory") } } } sourceSets { e2eTest { java { java { compileClasspath += main.output + test.output runtimeClasspath += main.output + test.output srcDir file('src/e2e/java') } resources.srcDir file('src/e2e/resources') } } jmh { java { java { compileClasspath += main.output + test.output runtimeClasspath += main.output + test.output srcDir file('src/jmh/java') } resources.srcDir file('src/jmh/resources') } } } java { toolchain { languageVersion.set(JavaLanguageVersion.of(17)) } } jmhJar { zip64 true } dependencies { implementation project(':third_party') implementation project(':protocol') implementation 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' implementation 'com.github.jknack:handlebars:4.3.1' implementation 'com.github.jknack:handlebars-markdown:4.2.1' implementation group: 'com.google.code.gson', name: 'gson', version: '2.9.0' implementation 'io.reactivex.rxjava3:rxjava:3.1.5' implementation "com.sparkjava:spark-core:2.9.3" implementation 'com.opencsv:opencsv:5.6' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.2' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.2' implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.2' implementation 'org.slf4j:slf4j-api:1.7.36' testImplementation 'org.slf4j:slf4j-jdk14:2.0.3' implementation 'com.google.guava:guava:31.1-jre' implementation 'com.google.inject:guice:5.1.0' implementation 'com.github.jnr:jnr-ffi:2.2.12' implementation 'org.apache.httpcomponents:httpcore:4.4.15' implementation 'org.apache.httpcomponents:httpclient:4.5.13' implementation 'com.github.ThatJavaNerd:JRAW:1.1.0' implementation group: 'com.h2database', name: 'h2', version: '2.1.210' implementation 'org.jsoup:jsoup:1.15.3' implementation group: 'com.github.crawler-commons', name: 'crawler-commons', version: '1.2' implementation 'org.mariadb.jdbc:mariadb-java-client:3.0.6' implementation group: 'net.sf.trove4j', name: 'trove4j', version: '3.0.3' implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.apache.opennlp:opennlp-tools:1.9.4' implementation 'io.prometheus:simpleclient:0.16.0' implementation 'io.prometheus:simpleclient_servlet:0.16.0' implementation 'io.prometheus:simpleclient_httpserver:0.16.0' implementation 'io.prometheus:simpleclient_hotspot:0.16.0' implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.3' implementation group: 'org.yaml', name: 'snakeyaml', version: '1.30' implementation 'com.syncthemall:boilerpipe:1.2.2' implementation 'com.github.luben:zstd-jni:1.5.2-2' implementation 'com.github.vladimir-bukhtoyarov:bucket4j-core:7.5.0' implementation 'de.rototor.jeuclid:jeuclid-core:3.1.14' implementation 'org.imgscalr:imgscalr-lib:4.2' implementation 'org.jclarion:image4j:0.7' implementation 'commons-net:commons-net:3.8.0' implementation 'org.eclipse.jgit:org.eclipse.jgit:5.12.0.202106070339-r' implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.jsch:5.12.0.202106070339-r' implementation 'com.jcraft:jsch:0.1.55' implementation group: 'org.apache.commons', name: 'commons-compress', version: '1.21' implementation 'edu.stanford.nlp:stanford-corenlp:4.4.0' implementation group: 'it.unimi.dsi', name: 'fastutil', version: '8.5.8' implementation 'org.roaringbitmap:RoaringBitmap:0.9.32' implementation group: 'mysql', name: 'mysql-connector-java', version: '8.0.29' implementation 'com.github.Marcono1234:gson-record-type-adapter-factory:0.2.0' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' testCompileOnly 'org.projectlombok:lombok:1.18.24' testImplementation 'org.projectlombok:lombok:1.18.24' testAnnotationProcessor 'org.projectlombok:lombok:1.18.24' testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.5.1' testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') testImplementation 'org.testcontainers:mariadb:1.17.4' testImplementation 'org.testcontainers:junit-jupiter:1.17.4' e2eTestImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0' e2eTestRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' e2eTestImplementation 'org.projectlombok:lombok:1.18.24' e2eTestAnnotationProcessor 'org.projectlombok:lombok:1.18.24' e2eTestImplementation 'org.testcontainers:nginx:1.17.4' e2eTestImplementation "org.testcontainers:junit-jupiter:1.17.2" e2eTestImplementation 'org.testcontainers:selenium:1.17.4' e2eTestImplementation 'org.seleniumhq.selenium:selenium-remote-driver:4.5.3' e2eTestImplementation 'org.seleniumhq.selenium:selenium-chrome-driver:4.5.3' implementation 'org.seleniumhq.selenium:selenium-chrome-driver:4.5.3' implementation 'org.seleniumhq.selenium:selenium-java:4.5.3' implementation 'org.sejda.imageio:webp-imageio:0.1.6' jmh 'org.openjdk.jmh:jmh-core:1.35' jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.35' implementation 'net.agkn:hll:1.6.0' } configurations { e2eTestImplementation.extendsFrom(testImplementation) } test { maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 maxHeapSize = "8G" useJUnitPlatform() } task fastTests(type: Test) { maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 maxHeapSize = "8G" useJUnitPlatform { excludeTags "slow" } } task e2eTest(type: Test) { maxParallelForks = 1 forkEvery = 1 maxHeapSize = "8G" dependsOn ':shadowJar' dependsOn 'downloadTestData' dependsOn 'downloadRDRModelData' dependsOn 'downloadSentenceModelData' dependsOn 'downloadTokenModelData' dependsOn 'downloadTermFreqData' dependsOn 'IP2LocationFile' classpath = sourceSets.e2eTest.runtimeClasspath testClassesDirs = sourceSets.e2eTest.output.classesDirs useJUnitPlatform { includeTags "e2e" } } task downloadTestData(type: Download) { src 'http://hammurabi.acc.umu.se/mirror/kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2022-05.zim' dest file('data/test/wikipedia_en_100_nopic.zim') overwrite false } task downloadRDRModelData(type: Download) { src (['https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.DICT', 'https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.RDR']) dest file('data/models/') overwrite false } task downloadSentenceModelData(type: Download) { src 'https://dlcdn.apache.org/opennlp/models/ud-models-1.0/opennlp-en-ud-ewt-sentence-1.0-1.9.3.bin' dest file('data/models/opennlp-sentence.bin') overwrite false } task downloadTokenModelData(type: Download) { src 'https://dlcdn.apache.org/opennlp/models/ud-models-1.0/opennlp-en-ud-ewt-tokens-1.0-1.9.3.bin' dest file('data/models/opennlp-tokens.bin') overwrite false } task downloadIP2LocationFile(type: Download) { src 'https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.CSV.ZIP' dest file('data/models/IP2LOCATION-LITE-DB1.CSV.ZIP') overwrite false } task IP2LocationFile(type: Copy) { dependsOn 'downloadIP2LocationFile' def zipFile = file('data/models/IP2LOCATION-LITE-DB1.CSV.ZIP') def outputDir = file("data/models/IP2LOC") from zipTree(zipFile) into outputDir } task downloadTermFreqData(type: Download) { src 'https://downloads.marginalia.nu/model/tfreq-new-algo3.bin' dest file('data/models/tfreq-new-algo3.bin') overwrite false }