From 7b5ec6b98fa47cae83bb2da2e032b29e3d718b28 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Thu, 19 Oct 2023 17:47:38 +0200 Subject: [PATCH] (executor-service) Embed dist/ in executor-service's docker image --- build.gradle | 32 --------- .../build.gradle | 16 ++--- .../website-adjacencies-calculator/readme.md | 0 .../adjacencies/AdjacenciesData.java | 0 .../adjacencies/AdjacenciesLoader.java | 0 .../marginalia/adjacencies/DomainAliases.java | 0 .../adjacencies/SparseBitVector.java | 0 .../WebsiteAdjacenciesCalculator.java | 0 .../adjacencies/SparseBitVectorTest.java | 0 .../executor-service/build.gradle | 5 +- docker-service-with-dist.gradle | 68 +++++++++++++++++++ settings.gradle | 2 +- 12 files changed, 77 insertions(+), 46 deletions(-) rename code/{tools => processes}/website-adjacencies-calculator/build.gradle (91%) rename code/{tools => processes}/website-adjacencies-calculator/readme.md (100%) rename code/{tools => processes}/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java (100%) rename code/{tools => processes}/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java (100%) rename code/{tools => processes}/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java (100%) rename code/{tools => processes}/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java (100%) rename code/{tools => processes}/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java (100%) rename code/{tools => processes}/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java (100%) create mode 100644 docker-service-with-dist.gradle diff --git a/build.gradle b/build.gradle index 5f637070..d7f74837 100644 --- a/build.gradle +++ b/build.gradle @@ -50,37 +50,6 @@ allprojects { } -tasks.register('dist', Copy) { - from subprojects.collect { it.tasks.withType(Tar) } - into "$buildDir/dist" - - // For local development, each processes that are to be triggerable - // from the control-service need to go here to end up somewhere the - // control-service can find them - - doLast { - copy { - from tarTree("$buildDir/dist/converter-process.tar") - into "$projectDir/run/dist/" - } - copy { - from tarTree("$buildDir/dist/crawler-process.tar") - into "$projectDir/run/dist/" - } - copy { - from tarTree("$buildDir/dist/loader-process.tar") - into "$projectDir/run/dist/" - } - copy { - from tarTree("$buildDir/dist/website-adjacencies-calculator.tar") - into "$projectDir/run/dist/" - } - copy { - from tarTree("$buildDir/dist/index-construction-process.tar") - into "$projectDir/run/dist/" - } - } -} idea { module { // Exclude these directories from being indexed by IntelliJ @@ -98,7 +67,6 @@ idea { excludeDirs.add(file("$projectDir/run/test-data")) } } - java { toolchain { languageVersion.set(JavaLanguageVersion.of(21)) diff --git a/code/tools/website-adjacencies-calculator/build.gradle b/code/processes/website-adjacencies-calculator/build.gradle similarity index 91% rename from code/tools/website-adjacencies-calculator/build.gradle rename to code/processes/website-adjacencies-calculator/build.gradle index bbee4c21..8e1c55ac 100644 --- a/code/tools/website-adjacencies-calculator/build.gradle +++ b/code/processes/website-adjacencies-calculator/build.gradle @@ -4,17 +4,18 @@ plugins { id 'application' id 'jvm-test-suite' } +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} application { mainClass = 'nu.marginalia.adjacencies.WebsiteAdjacenciesCalculator' applicationName = 'website-adjacencies-calculator' } -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(21)) - } -} +tasks.distZip.enabled = false dependencies { implementation project(':code:common:model') @@ -34,8 +35,3 @@ dependencies { testImplementation libs.bundles.junit testImplementation libs.mockito } - -configurations { - e2eTestImplementation.extendsFrom(testImplementation) - -} \ No newline at end of file diff --git a/code/tools/website-adjacencies-calculator/readme.md b/code/processes/website-adjacencies-calculator/readme.md similarity index 100% rename from code/tools/website-adjacencies-calculator/readme.md rename to code/processes/website-adjacencies-calculator/readme.md diff --git a/code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java similarity index 100% rename from code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java rename to code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java diff --git a/code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java similarity index 100% rename from code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java rename to code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesLoader.java diff --git a/code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java similarity index 100% rename from code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java rename to code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/DomainAliases.java diff --git a/code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java similarity index 100% rename from code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java rename to code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/SparseBitVector.java diff --git a/code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java similarity index 100% rename from code/tools/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java rename to code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java diff --git a/code/tools/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java b/code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java similarity index 100% rename from code/tools/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java rename to code/processes/website-adjacencies-calculator/src/test/java/nu/marginalia/adjacencies/SparseBitVectorTest.java diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index a899a32e..0047b93b 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -13,7 +13,7 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +apply from: "$rootProject.projectDir/docker-service-with-dist.gradle" java { toolchain { @@ -63,5 +63,4 @@ dependencies { testImplementation 'org.testcontainers:junit-jupiter:1.17.4' -} - +} \ No newline at end of file diff --git a/docker-service-with-dist.gradle b/docker-service-with-dist.gradle new file mode 100644 index 00000000..9954b596 --- /dev/null +++ b/docker-service-with-dist.gradle @@ -0,0 +1,68 @@ +ext { + dockerImage='openjdk:21-slim' + serviceJvmOpts='--enable-preview -ea ${wmsa_jvm_param} -Dservice-host=0.0.0.0 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false' + serviceToolOpts='--enable-preview -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5000' +} + +tasks.register('dockerFile') { + buildDir.mkdir() + + var df = new File(buildDir, "Dockerfile") + doLast { + df.text = """# +# I'm auto-generated, please don't make changes to me or commit me to git +# +# The template exists in docker-service.gradle +# +FROM ${dockerImage} + +ADD ${application.applicationName}.tar / +ADD crawler-process.tar /dist +ADD loader-process.tar /dist +ADD converter-process.tar /dist +ADD website-adjacencies-calculator.tar /dist +ADD index-construction-process.tar /dist + +RUN mkdir /wmsa + +ENV JAVA_TOOL_OPTIONS="${serviceToolOpts}" +ENV JAVA_OPTS="${serviceJvmOpts} " + +ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} +""" + } + it.outputs.file(df) +} + +dockerPrepare { + dependsOn tasks.dockerFile + + dependsOn project(':code:processes:website-adjacencies-calculator').distTar + dependsOn project(':code:processes:crawling-process').distTar + dependsOn project(':code:processes:loading-process').distTar + dependsOn project(':code:processes:converting-process').distTar + dependsOn project(':code:processes:index-constructor-process').distTar +} + +dockerfileZip { + dependsOn tasks.dockerFile +} + +docker { + dockerfile = tasks.dockerFile.outputs.files.singleFile + name = 'marginalia.nu/'+application.applicationName+':latest' + files tasks.distTar.outputs, \ + project(':code:processes:crawling-process').distTar.outputs, \ + project(':code:processes:loading-process').distTar.outputs, \ + project(':code:processes:converting-process').distTar.outputs, \ + project(':code:processes:index-constructor-process').distTar.outputs, \ + project(':code:processes:website-adjacencies-calculator').distTar.outputs + + tags 'latest' + + dependsOn project(':code:processes:crawling-process').distTar + dependsOn project(':code:processes:loading-process').distTar + dependsOn project(':code:processes:converting-process').distTar + dependsOn project(':code:processes:index-constructor-process').distTar + dependsOn project(':code:processes:website-adjacencies-calculator').distTar +} diff --git a/settings.gradle b/settings.gradle index a7315ff1..423077f7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -67,6 +67,7 @@ include 'code:processes:crawling-process' include 'code:processes:loading-process' include 'code:processes:index-constructor-process' include 'code:processes:test-data' +include 'code:processes:website-adjacencies-calculator' include 'code:process-models:crawling-model' include 'code:process-models:work-log' @@ -75,7 +76,6 @@ include 'code:process-models:processed-data' include 'code:tools:term-frequency-extractor' include 'code:tools:experiment-runner' -include 'code:tools:website-adjacencies-calculator' include 'code:tools:screenshot-capture-tool' include 'code:tools:load-test' include 'code:tools:stackexchange-converter'