(run) Add two nodes to the demo setup

build.gradle

@@ -86,16 +86,15 @@ idea {
         // Exclude these directories from being indexed by IntelliJ
         // as they tend to bring the IDE to its knees and use up all
         // Inotify spots in a hurry
-        excludeDirs.add(file("$projectDir/run/backup"))
+        excludeDirs.add(file("$projectDir/run/node-1"))
+        excludeDirs.add(file("$projectDir/run/node-2"))
         excludeDirs.add(file("$projectDir/run/model"))
         excludeDirs.add(file("$projectDir/run/dist"))
-        excludeDirs.add(file("$projectDir/run/samples"))
         excludeDirs.add(file("$projectDir/run/db"))
         excludeDirs.add(file("$projectDir/run/logs"))
         excludeDirs.add(file("$projectDir/run/install"))
         excludeDirs.add(file("$projectDir/run/data"))
         excludeDirs.add(file("$projectDir/run/conf"))
-        excludeDirs.add(file("$projectDir/run/vol"))
         excludeDirs.add(file("$projectDir/run/test-data"))
     }
 }

docker-compose.yml

@@ -21,16 +21,35 @@ x-p1: &partition-1
     - data:/wmsa/data
     - logs:/var/log/wmsa
     - dist:/dist
-    - index:/idx
-    - work:/work
-    - backup:/backup
-    - samples:/storage
+    - index-1:/idx
+    - work-1:/work
+    - backup-1:/backup
+    - samples-1:/storage
   networks:
     - wmsa
   depends_on:
     - mariadb
   environment:
     - "WMSA_SERVICE_NODE=1"
+x-p2: &partition-2
+  env_file:
+    - "run/env/service.env"
+  volumes:
+    - conf:/wmsa/conf:ro
+    - model:/wmsa/model
+    - data:/wmsa/data
+    - logs:/var/log/wmsa
+    - dist:/dist
+    - index-2:/idx
+    - work-2:/work
+    - backup-2:/backup
+    - samples-2:/storage
+  networks:
+    - wmsa
+  depends_on:
+    - mariadb
+  environment:
+    - "WMSA_SERVICE_NODE=2"
 
 services:
   index-service-1:
@@ -41,6 +60,14 @@ services:
     <<: *partition-1
     image: "marginalia.nu/executor-service"
     container_name: "executor-service-1"
+  index-service-2:
+    <<: *partition-2
+    image: "marginalia.nu/index-service"
+    container_name: "index-service-2"
+  executor-service-2:
+    <<: *partition-2
+    image: "marginalia.nu/executor-service"
+    container_name: "executor-service-2"
   search-service:
     <<: *service
     image: "marginalia.nu/search-service"
@@ -104,24 +131,6 @@ volumes:
       type: none
       o: bind
       device: run/db
-  index:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: run/index
-  work:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: run/work
-  backup:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: run/backup
   logs:
     driver: local
     driver_opts:
@@ -152,9 +161,51 @@ volumes:
       type: none
       o: bind
       device: run/dist
-  samples:
+  samples-1:
     driver: local
     driver_opts:
       type: none
       o: bind
-      device: run/samples
+      device: run/node-1/samples
+  index-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-1/index
+  work-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-1/work
+  backup-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-1/backup
+  samples-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-2/samples
+  index-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-2/index
+  work-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-2/work
+  backup-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-2/backup

run/download-samples.sh

@@ -3,7 +3,7 @@
 set -e
 
 SAMPLE_NAME=crawl-${1:-m}
-SAMPLE_DIR="samples/${SAMPLE_NAME}/"
+SAMPLE_DIR="node-1/samples/${SAMPLE_NAME}/"
 
 function download_model {
   model=$1
@@ -21,7 +21,7 @@ if [ -d ${SAMPLE_DIR} ]; then
     echo "${SAMPLE_DIR} already exists; remove it if you want to re-download the sample"
 fi
 
-mkdir -p samples/
+mkdir -p node-1/samples/
 SAMPLE_TARBALL=samples/${SAMPLE_NAME}.tar.gz
 download_model ${SAMPLE_TARBALL} https://downloads.marginalia.nu/${SAMPLE_TARBALL} || rm ${SAMPLE_TARBALL}
 

run/setup.sh

@@ -18,7 +18,8 @@ function download_model {
 
 pushd $(dirname $0)
 
-mkdir -p model logs db samples backup install work index data samples/export
+mkdir -p model logs db  install  data samples
+mkdir -p {node-1,node-2}/{work,index,backup,samples/export}
 
 download_model model/English.DICT https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.DICT
 download_model model/English.RDR https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.RDR