(install) Add two separate templates for the install script

One template is for the full Marginalia Search style install, and the other is for a barebones install with no Marginalia-related fluff.
2024-01-13 18:27:42 +01:00 · 2024-01-13 18:27:42 +01:00 · 4c62065e74
commit 4c62065e74
parent d28fc99119
7 changed files with 376 additions and 20 deletions
--- a/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java
+++ b/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java
@ -3,12 +3,15 @@ package nu.marginalia.nodecfg;
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.nodecfg.model.NodeConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;

 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;

 public class NodeConfigurationService {
+    private final Logger logger = LoggerFactory.getLogger(NodeConfigurationService.class);

    private final HikariDataSource dataSource;

@ -37,7 +40,7 @@ public class NodeConfigurationService {
        }
    }

-    public List<NodeConfiguration> getAll() throws SQLException {
+    public List<NodeConfiguration> getAll() {
        try (var conn = dataSource.getConnection();
             var qs = conn.prepareStatement("""
                     SELECT ID, DESCRIPTION, ACCEPT_QUERIES, AUTO_CLEAN, PRECESSION, KEEP_WARCS, DISABLED
@ -60,6 +63,10 @@ public class NodeConfigurationService {
            }
            return ret;
        }
+        catch (SQLException ex) {
+            logger.warn("Failed to get node configurations", ex);
+            return List.of();
+        }
    }

    public NodeConfiguration get(int nodeId) throws SQLException {
--- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlRendererFactory.java
+++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlRendererFactory.java
@ -24,13 +24,21 @@ public class ControlRendererFactory {

    @SneakyThrows
    public Renderer renderer(String template) {
-        Map<String, Object> globalContext = Map.of(
-                "nodes", nodeConfigurationService.getAll(),
-                "hideMarginaliaApp", Boolean.getBoolean("control.hideMarginaliaApp")
-        );
+
        var baseRenderer = rendererFactory.renderer(template);

-        return (context) -> baseRenderer.render(context, Map.of("global-context", globalContext));
+        // We might want to add some sort of caching here, as this is called for every request
+        // (but we can't cache the result forever, as the node list might change)
+
+        return (context) ->
+            baseRenderer.render(context,
+                    Map.of("global-context",
+                            Map.of(
+                            "nodes", nodeConfigurationService.getAll(),
+                            "hideMarginaliaApp", Boolean.getBoolean("control.hideMarginaliaApp")
+                            )
+                    )
+            );
    }

    public interface Renderer {
--- a/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/NodeConfigurationWatcher.java
+++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/svc/NodeConfigurationWatcher.java
@ -30,18 +30,13 @@ public class NodeConfigurationWatcher {
    private void pollConfiguration() {
        for (;;) {
            List<Integer> goodNodes = new ArrayList<>();
-            try {
-                for (var cfg : configurationService.getAll()) {

-                    if (!cfg.disabled() && cfg.acceptQueries()) {
-                        goodNodes.add(cfg.node());
-                    }
+            for (var cfg : configurationService.getAll()) {
+                if (!cfg.disabled() && cfg.acceptQueries()) {
+                    goodNodes.add(cfg.node());
                }
-                queryNodes = goodNodes;
-            }
-            catch (SQLException ex) {
-                logger.warn("Failed to update node configurations", ex);
            }
+            queryNodes = goodNodes;

            TimeUnit.SECONDS.sleep(10);
        }
--- a/run/install.sh
+++ b/run/install.sh
@ -34,8 +34,22 @@ if [ -e "${1}" ]; then
  exit 1
 fi

-INSTALL_DIR=${1}
+INSTALL_DIR=$(realpath ${1})

+echo "Would you like to set up a:"
+echo
+echo "1) barebones instance"
+echo "2) full Marginalia Search instance?"
+read -p "Enter 1 or 2: " INSTANCE_TYPE
+
+## Validate
+if [ "${INSTANCE_TYPE}" != "1" ] && [ "${INSTANCE_TYPE}" != "2" ]; then
+  echo
+  echo "ERROR: Invalid instance type, choose 1 or 2"
+  exit 1
+fi
+
+echo
 echo "We're going to set up a Mariadb database in docker, please enter some details"

 read -p "MariaDB user (e.g. marginalia): " MARIADB_USER
@ -52,6 +66,7 @@ if [ "${MARIADB_PASSWORD}" != "${MARIADB_PASSWORD2}" ]; then
  exit 1
 fi

+echo
 echo "Will install to ${INSTALL_DIR}"
 read -p "Press enter to continue, or Ctrl-C to abort"

@ -63,7 +78,7 @@ mkdir -p ${INSTALL_DIR}

 echo "** Copying files to ${INSTALL_DIR}"

-for dir in model data conf env; do
+for dir in model data conf conf/properties env; do
  if [ ! -d ${dir} ]; then
    echo "ERROR: ${dir} does not exist"
    exit 1
@ -73,6 +88,11 @@ for dir in model data conf env; do
  find  ${dir} -maxdepth 1 -type f -exec cp -v {} ${INSTALL_DIR}/{} \;
 done

+# for barebones, tell the control service to hide the marginalia app specific stuff
+if [ "${INSTANCE_TYPE}" == "1" ]; then
+  echo "control.hideMarginaliaApp=true" > ${INSTALL_DIR}/conf/properties/control-service.properties
+fi
+
 echo "** Copying settings files"
 cp prometheus.yml ${INSTALL_DIR}/

@ -80,6 +100,7 @@ echo "** Creating directories"
 mkdir -p ${INSTALL_DIR}/logs
 mkdir -p ${INSTALL_DIR}/db
 mkdir -p ${INSTALL_DIR}/index-1/{work,index,backup,storage,uploads}
+mkdir -p ${INSTALL_DIR}/index-2/{work,index,backup,storage,uploads}

 echo "** Updating settings files"

@ -93,6 +114,11 @@ export uval="\$\$MARIADB_USER"
 export pval="\$\$MARIADB_PASSWORD"

 export INSTALL_DIR
-envsubst < install/docker-compose.yml.template >${INSTALL_DIR}/docker-compose.yml
+
+if [ "${INSTANCE_TYPE}" == "1" ]; then
+  envsubst < install/docker-compose-barebones.yml.template >${INSTALL_DIR}/docker-compose.yml
+else
+  envsubst < install/docker-compose-marginalia.yml.template >${INSTALL_DIR}/docker-compose.yml
+fi

 popd
--- a/run/install/docker-compose-barebones.yml.template
+++ b/run/install/docker-compose-barebones.yml.template
@ -0,0 +1,253 @@
+x-svc: &service
+  env_file:
+    - "${INSTALL_DIR}/env/service.env"
+  volumes:
+    - conf:/wmsa/conf:ro
+    - model:/wmsa/model
+    - data:/wmsa/data
+    - logs:/var/log/wmsa
+  networks:
+    - wmsa
+  healthcheck:
+      test: curl -f http://localhost:80/internal/ping || exit 1
+      start_period: 1s
+      interval: 5s
+      timeout: 5s
+      retries: 60
+x-p1: &partition-1
+  env_file:
+    - "${INSTALL_DIR}/env/service.env"
+  healthcheck:
+      test: curl -f http://localhost:80/internal/ping || exit 1
+      start_period: 1s
+      interval: 5s
+      timeout: 5s
+      retries: 60
+  volumes:
+    - conf:/wmsa/conf:ro
+    - model:/wmsa/model
+    - data:/wmsa/data
+    - logs:/var/log/wmsa
+    - index-1:/idx
+    - work-1:/work
+    - backup-1:/backup
+    - storage-1:/storage
+    - uploads-1:/uploads
+  networks:
+    - wmsa
+  environment:
+    - "WMSA_SERVICE_NODE=1"
+x-p2: &partition-2
+  env_file:
+    - "${INSTALL_DIR}/env/service.env"
+  healthcheck:
+      test: curl -f http://localhost:80/internal/ping || exit 1
+      start_period: 1s
+      interval: 5s
+      timeout: 5s
+      retries: 60
+  volumes:
+    - conf:/wmsa/conf:ro
+    - model:/wmsa/model
+    - data:/wmsa/data
+    - logs:/var/log/wmsa
+    - index-2:/idx
+    - work-2:/work
+    - backup-2:/backup
+    - storage-2:/storage
+    - uploads-2:/uploads
+  networks:
+    - wmsa
+  environment:
+    - "WMSA_SERVICE_NODE=2"
+services:
+  index-service-1:
+    <<: *partition-1
+    image: "marginalia/index-service"
+    container_name: "index-service-1"
+    depends_on:
+        control-service:
+          condition: service_healthy
+  executor-service-1:
+    <<: *partition-1
+    image: "marginalia/executor-service"
+    container_name: "executor-service-1"
+    depends_on:
+        control-service:
+          condition: service_healthy
+  index-service-2:
+    <<: *partition-2
+    image: "marginalia/index-service"
+    container_name: "index-service-2"
+    depends_on:
+        control-service:
+          condition: service_healthy
+  executor-service-2:
+    <<: *partition-2
+    image: "marginalia/executor-service"
+    container_name: "executor-service-2"
+    depends_on:
+        control-service:
+          condition: service_healthy
+  query-service:
+    <<: *service
+    image: "marginalia/query-service"
+    container_name: "query-service"
+    depends_on:
+        control-service:
+          condition: service_healthy
+    expose:
+    - 80
+    labels:
+    - "traefik.enable=true"
+    - "traefik.http.routers.search-service.rule=PathPrefix(`/`)"
+    - "traefik.http.routers.search-service.entrypoints=search"
+    - "traefik.http.routers.search-service.middlewares=add-xpublic"
+    - "traefik.http.routers.search-service.middlewares=add-public"
+    - "traefik.http.middlewares.add-xpublic.headers.customrequestheaders.X-Public=1"
+    - "traefik.http.middlewares.add-public.addprefix.prefix=/public"
+  control-service:
+    <<: *service
+    image: "marginalia/control-service"
+    container_name: "control-service"
+    depends_on:
+        mariadb:
+          condition: service_healthy
+    expose:
+    - 80
+    labels:
+    - "traefik.enable=true"
+    - "traefik.http.routers.control-service.rule=PathPrefix(`/`)"
+    - "traefik.http.routers.control-service.entrypoints=control"
+    - "traefik.http.routers.control-service.middlewares=add-xpublic"
+    - "traefik.http.routers.control-service.middlewares=add-public"
+    - "traefik.http.middlewares.add-xpublic.headers.customrequestheaders.X-Public=1"
+    - "traefik.http.middlewares.add-public.addprefix.prefix=/public"
+  mariadb:
+    image: "mariadb:lts"
+    container_name: "mariadb"
+    env_file: "${INSTALL_DIR}/env/mariadb.env"
+    command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
+    ports:
+      - "127.0.0.1:3306:3306/tcp"
+    healthcheck:
+      test: mysqladmin ping -h 127.0.0.1 -u ${uval} --password=${pval}
+      start_period: 5s
+      interval: 5s
+      timeout: 5s
+      retries: 60
+    volumes:
+      - db:/var/lib/mysql
+    networks:
+      - wmsa
+  traefik:
+    image: "traefik:v2.10"
+    container_name: "traefik"
+    command:
+      #- "--log.level=DEBUG"
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.search.address=:80"
+      - "--entrypoints.control.address=:81"
+    ports:
+      - "127.0.0.1:8080:80"
+      - "127.0.0.1:8081:81"
+      - "127.0.0.1:8090:8080"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+    networks:
+      - wmsa
+networks:
+  wmsa:
+volumes:
+  db:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/db
+  logs:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/logs
+  model:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/model
+  conf:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/conf
+  data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/data
+  storage-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-1/storage
+  uploads-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-1/uploads
+  index-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-1/index
+  work-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-1/work
+  backup-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-1/backup
+  storage-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/storage
+  uploads-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/uploads
+  index-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/index
+  work-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/work
+  backup-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/backup
--- a/run/install/docker-compose-marginalia.yml.template
+++ b/run/install/docker-compose-marginalia.yml.template
@ -37,6 +37,29 @@ x-p1: &partition-1
    - wmsa
  environment:
    - "WMSA_SERVICE_NODE=1"
+x-p2: &partition-2
+  env_file:
+    - "${INSTALL_DIR}/env/service.env"
+  healthcheck:
+      test: curl -f http://localhost:80/internal/ping || exit 1
+      start_period: 1s
+      interval: 5s
+      timeout: 5s
+      retries: 60
+  volumes:
+    - conf:/wmsa/conf:ro
+    - model:/wmsa/model
+    - data:/wmsa/data
+    - logs:/var/log/wmsa
+    - index-2:/idx
+    - work-2:/work
+    - backup-2:/backup
+    - storage-2:/storage
+    - uploads-2:/uploads
+  networks:
+    - wmsa
+  environment:
+    - "WMSA_SERVICE_NODE=2"
 services:
  index-service-1:
    <<: *partition-1
@ -52,6 +75,20 @@ services:
    depends_on:
        control-service:
          condition: service_healthy
+  index-service-2:
+    <<: *partition-2
+    image: "marginalia/index-service"
+    container_name: "index-service-2"
+    depends_on:
+        control-service:
+          condition: service_healthy
+  executor-service-2:
+    <<: *partition-2
+    image: "marginalia/executor-service"
+    container_name: "executor-service-2"
+    depends_on:
+        control-service:
+          condition: service_healthy
  query-service:
    <<: *service
    image: "marginalia/query-service"
@ -231,3 +268,33 @@ volumes:
      type: none
      o: bind
      device: ${INSTALL_DIR}/index-1/backup
+  storage-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/storage
+  uploads-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/uploads
+  index-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/index
+  work-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/work
+  backup-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-2/backup
--- a/run/template/conf/properties/system.properties
+++ b/run/template/conf/properties/system.properties
@ -2,7 +2,7 @@ crawler.userAgentString = Mozilla/5.0 (compatible)
 crawler.userAgentIdentifier = GoogleBot
 crawler.poolSize = 256

-search.websiteUrl = https://localhost:8080/
+search.websiteUrl = http://localhost:8080/

 executor.uploadDir = /uploads
 converter.sideloadThreshold = 10000