feat(docker): implemented dbimport (#13308)

## Changes Proposed:

- Implemented dbimport with docker
- deprecated db_assembler
- Fixed deno scripts and integrated them with the CI

.dockerignore

@@ -9,8 +9,11 @@
 /env/docker/*
 !/env/docker/bin/.gitkeep
 !/env/docker/data/.gitkeep
+!/env/docker/etc/
+/env/docker/etc/*
 !/env/docker/etc/authserver.conf.dockerdist
 !/env/docker/etc/worldserver.conf.dockerdist
+!/env/docker/etc/dbimport.conf.dockerdist
 !/env/docker/logs/.gitkeep
 /.env*
 .idea

.github/workflows/docker_build.yml

@@ -68,7 +68,7 @@ jobs:
         run: |
           export DOCKER_USER_ID=$(id -u)
           export DOCKER_GROUP_ID=$(id -u)
-          docker-compose --profile dev --profile local build --parallel
+          ./acore.sh docker build
 
       - name: Deploy Dev
         #env:
@@ -97,7 +97,7 @@ jobs:
         run: |
           export DOCKER_USER_ID=$(id -u)
           export DOCKER_GROUP_ID=$(id -u)
-          docker-compose --profile build --profile prod build --parallel
+          ./acore.sh docker prod:build
           docker-compose run --no-deps --name build ac-build echo "image created"
           docker cp build:/azerothcore/var/ccache var/docker/
           echo "ccache exported"

.gitignore

@@ -20,8 +20,11 @@
 /env/docker/*
 !/env/docker/bin/.gitkeep
 !/env/docker/data/.gitkeep
+!/env/docker/etc/
+/env/docker/etc/*
 !/env/docker/etc/authserver.conf.dockerdist
 !/env/docker/etc/worldserver.conf.dockerdist
+!/env/docker/etc/dbimport.conf.dockerdist
 !/env/docker/logs/.gitkeep
 /.env*
 /apps/joiner

apps/bash_shared/deno.sh

@@ -1,4 +1,4 @@
-DENO_MIN_VERSION="1.9.1"
+DENO_MIN_VERSION="1.26.0"
 
 function denoInstall() {
 

apps/ci/ci-import-db.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -e
-
-sudo systemctl start mysql
-./acore.sh "db-assembler" "import-all"
-  
-if [ -s modules/mod-premium/sql/example_item_9017.sql ]
-then
-  echo "Import custom module item..."
-  # if the premium module is available insert the example item or else the worldserver dry run will fail
-  mysql -uroot -proot acore_world < modules/mod-premium/sql/example_item_9017.sql
-  echo "Done!"
-fi

apps/compiler/includes/functions.sh

@@ -111,6 +111,7 @@ function comp_compile() {
     echo "Generating confs..."
     cp -n "env/dist/etc/worldserver.conf.dockerdist" "env/dist/etc/worldserver.conf"
     cp -n "env/dist/etc/authserver.conf.dockerdist" "env/dist/etc/authserver.conf"
+    cp -n "env/dist/etc/dbimport.conf.dockerdist" "env/dist/etc/dbimport.conf"
   fi
 
   runHooks "ON_AFTER_BUILD"

apps/db_assembler/README.md

@@ -1,5 +1,7 @@
 ## Description
 
+**ATTENTION:** this tool is not supported anymore. It has been replaced by the **dbimport** tools integrated in AC server sources
+
 This script allows you to assemble all sql files into one so you can easily import it to your databases (or use the main script to import directly). By default, it creates the merged files in `/env/dist`.
 
 ## How to use:
@@ -15,7 +17,7 @@ Just run it to display the options.
 Note: You can even use actions directly by command lines specifying the option.
 Ex:
 
-    ./db_assembler.sh 1  
+    ./db_assembler.sh 1
 
 It will merge all sql files without an interactive menu.
 

apps/db_assembler/conf.dist.sh

@@ -0,0 +1,121 @@
+##############################################
+#
+#  DB ASSEMBLER / EXPORTER CONFIGURATIONS
+#
+##############################################
+
+#
+# Comma separated list of databases
+#
+# You can add another element here if you need
+# to support multiple databases
+#
+
+DBLIST=${DBLIST:-"AUTH,CHARACTERS,WORLD"}
+# convert from comma separated list to an array.
+# This is needed to support environment variables
+readarray -td, DATABASES <<<"$DBLIST";
+
+OUTPUT_FOLDER=${OUTPUT_FOLDER:-"$AC_PATH_ROOT/env/dist/sql/"}
+
+DBASM_WAIT_TIMEOUT=${DBASM_WAIT_TIMEOUT:-5}
+DBASM_WAIT_RETRIES=${DBASM_WAIT_RETRIES:-3}
+
+####### BACKUP
+# Set to true if you want to backup your azerothcore databases before importing the SQL with the db_assembler
+# Do not forget to stop your database software (mysql) before doing so
+
+BACKUP_ENABLE=false
+
+BACKUP_FOLDER="$AC_PATH_ROOT/env/dist/sql/backup/"
+
+#######
+
+# FULL DB
+DB_AUTH_PATHS=(
+    "$SRCPATH/data/sql/base/db_auth/"
+)
+
+DB_CHARACTERS_PATHS=(
+    "$SRCPATH/data/sql/base/db_characters"
+)
+
+DB_WORLD_PATHS=(
+    "$SRCPATH/data/sql/base/db_world/"
+)
+
+# UPDATES
+DB_AUTH_UPDATES_PATHS=(
+    "$SRCPATH/data/sql/updates/db_auth/"
+    "$SRCPATH/data/sql/updates/pending_db_auth/"
+)
+
+DB_CHARACTERS_UPDATES_PATHS=(
+    "$SRCPATH/data/sql/updates/db_characters/"
+    "$SRCPATH/data/sql/updates/pending_db_characters/"
+)
+
+DB_WORLD_UPDATES_PATHS=(
+    "$SRCPATH/data/sql/updates/db_world/"
+    "$SRCPATH/data/sql/updates/pending_db_world/"
+)
+
+# CUSTOM
+DB_AUTH_CUSTOM_PATHS=(
+    "$SRCPATH/data/sql/custom/db_auth/"
+)
+
+DB_CHARACTERS_CUSTOM_PATHS=(
+    "$SRCPATH/data/sql/custom/db_characters/"
+)
+
+DB_WORLD_CUSTOM_PATHS=(
+    "$SRCPATH/data/sql/custom/db_world/"
+)
+
+##############################################
+#
+#  DB EXPORTER/IMPORTER CONFIGURATIONS
+#
+##############################################
+
+#
+# Skip import of base sql files to avoid
+# table dropping
+#
+DB_SKIP_BASE_IMPORT_IF_EXISTS=true
+
+#
+# Example:
+#        "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe"
+#        "/usr/bin/mysql"
+#        "mysql"
+#
+
+DB_MYSQL_EXEC="mysql"
+DB_MYSQL_DUMP_EXEC="mysqldump"
+
+
+DB_AUTH_CONF=${DB_AUTH_CONF:-"MYSQL_USER='acore'; \
+                    MYSQL_PASS='acore'; \
+                    MYSQL_HOST='localhost';\
+                    MYSQL_PORT='3306';\
+                    "}
+
+DB_CHARACTERS_CONF=${DB_CHARACTERS_CONF:-"MYSQL_USER='acore'; \
+                    MYSQL_PASS='acore'; \
+                    MYSQL_HOST='localhost';\
+                    MYSQL_PORT='3306';\
+                    "}
+
+DB_WORLD_CONF=${DB_WORLD_CONF:-"MYSQL_USER='acore'; \
+                    MYSQL_PASS='acore'; \
+                    MYSQL_HOST='localhost';\
+                    MYSQL_PORT='3306';\
+                    "}
+
+DB_AUTH_NAME="acore_auth"
+
+DB_CHARACTERS_NAME="acore_characters"
+
+DB_WORLD_NAME="acore_world"

apps/docker/Dockerfile

@@ -179,6 +179,7 @@ RUN mkdir -p /azerothcore/env/etc/
 COPY --chown=$DOCKER_USER:$DOCKER_USER var/docker/ccache /azerothcore/var/ccache
 COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/authserver.conf.dockerdist /azerothcore/env/dist/etc/authserver.conf.dockerdist
 COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/worldserver.conf.dockerdist /azerothcore/env/dist/etc/worldserver.conf.dockerdist
+COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/dbimport.conf.dockerdist /azerothcore/env/dist/etc/dbimport.conf.dockerdist
 
 # install eluna
 RUN git clone --depth=1 --branch=master https://github.com/azerothcore/mod-eluna.git /azerothcore/modules/mod-eluna
@@ -189,8 +190,8 @@ ENV AC_CCACHE=true
 ENV CCACHE_CPP2=true
 ENV CSCRIPTPCH=OFF
 ENV CCOREPCH=OFF
-# ENV CTOOLS_BUILD=all
+ENV CTOOLS_BUILD=all
-ENV CTOOLS_BUILD=maps-only
+# ENV CTOOLS_BUILD=maps-only
 ENV CSCRIPTS=static
 RUN bash apps/docker/docker-build-prod.sh
 
@@ -225,6 +226,7 @@ RUN mkdir -p /azerothcore/env/dist/bin/lua_scripts
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/etc /azerothcore/env/dist/etc
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/worldserver /azerothcore/env/dist/bin/worldserver
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/lua_scripts /azerothcore/env/dist/bin/lua_scripts
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/dbimport /azerothcore/env/dist/bin/dbimport
 
 #================================================================
 #

apps/docker/docker-cmd.ts

@@ -3,7 +3,7 @@ import * as ink from "https://deno.land/x/ink/mod.ts";
 import {
   Input,
   Select,
-} from "https://deno.land/x/cliffy@v0.18.2/prompt/mod.ts";
+} from "https://deno.land/x/cliffy@v0.25.2/prompt/mod.ts";
 
 const program = new Command();
 

apps/installer/includes/includes.sh

@@ -16,7 +16,6 @@ if [ -f "$AC_PATH_INSTALLER/config.sh"  ]; then
 fi
 
 source "$AC_PATH_APPS/compiler/includes/includes.sh"
-source "$AC_PATH_APPS/db_assembler/includes/includes.sh"
 
 source "$AC_PATH_DEPS/semver_bash/semver.sh"
 

apps/installer/main.sh

@@ -11,16 +11,15 @@ options=(
     "pull (u): Update Repository"                   # 3
     "reset (r): Reset & Clean Repository"           # 4
     "compiler (c): Run compiler tool"               # 5
-    "db-assembler (a): Run db assembler tool"       # 6
+    "module-search (ms): Module Search by keyword" # 6
-    "module-search (ms): Module Search by keyword" # 7
+    "module-install (mi): Module Install by name"  # 7
-    "module-install (mi): Module Install by name"  # 8
+    "module-update (mu): Module Update by name"    # 8
-    "module-update (mu): Module Update by name"    # 9
+    "module-remove: (mr): Module Remove by name"   # 9
-    "module-remove: (mr): Module Remove by name"   # 10
+    "client-data: (gd): download client data from github repository (beta)"   # 10
-    "client-data: (gd): download client data from github repository (beta)"   # 11
+    "run-worldserver (rw): execute a simple restarter for worldserver" # 11
-    "run-worldserver (rw): execute a simple restarter for worldserver" # 12
+    "run-authserver (ra): execute a simple restarter for authserver" # 12
-    "run-authserver (ra): execute a simple restarter for authserver" # 13
+    "docker (dr): Run docker tools" # 13
-    "docker (dr): Run docker tools" # 14
+    "quit: Exit from this menu"                     # 14
-    "quit: Exit from this menu"                     # 15
     )
 
 function _switch() {
@@ -43,35 +42,32 @@ function _switch() {
         ""|"c"|"compiler"|"5")
             bash "$AC_PATH_APPS/compiler/compiler.sh" $_opt
             ;;
-        ""|"a"|"db-assembler"|"6")
+        ""|"ms"|"module-search"|"6")
-            bash "$AC_PATH_APPS/db_assembler/db_assembler.sh" $_opt
-            ;;
-        ""|"ms"|"module-search"|"7")
             inst_module_search "$_opt"
             ;;
-        ""|"mi"|"module-install"|"8")
+        ""|"mi"|"module-install"|"7")
             inst_module_install "$_opt"
             ;;
-        ""|"mu"|"module-update"|"9")
+        ""|"mu"|"module-update"|"8")
             inst_module_update "$_opt"
             ;;
-        ""|"mr"|"module-remove"|"10")
+        ""|"mr"|"module-remove"|"9")
             inst_module_remove "$_opt"
             ;;
-        ""|"gd"|"client-data"|"11")
+        ""|"gd"|"client-data"|"10")
             inst_download_client_data
             ;;
-        ""|"rw"|"run-worldserver"|"12")
+        ""|"rw"|"run-worldserver"|"11")
             inst_simple_restarter worldserver
             ;;
-        ""|"ra"|"run-authserver"|"13")
+        ""|"ra"|"run-authserver"|"12")
             inst_simple_restarter authserver
             ;;
-        ""|"dr"|"docker"|"14")
+        ""|"dr"|"docker"|"13")
             DOCKER=1 denoRunFile "$AC_PATH_APPS/docker/docker-cmd.ts" "${@:2}"
             exit
             ;;
-        ""|"quit"|"15")
+        ""|"quit"|"14")
             echo "Goodbye!"
             exit
             ;;

conf/dist/config.sh

@@ -145,124 +145,3 @@ export CPUPROFILESIGNAL=${CPUPROFILESIGNAL:-12}
 #export HEAPCHECK=${HEAPCHECK:-normal}
 
 
-##############################################
-#
-#  DB ASSEMBLER / EXPORTER CONFIGURATIONS
-#
-##############################################
-
-#
-# Comma separated list of databases
-#
-# You can add another element here if you need
-# to support multiple databases
-#
-
-DBLIST=${DBLIST:-"AUTH,CHARACTERS,WORLD"}
-# convert from comma separated list to an array.
-# This is needed to support environment variables
-readarray -td, DATABASES <<<"$DBLIST";
-
-OUTPUT_FOLDER=${OUTPUT_FOLDER:-"$AC_PATH_ROOT/env/dist/sql/"}
-
-DBASM_WAIT_TIMEOUT=${DBASM_WAIT_TIMEOUT:-5}
-DBASM_WAIT_RETRIES=${DBASM_WAIT_RETRIES:-3}
-
-####### BACKUP
-# Set to true if you want to backup your azerothcore databases before importing the SQL with the db_assembler
-# Do not forget to stop your database software (mysql) before doing so
-
-BACKUP_ENABLE=false
-
-BACKUP_FOLDER="$AC_PATH_ROOT/env/dist/sql/backup/"
-
-#######
-
-# FULL DB
-DB_AUTH_PATHS=(
-    "$SRCPATH/data/sql/base/db_auth/"
-)
-
-DB_CHARACTERS_PATHS=(
-    "$SRCPATH/data/sql/base/db_characters"
-)
-
-DB_WORLD_PATHS=(
-    "$SRCPATH/data/sql/base/db_world/"
-)
-
-# UPDATES
-DB_AUTH_UPDATES_PATHS=(
-    "$SRCPATH/data/sql/updates/db_auth/"
-    "$SRCPATH/data/sql/updates/pending_db_auth/"
-)
-
-DB_CHARACTERS_UPDATES_PATHS=(
-    "$SRCPATH/data/sql/updates/db_characters/"
-    "$SRCPATH/data/sql/updates/pending_db_characters/"
-)
-
-DB_WORLD_UPDATES_PATHS=(
-    "$SRCPATH/data/sql/updates/db_world/"
-    "$SRCPATH/data/sql/updates/pending_db_world/"
-)
-
-# CUSTOM
-DB_AUTH_CUSTOM_PATHS=(
-    "$SRCPATH/data/sql/custom/db_auth/"
-)
-
-DB_CHARACTERS_CUSTOM_PATHS=(
-    "$SRCPATH/data/sql/custom/db_characters/"
-)
-
-DB_WORLD_CUSTOM_PATHS=(
-    "$SRCPATH/data/sql/custom/db_world/"
-)
-
-##############################################
-#
-#  DB EXPORTER/IMPORTER CONFIGURATIONS
-#
-##############################################
-
-#
-# Skip import of base sql files to avoid
-# table dropping
-#
-DB_SKIP_BASE_IMPORT_IF_EXISTS=true
-
-#
-# Example:
-#        "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe"
-#        "/usr/bin/mysql"
-#        "mysql"
-#
-
-DB_MYSQL_EXEC="mysql"
-DB_MYSQL_DUMP_EXEC="mysqldump"
-
-
-DB_AUTH_CONF=${DB_AUTH_CONF:-"MYSQL_USER='acore'; \
-                    MYSQL_PASS='acore'; \
-                    MYSQL_HOST='localhost';\
-                    MYSQL_PORT='3306';\
-                    "}
-
-DB_CHARACTERS_CONF=${DB_CHARACTERS_CONF:-"MYSQL_USER='acore'; \
-                    MYSQL_PASS='acore'; \
-                    MYSQL_HOST='localhost';\
-                    MYSQL_PORT='3306';\
-                    "}
-
-DB_WORLD_CONF=${DB_WORLD_CONF:-"MYSQL_USER='acore'; \
-                    MYSQL_PASS='acore'; \
-                    MYSQL_HOST='localhost';\
-                    MYSQL_PORT='3306';\
-                    "}
-
-DB_AUTH_NAME="acore_auth"
-
-DB_CHARACTERS_NAME="acore_characters"
-
-DB_WORLD_NAME="acore_world"

docker-compose.yml

@@ -291,6 +291,12 @@ services:
       - ${DOCKER_VOL_TOOLS_MMAPS:-./var/extractors/mmaps}:/azerothcore/env/client/mmaps
     profiles: [prod, tools]
 
+  ac-db-import:
+    <<: *ac-shared-conf
+    image: acore/ac-wotlk-worldserver:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    command: ./env/dist/bin/dbimport
+    profiles: [db-import]
+
 volumes:
   ac-database:
   ac-bin:

env/docker/etc/dbimport.conf.dockerdist

@@ -0,0 +1,18 @@
+# Do NOT change those Dir configs
+# Files in LogsDir will reflect on your host directory: docker/worldserver/logs
+LogsDir = "/azerothcore/env/dist/logs"
+DataDir = "/azerothcore/env/dist/data"
+
+# Change this configuration accordingly with your docker setup
+# The format is "hostname;port;username;password;database":
+# - docker containers must be on the same docker network to be able to communicate
+# - the DB hostname will be the name of the database docker container
+LoginDatabaseInfo     = "ac-database;3306;root;password;acore_auth"
+WorldDatabaseInfo     = "ac-database;3306;root;password;acore_world"
+CharacterDatabaseInfo = "ac-database;3306;root;password;acore_characters"
+
+# Add more configuration overwrites by copying settings from worldserver.conf.dist
+LogLevel = 2
+
+# Disable idle connections automatic kick since it doesn't work well on macOS + Docker
+CloseIdleConnections = 0