feat(docker): production images, integrated ccache and many other improvements (#5551)

.devcontainer/devcontainer.json

@@ -33,7 +33,8 @@
     "mhutchie.git-graph",
     "github.vscode-pull-request-github",
     "eamodio.gitlens",
-    "cschlosser.doxdocgen"
+    "cschlosser.doxdocgen",
+    "sanaajani.taskrunnercode"
   ],
 
   // Use 'forwardPorts' to make a list of ports inside the container available locally.

.dockerignore

@@ -1,13 +1,20 @@
 /cmake-build-debug/*
 /build*/
 /var/*
+!/var/build/.gitkeep
+!/var/ccache/.gitkeep
 /env/dist/*
+!/env/dist/.gitkeep
 /env/user/*
-/env/docker/data/*
+/env/docker/*
-/env/docker/logs/*
+!/env/docker/bin/.gitkeep
-/env/docker/etc/*
+!/env/docker/data/.gitkeep
 !/env/docker/etc/authserver.conf.dockerdist
 !/env/docker/etc/worldserver.conf.dockerdist
+!/env/docker/logs/.gitkeep
 /.env*
 .idea
 !.gitkeep
+
+# do not ignore the ccache folder (used by the ci)
+!/var/docker/ccache

.github/workflows/core_build.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Cache
         uses: actions/cache@v2
         with:
-          path: /home/runner/.ccache
+          path: var/ccache
           key: ccache:${{ matrix.os }}:${{ matrix.compiler }}:${{ matrix.modules }}-modules:${{ github.ref }}:${{ github.sha }}
           restore-keys: |
             ccache:${{ matrix.os }}:${{ matrix.compiler }}:${{ matrix.modules }}-modules:${{ github.ref }}

.github/workflows/docker_build.yml

@@ -12,9 +12,6 @@ jobs:
     runs-on: ubuntu-20.04
     if: github.repository == 'azerothcore/azerothcore-wotlk'
     env:
-      DOCKER_EXTENDS_BIND: abstract-no-bind
-      DOCKER_BUILD_WORLD_TARGET: worldserver
-      DOCKER_BUILD_AUTH_TARGET: authserver
       COMPOSE_DOCKER_CLI_BUILD: 1
       DOCKER_BUILDKIT: 1
       BUILDKIT_INLINE_CACHE: 1
@@ -29,14 +26,37 @@ jobs:
 
       - name: Configure
         run: |
+          sudo rm -rf /usr/share/dotnet
           docker --version
           docker-compose --version
 
-      - name: Build
+    # TODO: make it work
+    #   - uses: whoan/docker-build-with-cache-action@v5
+    #     with:
+    #       image_tag: master
+    #       username: ${{ secrets.DOCKERHUB_USERNAME }}
+    #       password: ${{ secrets.DOCKERHUB_TOKEN }}
+    #       compose_file: docker-compose.yml
+
+      # before cache
+      # needed to avoid ccache injected into these images
+      - name: Build Dev
+        env:
+          #DOCKER_IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
+          DOCKER_EXTENDS_BIND: abstract-no-bind
+          DOCKER_CLIENT_TIMEOUT: 400
+          COMPOSE_HTTP_TIMEOUT: 400
+        run: |
+          export DOCKER_USER_ID=$(id -u)
+          export DOCKER_GROUP_ID=$(id -u)
+          docker-compose --profile dev --profile local build --parallel
+
+      - name: Deploy Dev
         #env:
         #  DOCKER_IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
+        if: ${{ steps.extract_branch.outputs.branch == 'master' }}
         run: |
-           docker-compose --profile all build
+          docker-compose --profile dev --profile local push
 
       - name: Login to Docker Hub
         if: ${{ steps.extract_branch.outputs.branch == 'master' }}
@@ -45,9 +65,33 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Deploy
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: var/docker/ccache
+          key: ccache:ubuntu-20.04:clang:without-modules:${{ github.ref }}:${{ github.sha }}
+          restore-keys: |
+            ccache:ubuntu-20.04:clang:without-modules:${{ github.ref }}
+            ccache:ubuntu-20.04:clang:without-modules
+
+      - name: Build Production images
+        env:
+          #DOCKER_IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
+          DOCKER_EXTENDS_BIND: abstract-no-bind
+          DOCKER_AC_BUILD_TARGET: build
+          DOCKER_CLIENT_TIMEOUT: 220
+          COMPOSE_HTTP_TIMEOUT: 220
+        run: |
+          export DOCKER_USER_ID=$(id -u)
+          export DOCKER_GROUP_ID=$(id -u)
+          docker-compose --profile build --profile prod build --parallel
+          docker-compose run --no-deps --name build ac-build echo "image created"
+          docker cp build:/azerothcore/var/ccache var/docker/
+          echo "ccache exported"
+
+      - name: Deploy Production images
         #env:
         #  DOCKER_IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         if: ${{ steps.extract_branch.outputs.branch == 'master' }}
         run: |
-          docker-compose --profile all push
+          docker-compose --profile prod push

.gitignore

@@ -9,13 +9,17 @@
 !/modules/*.sh
 /build*/
 /var/*
+!/var/build/.gitkeep
+!/var/ccache/.gitkeep
 /env/dist/*
+!/env/dist/.gitkeep
 /env/user/*
-/env/docker/data/*
+/env/docker/*
-/env/docker/logs/*
+!/env/docker/bin/.gitkeep
-/env/docker/etc/*
+!/env/docker/data/.gitkeep
 !/env/docker/etc/authserver.conf.dockerdist
 !/env/docker/etc/worldserver.conf.dockerdist
+!/env/docker/logs/.gitkeep
 /.env*
 /apps/joiner
 /deps/deno

.vscode/extensions.json

@@ -10,6 +10,7 @@
         "mhutchie.git-graph",
         "github.vscode-pull-request-github",
         "eamodio.gitlens",
-        "cschlosser.doxdocgen"
+        "cschlosser.doxdocgen",
+        "sanaajani.taskrunnercode"
     ]
 }

.vscode/tasks.json

@@ -2,7 +2,51 @@
     "version": "2.0.0",
     "tasks": [
       {
-        "label": "build",
+        "label": "AzerothCore: Dashboard",
+        "type": "shell",
+        "command": "./acore.sh",
+        "group": "none",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "AzerothCore: Import/update database",
+        "type": "shell",
+        "command": "./acore.sh db-assembler import-all",
+        "group": "build",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "AzerothCore: download client-data",
+        "type": "shell",
+        "command": "./acore.sh client-data",
+        "group": "none",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "AzerothCore: Clean build",
+        "type": "shell",
+        "command": "./acore.sh compiler clean",
+        "group": "build",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "AzerothCore: Build",
         "type": "shell",
         "command": "./acore.sh compiler build",
         "group": {
@@ -14,6 +58,28 @@
           "panel": "new"
         },
         "problemMatcher": []
+      },
+      {
+        "label": "AzerothCore: Run authserver (restarter)",
+        "type": "shell",
+        "command": "./acore.sh run-authserver",
+        "group": "none",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "AzerothCore: Run worldserver (restarter)",
+        "type": "shell",
+        "command": "./acore.sh run-worldserver",
+        "group": "none",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        },
+        "problemMatcher": []
       }
     ]
   }

apps/bash_shared/defines.sh

@@ -24,3 +24,5 @@ AC_PATH_CONF="$AC_PATH_ROOT/conf"
 AC_PATH_MODULES="$AC_PATH_ROOT/modules"
 
 AC_PATH_DEPS="$AC_PATH_ROOT/deps"
+
+AC_PATH_VAR="$AC_PATH_ROOT/var"

apps/ci/ci-compile.sh

@@ -3,9 +3,6 @@
 set -e
 
 echo "compile core"
-export CCACHE_CPP2=true
+export AC_CCACHE=true
-export CCACHE_MAXSIZE='500MB'
-export CCACHE_COMPRESS=1
-ccache -s
 ./acore.sh "compiler" "all"
-ccache -s
+

apps/compiler/compiler.sh

@@ -28,6 +28,7 @@ comp_options=(
     "configure: Run CMake"
     "compile: Compile only"
     "all: clean, configure and compile"
+    "ccacheClean: Clean ccache files, normally not needed"
     "quit: Close this menu")
 comp_functions=(
     "comp_build"
@@ -35,6 +36,7 @@ comp_functions=(
     "comp_configure"
     "comp_compile"
     "comp_all"
+    "comp_ccacheClean"
     "comp_quit")
 
 PS3='[ Please enter your choice ]: '

apps/compiler/includes/functions.sh

@@ -8,6 +8,40 @@ function comp_clean() {
   [ -d "$DIRTOCLEAN" ] && rm -rf $PATTERN
 }
 
+function comp_ccacheEnable() {
+    [ "$AC_CCACHE" != true ] && return
+
+    export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-'1000MB'}
+    #export CCACHE_DEPEND=true
+    export CCACHE_SLOPPINESS=${CCACHE_SLOPPINESS:-pch_defines,time_macros,include_file_mtime}
+    export CCACHE_CPP2=${CCACHE_CPP2:-true} # optimization for clang
+    export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+    export CCACHE_COMPRESSLEVEL=${CCACHE_COMPRESSLEVEL:-9}
+    #export CCACHE_NODIRECT=true
+
+    export CCUSTOMOPTIONS="$CCUSTOMOPTIONS -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+}
+
+function comp_ccacheClean() {
+    [ "$AC_CCACHE" != true ] && echo "ccache is disabled" && return
+
+    echo "Cleaning ccache"
+    ccache -C
+    ccache -s
+}
+
+function comp_ccacheResetStats() {
+    [ "$AC_CCACHE" != true ] && return
+
+    ccache -zc
+}
+
+function comp_ccacheShowStats() {
+    [ "$AC_CCACHE" != true ] && return
+
+    ccache -s
+}
+
 function comp_configure() {
   CWD=$(pwd)
 
@@ -16,6 +50,7 @@ function comp_configure() {
   echo "Build path: $BUILDPATH"
   echo "DEBUG info: $CDEBUG"
   echo "Compilation type: $CTYPE"
+  echo "CCache: $AC_CCACHE"
   # -DCMAKE_BUILD_TYPE=$CCTYPE disable optimization "slow and huge amount of ram"
   # -DWITH_COREDEBUG=$CDEBUG compiled with debug information
 
@@ -28,6 +63,8 @@ function comp_configure() {
     DCONF="-DCONF_DIR=$CONFDIR"
   fi
 
+  comp_ccacheEnable
+
   cmake $SRCPATH -DCMAKE_INSTALL_PREFIX=$BINPATH $DCONF -DSERVERS=$CSERVERS \
   -DSCRIPTS=$CSCRIPTS \
   -DBUILD_TESTING=$CBUILD_TESTING \
@@ -49,11 +86,21 @@ function comp_compile() {
 
   cd $BUILDPATH
 
+  comp_ccacheResetStats
+
   time make -j $MTHREADS
   make -j $MTHREADS install
 
+  comp_ccacheShowStats
+
   cd $CWD
 
+  if [ $DOCKER = 1 ]; then
+    echo "Generating confs..."
+    cp -n "env/dist/etc/worldserver.conf.dockerdist" "env/dist/etc/worldserver.conf"
+    cp -n "env/dist/etc/authserver.conf.dockerdist" "env/dist/etc/authserver.conf"
+  fi
+
   runHooks "ON_AFTER_BUILD"
 }
 

apps/docker/Dockerfile

@@ -1,3 +1,5 @@
+#syntax=docker/dockerfile:1.2
+
 #================================================================
 #
 # DEV: Stage used for the development environment
@@ -41,10 +43,11 @@ RUN addgroup --gid $GROUP_ID acore && \
     echo 'acore ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
 
 # must be created to set the correct permissions on them
+RUN mkdir -p /azerothcore/env/dist/bin
 RUN mkdir -p /azerothcore/env/dist/data
 RUN mkdir -p /azerothcore/env/dist/logs
 RUN mkdir -p /azerothcore/env/dist/etc
-RUN mkdir -p /azerothcore/var/build
+RUN mkdir -p /azerothcore/var/build/obj
 
 # Correct permissions for non-root operations
 RUN chown -R $DOCKER_USER:$DOCKER_USER /home/acore
@@ -157,9 +160,28 @@ CMD ./acore.sh run-worldserver
 #=================================================================
 FROM base as build
 
+ARG DOCKER_USER=acore
+
 LABEL description="AC Image used by the build stage to generate production images"
 
-RUN bash bin/acore-docker-build
+RUN mkdir -p /azerothcore/env/etc/
+
+# check if we have ccache files available outside
+COPY --chown=$DOCKER_USER:$DOCKER_USER var/docker/ccache /azerothcore/var/ccache
+COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/authserver.conf.dockerdist /azerothcore/env/dist/etc/authserver.conf.dockerdist
+COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/worldserver.conf.dockerdist /azerothcore/env/dist/etc/worldserver.conf.dockerdist
+
+# install eluna
+RUN git clone --depth=1 --branch=master --recursive https://github.com/azerothcore/mod-eluna-lua-engine.git /azerothcore/modules/mod-eluna-lua-engine
+
+ENV USER_CONF_PATH=/azerothcore/apps/docker/config-docker.sh
+ENV CTYPE=RelWithDebInfo
+ENV AC_CCACHE=true
+ENV CCACHE_CPP2=true
+ENV CSCRIPTPCH=OFF
+ENV CCOREPCH=OFF
+ENV CTOOLS=ON
+RUN bash apps/docker/docker-build-prod.sh
 
 #================================================================
 #
@@ -169,7 +191,7 @@ RUN bash bin/acore-docker-build
 #=================================================================
 FROM authserver-local as authserver
 
-LABEL description="AC Production ready authserver"
+LABEL description="AC Production: authserver"
 
 ARG DOCKER_USER=acore
 
@@ -184,14 +206,95 @@ COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/au
 #=================================================================
 FROM worldserver-local as worldserver
 
-LABEL description="AC Production ready worldserver"
+LABEL description="AC Production: worldserver"
 
 ARG DOCKER_USER=acore
 
+RUN mkdir -p /azerothcore/env/dist/bin/lua_scripts
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/etc /azerothcore/env/dist/etc
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/worldserver /azerothcore/env/dist/bin/worldserver
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/lua_scripts /azerothcore/env/dist/bin/lua_scripts
+
+#================================================================
+#
+# CLIENT DATA
+#
+#=================================================================
+
+FROM ubuntu:20.04 as client-data
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG DOCKER_USER=acore
+
+LABEL description="AC Production: client-data"
+
+RUN apt-get update && apt-get install -y tzdata curl unzip && rm -rf /var/lib/apt/lists/* ;
+
+# set timezone environment variable
+ENV TZ=Etc/UTC
+
+# set noninteractive mode so tzdata doesn't ask to set timezone on install
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN addgroup --gid $GROUP_ID acore && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID acore && \
+    passwd -d acore && \
+    echo 'acore ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
 
 ENV DATAPATH=/azerothcore/env/dist/data
+ENV DATAPATH_ZIP=/tmp/data.zip
 
-RUN /azerothcore/acore.sh client-data
+RUN --mount=type=bind,target=/azerothcore-temp,readwrite /azerothcore-temp/acore.sh client-data
+
+RUN apt-get remove --purge -y tzdata curl unzip && apt-get autoremove -y
+
+RUN chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
+
+USER $DOCKER_USER
+
+#================================================================
+#
+# TOOLS
+#
+#=================================================================
+
+FROM ubuntu:20.04 as tools
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG DOCKER_USER=acore
+
+LABEL description="AC Production: tools"
+
+# List of timezones: http://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+
+# set timezone environment variable
+ENV TZ=Etc/UTC
+
+# set noninteractive mode so tzdata doesn't ask to set timezone on install
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y libmysqlclient-dev libace-dev libssl-dev libbz2-dev sudo && rm -rf /var/lib/apt/lists/* ;
+
+# Create a non-root user
+RUN addgroup --gid $GROUP_ID acore && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID acore && \
+    passwd -d acore && \
+    echo 'acore ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+RUN mkdir -p /azerothcore/env/client/
+RUN chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
+
+USER $DOCKER_USER
+
+WORKDIR /azerothcore/env/client/
+
+RUN mkdir -p /azerothcore/env/client/dbc
+RUN mkdir -p /azerothcore/env/client/maps
+RUN mkdir -p /azerothcore/env/client/mmaps
+RUN mkdir -p /azerothcore/env/client/vmaps
+
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mapextractor /azerothcore/env/client/mapextractor
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mmaps_generator /azerothcore/env/client/mmaps_generator
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4assembler /azerothcore/env/client/vmap4assembler
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4extractor /azerothcore/env/client/vmap4extractor
 

apps/docker/docker-build-dev.sh

@@ -1,12 +1,10 @@
 #!/usr/bin/env bash
 
-cd /azerothcore
+CUR_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-bash acore.sh compiler build
+IMPORT_DB=$1
 
-echo "Generating confs..."
+source "$CUR_PATH/docker-build-prod.sh"
-cp -n "env/dist/etc/worldserver.conf.dockerdist" "env/dist/etc/worldserver.conf"
-cp -n "env/dist/etc/authserver.conf.dockerdist" "env/dist/etc/authserver.conf"
 
 echo "Fixing EOL..."
 # using -n (new file mode) should also fix the issue
@@ -17,4 +15,4 @@ do
     dos2unix -n $file $file
 done
 
-
+[[ $IMPORT_DB != 0 ]] && bash acore.sh db-assembler import-all || true

apps/docker/docker-build-prod.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cd /azerothcore
+
+bash acore.sh compiler build

apps/docker/docker-cmd.ts

@@ -7,6 +7,12 @@ import {
 
 const program = new Command();
 
+const env = {
+  COMPOSE_DOCKER_CLI_BUILD: "1",
+  DOCKER_BUILDKIT: "1",
+  BUILDKIT_INLINE_CACHE: "1",
+};
+
 program
   .name("acore.sh docker")
   .description("Shell scripts for docker")
@@ -16,85 +22,125 @@ shellCommandFactory(
   "start:app",
   "Startup the authserver and worldserver apps",
   ["docker-compose --profile app up"],
+  env,
 );
 
 shellCommandFactory(
   "start:app:d",
   "Startup the authserver and worldserver apps in detached mode",
   ["docker-compose --profile app up -d"],
+  env,
 );
 
 shellCommandFactory("build", "Build the authserver and worldserver", [
-  "docker-compose --profile all build",
+  "docker-compose --profile local build --parallel",
   "docker image prune -f",
-  "docker-compose run --rm ac-build bash bin/acore-docker-update",
+  "docker-compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
-]);
+], env);
-
-shellCommandFactory(
-  "build:clean",
-  "Clean and run build",
-  [
-    "docker-compose --profile all build",
-    "docker image prune -f",
-    `docker-compose run --rm ac-build bash acore.sh compiler clean`,
-    "docker-compose run --rm ac-build bash bin/acore-docker-update",
-  ],
-);
 
 shellCommandFactory(
   "build:nocache",
   "Build the authserver and worldserver without docker cache",
   [
-    "docker-compose --profile all build --no-cache",
+    "docker-compose --profile local build --no-cache --parallel",
     "docker image prune -f",
-    "docker-compose run --rm ac-build bash bin/acore-docker-update",
+    "docker-compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
   ],
+  env,
 );
 
 shellCommandFactory(
   "build:compile",
   "Run the compilation process only, without rebuilding all docker images and importing db",
   [
-    "docker-compose build ac-build",
+    "docker-compose build --parallel ac-build",
     "docker image prune -f",
-    "docker-compose run --rm ac-build bash acore.sh compiler build",
+    "docker-compose run --rm ac-build bash apps/docker/docker-build-dev.sh 0",
   ],
+  env,
+);
+
+shellCommandFactory(
+  "clean:build",
+  "Clean build files",
+  [
+    "docker image prune -f",
+    `docker-compose run --rm ac-build bash acore.sh compiler clean`,
+  ],
+  env,
 );
 
 shellCommandFactory(
   "client-data",
   "Download client data inside the ac-data volume",
-  ["docker-compose run --rm ac-worldserver bash acore.sh client-data"],
+  ["docker-compose run --rm ac-build bash acore.sh client-data"],
+  env,
 );
 
 shellCommandFactory(
   "db-import",
   "Create and upgrade the database with latest updates",
   ["docker-compose run --rm ac-build bash acore.sh db-assembler import-all"],
+  env,
 );
 
 shellCommandFactory(
   "dev:up",
-  "Start the dev server container",
+  "Start the dev server container in background",
-  ["docker-compose up ac-dev-server"],
+  ["docker-compose up -d ac-dev-server"],
+  env,
 );
 
 shellCommandFactory(
   "dev:build",
   "Build using the dev server, it uses volumes to compile which can be faster on linux & WSL",
   ["docker-compose run --rm ac-dev-server bash acore.sh compiler build"],
+  env,
 );
 
 shellCommandFactory(
   "dev:dash [args...]",
   "Execute acore dashboard within a running ac-dev-server",
   ["docker-compose run --rm ac-dev-server bash acore.sh"],
+  env,
 );
 
 shellCommandFactory(
   "dev:shell [args...]",
   "Open an interactive shell within the dev server",
   ["docker-compose run --rm ac-dev-server bash"],
+  env,
+);
+
+shellCommandFactory(
+  "prod:build",
+  "Build producion services",
+  [
+    "docker-compose --profile prod build --parallel",
+    "docker image prune -f",
+  ],
+  env,
+);
+
+shellCommandFactory(
+  "prod:pull",
+  "Pull production services from the remote registry",
+  ["docker-compose --profile prod pull"],
+  env,
+);
+
+shellCommandFactory(
+  "prod:up",
+  "Start production services (foreground)",
+  ["docker-compose --profile prod-app up"],
+  env,
+);
+
+shellCommandFactory(
+  "prod:up:d",
+  "Start production services (background)",
+  ["docker-compose --profile prod-app up -d"],
+  env,
 );
 
 program
@@ -125,7 +171,7 @@ program
 
     if (!services) {
       console.error("No services available!");
-      return
+      return;
     }
 
     services.pop();
@@ -144,8 +190,8 @@ program
     }
 
     if (!selService) {
-        console.log(`Service ${service} is not available`)
+      console.log(`Service ${service} is not available`);
-        return;
+      return;
     }
 
     command = `docker attach ${selService.split(" ")[0]}`;
@@ -185,7 +231,7 @@ while (true) {
     const command = await Input.prompt({
       message: "Enter the command:",
     });
-    console.log(command)
+    console.log(command);
     await program.parseAsync(command.split(" "));
   } else {
     await program.parseAsync(Deno.args);
@@ -204,6 +250,7 @@ function shellCommandFactory(
   name: string,
   description: string,
   commands: string[],
+  env?: { [key: string]: string },
 ): Command {
   return program
     .command(name)
@@ -231,6 +278,7 @@ function shellCommandFactory(
         const shellCmd = run({
           cmd,
           cwd: process.cwd(),
+          env: { ...process.env, ...env },
         });
 
         const status = await shellCmd.status();

apps/git_tools/subrepo-update.sh

@@ -15,7 +15,7 @@
 #######################
 
 set -e
-ROOT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../"
+ROOT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../"
 # update all submodules
 git submodule update --init --recursive
 git submodule foreach git pull origin master

apps/installer/includes/functions.sh

@@ -227,6 +227,7 @@ function inst_download_client_data {
 
     # first check if it's defined in env, otherwise use the default
     local path="${DATAPATH:-$AC_BINPATH_FULL}"
+    local zipPath="${DATAPATH_ZIP:-"$DATAPATH/data.zip"}"
 
     dataVersionFile="$path/data-version"
 
@@ -240,9 +241,9 @@ function inst_download_client_data {
         return
     fi
 
-    echo "Downloading client data in: $path/data.zip ..."
+    echo "Downloading client data in: $zipPath ..."
-    curl -L https://github.com/wowgaming/client-data/releases/download/$VERSION/data.zip > "$path/data.zip" \
+    curl -L https://github.com/wowgaming/client-data/releases/download/$VERSION/data.zip > "$zipPath" \
-        && echo "unzip downloaded file..." && unzip -q -o "$path/data.zip" -d "$path/" \
+        && echo "unzip downloaded file in $path..." && unzip -q -o "$zipPath" -d "$path/" \
-        && echo "Remove downloaded file" && rm "$path/data.zip" \
+        && echo "Remove downloaded file" && rm "$zipPath" \
         && echo "INSTALLED_VERSION=$VERSION" > "$dataVersionFile"
 }

apps/installer/includes/os_configs/debian.sh

@@ -6,12 +6,12 @@ DEBIAN_VERSION=$(lsb_release -sr)
 
 sudo apt-get update -y
 
+sudo apt-get install -y gdbserver gdb unzip curl libace-6.* libace-dev \
+                     libncurses-dev libreadline-dev clang g++ \
+                     gcc git cmake make ccache
+
 if [[ $DEBIAN_VERSION -eq "10" ]]; then
-  sudo apt-get install -y git cmake make gcc g++ clang default-libmysqlclient-dev \
+  sudo apt-get install -y default-libmysqlclient-dev libssl-dev libreadline-dev libncurses-dev mariadb-server
-  libssl-dev libbz2-dev libreadline-dev libncurses-dev mariadb-server \
-  libace-6.* libace-dev curl unzip gdb gdbserver
 else # Debian 8 and 9 should work using this
-  sudo apt-get install -y git cmake make gcc g++ clang libmysqlclient-dev \
+  sudo apt-get install -y libmysqlclient-dev libssl1.0-dev mysql-server
-  libssl1.0-dev libbz2-dev libreadline-dev libncurses-dev \
-  mysql-server libace-6.* libace-dev curl unzip gdb gdbserver
 fi

apps/installer/includes/os_configs/ubuntu.sh

@@ -8,12 +8,12 @@ UBUNTU_VERSION=$(lsb_release -sr);
 sudo apt-get update -y
 
 # shared deps
-sudo apt-get -y install make cmake clang curl unzip libmysqlclient-dev libace-dev
+sudo apt-get -y install make cmake clang curl unzip libmysqlclient-dev libace-dev ccache
 
 if [[ $CONTINUOUS_INTEGRATION || $DOCKER ]]; then
   sudo apt-get -y install build-essential libtool cmake-data openssl libgoogle-perftools-dev \
   libssl-dev libmysql++-dev libreadline6-dev zlib1g-dev libbz2-dev mysql-client \
-  libncurses5-dev ccache curl unzip
+  libncurses5-dev
 else
   sudo apt-get install -y git gcc g++ gdb gdbserver \
   libssl-dev libbz2-dev libreadline-dev libncurses-dev \

apps/installer/includes/os_configs/windows.sh

@@ -11,7 +11,7 @@ echo "!!README!!: Please install openssl and mysql libraries manually following
 # microsoft-build-tools
 # mysql 5.6
 
-choco install -y --skip-checksums cmake git git.install microsoft-build-tools
+choco install -y --skip-checksums cmake git git.install microsoft-build-tools ccache
 choco install -y --skip-checksums  mysql --version 5.6.12
 
 echo "!!README!!: Please remember to install openssl and mysql libraries manually following our wiki"

bin/README.md

@@ -1,3 +1,5 @@
+The following folder contains scripts aliases. Do not implement scripts here.
+
 All bash script here must be compatible with following environments:
 
 - *linux*: bash 4.x

bin/acore-docker-update

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-CUR_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-source "$CUR_PATH/acore-docker-build"
-
-bash acore.sh db-assembler import-all

conf/dist/config.sh

@@ -3,7 +3,7 @@
 SRCPATH="$AC_PATH_ROOT"
 
 # absolute path where build files must be stored
-BUILDPATH="$AC_PATH_ROOT/var/build/obj"
+BUILDPATH=${BUILDPATH:-"$AC_PATH_VAR/build/obj"}
 
 # absolute path where azerothcore will be installed
 # NOTE: on linux the binaries are stored in a subfolder (/bin)
@@ -27,6 +27,7 @@ BINPATH="$AC_PATH_ROOT/env/dist"
 # by the AC dashboard
 # default: the system will use binpath by default
 # DATAPATH="$BINPATH/bin"
+# DATAPATH_ZIP="$DATAPATH/data.zip"
 
 ##############################################
 #
@@ -45,7 +46,7 @@ CCOMPILERCXX="/usr/bin/clang++"
 
 
 # how many thread must be used for compilation ( leave zero to use all available )
-MTHREADS=0
+MTHREADS=${MTHREADS:-0}
 # enable/disable warnings during compilation
 CWARNINGS=ON
 # enable/disable some debug informations ( it's not a debug compilation )
@@ -63,12 +64,12 @@ CSCRIPTS=${CSCRIPTS:-ON}
 # compile unit tests
 CBUILD_TESTING=OFF
 # compile server
-CSERVERS=ON
+CSERVERS=${CSERVERS:-ON}
 # compile tools
-CTOOLS=OFF
+CTOOLS=${CTOOLS:-OFF}
 # use precompiled headers ( fatest compilation but not optimized if you change headers often )
-CSCRIPTPCH=ON
+CSCRIPTPCH=${CSCRIPTPCH:-ON}
-CCOREPCH=ON
+CCOREPCH=${CCOREPCH:-ON}
 # enable/disable extra logs
 CEXTRA_LOGS=0
 
@@ -79,7 +80,13 @@ CDISABLED_AC_MODULES=""
 # you can add your custom definitions here ( -D )
 # example:  CCUSTOMOPTIONS=" -DWITH_PERFTOOLS=ON -DENABLE_EXTRA_LOGS=ON"
 #
-CCUSTOMOPTIONS=""
+CCUSTOMOPTIONS=${CCUSTOMOPTIONS:-''}
+
+# Enable ccache to speedup
+# recompilations
+#
+AC_CCACHE=${AC_CCACHE:-false}
+export CCACHE_DIR=${CCACHE_DIR:-"$AC_PATH_VAR/ccache"}
 
 
 ##############################################

conf/dist/env.ac

@@ -8,8 +8,10 @@ DATAPATH=/azerothcore/env/dist/data
 #
 # COMPILER
 #
+
 CTYPE=RelWithDebInfo
 CSCRIPTS=ON
+AC_CCACHE=true
 
 #
 # DATABASE

docker-compose.yml

@@ -5,6 +5,16 @@ x-networks: &networks
   networks:
     - ac-network
 
+x-cache-from: &cache-from
+  cache_from:
+    - acore/ac-wotlk-authserver:${DOCKER_IMAGE_TAG:-master}
+    - acore/ac-wotlk-authserver-local:${DOCKER_IMAGE_TAG:-master}
+    - acore/ac-wotlk-worldserver:${DOCKER_IMAGE_TAG:-master}
+    - acore/ac-wotlk-worldserver-local:${DOCKER_IMAGE_TAG:-master}
+    - acore/ac-wotlk-dev-server:${DOCKER_IMAGE_TAG:-master}
+    - acore/ac-wotlk-tools:${DOCKER_IMAGE_TAG:-master}
+    #- acore/ac-wotlk-client-data-server:${DOCKER_IMAGE_TAG:-master}
+
 x-ac-shared-conf: &ac-shared-conf
   <<: *networks
   working_dir: /azerothcore
@@ -23,11 +33,9 @@ services:
     image: local/azerothcore/abstract-bind
     volumes:
       - .:/azerothcore/
-      # env dir shared between services
-      # we cannot use /env/dist to avoid permission issues
-      - ac-env:/azerothcore/env
       # expose some dist folder outside allowing the host to use them
       - ${DOCKER_VOL_CONF:-./conf}:/azerothcore/conf
+      - ${DOCKER_VOL_BIN:-ac-bin}:/azerothcore/env/dist/bin
       - ${DOCKER_VOL_ETC:-./env/docker/etc}:/azerothcore/env/dist/etc
       # [osxfs optimization]: https://stackoverflow.com/a/63437557/1964544
       - ${DOCKER_VOL_LOGS:-./env/docker/logs}:/azerothcore/env/dist/logs:delegated
@@ -42,7 +50,7 @@ services:
 
 #=======================
 #
-# Applications
+# DATABASE
 #
 #=======================
 
@@ -67,6 +75,11 @@ services:
       timeout: 10s
       retries: 40
 
+#=======================
+#
+# APP Services
+#
+#=======================
 
   ac-worldserver:
     <<: *ac-shared-conf
@@ -74,7 +87,7 @@ services:
     stdin_open: true
     tty: true
     command: ./acore.sh run-worldserver
-    image: acore/worldserver:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    image: acore/ac-wotlk-worldserver-local:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
     restart: unless-stopped
     env_file:
         ${DOCKER_AC_ENV_FILE:-conf/dist/env.ac}
@@ -82,44 +95,40 @@ services:
     privileged: true
     build:
       context: .
-      target: ${DOCKER_BUILD_WORLD_TARGET:-worldserver-local}
+      target: worldserver-local
       dockerfile: ./apps/docker/Dockerfile
       args:
         USER_ID: ${DOCKER_USER_ID:-1000}
         GROUP_ID: ${DOCKER_GROUP_ID:-1000}
         DOCKER_USER: ${DOCKER_USER:-acore}
-      cache_from:
+      <<: *cache-from
-        - acore/authserver:${DOCKER_IMAGE_TAG:-master}
-        - acore/dev-server:${DOCKER_IMAGE_TAG:-master}
     ports:
       - ${DOCKER_WORLD_EXTERNAL_PORT:-8085}:8085
       - ${DOCKER_SOAP_EXTERNAL_PORT:-7878}:7878
-    profiles: [all, app, worldserver]
+    profiles: [local, app, worldserver]
 
   ac-authserver:
     <<: *ac-shared-conf
     extends: ${DOCKER_EXTENDS_BIND:-abstract-bind}
     tty: true
     command: ./acore.sh run-authserver
-    image: acore/authserver:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    image: acore/ac-wotlk-authserver-local:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
     restart: unless-stopped
     env_file:
         ${DOCKER_AC_ENV_FILE:-conf/dist/env.ac}
     user: ${DOCKER_USER:-acore}
     build:
       context: .
-      target: ${DOCKER_BUILD_AUTH_TARGET:-authserver-local}
+      target: authserver-local
       dockerfile: ./apps/docker/Dockerfile
       args:
         USER_ID: ${DOCKER_USER_ID:-1000}
         GROUP_ID: ${DOCKER_GROUP_ID:-1000}
         DOCKER_USER: ${DOCKER_USER:-acore}
-      cache_from:
+      <<: *cache-from
-        - acore/worldserver:${DOCKER_IMAGE_TAG:-master}
-        - acore/dev-server:${DOCKER_IMAGE_TAG:-master}
     ports:
       - ${DOCKER_AUTH_EXTERNAL_PORT:-3724}:3724
-    profiles: [all, app, authserver]
+    profiles: [local, app, authserver]
 
 #======================
 #
@@ -129,33 +138,34 @@ services:
   ac-build:
     <<: *ac-shared-conf
     extends: ${DOCKER_EXTENDS_BIND:-abstract-bind}
-    image: acore/dev-server:${DOCKER_IMAGE_TAG:-master}
+    image: acore/ac-wotlk-dev-server:${DOCKER_IMAGE_TAG:-master}
     build:
       context: .
-      target: dev
+      target: ${DOCKER_AC_BUILD_TARGET:-dev}
       dockerfile: ./apps/docker/Dockerfile
       args:
         USER_ID: ${DOCKER_USER_ID:-1000}
         GROUP_ID: ${DOCKER_GROUP_ID:-1000}
         DOCKER_USER: ${DOCKER_USER:-acore}
-      cache_from:
+      <<: *cache-from
-        - acore/dev-server:${DOCKER_IMAGE_TAG:-master}
     env_file:
         ${DOCKER_AC_ENV_FILE:-conf/dist/env.ac}
     working_dir: /azerothcore/
     volumes:
-        - ac-build:/azerothcore/var/build
+        - ${DOCKER_VOL_BUILD:-ac-build}:/azerothcore/var/build
+        # with this conf you can use an external path for it (useful for CI)
+        - ${DOCKER_VOL_CCACHE:-ac-ccache}:/azerothcore/var/ccache
          # use internal copied files instead of volumes
         - /azerothcore/src
         - /azerothcore/data
         - /azerothcore/modules
-    profiles: [all, build]
+    profiles: [local, build]
 
   ac-dev-server:
     <<: *ac-shared-conf
     extends: ${DOCKER_EXTENDS_BIND:-abstract-bind}
     tty: true
-    image: acore/dev-server:${DOCKER_IMAGE_TAG:-master}
+    image: acore/ac-wotlk-dev-server:${DOCKER_IMAGE_TAG:-master}
     build:
       context: .
       target: dev
@@ -164,8 +174,7 @@ services:
         USER_ID: ${DOCKER_USER_ID:-1000}
         GROUP_ID: ${DOCKER_GROUP_ID:-1000}
         DOCKER_USER: ${DOCKER_USER:-acore}
-      cache_from:
+      <<: *cache-from
-        - acore/dev-server:${DOCKER_IMAGE_TAG:-master}
     security_opt:
       - seccomp:unconfined
     env_file:
@@ -179,16 +188,116 @@ services:
       - ${DOCKER_SOAP_EXTERNAL_PORT:-7878}:7878
     volumes:
       - ac-build-dev:/azerothcore/var/build
-    profiles: [all, dev]
+      - ac-ccache-dev:/azerothcore/var/ccache
+      # this is not the directory of the extracted data! It's the client folder used by the extractors
+      - ${DOCKER_CLIENT_DATA_FOLDER:-./var/client}:/azerothcore/env/dist/bin/Data
+    profiles: [dev]
+
+#======================
+#
+# Production services
+#
+#======================
+
+  ac-worldserver-prod:
+    <<: *ac-shared-conf
+    stdin_open: true
+    tty: true
+    command: ./acore.sh run-worldserver
+    image: acore/ac-wotlk-worldserver:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    restart: unless-stopped
+    env_file:
+      ${DOCKER_AC_ENV_FILE:-conf/dist/env.ac}
+    user: ${DOCKER_USER:-acore}
+    privileged: true
+    build:
+      context: .
+      target: worldserver
+      dockerfile: ./apps/docker/Dockerfile
+      args:
+        USER_ID: ${DOCKER_USER_ID:-1000}
+        GROUP_ID: ${DOCKER_GROUP_ID:-1000}
+        DOCKER_USER: ${DOCKER_USER:-acore}
+      <<: *cache-from
+    ports:
+      - ${DOCKER_WORLD_EXTERNAL_PORT:-8085}:8085
+      - ${DOCKER_SOAP_EXTERNAL_PORT:-7878}:7878
+    volumes:
+      - ${DOCKER_VOL_LOGS:-./env/docker/logs}:/azerothcore/env/dist/logs:delegated
+    volumes_from:
+      - ac-client-data-server:ro
     depends_on:
-      ac-database:
+      - ac-client-data-server
-        condition: service_healthy
+    profiles: [prod, prod-app, prod-worldserver]
+
+  ac-authserver-prod:
+    <<: *ac-shared-conf
+    tty: true
+    command: ./acore.sh run-authserver
+    image: acore/ac-wotlk-authserver:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    restart: unless-stopped
+    env_file:
+      ${DOCKER_AC_ENV_FILE:-conf/dist/env.ac}
+    user: ${DOCKER_USER:-acore}
+    build:
+      context: .
+      target: authserver
+      dockerfile: ./apps/docker/Dockerfile
+      args:
+        USER_ID: ${DOCKER_USER_ID:-1000}
+        GROUP_ID: ${DOCKER_GROUP_ID:-1000}
+        DOCKER_USER: ${DOCKER_USER:-acore}
+      <<: *cache-from
+    volumes:
+      - ${DOCKER_VOL_LOGS:-./env/docker/logs}:/azerothcore/env/dist/logs:delegated
+    ports:
+      - ${DOCKER_AUTH_EXTERNAL_PORT:-3724}:3724
+    profiles: [prod, prod-app, prod-authserver]
+
+  ac-client-data-server:
+    image: acore/ac-wotlk-client-data:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    user: ${DOCKER_USER:-acore}
+    build:
+      context: .
+      target: client-data
+      dockerfile: ./apps/docker/Dockerfile
+      args:
+        USER_ID: ${DOCKER_USER_ID:-1000}
+        GROUP_ID: ${DOCKER_GROUP_ID:-1000}
+        DOCKER_USER: ${DOCKER_USER:-acore}
+      <<: *cache-from
+    volumes:
+      - /azerothcore/env/dist/data
+    profiles: [prod, prod-app, clientdata]
+
+  ac-tools:
+    image: acore/ac-wotlk-tools:${DOCKER_IMAGE_TAG:-master} # name of the generated image after built locally
+    user: ${DOCKER_USER:-acore}
+    build:
+      context: .
+      target: tools
+      dockerfile: ./apps/docker/Dockerfile
+      args:
+        USER_ID: ${DOCKER_USER_ID:-1000}
+        GROUP_ID: ${DOCKER_GROUP_ID:-1000}
+        DOCKER_USER: ${DOCKER_USER:-acore}
+      <<: *cache-from
+    working_dir: /azerothcore/env/client/
+    volumes:
+      - ${DOCKER_CLIENT_DATA_FOLDER:-./var/client}:/azerothcore/env/client/Data
+      - ${DOCKER_VOL_TOOLS_DBC:-./var/extractors/dbc}:/azerothcore/env/client/dbc
+      - ${DOCKER_VOL_TOOLS_MAPS:-./var/extractors/maps}:/azerothcore/env/client/maps
+      - ${DOCKER_VOL_TOOLS_VMAPS:-./var/extractors/vmaps}:/azerothcore/env/client/vmaps
+      - ${DOCKER_VOL_TOOLS_MMAPS:-./var/extractors/mmaps}:/azerothcore/env/client/mmaps
+    profiles: [prod, tools]
 
 volumes:
   ac-database:
-  ac-env:
+  ac-bin:
   ac-build-dev:
   ac-build:
+  ac-ccache-dev:
+  ac-ccache:
   ac-proj:
   # not used, but you can use them by setting
   # the DOCKER_VOL_* env variabiles
@@ -196,6 +305,7 @@ volumes:
   ac-etc:
   ac-logs:
   ac-client-data:
+  ac-client-data-prod:
 
 networks:
   ac-network:

src/cmake/compiler/clang/settings.cmake

@@ -3,6 +3,12 @@
 # Copyright (C) 2008-2019 TrinityCore <https://www.trinitycore.org/>
 #
 
+if ((USE_COREPCH OR USE_SCRIPTPCH) AND (CMAKE_C_COMPILER_LAUNCHER STREQUAL "ccache" OR CMAKE_CXX_COMPILER_LAUNCHER STREQUAL "ccache"))
+  message(STATUS "Clang: disable pch timestamp when ccache and pch enabled")
+  # TODO: for ccache https://github.com/ccache/ccache/issues/539
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fno-pch-timestamp")
+endif()
+
 # Set build-directive (used in core to tell which buildtype we used)
 target_compile_definitions(acore-compile-option-interface
   INTERFACE