feat(docker): production images, integrated ccache and many other improvements (#5551)

apps/docker/Dockerfile

@@ -1,3 +1,5 @@
+#syntax=docker/dockerfile:1.2
+
 #================================================================
 #
 # DEV: Stage used for the development environment
@@ -41,10 +43,11 @@ RUN addgroup --gid $GROUP_ID acore && \
     echo 'acore ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
 
 # must be created to set the correct permissions on them
+RUN mkdir -p /azerothcore/env/dist/bin
 RUN mkdir -p /azerothcore/env/dist/data
 RUN mkdir -p /azerothcore/env/dist/logs
 RUN mkdir -p /azerothcore/env/dist/etc
-RUN mkdir -p /azerothcore/var/build
+RUN mkdir -p /azerothcore/var/build/obj
 
 # Correct permissions for non-root operations
 RUN chown -R $DOCKER_USER:$DOCKER_USER /home/acore
@@ -157,9 +160,28 @@ CMD ./acore.sh run-worldserver
 #=================================================================
 FROM base as build
 
+ARG DOCKER_USER=acore
+
 LABEL description="AC Image used by the build stage to generate production images"
 
-RUN bash bin/acore-docker-build
+RUN mkdir -p /azerothcore/env/etc/
+
+# check if we have ccache files available outside
+COPY --chown=$DOCKER_USER:$DOCKER_USER var/docker/ccache /azerothcore/var/ccache
+COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/authserver.conf.dockerdist /azerothcore/env/dist/etc/authserver.conf.dockerdist
+COPY --chown=$DOCKER_USER:$DOCKER_USER env/docker/etc/worldserver.conf.dockerdist /azerothcore/env/dist/etc/worldserver.conf.dockerdist
+
+# install eluna
+RUN git clone --depth=1 --branch=master --recursive https://github.com/azerothcore/mod-eluna-lua-engine.git /azerothcore/modules/mod-eluna-lua-engine
+
+ENV USER_CONF_PATH=/azerothcore/apps/docker/config-docker.sh
+ENV CTYPE=RelWithDebInfo
+ENV AC_CCACHE=true
+ENV CCACHE_CPP2=true
+ENV CSCRIPTPCH=OFF
+ENV CCOREPCH=OFF
+ENV CTOOLS=ON
+RUN bash apps/docker/docker-build-prod.sh
 
 #================================================================
 #
@@ -169,7 +191,7 @@ RUN bash bin/acore-docker-build
 #=================================================================
 FROM authserver-local as authserver
 
-LABEL description="AC Production ready authserver"
+LABEL description="AC Production: authserver"
 
 ARG DOCKER_USER=acore
 
@@ -184,14 +206,95 @@ COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/au
 #=================================================================
 FROM worldserver-local as worldserver
 
-LABEL description="AC Production ready worldserver"
+LABEL description="AC Production: worldserver"
 
 ARG DOCKER_USER=acore
 
+RUN mkdir -p /azerothcore/env/dist/bin/lua_scripts
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/etc /azerothcore/env/dist/etc
 COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/worldserver /azerothcore/env/dist/bin/worldserver
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/lua_scripts /azerothcore/env/dist/bin/lua_scripts
+
+#================================================================
+#
+# CLIENT DATA
+#
+#=================================================================
+
+FROM ubuntu:20.04 as client-data
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG DOCKER_USER=acore
+
+LABEL description="AC Production: client-data"
+
+RUN apt-get update && apt-get install -y tzdata curl unzip && rm -rf /var/lib/apt/lists/* ;
+
+# set timezone environment variable
+ENV TZ=Etc/UTC
+
+# set noninteractive mode so tzdata doesn't ask to set timezone on install
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN addgroup --gid $GROUP_ID acore && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID acore && \
+    passwd -d acore && \
+    echo 'acore ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
 
 ENV DATAPATH=/azerothcore/env/dist/data
+ENV DATAPATH_ZIP=/tmp/data.zip
 
-RUN /azerothcore/acore.sh client-data
+RUN --mount=type=bind,target=/azerothcore-temp,readwrite /azerothcore-temp/acore.sh client-data
+
+RUN apt-get remove --purge -y tzdata curl unzip && apt-get autoremove -y
+
+RUN chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
+
+USER $DOCKER_USER
+
+#================================================================
+#
+# TOOLS
+#
+#=================================================================
+
+FROM ubuntu:20.04 as tools
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG DOCKER_USER=acore
+
+LABEL description="AC Production: tools"
+
+# List of timezones: http://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+
+# set timezone environment variable
+ENV TZ=Etc/UTC
+
+# set noninteractive mode so tzdata doesn't ask to set timezone on install
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y libmysqlclient-dev libace-dev libssl-dev libbz2-dev sudo && rm -rf /var/lib/apt/lists/* ;
+
+# Create a non-root user
+RUN addgroup --gid $GROUP_ID acore && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID acore && \
+    passwd -d acore && \
+    echo 'acore ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+RUN mkdir -p /azerothcore/env/client/
+RUN chown -R $DOCKER_USER:$DOCKER_USER /azerothcore
+
+USER $DOCKER_USER
+
+WORKDIR /azerothcore/env/client/
+
+RUN mkdir -p /azerothcore/env/client/dbc
+RUN mkdir -p /azerothcore/env/client/maps
+RUN mkdir -p /azerothcore/env/client/mmaps
+RUN mkdir -p /azerothcore/env/client/vmaps
+
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mapextractor /azerothcore/env/client/mapextractor
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/mmaps_generator /azerothcore/env/client/mmaps_generator
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4assembler /azerothcore/env/client/vmap4assembler
+COPY --chown=$DOCKER_USER:$DOCKER_USER --from=build /azerothcore/env/dist/bin/vmap4extractor /azerothcore/env/client/vmap4extractor
 

apps/docker/docker-build-dev.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+CUR_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+IMPORT_DB=$1
+
+source "$CUR_PATH/docker-build-prod.sh"
+
+echo "Fixing EOL..."
+# using -n (new file mode) should also fix the issue
+# when the file is created with the default acore user but you
+# set a different user into the docker configurations
+for file in "env/dist/etc/"*
+do
+    dos2unix -n $file $file
+done
+
+[[ $IMPORT_DB != 0 ]] && bash acore.sh db-assembler import-all || true

apps/docker/docker-build-prod.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cd /azerothcore
+
+bash acore.sh compiler build

apps/docker/docker-cmd.ts

@@ -7,6 +7,12 @@ import {
 
 const program = new Command();
 
+const env = {
+  COMPOSE_DOCKER_CLI_BUILD: "1",
+  DOCKER_BUILDKIT: "1",
+  BUILDKIT_INLINE_CACHE: "1",
+};
+
 program
   .name("acore.sh docker")
   .description("Shell scripts for docker")
@@ -16,85 +22,125 @@ shellCommandFactory(
   "start:app",
   "Startup the authserver and worldserver apps",
   ["docker-compose --profile app up"],
+  env,
 );
 
 shellCommandFactory(
   "start:app:d",
   "Startup the authserver and worldserver apps in detached mode",
   ["docker-compose --profile app up -d"],
+  env,
 );
 
 shellCommandFactory("build", "Build the authserver and worldserver", [
-  "docker-compose --profile all build",
+  "docker-compose --profile local build --parallel",
   "docker image prune -f",
-  "docker-compose run --rm ac-build bash bin/acore-docker-update",
-]);
-
-shellCommandFactory(
-  "build:clean",
-  "Clean and run build",
-  [
-    "docker-compose --profile all build",
-    "docker image prune -f",
-    `docker-compose run --rm ac-build bash acore.sh compiler clean`,
-    "docker-compose run --rm ac-build bash bin/acore-docker-update",
-  ],
-);
+  "docker-compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
+], env);
 
 shellCommandFactory(
   "build:nocache",
   "Build the authserver and worldserver without docker cache",
   [
-    "docker-compose --profile all build --no-cache",
+    "docker-compose --profile local build --no-cache --parallel",
     "docker image prune -f",
-    "docker-compose run --rm ac-build bash bin/acore-docker-update",
+    "docker-compose run --rm ac-build bash apps/docker/docker-build-dev.sh",
   ],
+  env,
 );
 
 shellCommandFactory(
   "build:compile",
   "Run the compilation process only, without rebuilding all docker images and importing db",
   [
-    "docker-compose build ac-build",
+    "docker-compose build --parallel ac-build",
     "docker image prune -f",
-    "docker-compose run --rm ac-build bash acore.sh compiler build",
+    "docker-compose run --rm ac-build bash apps/docker/docker-build-dev.sh 0",
   ],
+  env,
+);
+
+shellCommandFactory(
+  "clean:build",
+  "Clean build files",
+  [
+    "docker image prune -f",
+    `docker-compose run --rm ac-build bash acore.sh compiler clean`,
+  ],
+  env,
 );
 
 shellCommandFactory(
   "client-data",
   "Download client data inside the ac-data volume",
-  ["docker-compose run --rm ac-worldserver bash acore.sh client-data"],
+  ["docker-compose run --rm ac-build bash acore.sh client-data"],
+  env,
 );
 
 shellCommandFactory(
   "db-import",
   "Create and upgrade the database with latest updates",
   ["docker-compose run --rm ac-build bash acore.sh db-assembler import-all"],
+  env,
 );
 
 shellCommandFactory(
   "dev:up",
-  "Start the dev server container",
-  ["docker-compose up ac-dev-server"],
+  "Start the dev server container in background",
+  ["docker-compose up -d ac-dev-server"],
+  env,
 );
 
 shellCommandFactory(
   "dev:build",
   "Build using the dev server, it uses volumes to compile which can be faster on linux & WSL",
   ["docker-compose run --rm ac-dev-server bash acore.sh compiler build"],
+  env,
 );
 
 shellCommandFactory(
   "dev:dash [args...]",
   "Execute acore dashboard within a running ac-dev-server",
   ["docker-compose run --rm ac-dev-server bash acore.sh"],
+  env,
 );
 
 shellCommandFactory(
   "dev:shell [args...]",
   "Open an interactive shell within the dev server",
   ["docker-compose run --rm ac-dev-server bash"],
+  env,
+);
+
+shellCommandFactory(
+  "prod:build",
+  "Build producion services",
+  [
+    "docker-compose --profile prod build --parallel",
+    "docker image prune -f",
+  ],
+  env,
+);
+
+shellCommandFactory(
+  "prod:pull",
+  "Pull production services from the remote registry",
+  ["docker-compose --profile prod pull"],
+  env,
+);
+
+shellCommandFactory(
+  "prod:up",
+  "Start production services (foreground)",
+  ["docker-compose --profile prod-app up"],
+  env,
+);
+
+shellCommandFactory(
+  "prod:up:d",
+  "Start production services (background)",
+  ["docker-compose --profile prod-app up -d"],
+  env,
 );
 
 program
@@ -125,7 +171,7 @@ program
 
     if (!services) {
       console.error("No services available!");
-      return
+      return;
     }
 
     services.pop();
@@ -144,8 +190,8 @@ program
     }
 
     if (!selService) {
-        console.log(`Service ${service} is not available`)
-        return;
+      console.log(`Service ${service} is not available`);
+      return;
     }
 
     command = `docker attach ${selService.split(" ")[0]}`;
@@ -185,7 +231,7 @@ while (true) {
     const command = await Input.prompt({
       message: "Enter the command:",
     });
-    console.log(command)
+    console.log(command);
     await program.parseAsync(command.split(" "));
   } else {
     await program.parseAsync(Deno.args);
@@ -204,6 +250,7 @@ function shellCommandFactory(
   name: string,
   description: string,
   commands: string[],
+  env?: { [key: string]: string },
 ): Command {
   return program
     .command(name)
@@ -231,6 +278,7 @@ function shellCommandFactory(
         const shellCmd = run({
           cmd,
           cwd: process.cwd(),
+          env: { ...process.env, ...env },
         });
 
         const status = await shellCmd.status();