feat(DB): New DB exporter and versioning tools for DB squashes (#20920)

apps/DatabaseExporter/DatabaseExporter.ps1

@@ -0,0 +1,219 @@
+########################################################################################
+# SETTINGS                                                                             #
+########################################################################################
+$mysql_host = "127.0.0.1"
+$mysql_user = "export"
+$mysql_password = "export"
+$mysql_database_auth = "acore_auth"
+$mysql_database_characters = "acore_characters"
+$mysql_database_world = "acore_world"
+########################################################################################
+# SETTINGS END                                                                         #
+########################################################################################
+
+# Set MySQL password as temporary env var
+$env:MYSQL_PWD = $mysql_password
+
+# Get the directory to sql\base directory
+$scriptDirectory = $PSScriptRoot
+$relativePath = "..\..\data\sql\base"
+$combinedPath = Join-Path -Path $scriptDirectory -ChildPath $relativePath
+$fullPath = Resolve-Path -Path $combinedPath
+
+# Define the output directory (using database name)
+$output_directory_auth = "$fullPath\db_auth"
+$output_directory_characters = "$fullPath\db_characters"
+$output_directory_world = "$fullPath\db_world"
+
+Write-Host " ___                     _    _      ___                "
+Write-Host "/   \ ___ ___  _ _  ___ | |_ | |_   / __| ___  _ _  ___ "
+Write-Host "| - ||_ // -_)| '_|/ _ \|  _||   \ | (__ / _ \| '_|/ -_)"
+Write-Host "|_|_|/__|\___||_|  \___/ \__||_||_| \___|\___/|_|  \___|"
+Write-Host "AzerothCore 3.3.5a  -  www.azerothcore.org"
+Write-Host ""
+Write-Host "Welcome to the AzerothCore Database Exporter for database squashes!"
+Write-Host ""
+Write-Host "You have configured:"
+Write-Host "Database Auth: '$mysql_database_auth'"
+Write-Host "Database Characters: '$mysql_database_characters'"
+Write-Host "Database World: '$mysql_database_world'"
+Write-Host "Output Dir Auth: '$output_directory_auth'"
+Write-Host "Output Dir Characters: '$output_directory_characters'"
+Write-Host "Output Dir World: '$output_directory_world'"
+Write-Host ""
+Write-Host "Make sure you read the entire process before you continue."
+Write-Host "https://github.com/azerothcore/azerothcore-wotlk/blob/master/data/sql/base/database-squash.md"
+Write-Host "https://github.com/azerothcore/azerothcore-wotlk/blob/master/apps/DatabaseExporter/databaseexporter.md"
+Write-Host ""
+
+# Check if the user wants to continue using the tool
+do {
+    $confirmation = Read-Host "Do you want to continue using the tool? (Y/N)"
+    
+    if ($confirmation -eq 'Y' -or $confirmation -eq 'y') {
+        # Continue the script
+        Write-Host "AzerothCore Database Exporter starts."
+        $continue = $true
+    }
+    elseif ($confirmation -eq 'N' -or $confirmation -eq 'n') {
+        # Exit the script
+        Write-Host "Exiting the AzerothCore Database Exporter."
+        exit
+    }
+    else {
+        Write-Host "Invalid input. Please enter Y or N."
+        $continue = $null
+    }
+} while ($continue -eq $null)
+
+# Remove the output directory if it exist
+if (Test-Path $output_directory_auth) {
+    Remove-Item -Path $output_directory_auth -Recurse -Force
+    Write-Host "Deleted directory $output_directory_auth"
+}
+if (Test-Path $output_directory_characters) {
+    Remove-Item -Path $output_directory_characters -Recurse -Force
+    Write-Host "Deleted directory $output_directory_characters"
+}
+if (Test-Path $output_directory_world) {
+    Remove-Item -Path $output_directory_world -Recurse -Force
+    Write-Host "Deleted directory $output_directory_world"
+}
+
+# Create the output directory if it doesn't exist
+if (-not (Test-Path -Path $output_directory_auth)) {
+    New-Item -ItemType Directory -Force -Path $output_directory_auth
+    Write-Host "Created directory $output_directory_auth"
+}
+if (-not (Test-Path -Path $output_directory_characters)) {
+    New-Item -ItemType Directory -Force -Path $output_directory_characters
+    Write-Host "Created directory $output_directory_characters"
+}
+if (-not (Test-Path -Path $output_directory_world)) {
+    New-Item -ItemType Directory -Force -Path $output_directory_world
+    Write-Host "Created directory $output_directory_world"
+}
+
+# Fix for dumping TIMESTAMP data
+$timezone = "+01:00"
+$mysqlCommand = "SET time_zone = '$timezone';"
+$mysqlExec = "mysql -h $mysql_host -u $mysql_user -p$mysql_password -e `"$mysqlCommand`""
+Invoke-Expression -Command $mysqlExec
+
+Write-Host ""
+Write-Host "#########################################################"
+Write-Host "EXPORT AUTH DATABASE START"
+Write-Host "#########################################################"
+Write-Host ""
+Write-Host "Please enter your password for user '$mysql_user'"
+
+# Export Auth Database
+# Connect to MySQL and get all the tables
+$tables_auth = mysql -h $mysql_host -u $mysql_user -D $mysql_database_auth -e "SHOW TABLES;" | Select-Object -Skip 1
+# Iterate through each table and export both the structure and contents into the same SQL file
+foreach ($table in $tables_auth) {
+    # Define the output file path for this table
+    $output_file = "$output_directory_auth\$table.sql"
+
+    # Clear the content of the output file if it exists, or create a new one
+    if (Test-Path $output_file) {
+        Clear-Content -Path $output_file
+    }
+
+    # Export the table structure (CREATE TABLE) and table data (INSERT) to the SQL file
+    $create_table_command = "mysqldump -h $mysql_host -u $mysql_user --skip-tz-utc $mysql_database_auth $table"
+    $create_table_output = Invoke-Expression -Command $create_table_command
+    Add-Content -Path $output_file -Value $create_table_output
+
+    # Format the INSERT values to be on seperate lines.
+    $content = Get-Content $output_file
+    $formattedContent = $content -replace 'VALUES \(', "VALUES`r`n("
+    $formattedContent = $formattedContent -replace '\),', "),`r`n"
+    $formattedContent | Set-Content $output_file
+
+    Write-Host "Exported structure and data for table $table to $output_file"
+}
+
+Write-Host ""
+Write-Host "#########################################################"
+Write-Host "EXPORT AUTH DATABASE END"
+Write-Host "#########################################################"
+Write-Host ""
+Write-Host "#########################################################"
+Write-Host "EXPORT CHARACTERS DATABASE START"
+Write-Host "#########################################################"
+Write-Host ""
+Write-Host "Please enter your password for user '$mysql_user'"
+
+# Export Characters Database
+# Connect to MySQL and get all the tables
+$tables_characters = mysql -h $mysql_host -u $mysql_user -D $mysql_database_characters -e "SHOW TABLES;" | Select-Object -Skip 1
+# Iterate through each table and export both the structure and contents into the same SQL file
+foreach ($table in $tables_characters) {
+    # Define the output file path for this table
+    $output_file = "$output_directory_characters\$table.sql"
+
+    # Clear the content of the output file if it exists, or create a new one
+    if (Test-Path $output_file) {
+        Clear-Content -Path $output_file
+    }
+
+    # Export the table structure (CREATE TABLE) and table data (INSERT) to the SQL file
+    $create_table_command = "mysqldump -h $mysql_host -u $mysql_user --skip-tz-utc $mysql_database_characters $table"
+    $create_table_output = Invoke-Expression -Command $create_table_command
+    Add-Content -Path $output_file -Value $create_table_output
+
+    # Format the INSERT values to be on seperate lines.
+    $content = Get-Content $output_file
+    $formattedContent = $content -replace 'VALUES \(', "VALUES`r`n("
+    $formattedContent = $formattedContent -replace '\),', "),`r`n"
+    $formattedContent | Set-Content $output_file
+
+    Write-Host "Exported structure and data for table $table to $output_file"
+}
+
+Write-Host ""
+Write-Host "#########################################################"
+Write-Host "EXPORT CHARACTERS DATABASE END"
+Write-Host "#########################################################"
+Write-Host ""
+Write-Host "#########################################################"
+Write-Host "EXPORT WORLD DATABASE START"
+Write-Host "#########################################################"
+Write-Host ""
+Write-Host "Please enter your password for user '$mysql_user'"
+
+# Export World Database
+# Connect to MySQL and get all the tables
+$tables_world = mysql -h $mysql_host -u $mysql_user -D $mysql_database_world -e "SHOW TABLES;" | Select-Object -Skip 1
+# Iterate through each table and export both the structure and contents into the same SQL file
+foreach ($table in $tables_world) {
+    # Define the output file path for this table
+    $output_file = "$output_directory_world\$table.sql"
+
+    # Clear the content of the output file if it exists, or create a new one
+    if (Test-Path $output_file) {
+        Clear-Content -Path $output_file
+    }
+
+    # Export the table structure (CREATE TABLE) and table data (INSERT) to the SQL file
+    $create_table_command = "mysqldump -h $mysql_host -u $mysql_user --skip-tz-utc $mysql_database_world $table"
+    $create_table_output = Invoke-Expression -Command $create_table_command
+    Add-Content -Path $output_file -Value $create_table_output
+
+    # Format the INSERT values to be on seperate lines.
+    $content = Get-Content $output_file
+    $formattedContent = $content -replace 'VALUES \(', "VALUES`r`n("
+    $formattedContent = $formattedContent -replace '\),', "),`r`n"
+    $formattedContent | Set-Content $output_file
+
+    Write-Host "Exported structure and data for table $table to $output_file"
+}
+
+Write-Host ""
+Write-Host "#########################################################"
+Write-Host "EXPORT WORLD DATABASE END"
+Write-Host "#########################################################"
+Write-Host ""
+Write-Host "Database Exporter completed."
+Write-Host "Have a nice day :)"

apps/DatabaseExporter/databaseexporter.md

@@ -0,0 +1,85 @@
+# The AzerothCore Database Exporter for Database Squashes
+
+> [!CAUTION]
+> These steps are only for project maintainers who intend to update base files.
+
+## Manual setting updates
+
+Update the settings in `DatabaseExporter.ps1` to reflect your setup by opening it with your preffered text editor.
+
+> [!NOTE]
+> Only update the settings within the SETTINGS block.
+
+These are the default settings:
+```ps
+########################################################################################
+# SETTINGS                                                                             #
+########################################################################################
+$mysql_host = "127.0.0.1"
+$mysql_user = "export"
+$mysql_password = "export"
+$mysql_database_auth = "acore_auth"
+$mysql_database_characters = "acore_characters"
+$mysql_database_world = "acore_world"
+########################################################################################
+# SETTINGS END                                                                         #
+########################################################################################
+```
+
+## Description of the tool
+
+This tool updates the base files automatically. Hence, it must run from this directory.
+
+This is how it works step-by-step:
+
+1. Check that all paths look correct.
+2. Accept to continue using the tool.
+3. The tool will delete the `db_auth` `db_characters` `db_world` directories in `..\..\data\sql\base\`
+4. The tool will create the `db_auth` `db_characters` `db_world` directories in `..\..\data\sql\base\`
+5. The tool will export the `db_auth` table into `..\..\data\sql\base\db_auth\`
+6. The tool will export the `db_characters` table into `..\..\data\sql\base\db_characters\`
+7. The tool will export the `db_world` table into `..\..\data\sql\base\db_world\`
+
+## Run the tool
+
+> [!IMPORTANT]
+> This tool CAN NOT be moved outside this directory. If you do it will create files in the wrong places.
+
+1. Make sure you have MySQL installed on your system and that the mysqldump tool is accessible by your PATH system variable. If it is not set you will encounter errors.
+
+    - Go into System Variables
+    - Open the PATH variable
+    - Add the path to your $\MySQL Server\bin\ - e.g. C:\Program Files\MySQL\MySQL Server 8.4\bin\
+
+2. If you haven't run PowerShell scripts before, you'll need to adjust the execution policy.
+
+    - Open PowerShell as an Administrator.
+    - Run the following command to allow running scripts:
+    ```ps
+    Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
+    ```
+    - This allows scripts to run on your system, but they need to be locally created or downloaded from trusted sources.
+
+3. Open PowerShell (PS)
+
+    - Press Win + X and select Windows PowerShell (Admin) / Terminal (Admin)
+
+4. Navigate to the script
+
+    - In PS, use the `cd` command to change the directory
+    ```ps
+    cd "C:\AzerothCore\apps\DatabaseExporter"
+    ```
+
+5. Run the script
+
+    - In PS, run the script
+    ```ps
+    .\DatabaseExporter.ps1
+    ```
+
+6. Follow the instructions given by the tool.
+
+7. Now refer back to the database-squash.md instructions. (Located in ..\..\data\sql\base\)
+
+Completed :)

apps/VersionUpdater/VersionUpdater.ps1

@@ -0,0 +1,132 @@
+# Get the directory to acore.json
+$scriptDirectory = $PSScriptRoot
+$relativePath = "..\.."
+$combinedPath = Join-Path -Path $scriptDirectory -ChildPath $relativePath
+$fullPath = Resolve-Path -Path $combinedPath
+$jsonFilePath = "$fullPath\acore.json"
+# Get the directory for SQL update
+$relativePathDbWorldUpdate = "..\..\data\sql\updates\db_world"
+$combinedPathDbWorldUpdate = Join-Path -Path $scriptDirectory -ChildPath $relativePathDbWorldUpdate
+$fullPathDbWorldUpdate = Resolve-Path -Path $combinedPathDbWorldUpdate
+
+Write-Host " ___                     _    _      ___                "
+Write-Host "/   \ ___ ___  _ _  ___ | |_ | |_   / __| ___  _ _  ___ "
+Write-Host "| - ||_ // -_)| '_|/ _ \|  _||   \ | (__ / _ \| '_|/ -_)"
+Write-Host "|_|_|/__|\___||_|  \___/ \__||_||_| \___|\___/|_|  \___|"
+Write-Host "AzerothCore 3.3.5a  -  www.azerothcore.org"
+Write-Host ""
+Write-Host "Welcome to the AzerothCore Version Updater for database squashes!"
+Write-Host ""
+Write-Host "You have configured:"
+Write-Host "acore.json Path: '$jsonFilePath'"
+Write-Host "World SQL Updates path: '$fullPathDbWorldUpdate'"
+Write-Host ""
+Write-Host "Make sure you read the entire process before you continue."
+Write-Host "https://github.com/azerothcore/azerothcore-wotlk/blob/master/data/sql/base/database-squash.md"
+Write-Host "https://github.com/azerothcore/azerothcore-wotlk/blob/master/apps/VersionUpdater/versionupdater.md"
+Write-Host ""
+
+# Check if the user wants to continue using the tool
+do {
+    $confirmation = Read-Host "Do you want to continue using the tool? (Y/N)"
+    
+    if ($confirmation -eq 'Y' -or $confirmation -eq 'y') {
+        # Continue the script
+        Write-Host "AzerothCore Version Updater starts."
+        Write-Host ""
+        $continue = $true
+    }
+    elseif ($confirmation -eq 'N' -or $confirmation -eq 'n') {
+        # Exit the script
+        Write-Host "Exiting the AzerothCore Version Updater."
+        exit
+    }
+    else {
+        Write-Host "Invalid input. Please enter Y or N."
+        $continue = $null
+    }
+} while ($continue -eq $null)
+
+# Read the JSON file and convert it to a PowerShell object
+$jsonContent = Get-Content -Path $jsonFilePath | ConvertFrom-Json
+
+# Get the current version
+$currentVersion = $jsonContent.version
+
+# Match version components (major.minor.patch and optional suffix like -dev or -alpha)
+if ($currentVersion -match '(\d+)\.(\d+)\.(\d+)(-.*)?') {
+    $major = $matches[1]
+    $minor = $matches[2]
+    $patch = $matches[3]
+    $suffix = $matches[4]
+
+    # Increment the major version
+    $major = [int]$major + 1
+
+    # Reset minor and patch version to 0 (if incrementing major)
+    $minor = 0
+    $patch = 0
+
+    # Reassemble the version with the suffix if it exists
+    $newVersion = "$major.$minor.$patch$suffix"
+    
+    # Update the version in the JSON object
+    $jsonContent.version = $newVersion
+} else {
+    Write-Host "Unknown error in $jsonFilePath. Exiting."
+    exit
+}
+
+# Convert the updated object back to JSON format
+$newJsonContent = $jsonContent | ConvertTo-Json -Depth 3
+
+# Write the updated content back to the file
+$newJsonContent | Set-Content -Path $jsonFilePath
+
+Write-Host "acore.json version updated to $newVersion"
+
+# Create the SQL Version update file.
+# Get today's date in the format YYYY_MM_DD
+$today = Get-Date -Format "yyyy_MM_dd"
+
+# Get the list of files in the directory that match the pattern "YYYY_MM_DD_versionNumber.sql"
+$existingFiles = Get-ChildItem -Path $fullPathDbWorldUpdate -Filter "$today*_*.sql"
+
+# If no files exist for today, start with version number 00
+if ($existingFiles.Count -eq 0) {
+    [int]$newVersionNumber = 0
+} else {
+    # Extract the version number from the existing files (e.g., YYYY_MM_DD_versionNumber.sql)
+    $maxVersionNumber = $existingFiles | ForEach-Object {
+        if ($_ -match "$today_(\d{2})\.sql") {
+            [int]$matches[1]
+        }
+    } | Measure-Object -Maximum | Select-Object -ExpandProperty Maximum
+
+    # Increment the version number by 1
+    [int]$newVersionNumber = $maxVersionNumber + 1
+}
+
+# Format the new version number as a two-digit number (e.g., 01, 02, etc.)
+$formattedVersionNumber = $newVersionNumber.ToString("D2")
+
+# Define the new filename using the date and incremented version number
+$newFileName = "$today" + "_$formattedVersionNumber.sql"
+$newFilePath = Join-Path -Path $fullPathDbWorldUpdate -ChildPath $newFileName
+
+# Define the SQL content to write to the file
+$tableName = '`version`'
+$db_version = '`db_version`'
+$db_version_content = "'ACDB 335.$major-dev'"
+$cache_id = '`cache_id`'
+$sqlContent = "UPDATE $tableName SET $db_version=$db_version_content, $cache_id=$major LIMIT 1;"
+
+# Write the content to the new SQL file
+$sqlContent | Set-Content -Path $newFilePath
+
+Write-Host "SQL file created: $newFilePath"
+Write-Host "SQL content: $sqlContent"
+
+Write-Host ""
+Write-Host "Version Updater completed."
+Write-Host "Have a nice day :)"

apps/VersionUpdater/versionupdater.md

@@ -0,0 +1,53 @@
+# The AzerothCore Version Updater for Database Squashes
+
+> [!CAUTION]
+> These steps are only for project maintainers who intend to update base files.
+
+## Description of the tool
+
+This tool updates the version in DB and acore.json automatically. Hence, it must run from this directory.
+
+This is how it works step-by-step:
+
+1. Check that all paths look correct.
+2. Accept to continue using the tool.
+3. The tool will update the acore.json file and increment it by 1.
+4. The tool will create a file with the proper UPDATE for world database in `..\..\data\sql\updates\db_world`.
+
+## Run the tool
+
+> [!IMPORTANT]
+> This tool CAN NOT be moved outside this directory. If you do it will create files in the wrong places.
+
+1. If you haven't run PowerShell scripts before, you'll need to adjust the execution policy.
+
+    - Open PowerShell as an Administrator.
+    - Run the following command to allow running scripts:
+    ```ps
+    Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
+    ```
+    - This allows scripts to run on your system, but they need to be locally created or downloaded from trusted sources.
+
+2. Open PowerShell (PS)
+
+    - Press Win + X and select Windows PowerShell (Admin) / Terminal (Admin)
+
+3. Navigate to the script
+
+    - In PS, use the `cd` command to change the directory
+    ```ps
+    cd "C:\AzerothCore\apps\VersionUpdater"
+    ```
+
+4. Run the script
+
+    - In PS, run the script
+    ```ps
+    .\VersionUpdater.ps1
+    ```
+
+5. Follow the instructions given by the tool.
+
+6. Now refer back to the database-squash.md instructions. (Located in ..\..\data\sql\base\)
+
+Completed :)

data/sql/base/database-squash.md

@@ -1,42 +1,31 @@
-New routines around handling database squashes since https://github.com/azerothcore/azerothcore-wotlk/pull/18197
+New process around handling database squashes since https://github.com/azerothcore/azerothcore-wotlk/pull/18197
 
 > [!CAUTION]
 > These steps are only for project maintainers who intend to update base files.
 
-During the DB squash procedure, we do NOT move files.
-The archive dir is NO longer used as part of the DB squash procedure, 
-but simply as a place where to move update files when they get too many.
-
-Moving files to the archive folder is NOT part of the squash procedure anymore.
-
-as the `updates` table in base files always will contain the entries from the updates dir they will never be run again on a clean setup. 
-
 How to do the squash.
 
 > [!IMPORTANT]
 > A squash needs to be done on a clean database. Drop all tables in Auth, Characters and World.
 
-1. Update the acore.json file. Increment version by one
+1. Update the acore.json and DB version by running VersionUpdater.ps1 (Located in ..\apps\VersionUpdater\)
-2. Create a new file in the updates/db_world/ dir, the file should be incremented containing
 
 > [!NOTE]
-> It should NOT be placed as a pending update and should only be done on world db. This is because we want the updated DB version to follow into the base files
+> Read the versionupdater.md file to use it properly.
+
+2. Drop all your databases, and run Worldserver to populate a clean database.
+3. Export the databases using the DatabaseExporter.ps1 (Located in ..\apps\DatabaseExporter\)
 
-```sql
-UPDATE `version` SET `db_version`='ACDB 335.11-dev', `cache_id`=11 LIMIT 1;
-```
 > [!NOTE]
-> Remember to increment the db_version and cache_id the same as acore.json
+> Read the databaseexporter.md file to use it properly.
 
-4. Drop all your databases, and run Worldserver to populate a clean database.
+6. Make a PR
-5. Export the databases using i.e HeidiSQL
 
 > [!IMPORTANT]
-> Set the following values
+> No DB PRs should be merged during this process!
-> Tables -> DROP + CREATE
-> Data -> Delete + insert (truncate existing data)
-> Max INSERT size -> 1024
-> This is so that no unexpected issues occur.
 
-6. Move the exported table files into the base directory to update the existing files.
+> [!NOTE]
-7. Make a PR
+> During the DB squash procedure, we do NOT move files.
+> The archive dir is NO longer used as part of the DB squash procedure, 
+> but simply as a place where to move update files when they get too many
+> as the `updates` table in base files always will contain the entries from the updates dir they will never be run again on a clean setup.