Core/Setup

* rewritten to be able to dynamicly load it's components
   - CLISetup -> checks for UtilityScripts (config, setup, dbc reader, etc.) -> checks for SetupScripts (individual sql/file generators)
   - each step may now have a help prompt attached. If none are provided, the containing script may provide it's help.
   - all Scripts are self contained modules. No more editing of 3+ files if some component is added/removed
 * removed intermediaries FileGen & SqlGen
 * functional changes
   - allow providing CLI arguments to siteconfig and account UtilityScript and skip the interactive prompts
   - set slot for consumable enchantment items so they are filtrable
   - zones dataset is now localized and generated from GlobalStrings.lua and DungeonMap.dbc. Related data dumps removed.
   - 'aowow' and 'prQueue' executables now have shebangs

    WARNING - command line options have been renamed!
This commit is contained in:
Sarjuuk
2024-06-20 16:32:20 +02:00
parent ab4cf67e80
commit bf184e7555
149 changed files with 9642 additions and 9191 deletions

View File

@@ -20,123 +20,404 @@ class CLISetup
'frFR' => LOCALE_FR,
'deDE' => LOCALE_DE,
'zhCN' => LOCALE_CN, 'enCN' => LOCALE_CN,
'esES' => LOCALE_ES, 'esMX' => LOCALE_ES,
'esES' => LOCALE_ES,
'ruRU' => LOCALE_RU
);
public const SQL_BATCH = 1000; // max. n items per sql insert
public const LOCK_OFF = 0;
public const LOCK_ON = 1;
public const LOCK_RESTORE = 2;
private static $lock = 1;
private static $lock = self::LOCK_ON;
private const ARGV_REQUIRED = 0x01;
private const ARGV_OPTIONAL = 0x02;
private const ARGV_ARRAY = 0x10;
public const ARGV_NONE = 0x00;
public const ARGV_REQUIRED = 0x01;
public const ARGV_OPTIONAL = 0x02;
public const ARGV_PARAM = 0x04; // parameter to another argument
public const ARGV_ARRAY = 0x10; // arg accepts list of values
public const OPT_GRP_SETUP = 0;
public const OPT_GRP_UTIL = 1;
public const OPT_GRP_MISC = 2;
private const GLOBALSTRINGS_LUA = '%s%sinterface/framexml/globalstrings.lua';
private static $opts = [];
private static $optGroups = ['AoWoW Setup', 'Utility Functions', 'Additional Options', 'Additional arguments specific to --build=simpleImg', 'Additional arguments specific to --build=complexImg'];
private static $optDefs = array( // cmd => [groupId, aliasses[], flags, description, appendix]
'setup' => [0, ['s', 'firstrun'], 0x00, 'Step by step initial setup. Resumes if interrupted.', '' ],
'update' => [0, ['u'], 0x00, 'Apply new sql updates fetched from Github and run --sync as needed.', '' ],
'dbconfig' => [1, [], 0x00, 'Set up DB connection.', '' ],
'siteconfig' => [1, [], 0x00, 'Set up site variables.', '' ],
'account' => [1, [], 0x00, 'Create an account with admin privileges.', '' ],
'sql' => [1, [], 0x12, 'Generate DB content from your world tables.', '=<subScriptList,>' ],
'build' => [1, [], 0x12, 'Compile image files and data dumps.', '=<subScriptList,>' ],
'sync' => [1, [], 0x12, 'Regenerate tables/files that depend on given world DB table.', '=<worldTableList,>'],
'dbc' => [1, [], 0x11, 'Extract dbc files from mpqDataDir into sql table. Structure must be defined in setup/dbc.class.php.', '=<dbcfileList,>' ],
'delete' => [2, ['d'], 0x00, 'Delete dbc_* tables generated by this prompt when done.', '' ],
'log' => [2, [], 0x01, 'Write CLI ouput to file.', '=logfile' ],
'help' => [2, ['h'], 0x00, 'Display contextual help, if available.', '' ],
'force' => [2, ['f'], 0x00, 'Force existing files to be overwritten.', '' ],
'locales' => [2, [], 0x12, 'Limit setup to enUS, frFR, deDE, zhCN, esES and/or ruRU. (does not override config settings)', '=<regionCodes,>' ],
'mpqDataDir' => [2, [], 0x02, 'Manually point to directory with extracted mpq files. This is limited to setup/ (default: setup/mpqData/)', '=path/' ],
'icons' => [3, ['1'], 0x00, 'Generate icons for spells, items, classes, races, ect.', '' ],
'glyphs' => [3, ['2'], 0x00, 'Generate decorative glyph symbols displayed on related item and spell pages.', '' ],
'pagetexts' => [3, ['3'], 0x00, 'Generate images contained in text on readable items and gameobjects.', '' ],
'loadingscreens' => [3, ['4'], 0x00, 'Generate loading screen images (not used on page; skipped by default)', '' ],
'talentbgs' => [4, ['1'], 0x00, 'Generate backgrounds for the talent calculator.', '' ],
'maps' => [4, ['2'], 0x00, 'Generate zone and continental maps.', '' ],
'spawn-maps' => [4, ['3'], 0x00, 'Fallback to generate alpha masks for each zone to match creature and gameobject spawn points.', '' ],
'artwork' => [4, ['4'], 0x00, 'Generate images from /glues/credits (not used on page; skipped by default))', '' ],
'area-maps' => [4, ['5'], 0x00, 'Generate additional area maps with highlighting for subzones (optional; skipped by default)', '' ]
private static $optGroups = ['AoWoW Setup', 'Utility Functions', 'Additional Options'];
private static $optDefs = array( // cmd => [groupId, aliases[], argvFlags, description, appendix]
'delete' => [self::OPT_GRP_MISC, ['d'], self::ARGV_NONE, 'Delete dbc_* tables generated by this prompt when done. (not recommended)', '' ],
'log' => [self::OPT_GRP_MISC, [], self::ARGV_REQUIRED, 'Write CLI ouput to file.', '=logfile' ],
'help' => [self::OPT_GRP_MISC, ['h'], self::ARGV_NONE, 'Display contextual help, if available.', '' ],
'force' => [self::OPT_GRP_MISC, ['f'], self::ARGV_NONE, 'Force existing files to be overwritten.', '' ],
'locales' => [self::OPT_GRP_MISC, [], self::ARGV_ARRAY | self::ARGV_OPTIONAL, 'Limit setup to enUS, frFR, deDE, zhCN, esES and/or ruRU. (does not override config settings)', '=<regionCodes,>'],
'datasrc' => [self::OPT_GRP_MISC, [], self::ARGV_OPTIONAL, 'Manually point to directory with extracted mpq files. This is limited to setup/ (default: setup/mpqdata/)', '=path/' ],
);
private static $utilScriptRefs = [];
private static $setupScriptRefs = [];
private static $tmpStore = [];
private static $gsFiles = [];
/**************************/
/* command line arguments */
/**************************/
public static function registerUtility(UtilityScript $us) : void
{
if (isset(self::$optDefs[$us::COMMAND]) || isset(self::$utilScriptRefs[$us::COMMAND]))
{
CLI::write(' Utility function '.CLI::bold($us::COMMAND).' already defined.', CLI::LOG_ERROR);
return;
}
self::$optDefs[$us::COMMAND] = [$us->optGroup, $us->argvOpts, $us->argvFlags, $us::DESCRIPTION, $us::APPENDIX];
self::$utilScriptRefs[$us::COMMAND] = $us;
}
public static function registerSetup(string $invoker, SetupScript $ss) : void
{
if (isset(self::$optDefs[$invoker]) || isset(self::$utilScriptRefs[$invoker]))
{
CLI::write(' Utility function '.CLI::bold($invoker).' not defined. Can\'t attach Subscript '.CLI::bold($ss->getName()).', invoker is missing. Skipping...', CLI::LOG_ERROR);
return;
}
if (isset(self::$setupScriptRefs[$invoker][$ss->getName()]))
{
CLI::write(' Subscript function '.CLI::bold($ss->getName()).' already defined for invoker '.CLI::bold($invoker).'. Skipping...', CLI::LOG_ERROR);
return;
}
if ($childArgs = $ss->getSubCommands())
{
if ($duplicates = array_intersect(array_keys($childArgs), array_keys(self::$optDefs)))
{
CLI::write(' Subscript function '.CLI::bold($ss->getName()).'\'s child arguments --'.implode(', --', $duplicates).' are already defined. Skipping...', CLI::LOG_ERROR);
return;
}
$newIdx = count(self::$optGroups);
self::$optGroups[] = '--' . $invoker . '=' . $ss->getName();
foreach ($childArgs as $cmd => [$aliases, $argFlags, $description])
self::$optDefs[$cmd] = [$newIdx, $aliases, $argFlags, $description, ''];
}
// checks done ... store SetupScript
if (self::checkDependencies($ss))
{
self::$setupScriptRefs[] = [$invoker, $ss->getName(), $ss];
// recheck temp stored dependencies
foreach (self::$tmpStore as $idx => [$invoker, $ts])
{
if (!self::checkDependencies($ts))
continue;
self::$setupScriptRefs[] = [$invoker, $ts->getName(), $ts];
unset(self::$tmpStore[$idx]);
}
}
else // if dependencies haven't been stored yet, put aside for later use
self::$tmpStore[] = [$invoker, $ss];
}
private static function checkDependencies(SetupScript &$ss) : bool
{
if ($ss->isOptional) // optional scripts should no depend on anything
return true;
[$sDep, $bDep] = $ss->getSelfDependencies();
return ((!$sDep || $sDep == array_intersect($sDep, array_column(array_filter(self::$setupScriptRefs, function($x) { return $x[0] == 'sql'; }), 1))) &&
(!$bDep || $bDep == array_intersect($bDep, array_column(array_filter(self::$setupScriptRefs, function($x) { return $x[0] == 'build'; }), 1))));
}
public static function loadScripts() : void
{
foreach (glob('setup/tools/clisetup/*.us.php') as $file)
include_once $file;
if (self::$tmpStore)
{
CLI::write('Some SubScripts have unresolved dependencies and have not been loaded', CLI::LOG_ERROR);
CLI::write();
$tbl = [['Name', '--sql dep.', '--build dep.']];
foreach (self::$tmpStore as [$_, $ssRef])
{
[$sDep, $bDep] = $ssRef->getSelfDependencies();
$missS = array_intersect($sDep, array_column(array_filter(self::$setupScriptRefs, function($x) { return $x[0] == 'sql'; }), 1));
$missB = array_intersect($sDep, array_column(array_filter(self::$setupScriptRefs, function($x) { return $x[0] == 'build'; }), 1));
array_walk($sDep, function (&$x) use($missS) { $x = in_array($x, $missS) ? $x : CLI::red($x); });
array_walk($bDep, function (&$x) use($missB) { $x = in_array($x, $missB) ? $x : CLI::red($x); });
$tbl[] = [$ssRef->getName(), implode(', ', $sDep), implode(', ', $bDep)];
}
CLI::writeTable($tbl);
}
// link SubScipts back to UtilityScript after all UtilityScripts have been loaded
foreach (self::$utilScriptRefs as $name => $us)
if (in_array('TrSubScripts', class_uses($us)))
$us->assignGenerators($name);
self::evalOpts();
}
public static function getSubScripts(string $invoker = '') : generator
{
foreach (self::$setupScriptRefs as [$src, $name, $ref])
if (!$invoker || $src == $invoker)
yield $name => [$src, $ref];
}
public static function init() : void
{
$short = '';
$long = [];
$alias = [];
foreach (self::$optDefs as $opt => [$idx, $aliasses, $flags, , ])
{
if ($flags & self::ARGV_REQUIRED)
$opt .= ':';
else if ($flags & self::ARGV_OPTIONAL)
$opt .= '::';
$long[] = $opt;
foreach ($aliasses as $a)
{
if ($flags & self::ARGV_REQUIRED) // neither should be set with shortOpts
$_a = $a.':';
else if ($flags & self::ARGV_OPTIONAL)
$_a = $a.'::';
else
$_a = $a;
$alias[$a] = $opt;
if (strlen($a) == 1)
$short .= $_a;
else
$long[] = $_a;
}
}
if ($opts = getopt($short, $long))
foreach ($opts as $o => $v)
self::$opts[$alias[$o] ?? $o] = (self::$optDefs[$alias[$o] ?? $o][2] & self::ARGV_ARRAY) ? ($v ? explode(',', $v) : []) : ($v ?: true);
self::evalOpts();
// optional logging
if (isset(self::$opts['log']))
CLI::initLogFile(trim(self::$opts['log']));
// alternative data source (no quotes, use forward slash)
if (isset(self::$opts['mpqDataDir']))
self::$srcDir = CLI::nicePath(self::$opts['mpqDataDir']);
if (isset(self::$opts['datasrc']))
self::$srcDir = CLI::nicePath(self::$opts['datasrc']);
// optional limit handled locales
if (isset(self::$opts['locales']))
{
// engb and enus are identical for all intents and purposes
$from = ['engb', 'esmx', 'encn'];
$to = ['enus', 'eses', 'zhcn'];
$opts['locales'] = str_ireplace($from, $to, strtolower($opts['locales']));
$from = ['engb', 'encn'];
$to = ['enus', 'zhcn'];
self::$locales = array_intersect(Util::$localeStrings, explode(',', $opts['locales']));
self::$opts['locales'] = str_ireplace($from, $to, self::$opts['locales']);
self::$locales = array_intersect(Util::$localeStrings, array_map('strtolower', self::$opts['locales']));
}
if (!self::$locales)
self::$locales = array_filter(Util::$localeStrings);
// restrict actual locales
foreach (self::$locales as $idx => $str)
foreach (self::$locales as $idx => $_)
{
if (!($l = Cfg::get('LOCALES')) || ($l & (1 << $idx)))
self::$localeIds[] = $idx;
else
unset(self::$locales[$idx]);
}
if (!self::$localeIds)
CLI::write('No valid locale specified. Check your config or --locales parameter, if used', CLI::LOG_ERROR);
// get site status
if (DB::isConnected(DB_AOWOW))
self::$lock = (int)Cfg::get('MAINTENANCE');
self::$lock = Cfg::get('MAINTENANCE');
else
self::$lock = self::LOCK_ON;
}
public static function getOpt(...$args)
public static function writeCLIHelp(bool $full = false) : void
{
$cmd = self::getOpt(1 << self::OPT_GRP_SETUP | 1 << self::OPT_GRP_UTIL);
if (!$cmd || !self::$utilScriptRefs[$cmd[0]]->writeCLIHelp())
{
$lines = [];
foreach (self::$optGroups as $idx => $og)
{
if (!$full && $idx > self::OPT_GRP_SETUP)
continue;
$lines[] = [$og, ''];
foreach (self::$optDefs as $opt => [$group, $alias, , $desc, $app])
{
if ($group != $idx)
continue;
$cmd = ' --'.$opt;
foreach ($alias as $a)
$cmd .= ' | '.(strlen($a) == 1 ? '-'.$a : '--'.$a);
$lines[] = [$cmd.$app, $desc];
}
}
CLI::writeTable($lines);
CLI::write();
}
}
// called from Setup
public static function runInitial() : void
{
global $argc, $argv; // todo .. find better way? argv, argc are effectivley already global
// get arguments present in argGroup 1 or 2, if set. Pick first.
$cmd = self::getOpt(1 << self::OPT_GRP_SETUP | 1 << self::OPT_GRP_UTIL)[0];
$us = &self::$utilScriptRefs[$cmd];
$inOut = [null, null, null, null];
$allOk = true;
$i = 0;
if ($us::USE_CLI_ARGS)
foreach ($argv as $n => $arg)
{
if (!$n || ($arg && $arg[0] == '-')) // not parent; not handled by getOpt()
continue;
$inOut[$i++] = $arg;
if ($i > 3)
break;
}
if ($dbError = array_filter($us::REQUIRED_DB, function ($x) { return !DB::isConnected($x); }))
{
CLI::write('Database on index '.implode(', ', $dbError).' not yet set up!', CLI::LOG_ERROR);
CLI::write('Please use '.CLI::bold('"php aowow --db"').' for setup', CLI::LOG_BLANK);
CLI::write();
return;
}
if ($us::LOCK_SITE != self::LOCK_OFF)
self::siteLock(self::LOCK_ON);
if ($us::NOTE_START)
CLI::write($us::NOTE_START);
if (!$us->run($inOut))
$allOk = false;
$error = [];
if ($allOk && !$us->test($error))
{
if ($us::NOTE_ERROR)
CLI::write($us::NOTE_ERROR, CLI::LOG_ERROR);
foreach ($error as $e)
CLI::write($e, CLI::LOG_BLANK);
CLI::write();
$allOk = false;
}
if ($allOk)
if ($ff = $us->followupFn)
if (array_filter($inOut))
self::run($ff, $inOut);
self::siteLock($us::LOCK_SITE == self::LOCK_RESTORE ? self::LOCK_RESTORE : self::LOCK_OFF);
// end
if ($us::NOTE_END_OK && $allOk)
CLI::write($us::NOTE_END_OK, CLI::LOG_OK);
else if($us::NOTE_END_FAIL && !$allOk)
CLI::write($us::NOTE_END_FAIL, CLI::LOG_ERROR);
}
// consecutive calls
public static function run(string $cmd, &$args) : bool
{
if (!isset(self::$utilScriptRefs[$cmd]))
return false;
$us = &self::$utilScriptRefs[$cmd];
if ($dbError = array_filter($us::REQUIRED_DB, function ($x) { return !DB::isConnected($x); }))
{
CLI::write('Database on index '.implode(', ', $dbError).' not yet set up!', CLI::LOG_ERROR);
CLI::write('Please use '.CLI::bold('"php aowow --db"').' for setup', CLI::LOG_BLANK);
CLI::write();
return false;
}
if ($us::PROMPT)
{
CLI::write($us::PROMPT, -1, false);
CLI::write();
if (!CLI::read(['x' => ['Press any key to continue', true, true]], $_)) // we don't actually care about the input
return false;
}
$args = array_pad($args, 4, []);
$success = $us->run($args);
$error = [];
if ($us::NOTE_ERROR && $success && !$us->test($error))
{
CLI::write($us::NOTE_ERROR, CLI::LOG_ERROR);
foreach ($error as $e)
CLI::write($e, CLI::LOG_BLANK);
CLI::write();
return false;
}
if ($success)
if ($ff = $us->followupFn)
if (array_filter($args))
if (!self::run($ff, $args))
$success = false;
return $success;
}
/**************************/
/* command line arguments */
/**************************/
public static function evalOpts() : void
{
$short = '';
$long = [];
$alias = [];
foreach (self::$optDefs as $opt => [, $aliases, $flags, , ])
{
foreach ($aliases as $i => $a)
{
if (isset($alias[$a]))
$alias[$a][] = $opt;
else
$alias[$a] = [$opt];
if ($flags & self::ARGV_REQUIRED)
$a .= ':';
else if ($flags & self::ARGV_OPTIONAL)
$a .= '::';
if (strlen($aliases[$i]) == 1)
$short .= $a;
else
$long[] = $a;
}
if ($flags & self::ARGV_REQUIRED)
$opt .= ':';
else if ($flags & self::ARGV_OPTIONAL)
$opt .= '::';
$long[] = $opt;
}
if ($opts = getopt($short, $long))
{
foreach ($opts as $o => $v)
{
if (!isset($alias[$o]))
self::$opts[$o] = (self::$optDefs[$o][2] & self::ARGV_ARRAY) ? ($v ? explode(',', $v) : []) : ($v ?: true);
else
foreach ($alias[$o] as $a)
self::$opts[$a] = (self::$optDefs[$a][2] & self::ARGV_ARRAY) ? ($v ? explode(',', $v) : []) : ($v ?: true);
}
}
}
public static function getOpt(/* string|int */ ...$args) // : bool|array|string
{
if (!$args)
return false;
@@ -165,39 +446,12 @@ class CLISetup
return $result;
}
public static function optHelp(int $groupMask = 0x0) : void
{
$lines = [];
foreach (self::$optGroups as $idx => $og)
{
if ($groupMask && !($groupMask & (1 << $idx)))
continue;
$lines[] = [$og, ''];
foreach (self::$optDefs as $opt => [$group, $alias, , $desc, $app])
{
if ($group != $idx)
continue;
$cmd = ' --'.$opt;
foreach ($alias as $a)
$cmd .= ' | '.(strlen($a) == 1 ? '-'.$a : '--'.$a);
$lines[] = [$cmd.$app, $desc];
}
}
CLI::writeTable($lines);
}
/*******************/
/* web page access */
/*******************/
public static function siteLock(int $mode = self::LOCK_RESTORE) : void
private static function siteLock(int $mode = self::LOCK_RESTORE) : void
{
if (DB::isConnected(DB_AOWOW))
Cfg::set('MAINTENANCE', $mode == self::LOCK_RESTORE ? self::$lock : $mode);
@@ -216,8 +470,7 @@ class CLISetup
*/
private static function buildFileList() : bool
{
CLI::write();
CLI::write('indexing game data from '.self::$srcDir.' for first time use...');
CLI::write('indexing game data from '.self::$srcDir.' for first time use...', CLI::LOG_INFO, true, true);
$setupDirs = glob('setup/*');
foreach ($setupDirs as $sd)
@@ -246,8 +499,7 @@ class CLISetup
self::$mpqFiles[strtolower($_)] = $_;
}
CLI::write('done');
CLI::write();
CLI::write('indexing game data from '.self::$srcDir.' for first time use... done!', CLI::LOG_INFO);
}
catch (UnexpectedValueException $e)
{
@@ -258,7 +510,7 @@ class CLISetup
return true;
}
public static function fileExists(&$file)
public static function fileExists(string &$file) : bool
{
// read mpq source file structure to tree
if (!self::$mpqFiles)
@@ -281,7 +533,7 @@ class CLISetup
return false;
}
public static function filesInPath($path, $useRegEx = false)
public static function filesInPath(string $path, bool $useRegEx = false) : array
{
$result = [];
@@ -304,12 +556,82 @@ class CLISetup
return $result;
}
public static function filesInPathLocalized(string $pathPattern, ?bool &$status = true, bool $matchAll = true) : array
{
$result = [];
foreach (self::$expectedPaths as $xp => $locId)
{
if (!in_array($locId, self::$localeIds))
continue;
if (isset($result[$locId]))
continue;
if ($xp) // if in subDir add trailing slash
$xp .= '/';
$path = sprintf($pathPattern, $xp);
if (self::fileExists($path))
{
$result[$locId] = $path;
continue;
}
}
if (!$matchAll && !$result)
$status = false;
if ($matchAll && array_diff(self::$localeIds, array_keys($result)))
$status = false;
return $result;
}
public static function loadGlobalStrings() : bool
{
CLI::write('loading required GlobalStrings', CLI::LOG_INFO);
// try to load globalstrings for all selected locales
foreach (self::$expectedPaths as $xp => $lId)
{
if (isset(self::$gsFiles[$lId]))
continue;
if ($xp)
$xp .= '/';
$gsFile = sprintf(self::GLOBALSTRINGS_LUA, self::$srcDir, $xp);
if (self::fileExists($gsFile))
self::$gsFiles[$lId] = file($gsFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
}
if ($missing = array_diff(self::$localeIds, array_keys(self::$gsFiles)))
{
ClI::write('GlobalStrings.lua not found for locale '. Lang::concat($missing), CLI::LOG_WARN);
return false;
}
return true;
}
public static function searchGlobalStrings(string $pattern) : generator
{
if (!self::$gsFiles)
return;
foreach (self::$gsFiles as $lId => $globalStrings)
foreach ($globalStrings as $gs)
if (preg_match($pattern, $gs, $result))
yield $lId => $result;
}
/*****************/
/* file handling */
/*****************/
public static function writeFile($file, $content)
public static function writeFile(string $file, string $content) : bool
{
if (Util::writeFile($file, $content))
{
@@ -322,17 +644,23 @@ class CLISetup
return false;
}
public static function writeDir($dir)
public static function writeDir(string $dir, bool &$exist = true) : bool
{
if (Util::writeDir($dir))
if (Util::writeDir($dir, $exist))
return true;
CLI::write(error_get_last()['message'].' '.CLI::bold($dir), CLI::LOG_ERROR);
return false;
}
public static function loadDBC($name)
public static function loadDBC( string $name) : bool
{
if (!DB::isConnected(DB_AOWOW))
{
CLI::write('CLISetup::loadDBC() - not connected to DB. Cannot write results!', CLI::LOG_ERROR);
return false;
}
if (DB::Aowow()->selectCell('SHOW TABLES LIKE ?', 'dbc_'.$name) && DB::Aowow()->selectCell('SELECT count(1) FROM ?#', 'dbc_'.$name))
return true;