util.php 16.4 KB
Newer Older
David Monllaó's avatar
David Monllaó committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.

/**
18
 * CLI tool with utilities to manage parallel Behat integration in Moodle
19 20 21
 *
 * All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as
 * $CFG->dataroot and $CFG->prefix
David Monllaó's avatar
David Monllaó committed
22 23 24 25 26 27 28
 *
 * @package    tool_behat
 * @copyright  2012 David Monllaó
 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
 */


29 30 31 32
if (isset($_SERVER['REMOTE_ADDR'])) {
    die(); // No access from web!.
}

33 34 35 36
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
37
define('ABORT_AFTER_CONFIG', true);
38

39 40
require_once(__DIR__ . '/../../../../lib/clilib.php');

41
// CLI options.
David Monllaó's avatar
David Monllaó committed
42 43
list($options, $unrecognized) = cli_get_params(
    array(
44 45 46 47 48 49 50 51 52
        'help'        => false,
        'install'     => false,
        'drop'        => false,
        'enable'      => false,
        'disable'     => false,
        'diag'        => false,
        'parallel'    => 0,
        'maxruns'     => false,
        'updatesteps' => false,
53 54
        'fromrun'     => 1,
        'torun'       => 0,
55
        'optimize-runs' => '',
56
        'add-core-features-to-theme' => false,
David Monllaó's avatar
David Monllaó committed
57 58
    ),
    array(
59 60
        'h' => 'help',
        'j' => 'parallel',
61
        'm' => 'maxruns',
62 63
        'o' => 'optimize-runs',
        'a' => 'add-core-features-to-theme',
David Monllaó's avatar
David Monllaó committed
64 65 66
    )
);

67
// Checking util.php CLI script usage.
David Monllaó's avatar
David Monllaó committed
68
$help = "
69
Behat utilities to manage the test environment
David Monllaó's avatar
David Monllaó committed
70

71 72 73
Usage:
  php util.php [--install|--drop|--enable|--disable|--diag|--updatesteps|--help] [--parallel=value [--maxruns=value]]

David Monllaó's avatar
David Monllaó committed
74
Options:
75 76 77 78 79
--install      Installs the test environment for acceptance tests
--drop         Drops the database tables and the dataroot contents
--enable       Enables test environment and updates tests list
--disable      Disables test environment
--diag         Get behat test environment status code
80
--updatesteps  Update feature step file.
81

82 83 84 85
-j, --parallel Number of parallel behat run operation
-m, --maxruns Max parallel processes to be executed at one time.
-o, --optimize-runs Split features with specified tags in all parallel runs.
-a, --add-core-features-to-theme Add all core features to specified theme's
86

87
-h, --help     Print out this help
David Monllaó's avatar
David Monllaó committed
88 89

Example from Moodle root directory:
90
\$ php admin/tool/behat/cli/util.php --enable --parallel=4
91

92
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
David Monllaó's avatar
David Monllaó committed
93 94 95 96 97 98 99
";

if (!empty($options['help'])) {
    echo $help;
    exit(0);
}

100
$cwd = getcwd();
101

102 103 104 105 106 107 108 109 110 111
// If Behat parallel site is being initiliased, then define a param to be used to ignore single run install.
if (!empty($options['parallel'])) {
    define('BEHAT_PARALLEL_UTIL', true);
}

require_once(__DIR__ . '/../../../../config.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');

112
// For drop option check if parallel site.
113
if ((empty($options['parallel'])) && ($options['drop']) || $options['updatesteps']) {
114
    $options['parallel'] = behat_config_manager::get_behat_run_config_value('parallel');
115 116
}

117 118
// If not a parallel site then open single run.
if (empty($options['parallel'])) {
119 120 121
    // Set run config value for single run.
    behat_config_manager::set_behat_run_config_value('singlerun', 1);

122 123 124 125 126 127 128 129 130 131 132 133 134
    chdir(__DIR__);
    // Check if behat is initialised, if not exit.
    passthru("php util_single_run.php --diag", $status);
    if ($status) {
        exit ($status);
    }
    $cmd = commands_to_execute($options);
    $processes = cli_execute_parallel(array($cmd), __DIR__);
    $status = print_sequential_output($processes, false);
    chdir($cwd);
    exit($status);
}

135 136 137 138 139
// Default torun is maximum parallel runs.
if (empty($options['torun'])) {
    $options['torun'] = $options['parallel'];
}

140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
$status = false;
$cmds = commands_to_execute($options);

// Start executing commands either sequential/parallel for options provided.
if ($options['diag'] || $options['enable'] || $options['disable']) {
    // Do it sequentially as it's fast and need to be displayed nicely.
    foreach (array_chunk($cmds, 1, true) as $cmd) {
        $processes = cli_execute_parallel($cmd, __DIR__);
        print_sequential_output($processes);
    }

} else if ($options['drop']) {
    $processes = cli_execute_parallel($cmds, __DIR__);
    $exitcodes = print_combined_drop_output($processes);
    foreach ($exitcodes as $exitcode) {
        $status = (bool)$status || (bool)$exitcode;
    }
157

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
    // Remove run config file.
    $behatrunconfigfile = behat_config_manager::get_behat_run_config_file_path();
    if (file_exists($behatrunconfigfile)) {
        if (!unlink($behatrunconfigfile)) {
            behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete behat run config file');
        }
    }

    // Remove test file path.
    if (file_exists(behat_util::get_test_file_path())) {
        if (!unlink(behat_util::get_test_file_path())) {
            behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete test file enable info');
        }
    }

173 174 175 176 177 178 179 180
} else if ($options['install']) {
    // This is intensive compared to behat itself so run them in chunk if option maxruns not set.
    if ($options['maxruns']) {
        foreach (array_chunk($cmds, $options['maxruns'], true) as $chunk) {
            $processes = cli_execute_parallel($chunk, __DIR__);
            $exitcodes = print_combined_install_output($processes);
            foreach ($exitcodes as $name => $exitcode) {
                if ($exitcode != 0) {
181 182 183
                    echo "Failed process [[$name]]" . PHP_EOL;
                    echo $processes[$name]->getOutput();
                    echo PHP_EOL;
184
                    echo $processes[$name]->getErrorOutput();
185
                    echo PHP_EOL . PHP_EOL;
186 187 188 189 190 191 192 193 194
                }
                $status = (bool)$status || (bool)$exitcode;
            }
        }
    } else {
        $processes = cli_execute_parallel($cmds, __DIR__);
        $exitcodes = print_combined_install_output($processes);
        foreach ($exitcodes as $name => $exitcode) {
            if ($exitcode != 0) {
195 196 197
                echo "Failed process [[$name]]" . PHP_EOL;
                echo $processes[$name]->getOutput();
                echo PHP_EOL;
198
                echo $processes[$name]->getErrorOutput();
199
                echo PHP_EOL . PHP_EOL;
200 201 202 203 204
            }
            $status = (bool)$status || (bool)$exitcode;
        }
    }

205 206 207
} else if ($options['updatesteps']) {
    // Rewrite config file to ensure we have all the features covered.
    if (empty($options['parallel'])) {
208
        behat_config_manager::update_config_file('', true, '', $options['add-core-features-to-theme'], false, false);
209 210 211 212
    } else {
        // Update config file, ensuring we have up-to-date behat.yml.
        for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
            $CFG->behatrunprocess = $i;
213 214

            // Update config file for each run.
215
            behat_config_manager::update_config_file('', true, $options['optimize-runs'], $options['add-core-features-to-theme'],
216
                $options['parallel'], $i);
217 218 219 220 221 222 223 224 225 226 227
        }
        unset($CFG->behatrunprocess);
    }

    // Do it sequentially as it's fast and need to be displayed nicely.
    foreach (array_chunk($cmds, 1, true) as $cmd) {
        $processes = cli_execute_parallel($cmd, __DIR__);
        print_sequential_output($processes);
    }
    exit(0);

228 229 230 231
} else {
    // We should never reach here.
    echo $help;
    exit(1);
232
}
233

234 235 236 237 238 239 240 241 242
// Ensure we have success status to show following information.
if ($status) {
    echo "Unknown failure $status" . PHP_EOL;
    exit((int)$status);
}

// Show command o/p (only one per time).
if ($options['install']) {
    echo "Acceptance tests site installed for sites:".PHP_EOL;
243

244
    // Display all sites which are installed/drop/diabled.
245
    for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
246 247 248 249 250 251
        if (empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) {
            echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL;
        } else {
            echo $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'] . PHP_EOL;
        }

252 253 254
    }
} else if ($options['drop']) {
    echo "Acceptance tests site dropped for " . $options['parallel'] . " parallel sites" . PHP_EOL;
255

256 257 258
} else if ($options['enable']) {
    echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:" . PHP_EOL;
    echo behat_command::get_behat_command(true, true);
259 260 261 262 263 264 265 266 267 268 269 270 271

    // Save fromrun and to run information.
    if (isset($options['fromrun'])) {
        behat_config_manager::set_behat_run_config_value('fromrun', $options['fromrun']);
    }

    if (isset($options['torun'])) {
        behat_config_manager::set_behat_run_config_value('torun', $options['torun']);
    }
    if (isset($options['parallel'])) {
        behat_config_manager::set_behat_run_config_value('parallel', $options['parallel']);
    }

272
    echo PHP_EOL;
273

274 275
} else if ($options['disable']) {
    echo "Acceptance tests environment disabled for " . $options['parallel'] . " parallel sites" . PHP_EOL;
276

277 278
} else if ($options['diag']) {
    // Valid option, so nothing to do.
279 280
} else {
    echo $help;
281 282
    chdir($cwd);
    exit(1);
David Monllaó's avatar
David Monllaó committed
283 284
}

285 286 287 288 289 290 291 292 293 294
chdir($cwd);
exit(0);

/**
 * Create commands to be executed for parallel run.
 *
 * @param array $options options provided by user.
 * @return array commands to be executed.
 */
function commands_to_execute($options) {
295
    $removeoptions = array('maxruns', 'fromrun', 'torun');
296 297 298 299 300 301 302 303 304
    $cmds = array();
    $extraoptions = $options;
    $extra = "";

    // Remove extra options not in util_single_run.php.
    foreach ($removeoptions as $ro) {
        $extraoptions[$ro] = null;
        unset($extraoptions[$ro]);
    }
305

306 307 308 309
    foreach ($extraoptions as $option => $value) {
        if ($options[$option]) {
            $extra .= " --$option";
            if ($value) {
310
                $extra .= "=\"$value\"";
311 312
            }
        }
313
    }
314 315 316 317 318

    if (empty($options['parallel'])) {
        $cmds = "php util_single_run.php " . $extra;
    } else {
        // Create commands which has to be executed for parallel site.
319
        for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
320 321 322
            $prefix = BEHAT_PARALLEL_SITE_NAME . $i;
            $cmds[$prefix] = "php util_single_run.php " . $extra . " --run=" . $i . " 2>&1";
        }
323
    }
324
    return $cmds;
325
}
326

327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
/**
 * Print drop output merging each run.
 *
 * @param array $processes list of processes.
 * @return array exit codes of each process.
 */
function print_combined_drop_output($processes) {
    $exitcodes = array();
    $maxdotsonline = 70;
    $remainingprintlen = $maxdotsonline;
    $progresscount = 0;
    echo "Dropping tables:" . PHP_EOL;

    while (count($exitcodes) != count($processes)) {
        usleep(10000);
        foreach ($processes as $name => $process) {
            if ($process->isRunning()) {
                $op = $process->getIncrementalOutput();
                if (trim($op)) {
                    $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
                    $strlentoprint = strlen($update);

                    // If not enough dots printed on line then just print.
                    if ($strlentoprint < $remainingprintlen) {
                        echo $update;
                        $remainingprintlen = $remainingprintlen - $strlentoprint;
                    } else if ($strlentoprint == $remainingprintlen) {
                        $progresscount += $maxdotsonline;
                        echo $update . " " . $progresscount . PHP_EOL;
                        $remainingprintlen = $maxdotsonline;
                    } else {
                        while ($part = substr($update, 0, $remainingprintlen) > 0) {
                            $progresscount += $maxdotsonline;
                            echo $part . " " . $progresscount . PHP_EOL;
                            $update = substr($update, $remainingprintlen);
                            $remainingprintlen = $maxdotsonline;
                        }
                    }
                }
            } else {
                // Process exited.
                $process->clearOutput();
                $exitcodes[$name] = $process->getExitCode();
            }
        }
372 373
    }

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
    echo PHP_EOL;
    return $exitcodes;
}

/**
 * Print install output merging each run.
 *
 * @param array $processes list of processes.
 * @return array exit codes of each process.
 */
function print_combined_install_output($processes) {
    $exitcodes = array();
    $line = array();

    // Check what best we can do to accommodate  all parallel run o/p on single line.
    // Windows command line has length of 80 chars, so default we will try fit o/p in 80 chars.
    if (defined('BEHAT_MAX_CMD_LINE_OUTPUT') && BEHAT_MAX_CMD_LINE_OUTPUT) {
        $lengthofprocessline = (int)max(10, BEHAT_MAX_CMD_LINE_OUTPUT / count($processes));
    } else {
        $lengthofprocessline = (int)max(10, 80 / count($processes));
394 395
    }

396 397 398 399 400
    echo "Installing behat site for " . count($processes) . " parallel behat run" . PHP_EOL;

    // Show process name in first row.
    foreach ($processes as $name => $process) {
        // If we don't have enough space to show full run name then show runX.
401
        if ($lengthofprocessline < strlen($name) + 2) {
402
            $name = substr($name, -5);
403
        }
404 405
        // One extra padding as we are adding | separator for rest of the data.
        $line[$name] = str_pad('[' . $name . '] ', $lengthofprocessline + 1);
406
    }
407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
    ksort($line);
    $tableheader = array_keys($line);
    echo implode("", $line) . PHP_EOL;

    // Now print o/p from each process.
    while (count($exitcodes) != count($processes)) {
        usleep(50000);
        $poutput = array();
        // Create child process.
        foreach ($processes as $name => $process) {
            if ($process->isRunning()) {
                $output = $process->getIncrementalOutput();
                if (trim($output)) {
                    $poutput[$name] = explode(PHP_EOL, $output);
                }
            } else {
                // Process exited.
                $exitcodes[$name] = $process->getExitCode();
            }
        }
        ksort($poutput);
428

429 430 431 432 433 434
        // Get max depth of o/p before displaying.
        $maxdepth = 0;
        foreach ($poutput as $pout) {
            $pdepth = count($pout);
            $maxdepth = $pdepth >= $maxdepth ? $pdepth : $maxdepth;
        }
435

436 437 438 439 440 441 442 443 444 445 446 447 448 449
        // Iterate over each process to get line to print.
        for ($i = 0; $i <= $maxdepth; $i++) {
            $pline = "";
            foreach ($tableheader as $name) {
                $po = empty($poutput[$name][$i]) ? "" : substr($poutput[$name][$i], 0, $lengthofprocessline - 1);
                $po = str_pad($po, $lengthofprocessline);
                $pline .= "|". $po;
            }
            if (trim(str_replace("|", "", $pline))) {
                echo $pline . PHP_EOL;
            }
        }
        unset($poutput);
        $poutput = null;
450

451 452 453
    }
    echo PHP_EOL;
    return $exitcodes;
David Monllaó's avatar
David Monllaó committed
454 455
}

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
/**
 * Print install output merging showing one run at a time.
 * If any process fail then exit.
 *
 * @param array $processes list of processes.
 * @param bool $showprefix show prefix.
 * @return bool exitcode.
 */
function print_sequential_output($processes, $showprefix = true) {
    $status = false;
    foreach ($processes as $name => $process) {
        $shownname = false;
        while ($process->isRunning()) {
            $op = $process->getIncrementalOutput();
            if (trim($op)) {
                // Show name of the run once for sequential.
                if ($showprefix && !$shownname) {
                    echo '[' . $name . '] ';
                    $shownname = true;
                }
                echo $op;
            }
        }
        // If any error then exit.
        $exitcode = $process->getExitCode();
        if ($exitcode != 0) {
            exit($exitcode);
        }
        $status = $status || (bool)$exitcode;
    }
    return $status;
}