Commit 3c71c15c authored by Rajesh Taneja's avatar Rajesh Taneja
Browse files

MDL-39752 behat: Modified following for parallel run:

1. Create behat datadir within behat_dataroot not at same level
2. Define suffix for link and not use hard-coded values
3. Renamed ns_parallel to run.php
4. Rename variables to best understand them
5. Added support for each run to specify db, prefix, rerun and profile.
6. Showing number of steps in each line of parallel run.
parent 08e7f97e
......@@ -40,92 +40,109 @@ define('CACHE_DISABLE_ALL', true);
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
list($options, $unrecognized) = cli_get_params(
array(
'parallel' => 0,
'suffix' => '',
'maxruns' => false,
'help' => false,
),
array(
'j' => 'parallel',
'm' => 'maxruns',
'h' => 'help',
)
);
// Checking run.php CLI script usage.
$help = "
Behat utilities to initialise behat tests
$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']);
$suffixarg = $options['suffix'] ? "--suffix={$options['suffix']} --parallel=$nproc" : '';
Options:
-j, --parallel Number of parallel behat run to initialise
-m, --maxruns Max parallel processes to be executed at one time.
-h, --help Print out this help
if ($nproc && !$suffixarg) {
foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) {
if (file_exists($dir) && is_link($dir) && preg_match('#/behat\d+$#', $dir)) {
unlink($dir);
}
}
$cmds = array();
for ($i = 1; $i <= $nproc; $i++) {
$cmds[] = "php ".__FILE__." --suffix=$i --parallel=$nproc 2>&1";
}
// This is intensive compared to behat itself so halve the parallelism.
foreach (array_chunk($cmds, max(1, floor($nproc/2)), true) as $chunk) {
ns_parallel_popen($chunk, true);
}
Example from Moodle root directory:
\$ php admin/tool/behat/cli/init.php --parallel=2
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
if (!empty($options['help'])) {
echo $help;
exit(0);
}
// Check which util file to call.
$utilfile = 'util.php';
$paralleloption = "";
// If parallel run then use utilparallel.
if ($options['parallel']) {
$utilfile = 'utilparallel.php';
$paralleloption = " --parallel=".$options['parallel'];
}
// Changing the cwd to admin/tool/behat/cli.
$cwd = getcwd();
chdir(__DIR__);
$output = null;
exec("php util.php --diag $suffixarg", $output, $code);
if ($code == 0) {
echo "Behat test environment already installed\n";
} else if ($code == BEHAT_EXITCODE_INSTALL) {
exec("php $utilfile --diag $paralleloption", $output, $code);
// Check if composer needs to be updated.
if (($code == BEHAT_EXITCODE_INSTALL) || $code == BEHAT_EXITCODE_REINSTALL || $code == BEHAT_EXITCODE_COMPOSER) {
testing_update_composer_dependencies();
}
if ($code == 0) {
echo "Behat test environment already installed\n";
} else if ($code == BEHAT_EXITCODE_INSTALL) {
// Behat and dependencies are installed and we need to install the test site.
chdir(__DIR__);
passthru("php util.php --install $suffixarg", $code);
passthru("php $utilfile --install $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
} else if ($code == BEHAT_EXITCODE_REINSTALL) {
testing_update_composer_dependencies();
// Test site data is outdated.
chdir(__DIR__);
passthru("php util.php --drop $suffixarg", $code);
passthru("php $utilfile --drop $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
passthru("php util.php --install $suffixarg", $code);
passthru("php $utilfile --install $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
} else if ($code == BEHAT_EXITCODE_COMPOSER) {
// Missing Behat dependencies.
testing_update_composer_dependencies();
// Returning to admin/tool/behat/cli.
chdir(__DIR__);
passthru("php util.php --install $suffixarg", $code);
passthru("php $utilfile --install $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
} else {
// Generic error, we just output it.
echo implode("\n", $output)."\n";
chdir($cwd);
exit($code);
}
// Enable editing mode according to config.php vars.
passthru("php util.php --enable $suffixarg", $code);
passthru("php $utilfile --enable $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
......
......@@ -22,48 +22,106 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
if (isset($_SERVER['REMOTE_ADDR'])) {
die(); // No access from web!
}
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('ABORT_AFTER_CONFIG', true);
define('CACHE_DISABLE_ALL', true);
define('NO_OUTPUT_BUFFERING', true);
require_once(__DIR__ .'/../../../../config.php');
require_once(__DIR__.'/../../../../lib/clilib.php');
require_once(__DIR__.'/../../../../lib/behat/lib.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php');
error_reporting(E_ALL | E_STRICT);
ini_set('display_errors', '1');
ini_set('log_errors', '1');
require_once __DIR__ .'/../../../../config.php';
require_once __DIR__.'/../../../../lib/clilib.php';
require_once __DIR__.'/../../../../lib/behat/lib.php';
list($options, $unrecognised) = cli_get_params(
array(
'stop-on-failure' => 0,
'parallel' => 0,
'verbose' => false,
'replace' => false,
'verbose' => false,
'replace' => false,
'help' => false,
'tags' => '',
'profile' => '',
),
array(
'h' => 'help',
't' => 'tags',
'p' => 'profile',
)
);
// Checking run.php CLI script usage.
$help = "
Behat utilities to run behat tests in parallel
Options:
-t, --tags{{color_green}} Tags to execute.
-p, --profile{{color_green}} Profile to execute.
--stop-on-failure{{color_green}} Stop on failure in any parallel run.
--verbose{{color_green}} Verbose output
--replace{{color_green}} Replace args string with run process number, useful for output.
-h, --help{{color_green}} Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/run.php --parallel=2
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
if (!empty($options['help'])) {
echo $help;
exit(0);
}
// Ensure we have parallel runs initialised and it's >= 1.
$parallelrun = behat_config_manager::get_parallel_test_runs(1);
if (empty($options['parallel']) && $dirs = glob("{$CFG->dirroot}/behat*")) {
sort($dirs);
if ($max = preg_filter('#.*behat(\d+)#', '$1', end($dirs))) {
$options['parallel'] = $max;
// Capture signals and ensure we clean symlinks.
pcntl_signal(SIGTERM, "signal_handler");
pcntl_signal(SIGINT, "signal_handler");
/**
* Signal handler for terminal exit.
*
* @param $signal
*/
function signal_handler($signal) {
switch ($signal) {
case SIGTERM:
case SIGKILL:
case SIGINT:
// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();
exit(1);
}
}
// If empty parallelrun then just check with user if it's a run single behat test.
if (empty($parallelrun)) {
if (cli_input("This is not a parallel site, do you want to run single behat run? (Y/N)", 'n', array('y', 'n')) == 'y') {
$runtestscommand = behat_command::get_behat_command();
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
exec("php $runtestscommand", $output, $code);
echo implode("\n", $output) . "\n";
exit($code);
} else {
exit(1);
}
}
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links()) {
exit(1);
}
$suffix = '';
$time = microtime(true);
$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']);
array_walk($unrecognised, function (&$v) {
if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) {
$v = $x;
......@@ -73,13 +131,6 @@ array_walk($unrecognised, function (&$v) {
});
$extraopts = implode(' ', $unrecognised);
if (empty($nproc)) {
fwrite(STDERR, "Invalid or missing --parallel parameter, must be >= 1.\n");
exit(1);
}
$checkfail = array();
$outputs = array();
$handles = array();
......@@ -88,13 +139,51 @@ $exits = array();
$unused = null;
$linelencnt = 0;
$procs = array();
$behatdataroot = $CFG->behat_dataroot;
$tags = '';
// Options parameters to be added to each run.
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts;
if ($options['profile']) {
$profile = $options['profile'];
if (empty($CFG->behat_config[$profile]['filters']['tags'])) {
echo "Invaid profile passed: " . $profile;
exit(1);
}
$tags = $CFG->behat_config[$profile]['filters']['tags'];
$myopts .= '--profile=\'' . $profile . "'";
} else if ($options['tags']) {
$tags = $options['tags'];
$myopts .= '--tags=' . $tags;
}
// Update config file if tags defined.
if ($tags) {
// Hack to set proper dataroot and wwroot.
$behatdataroot = $CFG->behat_dataroot;
$behatwwwroot = $CFG->behat_wwwroot;
for ($i = 1; $i <= $parallelrun; $i++) {
$CFG->behatrunprocess = $i;
$CFG->behat_dataroot = $behatdataroot . $i;
$CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i;
behat_config_manager::update_config_file('', true, $tags);
}
$CFG->behat_dataroot = $behatdataroot;
$CFG->behat_wwwroot = $behatwwwroot;
unset($CFG->behatrunprocess);
}
for ($i = 1; $i <= $parallelrun; $i++) {
$CFG->behatrunprocess = $i;
$behatcommand = behat_command::get_behat_command();
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Command to execute behat run.
$cmd = $behatcommand .' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . "] ". $cmd . "\n";
for ($i = 1; $i <= $nproc; $i++) {
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts;
$dirroot = dirname($CFG->behat_dataroot)."/behat$i";
$cmd = "exec {$CFG->dirroot}/vendor/bin/behat --config $dirroot/behat/behat.yml $myopts";
list($handle, $pipes) = ns_proc_open($cmd, true);
list($handle, $pipes) = cli_execute($cmd, true);
@fclose($pipes[0]);
unset($pipes[0]);
$exits[$i] = 1;
......@@ -108,7 +197,7 @@ for ($i = 1; $i <= $nproc; $i++) {
stream_set_blocking($pipes[2], 0);
}
$progresscount = 0;
while (!empty($procs)) {
usleep(10000);
......@@ -120,9 +209,11 @@ while (!empty($procs)) {
unset($procs[$i]);
unset($handles[$i][0]);
$last = array_pop($outputs[$i]);
for ($l=2; $l>=1; $l--)
while ($part = @fread($handles[$i][$l], 8192))
for ($l = 2; $l >= 1; $l--) {
while ($part = @fread($handles[$i][$l], 8192)) {
$last .= $part;
}
}
$outputs[$i] = array_merge($outputs[$i], explode("\n", $last));
}
}
......@@ -151,7 +242,8 @@ while (!empty($procs)) {
if (!$checkfail[$i]) {
foreach ($newlines as $l => $line) {
unset($newlines[$l]);
if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) {
if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 &&
preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) {
$checkfail[$i] = true;
break;
}
......@@ -181,23 +273,28 @@ while (!empty($procs)) {
$linelencnt += strlen($part);
echo $part;
if ($linelencnt >= 70) {
echo "\n";
$progresscount += 70;
echo " $progresscount\n";
$linelencnt = 0;
}
}
}
echo "\n\n";
$exits = array_filter($exits, function ($v) {return $v !== 0;});
$exits = array_filter($exits,
function ($v) {
return $v !== 0;
}
);
if ($exits || $options['verbose']) {
echo "Exit codes: ".implode(" ", $exits)."\n\n";
foreach ($outputs as $i => $output) {
unset($outputs[$i]);
if (!end($output)) array_pop($output);
$prefix = "[behat$i] ";
if (!end($output)) {
array_pop($output);
}
$prefix = "[" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . "] ";
array_walk($output, function (&$l) use ($prefix) {
$l = $prefix.$l;
});
......@@ -206,7 +303,10 @@ if ($exits || $options['verbose']) {
$failed = true;
}
$time = round(microtime(true) - $time, 1);
echo "Finished in {$time}s\n";
// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();
exit(!empty($failed) ? 1 : 0);
......@@ -34,18 +34,18 @@ if (isset($_SERVER['REMOTE_ADDR'])) {
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
// CLI options.
list($options, $unrecognized) = cli_get_params(
array(
'help' => false,
'install' => false,
'parallel' => 0,
'suffix' => '',
'run' => '',
'drop' => false,
'enable' => false,
'disable' => false,
'diag' => false
'diag' => false,
'tags' => '',
),
array(
'h' => 'help'
......@@ -65,10 +65,9 @@ Options:
--drop Drops the database tables and the dataroot contents
--enable Enables test environment and updates tests list
--disable Disables test environment
--parallel Run operation for all parallel behat environments.
--diag Get behat test environment status code
-h, --help Print out this help
-h, --help Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/util.php --enable
......@@ -81,32 +80,16 @@ if (!empty($options['help'])) {
exit(0);
}
if (!empty($options['parallel']) && empty($options['suffix'])) {
foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) {
if (file_exists($dir) && is_dir($dir)) {
unlink($dir);
}
}
$cmds = array();
$extra = preg_filter('#(.*)\s*--parallel=\d+\s*(.*?)#', '$1 $2', implode(' ', array_slice($argv, 1)));
for ($i = 1; $i <= $options['parallel']; $i++) {
$cmds[] = "php ".__FILE__." $extra --suffix=$i 2>&1";
}
// This is intensive compared to behat itself so halve the parallelism.
foreach (array_chunk($cmds, min(1, floor($options['parallel']/2)), true) as $chunk) {
ns_parallel_popen($chunk, true);
}
exit(0);
}
// Checking $CFG->behat_* vars and values.
// Describe this script.
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
define('BEHAT_SUFFIX', $options['suffix']);
// Set run value, to be used by setup for configuring proper CFG variables.
if ($options['run']) {
define('BEHAT_CURRENT_RUN', $options['run']);
}
// Only load CFG from config.php, stop ASAP in lib/setup.php.
define('ABORT_AFTER_CONFIG', true);
......@@ -139,29 +122,69 @@ if ($unrecognized) {
// Behat utilities.
require_once($CFG->libdir . '/behat/classes/util.php');
require_once($CFG->libdir . '/behat/classes/behat_command.php');
require_once($CFG->libdir . '/behat/classes/behat_config_manager.php');
// Ensure run option is <= parallel run installed.
if ($options['run']) {
if (!$options['parallel']) {
$options['parallel'] = behat_config_manager::get_parallel_test_runs();
}
if (empty($options['parallel']) || $options['run'] > $options['parallel']) {
echo "Parallel runs can't be more then ".$options['parallel'].PHP_EOL;
exit(1);
}
$CFG->behatrunprocess = $options['run'];
}
// Run command (only one per time).
if ($options['install']) {
behat_util::install_site();
mtrace("Acceptance tests site installed");
// This is only displayed once for parallel install.
if (empty($options['run'])) {
mtrace("Acceptance tests site installed");
}
} else if ($options['drop']) {
// Ensure no tests are running.
test_lock::acquire('behat');
behat_util::drop_site();
mtrace("Acceptance tests site dropped");
// This is only displayed once for parallel install.
if (empty($options['run'])) {
mtrace("Acceptance tests site dropped");
}
} else if ($options['enable']) {
behat_util::start_test_mode();
$runtestscommand = behat_command::get_behat_command(true) .
' --config ' . behat_config_manager::get_behat_cli_config_filepath();
mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:\n " . $runtestscommand);
// This is only displayed once for parallel install.
if (empty($options['run'])) {
$runtestscommand = behat_command::get_behat_command(true, !empty($options['run']));
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:\n " . $runtestscommand);
} else {
// Save parallel site info for enable and install options.
$filepath = behat_config_manager::get_parallel_test_file_path();
if (!file_put_contents($filepath, $options['parallel'])) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created');
}
}
} else if ($options['disable']) {
behat_util::stop_test_mode();
mtrace("Acceptance tests environment disabled");
// This is only displayed once for parallel install.
if (empty($options['run'])) {
mtrace("Acceptance tests environment disabled");
}
} else if ($options['diag']) {
$code = behat_util::get_behat_status();
exit($code);
} else {
echo $help;
exit(1);
}
exit(0);
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* CLI tool with utilities to manage parallel Behat integration in Moodle
*
* All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as
* $CFG->dataroot and $CFG->prefix
*
* @package tool_behat
* @copyright 2015 Rajesh Taneja
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
if (isset($_SERVER['REMOTE_ADDR'])) {
die(); // No access from web!.
}
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
// CLI options.
list($options, $unrecognized) = cli_get_params(
array(
'help' => false,
'install' => false,
'drop' => false,
'enable' => false,
'disable' => false,
'diag' => false,
'parallel' => 0,
'maxruns' => false