Commit 8cf4591a authored by Dan Poltawski's avatar Dan Poltawski Committed by Andrew Nicols
Browse files

Merge branch 'wip-mdl-54628' of https://github.com/rajeshtaneja/moodle

parents ac08d455 36cc9f8e
......@@ -54,6 +54,7 @@ list($options, $unrecognised) = cli_get_params(
'fromrun' => 1,
'torun' => 0,
'single-run' => false,
'rerun' => 0,
),
array(
'h' => 'help',
......@@ -77,6 +78,7 @@ Options:
--replace Replace args string with run process number, useful for output.
--fromrun Execute run starting from (Used for parallel runs on different vms)
--torun Execute run till (Used for parallel runs on different vms)
--rerun Re-run scenarios that failed during last execution.
-h, --help Print out this help
......@@ -91,11 +93,31 @@ if (!empty($options['help'])) {
exit(0);
}
$parallelrun = behat_config_manager::get_parallel_test_runs($options['fromrun']);
$parallelrun = behat_config_manager::get_behat_run_config_value('parallel');
// Default torun is maximum parallel runs.
if (empty($options['torun'])) {
$options['torun'] = $parallelrun;
// Check if the options provided are valid to run behat.
if ($parallelrun === false) {
// Parallel run should not have fromrun or torun options greater than 1.
if (($options['fromrun'] > 1) || ($options['torun'] > 1)) {
echo "Test site is not initialized for parallel run." . PHP_EOL;
exit(1);
}
} else {
// Ensure fromrun is within limits of initialized test site.
if (!empty($options['fromrun']) && ($options['fromrun'] > $parallelrun)) {
echo "From run (" . $options['fromrun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL;
exit(1);
}
// Default torun is maximum parallel runs and should be less than equal to parallelruns.
if (empty($options['torun'])) {
$options['torun'] = $parallelrun;
} else {
if ($options['torun'] > $parallelrun) {
echo "To run (" . $options['torun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL;
exit(1);
}
}
}
// Capture signals and ensure we clean symlinks.
......@@ -127,7 +149,7 @@ if ($options['profile']) {
exit(1);
}
$extraopts[] = '--profile="' . $profile . '"';
$extraopts['profile'] = '--profile="' . $profile . '"';
// By default, profile tags will be used.
if (!empty($CFG->behat_config[$profile]['filters']['tags'])) {
$tags = $CFG->behat_config[$profile]['filters']['tags'];
......@@ -137,117 +159,156 @@ if ($options['profile']) {
// Command line tags have precedence (std behat behavior).
if ($options['tags']) {
$tags = $options['tags'];
$extraopts[] = '--tags="' . $tags . '"';
$extraopts['tags'] = '--tags="' . $tags . '"';
}
// Add suite option if specified.
if ($options['suite']) {
$extraopts[] = '--suite="' . $options['suite'] . '"';
$extraopts['suite'] = '--suite="' . $options['suite'] . '"';
}
// Feature should be added to last, for behat command.
if ($options['feature']) {
$extraopts[] = $options['feature'];
$extraopts['feature'] = $options['feature'];
// Only run 1 process as process.
// Feature file is picked from absolute path provided, so no need to check for behat.yml.
$options['torun'] = $options['fromrun'];
}
// Set of options to pass to behat.
$extraopts = implode(' ', $extraopts);
$extraoptstr = implode(' ', $extraopts);
// If rerun is passed then ensure we just run the failed processes.
$lastfailedstatus = 0;
$lasttorun = $options['torun'];
$lastfromrun = $options['fromrun'];
if ($options['rerun']) {
// Get last combined failed status.
$lastfailedstatus = behat_config_manager::get_behat_run_config_value('lastcombinedfailedstatus');
$lasttorun = behat_config_manager::get_behat_run_config_value('lasttorun');
$lastfromrun = behat_config_manager::get_behat_run_config_value('lastfromrun');
if ($lastfailedstatus !== false) {
$extraoptstr .= ' --rerun';
}
// If torun is less than last torun, then just set this to min last to run and similar for fromrun.
if ($options['torun'] < $lasttorun) {
$options['torun'];
}
if ($options['fromrun'] > $lastfromrun) {
$options['fromrun'];
}
unset($options['rerun']);
}
// If empty parallelrun then just check with user if it's a run single behat test.
$cmds = array();
$exitcodes = array();
$status = 0;
$verbose = empty($options['verbose']) ? false : true;
// Execute behat run commands.
if (empty($parallelrun)) {
$cwd = getcwd();
chdir(__DIR__);
$runtestscommand = behat_command::get_behat_command(false, false, true);
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
$runtestscommand .= ' ' . $extraopts;
$runtestscommand .= ' ' . $extraoptstr;
echo "Running single behat site:" . PHP_EOL;
passthru("php $runtestscommand", $code);
passthru("php $runtestscommand", $status);
chdir($cwd);
exit($code);
}
$cmds = array();
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
} else {
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
$CFG->behatrunprocess = $i;
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
// Options parameters to be added to each run.
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts;
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
$lastfailed = 1 & $lastfailedstatus >> ($i - 1);
$behatcommand = behat_command::get_behat_command(false, false, true);
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Command to execute behat run.
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
}
// Bypass if not failed in last run.
if ($lastfailedstatus && !$lastfailed && ($i <= $lasttorun) && ($i >= $lastfromrun)) {
continue;
}
if (empty($cmds)) {
echo "No commands to execute " . PHP_EOL;
exit(1);
}
$CFG->behatrunprocess = $i;
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
// Options parameters to be added to each run.
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraoptstr) : $extraoptstr;
// Execute all commands, relative to moodle root directory.
$processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../");
$stoponfail = empty($options['stop-on-failure']) ? false : true;
$behatcommand = behat_command::get_behat_command(false, false, true);
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Print header.
print_process_start_info($processes);
// Command to execute behat run.
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
}
// Print combined run o/p from processes.
$exitcodes = print_combined_run_output($processes, $stoponfail);
$time = round(microtime(true) - $time, 1);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
if (empty($cmds)) {
echo "No commands to execute " . PHP_EOL;
exit(1);
}
ksort($exitcodes);
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
// Print exit info from each run.
// Status bits contains pass/fail status of parallel runs.
$status = 0;
$processcounter = 0;
foreach ($exitcodes as $exitcode) {
if ($exitcode) {
$status |= (1 << $processcounter);
// Save torun and from run, so it can be used to detect if it was executed in last run.
behat_config_manager::set_behat_run_config_value('lasttorun', $options['torun']);
behat_config_manager::set_behat_run_config_value('lastfromrun', $options['fromrun']);
// Execute all commands, relative to moodle root directory.
$processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../");
$stoponfail = empty($options['stop-on-failure']) ? false : true;
// Print header.
print_process_start_info($processes);
// Print combined run o/p from processes.
$exitcodes = print_combined_run_output($processes, $stoponfail);
// Time to finish run.
$time = round(microtime(true) - $time, 1);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
ksort($exitcodes);
// Print exit info from each run.
// Status bits contains pass/fail status of parallel runs.
foreach ($exitcodes as $name => $exitcode) {
if ($exitcode) {
$runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
$status |= (1 << ($runno - 1));
}
}
$processcounter++;
// Print each process information.
print_each_process_info($processes, $verbose, $status);
}
// Run finished. Show exit code and output from individual process.
$verbose = empty($options['verbose']) ? false : true;
$verbose = $verbose || !empty($status);
// Show exit code from each process, if any process failed.
if ($verbose) {
// Echo exit codes.
echo "Exit codes for each behat run: " . PHP_EOL;
foreach ($exitcodes as $run => $exitcode) {
echo $run . ": " . $exitcode . PHP_EOL;
}
// Show exit code from each process, if any process failed and how to rerun failed process.
if ($verbose || $status) {
// Save final exit code containing which run failed.
behat_config_manager::set_behat_run_config_value('lastcombinedfailedstatus', $status);
// Show failed re-run commands.
if ($status) {
echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
foreach ($cmds as $name => $cmd) {
if (!empty($exitcodes[$name])) {
echo "[" . $name . "] " . $cmd . PHP_EOL;
if (!empty($cmds)) {
// Echo exit codes.
echo "Exit codes for each behat run: " . PHP_EOL;
foreach ($exitcodes as $run => $exitcode) {
echo $run . ": " . $exitcode . PHP_EOL;
}
}
echo "To re-run failed processes, you can use following command:" . PHP_EOL;
unset($extraopts['fromrun']);
unset($extraopts['torun']);
$extraopts['rerun'] = '--rerun';
$extraoptstr = implode(' ', $extraopts);
echo behat_command::get_behat_command(true, true, true) . " " . $extraoptstr . PHP_EOL;
}
echo PHP_EOL;
}
print_each_process_info($processes, $verbose);
// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();
......@@ -364,23 +425,47 @@ function print_combined_run_output($processes, $stoponfail = false) {
* @param array $processes list of processes to loop though.
* @param bool $verbose Show verbose output for each process.
*/
function print_each_process_info($processes, $verbose = false) {
function print_each_process_info($processes, $verbose = false, $status = 0) {
foreach ($processes as $name => $process) {
echo "**************** [" . $name . "] ****************" . PHP_EOL;
if ($verbose) {
echo $process->getOutput();
echo $process->getErrorOutput();
} else {
$op = explode(PHP_EOL, $process->getOutput());
foreach ($op as $line) {
// Don't print progress .
if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) &&
(strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) &&
(strpos($line, 'Browser specific fixes ') !== 0)) {
echo $line . PHP_EOL;
}
} else if ($status) {
// Only show failed o/p.
$runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
if ((1 << ($runno - 1)) & $status) {
echo $process->getOutput();
echo $process->getErrorOutput();
} else {
echo get_status_lines_from_run_op($process);
}
} else {
echo get_status_lines_from_run_op($process);
}
echo PHP_EOL;
}
}
/**
* Extract status information from behat o/p and return.
* @param Symfony\Component\Process\Process $process
* @return string
*/
function get_status_lines_from_run_op(Symfony\Component\Process\Process $process) {
$statusstr = '';
$op = explode(PHP_EOL, $process->getOutput());
foreach ($op as $line) {
// Don't print progress .
if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) &&
(strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) &&
(strpos($line, 'Browser specific fixes ') !== 0)) {
$statusstr .= $line . PHP_EOL;
}
}
return $statusstr;
}
......@@ -36,11 +36,7 @@ define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
define('ABORT_AFTER_CONFIG', true);
require_once(__DIR__ . '/../../../../config.php');
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');
// CLI options.
list($options, $unrecognized) = cli_get_params(
......@@ -103,14 +99,26 @@ if (!empty($options['help'])) {
$cwd = getcwd();
// If Behat parallel site is being initiliased, then define a param to be used to ignore single run install.
if (!empty($options['parallel'])) {
define('BEHAT_PARALLEL_UTIL', true);
}
require_once(__DIR__ . '/../../../../config.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');
// For drop option check if parallel site.
if ((empty($options['parallel'])) && ($options['drop']) || $options['updatesteps']) {
// Get parallel run info from first run.
$options['parallel'] = behat_config_manager::get_parallel_test_runs($options['fromrun']);
$options['parallel'] = behat_config_manager::get_behat_run_config_value('parallel');
}
// If not a parallel site then open single run.
if (empty($options['parallel'])) {
// Set run config value for single run.
behat_config_manager::set_behat_run_config_value('singlerun', 1);
chdir(__DIR__);
// Check if behat is initialised, if not exit.
passthru("php util_single_run.php --diag", $status);
......@@ -147,6 +155,21 @@ if ($options['diag'] || $options['enable'] || $options['disable']) {
$status = (bool)$status || (bool)$exitcode;
}
// Remove run config file.
$behatrunconfigfile = behat_config_manager::get_behat_run_config_file_path();
if (file_exists($behatrunconfigfile)) {
if (!unlink($behatrunconfigfile)) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete behat run config file');
}
}
// Remove test file path.
if (file_exists(behat_util::get_test_file_path())) {
if (!unlink(behat_util::get_test_file_path())) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete test file enable info');
}
}
} else if ($options['install']) {
// This is intensive compared to behat itself so run them in chunk if option maxruns not set.
if ($options['maxruns']) {
......@@ -233,6 +256,19 @@ if ($options['install']) {
} else if ($options['enable']) {
echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:" . PHP_EOL;
echo behat_command::get_behat_command(true, true);
// Save fromrun and to run information.
if (isset($options['fromrun'])) {
behat_config_manager::set_behat_run_config_value('fromrun', $options['fromrun']);
}
if (isset($options['torun'])) {
behat_config_manager::set_behat_run_config_value('torun', $options['torun']);
}
if (isset($options['parallel'])) {
behat_config_manager::set_behat_run_config_value('parallel', $options['parallel']);
}
echo PHP_EOL;
} else if ($options['disable']) {
......
......@@ -143,7 +143,7 @@ if ($options['run']) {
$run = $options['run'];
// If parallel option is not passed, then try get it form config.
if (!$options['parallel']) {
$parallel = behat_config_manager::get_parallel_test_runs();
$parallel = behat_config_manager::get_behat_run_config_value('parallel');
} else {
$parallel = $options['parallel'];
}
......@@ -176,10 +176,7 @@ if ($options['install']) {
} else if ($options['enable']) {
if (!empty($parallel)) {
// Save parallel site info for enable and install options.
$filepath = behat_config_manager::get_parallel_test_file_path();
if (!file_put_contents($filepath, $parallel)) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created');
}
behat_config_manager::set_behat_run_config_value('behatsiteenabled', 1);
}
// Enable test mode.
......@@ -200,7 +197,7 @@ if ($options['install']) {
}
} else if ($options['disable']) {
behat_util::stop_test_mode();
behat_util::stop_test_mode($run);
// This is only displayed once for parallel install.
if (empty($run)) {
mtrace("Acceptance tests environment disabled");
......
......@@ -42,6 +42,22 @@ class behat_command {
*/
const DOCS_URL = 'http://docs.moodle.org/dev/Acceptance_testing';
/**
* Ensures the behat dir exists in moodledata
*
* @return string Full path
*/
public static function get_parent_behat_dir() {
global $CFG;
// If not set then return empty string.
if (!isset($CFG->behat_dataroot_parent)) {
return "";
}
return $CFG->behat_dataroot_parent;
}
/**
* Ensures the behat dir exists in moodledata
* @param int $runprocess run process for which behat dir is returned.
......@@ -55,12 +71,11 @@ class behat_command {
return "";
}
if (empty($runprocess)) {
$behatdir = $CFG->behat_dataroot . '/behat';
} else if (isset($CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'])) {
// If $CFG->behat_parallel_run starts with index 0 and $runprocess for parallel run starts with 1.
if (!empty($runprocess) && isset($CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'])) {
$behatdir = $CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'] . '/behat';;
} else {
$behatdir = $CFG->behat_dataroot . $runprocess . '/behat';
$behatdir = $CFG->behat_dataroot . '/behat';
}
if (!is_dir($behatdir)) {
......
......@@ -87,7 +87,7 @@ class behat_config_manager {
// Behat must have a separate behat.yml to have access to the whole set of features and steps definitions.
if ($testsrunner === true) {
$configfilepath = behat_command::get_behat_dir() . '/behat.yml';
$configfilepath = behat_command::get_behat_dir($run) . '/behat.yml';
} else {
// Alternative for steps definitions filtering, one for each user.
$configfilepath = self::get_steps_list_config_filepath();
......@@ -117,7 +117,7 @@ class behat_config_manager {
// Get number of parallel runs if not passed.
if (empty($parallelruns) && ($parallelruns !== false)) {
$parallelruns = self::get_parallel_test_runs();
$parallelruns = self::get_behat_run_config_value('parallel');
}
// Behat config file specifing the main context class,
......@@ -212,31 +212,49 @@ class behat_config_manager {
* Returns the path to the parallel run file which specifies if parallel test environment is enabled
* and how many parallel runs to execute.
*
* @param int $runprocess run process for which behat dir is returned.
* @return string
*/
public final static function get_parallel_test_file_path($runprocess = 0) {
return behat_command::get_behat_dir($runprocess) . '/parallel_environment_enabled.txt';
public final static function get_behat_run_config_file_path() {
return behat_command::get_parent_behat_dir() . '/run_environment.json';
}
/**
* Returns number of parallel runs for which site is initialised.
* Get config for parallel run.
*
* @param int $runprocess run process for which behat dir is returned.
* @return int
* @param string $key Key to store
* @return string|int|array value which is stored.
*/
public final static function get_parallel_test_runs($runprocess = 0) {
public final static function get_behat_run_config_value($key) {
$parallelrunconfigfile = self::get_behat_run_config_file_path();
$parallelrun = 0;
// Get parallel run info from first file and last file.
$parallelrunconfigfile = self::get_parallel_test_file_path($runprocess);
if (file_exists($parallelrunconfigfile)) {
if ($parallel = file_get_contents($parallelrunconfigfile)) {
$parallelrun = (int) $parallel;
if ($parallelrunconfigs = @json_decode(file_get_contents($parallelrunconfigfile), true)) {
if (isset($parallelrunconfigs[$key])) {
return $parallelrunconfigs[$key];
}
}
}
return $parallelrun;
return false;
}
/**
* Save/update config for parallel run.
*
* @param string $key Key to store
* @param string|int|array $value to store.
*/
public final static function set_behat_run_config_value($key, $value) {
$parallelrunconfigs = array();
$parallelrunconfigfile = self::get_behat_run_config_file_path();
// Get any existing config first.
if (file_exists($parallelrunconfigfile)) {
$parallelrunconfigs = @json_decode(file_get_contents($parallelrunconfigfile), true);
}
$parallelrunconfigs[$key] = $value;
@file_put_contents($parallelrunconfigfile, json_encode($parallelrunconfigs, JSON_PRETTY_PRINT));
}
/**
......@@ -247,8 +265,8 @@ class behat_config_manager {
public final static function drop_parallel_site_links() {
global $CFG;
// Get parallel test runs from first run.
$parallelrun = self::get_parallel_test_runs(1);
// Get parallel test runs.
$parallelrun = self::get_behat_run_config_value('parallel');
if (empty($parallelrun)) {
return false;
......
......@@ -147,7 +147,7 @@ class behat_config_util {
public function get_number_of_parallel_run() {
// Get number of parallel runs if not passed.
if (empty($this->parallelruns) && ($this->parallelruns !== false)) {
$this->parallelruns = behat_config_manager::get_parallel_test_runs();
$this->parallelruns = behat_config_manager::get_behat_run_config_value('parallel');
}
return $this->parallelruns;
......
......@@ -139,8 +139,23 @@ class behat_util extends testing_util {
}
self::reset_dataroot();
self::drop_dataroot();
self::drop_database(true);
self::drop_dataroot();
}