Skip to content
GitLab
Unverified Commit 36cc9f8e authored by Rajesh Taneja's avatar Rajesh Taneja
Browse files

MDL-54628 behat: Set behat dir for each run under behat_dataroot 

Before this patch behat_dataroot for parallel runs
were created at same level as ->behat_dataroot
After this patch, it will be created 1 level under
->behat_dataroot with BEHAT_PARALLEL_SITE_NAME
prefix and run number as suffix. This will allow
common files as test enabled and parallel run info
to be saved in parent directory and access it easily.
parent 1d37f50e
Hide whitespace changes
Inline Side-by-side
admin/tool/behat/cli/run.php
View file @ 36cc9f8e
......@@ -93,11 +93,31 @@ if (!empty($options['help'])) {
exit(0);
}
$parallelrun = behat_config_manager::get_parallel_test_runs($options['fromrun']);
$parallelrun = behat_config_manager::get_behat_run_config_value('parallel');
// Default torun is maximum parallel runs.
if (empty($options['torun'])) {
$options['torun'] = $parallelrun;
// Check if the options provided are valid to run behat.
if ($parallelrun === false) {
// Parallel run should not have fromrun or torun options greater than 1.
if (($options['fromrun'] > 1) || ($options['torun'] > 1)) {
echo "Test site is not initialized for parallel run." . PHP_EOL;
exit(1);
}
} else {
// Ensure fromrun is within limits of initialized test site.
if (!empty($options['fromrun']) && ($options['fromrun'] > $parallelrun)) {
echo "From run (" . $options['fromrun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL;
exit(1);
}
// Default torun is maximum parallel runs and should be less than equal to parallelruns.
if (empty($options['torun'])) {
$options['torun'] = $parallelrun;
} else {
if ($options['torun'] > $parallelrun) {
echo "To run (" . $options['torun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL;
exit(1);
}
}
}
// Capture signals and ensure we clean symlinks.
......@@ -158,7 +178,36 @@ if ($options['feature']) {
// Set of options to pass to behat.
$extraoptstr = implode(' ', $extraopts);
// If empty parallelrun then just check with user if it's a run single behat test.
// If rerun is passed then ensure we just run the failed processes.
$lastfailedstatus = 0;
$lasttorun = $options['torun'];
$lastfromrun = $options['fromrun'];
if ($options['rerun']) {
// Get last combined failed status.
$lastfailedstatus = behat_config_manager::get_behat_run_config_value('lastcombinedfailedstatus');
$lasttorun = behat_config_manager::get_behat_run_config_value('lasttorun');
$lastfromrun = behat_config_manager::get_behat_run_config_value('lastfromrun');
if ($lastfailedstatus !== false) {
$extraoptstr .= ' --rerun';
}
// If torun is less than last torun, then just set this to min last to run and similar for fromrun.
if ($options['torun'] < $lasttorun) {
$options['torun'];
}
if ($options['fromrun'] > $lastfromrun) {
$options['fromrun'];
}
unset($options['rerun']);
}
$cmds = array();
$exitcodes = array();
$status = 0;
$verbose = empty($options['verbose']) ? false : true;
// Execute behat run commands.
if (empty($parallelrun)) {
$cwd = getcwd();
chdir(__DIR__);
......@@ -166,125 +215,100 @@ if (empty($parallelrun)) {
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
$runtestscommand .= ' ' . $extraoptstr;
echo "Running single behat site:" . PHP_EOL;
passthru("php $runtestscommand", $code);
passthru("php $runtestscommand", $status);
chdir($cwd);
exit($code);
}
} else {
// If rerun is passed then ensure we just run the failed processes.
$lastfailedstatus = 0;
if ($options['rerun']) {
$lastfailedstatus = get_last_failed_status($options['fromrun'], $options['torun']);
unset($options['rerun']);
$extraoptstr .= ' --rerun';
}
$cmds = array();
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
// Bypass if not failed in last run.
if ($lastfailedstatus && !($i & $lastfailedstatus)) {
continue;
}
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
$lastfailed = 1 & $lastfailedstatus >> ($i - 1);
$CFG->behatrunprocess = $i;
// Bypass if not failed in last run.
if ($lastfailedstatus && !$lastfailed && ($i <= $lasttorun) && ($i >= $lastfromrun)) {
continue;
}
// Options parameters to be added to each run.
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraoptstr) : $extraoptstr;
$CFG->behatrunprocess = $i;
$behatcommand = behat_command::get_behat_command(false, false, true);
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Options parameters to be added to each run.
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraoptstr) : $extraoptstr;
// Command to execute behat run.
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
$behatcommand = behat_command::get_behat_command(false, false, true);
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Remove any old last failed status files.
$filepath = behat_command::get_last_failed_test_status_file_path($options['fromrun']);
if (file_exists($filepath)) {
@unlink($filepath);
// Command to execute behat run.
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
}
}
if (empty($cmds)) {
echo "No commands to execute " . PHP_EOL;
exit(1);
}
if (empty($cmds)) {
echo "No commands to execute " . PHP_EOL;
exit(1);
}
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
// Execute all commands, relative to moodle root directory.
$processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../");
$stoponfail = empty($options['stop-on-failure']) ? false : true;
// Save torun and from run, so it can be used to detect if it was executed in last run.
behat_config_manager::set_behat_run_config_value('lasttorun', $options['torun']);
behat_config_manager::set_behat_run_config_value('lastfromrun', $options['fromrun']);
// Print header.
print_process_start_info($processes);
// Execute all commands, relative to moodle root directory.
$processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../");
$stoponfail = empty($options['stop-on-failure']) ? false : true;
// Print combined run o/p from processes.
$exitcodes = print_combined_run_output($processes, $stoponfail);
$time = round(microtime(true) - $time, 1);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
// Print header.
print_process_start_info($processes);
ksort($exitcodes);
// Print combined run o/p from processes.
$exitcodes = print_combined_run_output($processes, $stoponfail);
// Time to finish run.
$time = round(microtime(true) - $time, 1);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
ksort($exitcodes);
// Print exit info from each run.
// Status bits contains pass/fail status of parallel runs.
$status = 0;
foreach ($exitcodes as $name => $exitcode) {
if ($exitcode) {
$runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
$status |= (1 << ($runno - 1));
// Print exit info from each run.
// Status bits contains pass/fail status of parallel runs.
foreach ($exitcodes as $name => $exitcode) {
if ($exitcode) {
$runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
$status |= (1 << ($runno - 1));
}
}
}
// Run finished. Show exit code and output from individual process.
$verbose = empty($options['verbose']) ? false : true;
// Print each process information.
print_each_process_info($processes, $verbose, $status);
}
// Show exit code from each process, if any process failed.
// Show exit code from each process, if any process failed and how to rerun failed process.
if ($verbose || $status) {
// Echo exit codes.
echo "Exit codes for each behat run: " . PHP_EOL;
foreach ($exitcodes as $run => $exitcode) {
echo $run . ": " . $exitcode . PHP_EOL;
}
// Save final exit code containing which run failed.
behat_config_manager::set_behat_run_config_value('lastcombinedfailedstatus', $status);
// Show failed re-run commands.
if ($status) {
echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
foreach ($cmds as $name => $cmd) {
if (!empty($exitcodes[$name])) {
// Show rerun command only for the failed runs.
$runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
if ((1 << ($runno - 1)) & $status) {
$extraopts['fromrun'] = '--fromrun=' . $runno;
$extraopts['torun'] = '--torun=' . $runno;
$extraopts['rerun'] = '--rerun';
$extraoptstr = implode(' ', $extraopts);
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $runno, $extraoptstr) : $extraoptstr;
$behatcommand = behat_command::get_behat_command(true, true, true);
echo "[" . $name . "] " . $behatcommand . ' ' . $myopts . PHP_EOL;
}
// Save information about this failure.
$filepath = behat_command::get_last_failed_test_status_file_path($runno);
if (!file_put_contents($filepath, $status)) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created');
}
if (!empty($cmds)) {
// Echo exit codes.
echo "Exit codes for each behat run: " . PHP_EOL;
foreach ($exitcodes as $run => $exitcode) {
echo $run . ": " . $exitcode . PHP_EOL;
}
}
echo "To re-run failed processes, you can use following command:" . PHP_EOL;
unset($extraopts['fromrun']);
unset($extraopts['torun']);
$extraopts['rerun'] = '--rerun';
$extraoptstr = implode(' ', $extraopts);
echo behat_command::get_behat_command(true, true, true) . " " . $extraoptstr . PHP_EOL;
}
echo PHP_EOL;
}
print_each_process_info($processes, $verbose, $status);
// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();
......@@ -445,26 +469,3 @@ function get_status_lines_from_run_op(Symfony\Component\Process\Process $process
return $statusstr;
}
/**
* Return last failed status of parallel runs.
*
* @param int $fromrun starting run.
* @param int $torun end run.
* @return int status of last failure.
*/
function get_last_failed_status($fromrun, $torun) {
$lastfailedstatus = 0;
for ($i = $fromrun; $i <= $torun; $i++) {
$filepath = behat_command::get_last_failed_test_status_file_path($i);
if (file_exists($filepath)) {
if ($lastfailedstatus = file_get_contents($filepath)) {
$lastfailedstatus = (int)$lastfailedstatus;
break;
}
}
}
return $lastfailedstatus;
}
admin/tool/behat/cli/util.php
View file @ 36cc9f8e
......@@ -36,11 +36,7 @@ define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
define('ABORT_AFTER_CONFIG', true);
require_once(__DIR__ . '/../../../../config.php');
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');
// CLI options.
list($options, $unrecognized) = cli_get_params(
......@@ -103,14 +99,26 @@ if (!empty($options['help'])) {
$cwd = getcwd();
// If Behat parallel site is being initiliased, then define a param to be used to ignore single run install.
if (!empty($options['parallel'])) {
define('BEHAT_PARALLEL_UTIL', true);
}
require_once(__DIR__ . '/../../../../config.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');
// For drop option check if parallel site.
if ((empty($options['parallel'])) && ($options['drop']) || $options['updatesteps']) {
// Get parallel run info from first run.
$options['parallel'] = behat_config_manager::get_parallel_test_runs($options['fromrun']);
$options['parallel'] = behat_config_manager::get_behat_run_config_value('parallel');
}
// If not a parallel site then open single run.
if (empty($options['parallel'])) {
// Set run config value for single run.
behat_config_manager::set_behat_run_config_value('singlerun', 1);
chdir(__DIR__);
// Check if behat is initialised, if not exit.
passthru("php util_single_run.php --diag", $status);
......@@ -147,6 +155,21 @@ if ($options['diag'] || $options['enable'] || $options['disable']) {
$status = (bool)$status || (bool)$exitcode;
}
// Remove run config file.
$behatrunconfigfile = behat_config_manager::get_behat_run_config_file_path();
if (file_exists($behatrunconfigfile)) {
if (!unlink($behatrunconfigfile)) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete behat run config file');
}
}
// Remove test file path.
if (file_exists(behat_util::get_test_file_path())) {
if (!unlink(behat_util::get_test_file_path())) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete test file enable info');
}
}
} else if ($options['install']) {
// This is intensive compared to behat itself so run them in chunk if option maxruns not set.
if ($options['maxruns']) {
......@@ -233,6 +256,19 @@ if ($options['install']) {
} else if ($options['enable']) {
echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:" . PHP_EOL;
echo behat_command::get_behat_command(true, true);
// Save fromrun and to run information.
if (isset($options['fromrun'])) {
behat_config_manager::set_behat_run_config_value('fromrun', $options['fromrun']);
}
if (isset($options['torun'])) {
behat_config_manager::set_behat_run_config_value('torun', $options['torun']);
}
if (isset($options['parallel'])) {
behat_config_manager::set_behat_run_config_value('parallel', $options['parallel']);
}
echo PHP_EOL;
} else if ($options['disable']) {
......
admin/tool/behat/cli/util_single_run.php
View file @ 36cc9f8e
......@@ -143,7 +143,7 @@ if ($options['run']) {
$run = $options['run'];
// If parallel option is not passed, then try get it form config.
if (!$options['parallel']) {
$parallel = behat_config_manager::get_parallel_test_runs();
$parallel = behat_config_manager::get_behat_run_config_value('parallel');
} else {
$parallel = $options['parallel'];
}
......@@ -176,10 +176,7 @@ if ($options['install']) {
} else if ($options['enable']) {
if (!empty($parallel)) {
// Save parallel site info for enable and install options.
$filepath = behat_config_manager::get_parallel_test_file_path();
if (!file_put_contents($filepath, $parallel)) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created');
}
behat_config_manager::set_behat_run_config_value('behatsiteenabled', 1);
}
// Enable test mode.
......@@ -200,7 +197,7 @@ if ($options['install']) {
}
} else if ($options['disable']) {
behat_util::stop_test_mode();
behat_util::stop_test_mode($run);
// This is only displayed once for parallel install.
if (empty($run)) {
mtrace("Acceptance tests environment disabled");
......
lib/behat/classes/behat_command.php
View file @ 36cc9f8e
......@@ -42,6 +42,22 @@ class behat_command {
*/
const DOCS_URL = 'http://docs.moodle.org/dev/Acceptance_testing';
/**
* Ensures the behat dir exists in moodledata
*
* @return string Full path
*/
public static function get_parent_behat_dir() {
global $CFG;
// If not set then return empty string.
if (!isset($CFG->behat_dataroot_parent)) {
return "";
}
return $CFG->behat_dataroot_parent;
}
/**
* Ensures the behat dir exists in moodledata
* @param int $runprocess run process for which behat dir is returned.
......@@ -55,12 +71,11 @@ class behat_command {
return "";
}
if (empty($runprocess)) {
$behatdir = $CFG->behat_dataroot . '/behat';
} else if (isset($CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'])) {
// If $CFG->behat_parallel_run starts with index 0 and $runprocess for parallel run starts with 1.
if (!empty($runprocess) && isset($CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'])) {
$behatdir = $CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'] . '/behat';;
} else {
$behatdir = $CFG->behat_dataroot . $runprocess . '/behat';
$behatdir = $CFG->behat_dataroot . '/behat';
}
if (!is_dir($behatdir)) {
......@@ -218,17 +233,6 @@ class behat_command {
return true;
}
/**
* Returns the path to the parallel run file which specifies if parallel test environment is enabled
* and how many parallel runs to execute.
*
* @param int $runprocess run process for which behat dir is returned.
* @return string
*/
public final static function get_last_failed_test_status_file_path($runprocess = 0) {
return self::get_behat_dir($runprocess) . '/lastfailed.txt';
}
/**
* Outputs a message.
*
......
lib/behat/classes/behat_config_manager.php
View file @ 36cc9f8e
......@@ -87,7 +87,7 @@ class behat_config_manager {
// Behat must have a separate behat.yml to have access to the whole set of features and steps definitions.
if ($testsrunner === true) {
$configfilepath = behat_command::get_behat_dir() . '/behat.yml';
$configfilepath = behat_command::get_behat_dir($run) . '/behat.yml';
} else {
// Alternative for steps definitions filtering, one for each user.
$configfilepath = self::get_steps_list_config_filepath();
......@@ -117,7 +117,7 @@ class behat_config_manager {
// Get number of parallel runs if not passed.
if (empty($parallelruns) && ($parallelruns !== false)) {
$parallelruns = self::get_parallel_test_runs();
$parallelruns = self::get_behat_run_config_value('parallel');
}
// Behat config file specifing the main context class,
......@@ -212,31 +212,49 @@ class behat_config_manager {
* Returns the path to the parallel run file which specifies if parallel test environment is enabled
* and how many parallel runs to execute.
*
* @param int $runprocess run process for which behat dir is returned.
* @return string
*/
public final static function get_parallel_test_file_path($runprocess = 0) {
return behat_command::get_behat_dir($runprocess) . '/parallel_environment_enabled.txt';
public final static function get_behat_run_config_file_path() {
return behat_command::get_parent_behat_dir() . '/run_environment.json';
}
/**
* Returns number of parallel runs for which site is initialised.
* Get config for parallel run.
*
* @param int $runprocess run process for which behat dir is returned.
* @return int
* @param string $key Key to store
* @return string|int|array value which is stored.
*/
public final static function get_parallel_test_runs($runprocess = 0) {
public final static function get_behat_run_config_value($key) {
$parallelrunconfigfile = self::get_behat_run_config_file_path();
$parallelrun = 0;
// Get parallel run info from first file and last file.
$parallelrunconfigfile = self::get_parallel_test_file_path($runprocess);
if (file_exists($parallelrunconfigfile)) {
if ($parallel = file_get_contents($parallelrunconfigfile)) {
$parallelrun = (int) $parallel;
if ($parallelrunconfigs = @json_decode(file_get_contents($parallelrunconfigfile), true)) {
if (isset($parallelrunconfigs[$key])) {
return $parallelrunconfigs[$key];
}
}
}
return $parallelrun;
return false;
}
/**
* Save/update config for parallel run.
*
* @param string $key Key to store
* @param string|int|array $value to store.
*/
public final static function set_behat_run_config_value($key, $value) {
$parallelrunconfigs = array();
$parallelrunconfigfile = self::get_behat_run_config_file_path();
// Get any existing config first.
if (file_exists($parallelrunconfigfile)) {
$parallelrunconfigs = @json_decode(file_get_contents($parallelrunconfigfile), true);
}
$parallelrunconfigs[$key] = $value;
@file_put_contents($parallelrunconfigfile, json_encode($parallelrunconfigs, JSON_PRETTY_PRINT));
}
/**
......@@ -247,8 +265,8 @@ class behat_config_manager {
public final static function drop_parallel_site_links() {
global $CFG;
// Get parallel test runs from first run.
$parallelrun = self::get_parallel_test_runs(1);
// Get parallel test runs.
$parallelrun = self::get_behat_run_config_value('parallel');
if (empty($parallelrun)) {
return false;
......
lib/behat/classes/behat_config_util.php
View file @ 36cc9f8e
......@@ -147,7 +147,7 @@ class behat_config_util {
public function get_number_of_parallel_run() {
// Get number of parallel runs if not passed.
if (empty($this->parallelruns) && ($this->parallelruns !== false)) {
$this->parallelruns = behat_config_manager::get_parallel_test_runs();
$this->parallelruns = behat_config_manager::get_behat_run_config_value('parallel');
}
return $this->parallelruns;
......
lib/behat/classes/util.php
View file @ 36cc9f8e
......@@ -139,8 +139,23 @@ class behat_util extends testing_util {
}
self::reset_dataroot();
self::drop_dataroot();
self::drop_database(true);
self::drop_dataroot();
}
/**
* Delete files and directories under dataroot.
*/
public static function drop_dataroot() {
global $CFG;
// As behat directory is now created under default $CFG->behat_dataroot_parent, so remove the whole dir.
if ($CFG->behat_dataroot !== $CFG->behat_dataroot_parent) {
remove_dir($CFG->behat_dataroot, false);
} else {
// It should never come here.
throw new moodle_exception("Behat dataroot should not be same as parent behat data root.");
}
}
/**
......@@ -290,6 +305,7 @@ class behat_util extends testing_util {
}
$testenvfile = self::get_test_file_path();
behat_config_manager::set_behat_run_config_value('behatsiteenabled', 0);
if (!self::is_test_mode_enabled()) {
echo "Test environment was already disabled\n";
......@@ -322,8 +338,8 @@ class behat_util extends testing_util {
* Returns the path to the file which specifies if test environment is enabled
* @return string
*/
protected final static function get_test_file_path() {
return behat_command::get_behat_dir() . '/test_environment_enabled.txt';
public final static function get_test_file_path() {
return behat_command::get_parent_behat_dir() . '/test_environment_enabled.txt';
}
/**
......