mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-01-19 14:27:22 +01:00
Code Issues Projects Releases Wiki Activity

MDL-49456 behat: Fixed following issues in parallel run

Browse Source 
1. Parallel run without feature should exit with success
2. Updated exit codes to be more clear for each process
3. Fixed wwwroot and dataroot while executing with tags
4. Added support for running scenario/feature
5. Added support for or'ing tags
6. Added support for running single behat site
7. Fixed wrong example in help
8. Use parallel run only when parallel > 1
9. Exit showing behat site is not installed
10. Exit with error, if not a valid option
11. Added usage in help to explain how to use command
12. run.php should be able to run from any path
13. Pass all arguments to single run
This commit is contained in:
Rajesh Taneja 2015-04-07 11:48:35 +08:00
parent b6a76cd7cd
commit 72bac0d3f2
5 changed files with 115 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
admin/tool/behat/cli/init.php
View File

@ -59,6 +59,9 @@ list($options, $unrecognized) = cli_get_params(
$help = "
Behat utilities to initialise behat tests
Usage:
php init.php [--parallel=value [--maxruns=value] [--fromrun=value --torun=value]] [--help]
Options:
-j, --parallel Number of parallel behat run to initialise
-m, --maxruns Max parallel processes to be executed at one time.
@ -82,7 +85,7 @@ if (!empty($options['help'])) {
$utilfile = 'util_single_run.php';
$paralleloption = "";
// If parallel run then use utilparallel.
if ($options['parallel']) {
if ($options['parallel'] && $options['parallel'] > 1) {
$utilfile = 'util.php';
$paralleloption = "";
foreach ($options as $option => $value) {

108
admin/tool/behat/cli/run.php
View File

@ -50,24 +50,29 @@ list($options, $unrecognised) = cli_get_params(
'help' => false,
'tags' => '',
'profile' => '',
'feature' => '',
'fromrun' => 1,
'torun' => 0,
'single-run' => false,
),
array(
'h' => 'help',
't' => 'tags',
'p' => 'profile',
's' => 'single-run',
)
);
// Checking run.php CLI script usage.
$help = "
Behat utilities to run behat tests in parallel
Usage:
php run.php [--BEHAT_OPTION=\"value\"] [--feature=\"value\"] [--replace] [--fromrun=value --torun=value] [--help]
Options:
-t, --tags Tags to execute.
-p, --profile Profile to execute.
--stop-on-failure Stop on failure in any parallel run.
--verbose Verbose output
--BEHAT_OPTION Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html
--feature Only execute specified feature file (Absolute path of feature file).
--replace Replace args string with run process number, useful for output.
--fromrun Execute run starting from (Used for parallel runs on different vms)
--torun Execute run till (Used for parallel runs on different vms)
@ -75,7 +80,7 @@ Options:
-h, --help Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/run.php --parallel=2
\$ php admin/tool/behat/cli/run.php --tags=\"@javascript\"
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
@ -101,25 +106,6 @@ if (extension_loaded('pcntl')) {
}
}
// If empty parallelrun then just check with user if it's a run single behat test.
if (empty($parallelrun)) {
if (cli_input("This is not a parallel site, do you want to run single behat run? (Y/N)", 'n', array('y', 'n')) == 'y') {
$runtestscommand = behat_command::get_behat_command();
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
exec("php $runtestscommand", $output, $code);
echo implode(PHP_EOL, $output) . PHP_EOL;
exit($code);
} else {
exit(1);
}
}
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
$time = microtime(true);
array_walk($unrecognised, function (&$v) {
if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) {
@ -151,6 +137,30 @@ if ($options['tags']) {
$extraopts[] = '--tags="' . $tags . '"';
}
// Feature should be added to last, for behat command.
if ($options['feature']) {
$extraopts[] = $options['feature'];
// Only run 1 process as process.
// Feature file is picked from absolute path provided, so no need to check for behat.yml.
$options['torun'] = $options['fromrun'];
}
// Set of options to pass to behat.
$extraopts = implode(' ', $extraopts);
// If empty parallelrun then just check with user if it's a run single behat test.
if (empty($parallelrun)) {
$cwd = getcwd();
chdir(__DIR__);
$runtestscommand = '../../../../' . behat_command::get_behat_command();
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
$runtestscommand .= ' ' . $extraopts;
echo "Running single behat site:" . PHP_EOL;
passthru("php $runtestscommand", $code);
chdir($cwd);
exit($code);
}
// Update config file if tags defined.
if ($tags) {
// Hack to set proper dataroot and wwwroot.
@ -158,12 +168,17 @@ if ($tags) {
$behatwwwroot = $CFG->behat_wwwroot;
for ($i = 1; $i <= $parallelrun; $i++) {
$CFG->behatrunprocess = $i;
$CFG->behat_dataroot = $behatdataroot . $i;
if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) {
$CFG->behat_wwwroot = $CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'];
if (!empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) {
$CFG->behat_wwwroot = $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'];
} else {
$CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i;
}
if (!empty($CFG->behat_parallel_run[$i - 1]['behat_dataroot'])) {
$CFG->behat_dataroot = $CFG->behat_parallel_run[$i - 1]['behat_dataroot'];
} else {
$CFG->behat_dataroot = $behatdataroot . $i;
}
behat_config_manager::update_config_file('', true, $tags);
}
$CFG->behat_dataroot = $behatdataroot;
@ -172,7 +187,6 @@ if ($tags) {
}
$cmds = array();
$extraopts = implode(' ', $extraopts);
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
@ -185,7 +199,7 @@ for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Command to execute behat run.
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = '../../../../' . $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
}
@ -194,8 +208,14 @@ if (empty($cmds)) {
exit(1);
}
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
// Execute all commands.
$processes = cli_execute_parallel($cmds);
$processes = cli_execute_parallel($cmds, __DIR__);
$stoponfail = empty($options['stop-on-failure']) ? false : true;
// Print header.
@ -213,21 +233,31 @@ foreach ($exitcodes as $exitcode) {
$status = (bool)$status || (bool)$exitcode;
}
// Run finished. Show exit code and output from individual process.
$verbose = empty($options['verbose']) ? false : true;
$verbose = $verbose || $status;
// Show exit code from each process, if any process failed.
if ($status) {
echo "Exit codes: " . implode(" ", $exitcodes) . PHP_EOL;
echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
foreach ($cmds as $name => $cmd) {
if (!empty($exitcodes[$name])) {
echo "[" . $name . "] " . $cmd . PHP_EOL;
if ($verbose) {
// Echo exit codes.
echo "Exit codes for each behat run: " . PHP_EOL;
ksort($exitcodes);
foreach ($exitcodes as $run => $exitcode) {
echo $run . ": " . $exitcode . PHP_EOL;
}
// Show failed re-run commands.
if ($status) {
echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
foreach ($cmds as $name => $cmd) {
if (!empty($exitcodes[$name])) {
echo "[" . $name . "] " . $cmd . PHP_EOL;
}
}
}
echo PHP_EOL;
}
// Run finished. Show exit code and output from individual process.
$verbose = empty($options['verbose']) ? false : true;
$verbose = $verbose || $status;
print_each_process_info($processes, $verbose);
// Remove site symlink if necessary.

9
admin/tool/behat/cli/util.php
View File

@ -68,15 +68,18 @@ list($options, $unrecognized) = cli_get_params(
$help = "
Behat utilities to manage the test environment
Usage:
php util.php [--install|--drop|--enable|--disable|--diag|--updatesteps|--help] [--parallel=value [--maxruns=value]]
Options:
--install Installs the test environment for acceptance tests
--drop Drops the database tables and the dataroot contents
--enable Enables test environment and updates tests list
--disable Disables test environment
--diag Get behat test environment status code
--updatesteps Update feature step file.
-j, --parallel Number of parallel behat run operation
-m, --maxruns Max parallel processes to be executed at one time.
--updatesteps Update feature step file.
-h, --help Print out this help
@ -205,8 +208,12 @@ if ($options['install']) {
} else if ($options['disable']) {
echo "Acceptance tests environment disabled for " . $options['parallel'] . " parallel sites" . PHP_EOL;
} else if ($options['diag']) {
// Valid option, so nothing to do.
} else {
echo $help;
chdir($cwd);
exit(1);
}
chdir($cwd);

3
admin/tool/behat/cli/util_single_run.php
View File

@ -61,6 +61,9 @@ if ($options['install'] or $options['drop']) {
$help = "
Behat utilities to manage the test environment
Usage:
php util_single_run.php [--install|--drop|--enable|--disable|--diag|--updatesteps|--help]
Options:
--install Installs the test environment for acceptance tests
--drop Drops the database tables and the dataroot contents

34
lib/behat/classes/behat_config_manager.php
View File

@ -130,21 +130,44 @@ class behat_config_manager {
return $features;
}
$newfeaturelist = array();
$tagstosearch = explode('&&', $tags);
// Split tags in and and or.
$tags = explode('&&', $tags);
$andtags = array();
$ortags = array();
foreach ($tags as $tag) {
// Explode all tags seperated by , and add it to ortags.
$ortags = array_merge($ortags, explode(',', $tag));
// And tags will be the first one before comma(,).
$andtags[] = preg_replace('/,.*/', '', $tag);
}
foreach ($features as $featurefile) {
$contents = file_get_contents($featurefile);
$includefeature = true;
foreach ($tagstosearch as $tag) {
foreach ($andtags as $tag) {
// If negitive tag, then ensure it don't exist.
if (strpos($tag, '~') !== false) {
$tag = substr($tag, 1);
if ($contents && strpos($contents, $tag) !== false) {
$includefeature = false;
break;
}
} else if ($contents && strpos($contents, $tag) === false) {
$includefeature = false;
break;
}
}
// If feature not included then check or tags.
if (!$includefeature && !empty($ortags)) {
foreach ($ortags as $tag) {
if ($contents && (strpos($tag, '~') === false) && (strpos($contents, $tag) !== false)) {
$includefeature = true;
break;
}
}
}
if ($includefeature) {
$newfeaturelist[] = $featurefile;
}
@ -354,7 +377,12 @@ class behat_config_manager {
// Pull out the features for just this worker.
if (count($features)) {
$features = array_chunk($features, ceil(count($features) / max(1, $parallelruns)));
$features = $features[$CFG->behatrunprocess - 1];
// Check if there is any feature file for this process.
if (!empty($features[$CFG->behatrunprocess - 1])) {
$features = $features[$CFG->behatrunprocess - 1];
} else {
$features = null;
}
}
}
// Set proper selenium2 wd_host if defined.