mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-03-22 08:30:04 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'wip-mdl-39752' of https://github.com/rajeshtaneja/moodle

Browse Source
This commit is contained in:
Dan Poltawski 2015-03-10 08:08:51 +00:00
commit a0941ad0af
15 changed files with 1613 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
admin/tool/behat/cli/init.php
View File

@ -40,61 +40,126 @@ define('CACHE_DISABLE_ALL', true);
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
list($options, $unrecognized) = cli_get_params(
array(
'parallel' => 0,
'maxruns' => false,
'help' => false,
'fromrun' => 1,
'torun' => 0,
),
array(
'j' => 'parallel',
'm' => 'maxruns',
'h' => 'help',
)
);
// Checking run.php CLI script usage.
$help = "
Behat utilities to initialise behat tests
Options:
-j, --parallel Number of parallel behat run to initialise
-m, --maxruns Max parallel processes to be executed at one time.
--fromrun Execute run starting from (Used for parallel runs on different vms)
--torun Execute run till (Used for parallel runs on different vms)
-h, --help Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/init.php --parallel=2
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
if (!empty($options['help'])) {
echo $help;
exit(0);
}
// Check which util file to call.
$utilfile = 'util_single_run.php';
$paralleloption = "";
// If parallel run then use utilparallel.
if ($options['parallel']) {
$utilfile = 'util.php';
$paralleloption = "";
foreach ($options as $option => $value) {
if ($value) {
$paralleloption .= " --$option=\"$value\"";
}
}
}
// Changing the cwd to admin/tool/behat/cli.
chdir(__DIR__);
$cwd = getcwd();
$output = null;
exec("php util.php --diag", $output, $code);
// If behat dependencies not downloaded then do it first, else symfony/process can't be used.
if ($options['parallel'] && !file_exists(__DIR__ . "/../../../../vendor/autoload.php")) {
$code = BEHAT_EXITCODE_COMPOSER;
} else {
chdir(__DIR__);
exec("php $utilfile --diag $paralleloption", $output, $code);
}
// Check if composer needs to be updated.
if (($code == BEHAT_EXITCODE_INSTALL) || $code == BEHAT_EXITCODE_REINSTALL || $code == BEHAT_EXITCODE_COMPOSER) {
testing_update_composer_dependencies();
}
if ($code == 0) {
echo "Behat test environment already installed\n";
} else if ($code == BEHAT_EXITCODE_INSTALL) {
testing_update_composer_dependencies();
// Behat and dependencies are installed and we need to install the test site.
chdir(__DIR__);
passthru("php util.php --install", $code);
passthru("php $utilfile --install $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
} else if ($code == BEHAT_EXITCODE_REINSTALL) {
testing_update_composer_dependencies();
// Test site data is outdated.
chdir(__DIR__);
passthru("php util.php --drop", $code);
passthru("php $utilfile --drop $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
passthru("php util.php --install", $code);
chdir(__DIR__);
passthru("php $utilfile --install $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
} else if ($code == BEHAT_EXITCODE_COMPOSER) {
// Missing Behat dependencies.
testing_update_composer_dependencies();
// Returning to admin/tool/behat/cli.
chdir(__DIR__);
passthru("php util.php --install", $code);
passthru("php $utilfile --install $paralleloption", $code);
if ($code != 0) {
chdir($cwd);
exit($code);
}
} else {
// Generic error, we just output it.
echo implode("\n", $output)."\n";
chdir($cwd);
exit($code);
}
// Enable editing mode according to config.php vars.
passthru("php util.php --enable", $code);
chdir(__DIR__);
passthru("php $utilfile --enable $paralleloption", $code);
if ($code != 0) {
echo "Error enabling site" . PHP_EOL;
chdir($cwd);
exit($code);
}

361
admin/tool/behat/cli/run.php Normal file
View File

@ -0,0 +1,361 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Wrapper to run previously set-up behat tests in parallel.
*
* @package tool_behat
* @copyright 2014 NetSpot Pty Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
if (isset($_SERVER['REMOTE_ADDR'])) {
die(); // No access from web!
}
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('ABORT_AFTER_CONFIG', true);
define('CACHE_DISABLE_ALL', true);
define('NO_OUTPUT_BUFFERING', true);
require_once(__DIR__ .'/../../../../config.php');
require_once(__DIR__.'/../../../../lib/clilib.php');
require_once(__DIR__.'/../../../../lib/behat/lib.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php');
error_reporting(E_ALL | E_STRICT);
ini_set('display_errors', '1');
ini_set('log_errors', '1');
list($options, $unrecognised) = cli_get_params(
array(
'stop-on-failure' => 0,
'verbose' => false,
'replace' => false,
'help' => false,
'tags' => '',
'profile' => '',
'fromrun' => 1,
'torun' => 0,
),
array(
'h' => 'help',
't' => 'tags',
'p' => 'profile',
)
);
// Checking run.php CLI script usage.
$help = "
Behat utilities to run behat tests in parallel
Options:
-t, --tags Tags to execute.
-p, --profile Profile to execute.
--stop-on-failure Stop on failure in any parallel run.
--verbose Verbose output
--replace Replace args string with run process number, useful for output.
--fromrun Execute run starting from (Used for parallel runs on different vms)
--torun Execute run till (Used for parallel runs on different vms)
-h, --help Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/run.php --parallel=2
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
if (!empty($options['help'])) {
echo $help;
exit(0);
}
$parallelrun = behat_config_manager::get_parallel_test_runs($options['fromrun']);
// Default torun is maximum parallel runs.
if (empty($options['torun'])) {
$options['torun'] = $parallelrun;
}
// Capture signals and ensure we clean symlinks.
if (extension_loaded('pcntl')) {
$disabled = explode(',', ini_get('disable_functions'));
if (!in_array('pcntl_signal', $disabled)) {
pcntl_signal(SIGTERM, "signal_handler");
pcntl_signal(SIGINT, "signal_handler");
}
}
// If empty parallelrun then just check with user if it's a run single behat test.
if (empty($parallelrun)) {
if (cli_input("This is not a parallel site, do you want to run single behat run? (Y/N)", 'n', array('y', 'n')) == 'y') {
$runtestscommand = behat_command::get_behat_command();
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
exec("php $runtestscommand", $output, $code);
echo implode(PHP_EOL, $output) . PHP_EOL;
exit($code);
} else {
exit(1);
}
}
// Create site symlink if necessary.
if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
exit(1);
}
$time = microtime(true);
array_walk($unrecognised, function (&$v) {
if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) {
$v = $x;
} else if (!preg_match("#^-#", $v)) {
$v = escapeshellarg($v);
}
});
$extraopts = implode(' ', $unrecognised);
$tags = '';
if ($options['profile']) {
$profile = $options['profile'];
if (empty($CFG->behat_config[$profile]['filters']['tags'])) {
echo "Invaid profile passed: " . $profile;
exit(1);
}
$tags = $CFG->behat_config[$profile]['filters']['tags'];
$extraopts .= '--profile=\'' . $profile . "'";
} else if ($options['tags']) {
$tags = $options['tags'];
$extraopts .= '--tags="' . $tags . '"';
}
// Update config file if tags defined.
if ($tags) {
// Hack to set proper dataroot and wwwroot.
$behatdataroot = $CFG->behat_dataroot;
$behatwwwroot = $CFG->behat_wwwroot;
for ($i = 1; $i <= $parallelrun; $i++) {
$CFG->behatrunprocess = $i;
$CFG->behat_dataroot = $behatdataroot . $i;
if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) {
$CFG->behat_wwwroot = $CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'];
} else {
$CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i;
}
behat_config_manager::update_config_file('', true, $tags);
}
$CFG->behat_dataroot = $behatdataroot;
$CFG->behat_wwwroot = $behatwwwroot;
unset($CFG->behatrunprocess);
}
$cmds = array();
echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
$CFG->behatrunprocess = $i;
// Options parameters to be added to each run.
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts;
$behatcommand = behat_command::get_behat_command();
$behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
// Command to execute behat run.
$cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
}
if (empty($cmds)) {
echo "No commands to execute " . PHP_EOL;
exit(1);
}
// Execute all commands.
$processes = cli_execute_parallel($cmds);
$stoponfail = empty($options['stop-on-failure']) ? false : true;
// Print header.
print_process_start_info($processes);
// Print combined run o/p from processes.
$exitcodes = print_combined_run_output($processes, $stoponfail);
$time = round(microtime(true) - $time, 1);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
// Print exit info from each run.
$status = false;
foreach ($exitcodes as $exitcode) {
$status = (bool)$status || (bool)$exitcode;
}
// Show exit code from each process, if any process failed.
if ($status) {
echo "Exit codes: " . implode(" ", $exitcodes) . PHP_EOL;
echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
foreach ($cmds as $name => $cmd) {
if (!empty($exitcodes[$name])) {
echo "[" . $name . "] " . $cmd . PHP_EOL;
}
}
echo PHP_EOL;
}
// Run finished. Show exit code and output from individual process.
$verbose = empty($options['verbose']) ? false : true;
$verbose = $verbose || $status;
print_each_process_info($processes, $verbose);
// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();
exit((int) $status);
/**
* Signal handler for terminal exit.
*
* @param int $signal signal number.
*/
function signal_handler($signal) {
switch ($signal) {
case SIGTERM:
case SIGKILL:
case SIGINT:
// Remove site symlink if necessary.
behat_config_manager::drop_parallel_site_links();
exit(1);
}
}
/**
* Prints header from the first process.
*
* @param array $processes list of processes to loop though.
*/
function print_process_start_info($processes) {
$printed = false;
// Keep looping though processes, till we get first process o/p.
while (!$printed) {
usleep(10000);
foreach ($processes as $name => $process) {
// Exit if any process has stopped.
if (!$process->isRunning()) {
$printed = true;
break;
}
$op = explode(PHP_EOL, $process->getOutput());
if (count($op) >= 3) {
foreach ($op as $line) {
if (trim($line) && (strpos($line, '.') !== 0)) {
echo $line . PHP_EOL;
}
}
$printed = true;
}
}
}
}
/**
* Loop though all processes and print combined o/p
*
* @param array $processes list of processes to loop though.
* @param bool $stoponfail Stop all processes and exit if failed.
* @return array list of exit codes from all processes.
*/
function print_combined_run_output($processes, $stoponfail = false) {
$exitcodes = array();
$maxdotsonline = 70;
$remainingprintlen = $maxdotsonline;
$progresscount = 0;
while (count($exitcodes) != count($processes)) {
usleep(10000);
foreach ($processes as $name => $process) {
if ($process->isRunning()) {
$op = $process->getIncrementalOutput();
if (trim($op)) {
$update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
// Exit process if anything fails.
if ($stoponfail && (strpos($update, 'F') !== false)) {
$process->stop(0);
}
$strlentoprint = strlen($update);
// If not enough dots printed on line then just print.
if ($strlentoprint < $remainingprintlen) {
echo $update;
$remainingprintlen = $remainingprintlen - $strlentoprint;
} else if ($strlentoprint == $remainingprintlen) {
$progresscount += $maxdotsonline;
echo $update ." " . $progresscount . PHP_EOL;
$remainingprintlen = $maxdotsonline;
} else {
while ($part = substr($update, 0, $remainingprintlen) > 0) {
$progresscount += $maxdotsonline;
echo $part . " " . $progresscount . PHP_EOL;
$update = substr($update, $remainingprintlen);
$remainingprintlen = $maxdotsonline;
}
}
}
} else {
$exitcodes[$name] = $process->getExitCode();
if ($stoponfail && ($exitcodes[$name] != 0)) {
foreach ($processes as $l => $p) {
$exitcodes[$l] = -1;
$process->stop(0);
}
}
}
}
}
echo PHP_EOL;
return $exitcodes;
}
/**
* Loop though all processes and print combined o/p
*
* @param array $processes list of processes to loop though.
* @param bool $verbose Show verbose output for each process.
*/
function print_each_process_info($processes, $verbose = false) {
foreach ($processes as $name => $process) {
echo "**************** [" . $name . "] ****************" . PHP_EOL;
if ($verbose) {
echo $process->getOutput();
echo $process->getErrorOutput();
} else {
$op = explode(PHP_EOL, $process->getOutput());
foreach ($op as $line) {
// Don't print progress .
if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) &&
(strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) &&
(strpos($line, 'Browser specific fixes ') !== 0)) {
echo $line . PHP_EOL;
}
}
}
echo PHP_EOL;
}
}

413
admin/tool/behat/cli/util.php
View File

@ -15,7 +15,7 @@
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* CLI tool with utilities to manage Behat integration in Moodle
* CLI tool with utilities to manage parallel Behat integration in Moodle
*
* All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as
* $CFG->dataroot and $CFG->prefix
@ -30,45 +30,58 @@ if (isset($_SERVER['REMOTE_ADDR'])) {
die(); // No access from web!.
}
// Basic functions.
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
define('ABORT_AFTER_CONFIG', true);
require_once(__DIR__ . '/../../../../config.php');
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');
// CLI options.
list($options, $unrecognized) = cli_get_params(
array(
'help' => false,
'install' => false,
'drop' => false,
'enable' => false,
'disable' => false,
'diag' => false
'help' => false,
'install' => false,
'drop' => false,
'enable' => false,
'disable' => false,
'diag' => false,
'parallel' => 0,
'maxruns' => false,
'updatesteps' => false,
'fromrun' => 1,
'torun' => 0,
),
array(
'h' => 'help'
'h' => 'help',
'j' => 'parallel',
'm' => 'maxruns'
)
);
if ($options['install'] or $options['drop']) {
define('CACHE_DISABLE_ALL', true);
}
// Checking util.php CLI script usage.
$help = "
Behat utilities to manage the test environment
Options:
--install Installs the test environment for acceptance tests
--drop Drops the database tables and the dataroot contents
--enable Enables test environment and updates tests list
--disable Disables test environment
--diag Get behat test environment status code
--install Installs the test environment for acceptance tests
--drop Drops the database tables and the dataroot contents
--enable Enables test environment and updates tests list
--disable Disables test environment
--diag Get behat test environment status code
-j, --parallel Number of parallel behat run operation
-m, --maxruns Max parallel processes to be executed at one time.
--updatesteps Update feature step file.
-h, --help Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/util.php --enable
\$ php admin/tool/behat/cli/util.php --enable --parallel=4
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
@ -78,66 +91,324 @@ if (!empty($options['help'])) {
exit(0);
}
// Describe this script.
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
$cwd = getcwd();
// Only load CFG from config.php, stop ASAP in lib/setup.php.
define('ABORT_AFTER_CONFIG', true);
require_once(__DIR__ . '/../../../../config.php');
// Remove error handling overrides done in config.php.
$CFG->debug = (E_ALL | E_STRICT);
$CFG->debugdisplay = 1;
error_reporting($CFG->debug);
ini_set('display_errors', '1');
ini_set('log_errors', '1');
// Finish moodle init.
define('ABORT_AFTER_CONFIG_CANCEL', true);
require("$CFG->dirroot/lib/setup.php");
raise_memory_limit(MEMORY_HUGE);
require_once($CFG->libdir.'/adminlib.php');
require_once($CFG->libdir.'/upgradelib.php');
require_once($CFG->libdir.'/clilib.php');
require_once($CFG->libdir.'/installlib.php');
require_once($CFG->libdir.'/testing/classes/test_lock.php');
if ($unrecognized) {
$unrecognized = implode("\n ", $unrecognized);
cli_error(get_string('cliunknowoption', 'admin', $unrecognized));
// For drop option check if parallel site.
if ((empty($options['parallel'])) && $options['drop']) {
// Get parallel run info from first run.
$options['parallel'] = behat_config_manager::get_parallel_test_runs($options['fromrun']);
}
// Behat utilities.
require_once($CFG->libdir . '/behat/classes/util.php');
require_once($CFG->libdir . '/behat/classes/behat_command.php');
// If not a parallel site then open single run.
if (empty($options['parallel'])) {
chdir(__DIR__);
// Check if behat is initialised, if not exit.
passthru("php util_single_run.php --diag", $status);
if ($status) {
exit ($status);
}
$cmd = commands_to_execute($options);
$processes = cli_execute_parallel(array($cmd), __DIR__);
$status = print_sequential_output($processes, false);
chdir($cwd);
exit($status);
}
// Default torun is maximum parallel runs.
if (empty($options['torun'])) {
$options['torun'] = $options['parallel'];
}
$status = false;
$cmds = commands_to_execute($options);
// Start executing commands either sequential/parallel for options provided.
if ($options['diag'] || $options['enable'] || $options['disable']) {
// Do it sequentially as it's fast and need to be displayed nicely.
foreach (array_chunk($cmds, 1, true) as $cmd) {
$processes = cli_execute_parallel($cmd, __DIR__);
print_sequential_output($processes);
}
// Run command (only one per time).
if ($options['install']) {
behat_util::install_site();
mtrace("Acceptance tests site installed");
} else if ($options['drop']) {
// Ensure no tests are running.
test_lock::acquire('behat');
behat_util::drop_site();
mtrace("Acceptance tests site dropped");
$processes = cli_execute_parallel($cmds, __DIR__);
$exitcodes = print_combined_drop_output($processes);
foreach ($exitcodes as $exitcode) {
$status = (bool)$status || (bool)$exitcode;
}
} else if ($options['install']) {
// This is intensive compared to behat itself so run them in chunk if option maxruns not set.
if ($options['maxruns']) {
foreach (array_chunk($cmds, $options['maxruns'], true) as $chunk) {
$processes = cli_execute_parallel($chunk, __DIR__);
$exitcodes = print_combined_install_output($processes);
foreach ($exitcodes as $name => $exitcode) {
if ($exitcode != 0) {
echo "Failed process [[$name]]" . PHP_EOL;
echo $processes[$name]->getOutput();
echo PHP_EOL;
echo $processes[$name]->getErrorOutput();
echo PHP_EOL . PHP_EOL;
}
$status = (bool)$status || (bool)$exitcode;
}
}
} else {
$processes = cli_execute_parallel($cmds, __DIR__);
$exitcodes = print_combined_install_output($processes);
foreach ($exitcodes as $name => $exitcode) {
if ($exitcode != 0) {
echo "Failed process [[$name]]" . PHP_EOL;
echo $processes[$name]->getOutput();
echo PHP_EOL;
echo $processes[$name]->getErrorOutput();
echo PHP_EOL . PHP_EOL;
}
$status = (bool)$status || (bool)$exitcode;
}
}
} else {
// We should never reach here.
echo $help;
exit(1);
}
// Ensure we have success status to show following information.
if ($status) {
echo "Unknown failure $status" . PHP_EOL;
exit((int)$status);
}
// Show command o/p (only one per time).
if ($options['install']) {
echo "Acceptance tests site installed for sites:".PHP_EOL;
// Display all sites which are installed/drop/diabled.
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
if (empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) {
echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL;
} else {
echo $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'] . PHP_EOL;
}
}
} else if ($options['drop']) {
echo "Acceptance tests site dropped for " . $options['parallel'] . " parallel sites" . PHP_EOL;
} else if ($options['enable']) {
behat_util::start_test_mode();
$runtestscommand = behat_command::get_behat_command(true) .
' --config ' . behat_config_manager::get_behat_cli_config_filepath();
mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:\n " . $runtestscommand);
echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:" . PHP_EOL;
echo behat_command::get_behat_command(true, true);
echo PHP_EOL;
} else if ($options['disable']) {
behat_util::stop_test_mode();
mtrace("Acceptance tests environment disabled");
} else if ($options['diag']) {
$code = behat_util::get_behat_status();
exit($code);
echo "Acceptance tests environment disabled for " . $options['parallel'] . " parallel sites" . PHP_EOL;
} else {
echo $help;
}
chdir($cwd);
exit(0);
/**
* Create commands to be executed for parallel run.
*
* @param array $options options provided by user.
* @return array commands to be executed.
*/
function commands_to_execute($options) {
$removeoptions = array('maxruns', 'fromrun', 'torun');
$cmds = array();
$extraoptions = $options;
$extra = "";
// Remove extra options not in util_single_run.php.
foreach ($removeoptions as $ro) {
$extraoptions[$ro] = null;
unset($extraoptions[$ro]);
}
foreach ($extraoptions as $option => $value) {
if ($options[$option]) {
$extra .= " --$option";
if ($value) {
$extra .= "=$value";
}
}
}
if (empty($options['parallel'])) {
$cmds = "php util_single_run.php " . $extra;
} else {
// Create commands which has to be executed for parallel site.
for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
$prefix = BEHAT_PARALLEL_SITE_NAME . $i;
$cmds[$prefix] = "php util_single_run.php " . $extra . " --run=" . $i . " 2>&1";
}
}
return $cmds;
}
/**
* Print drop output merging each run.
*
* @param array $processes list of processes.
* @return array exit codes of each process.
*/
function print_combined_drop_output($processes) {
$exitcodes = array();
$maxdotsonline = 70;
$remainingprintlen = $maxdotsonline;
$progresscount = 0;
echo "Dropping tables:" . PHP_EOL;
while (count($exitcodes) != count($processes)) {
usleep(10000);
foreach ($processes as $name => $process) {
if ($process->isRunning()) {
$op = $process->getIncrementalOutput();
if (trim($op)) {
$update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
$strlentoprint = strlen($update);
// If not enough dots printed on line then just print.
if ($strlentoprint < $remainingprintlen) {
echo $update;
$remainingprintlen = $remainingprintlen - $strlentoprint;
} else if ($strlentoprint == $remainingprintlen) {
$progresscount += $maxdotsonline;
echo $update . " " . $progresscount . PHP_EOL;
$remainingprintlen = $maxdotsonline;
} else {
while ($part = substr($update, 0, $remainingprintlen) > 0) {
$progresscount += $maxdotsonline;
echo $part . " " . $progresscount . PHP_EOL;
$update = substr($update, $remainingprintlen);
$remainingprintlen = $maxdotsonline;
}
}
}
} else {
// Process exited.
$process->clearOutput();
$exitcodes[$name] = $process->getExitCode();
}
}
}
echo PHP_EOL;
return $exitcodes;
}
/**
* Print install output merging each run.
*
* @param array $processes list of processes.
* @return array exit codes of each process.
*/
function print_combined_install_output($processes) {
$exitcodes = array();
$line = array();
// Check what best we can do to accommodate all parallel run o/p on single line.
// Windows command line has length of 80 chars, so default we will try fit o/p in 80 chars.
if (defined('BEHAT_MAX_CMD_LINE_OUTPUT') && BEHAT_MAX_CMD_LINE_OUTPUT) {
$lengthofprocessline = (int)max(10, BEHAT_MAX_CMD_LINE_OUTPUT / count($processes));
} else {
$lengthofprocessline = (int)max(10, 80 / count($processes));
}
echo "Installing behat site for " . count($processes) . " parallel behat run" . PHP_EOL;
// Show process name in first row.
foreach ($processes as $name => $process) {
// If we don't have enough space to show full run name then show runX.
if ($lengthofprocessline < strlen($name + 2)) {
$name = substr($name, -5);
}
// One extra padding as we are adding | separator for rest of the data.
$line[$name] = str_pad('[' . $name . '] ', $lengthofprocessline + 1);
}
ksort($line);
$tableheader = array_keys($line);
echo implode("", $line) . PHP_EOL;
// Now print o/p from each process.
while (count($exitcodes) != count($processes)) {
usleep(50000);
$poutput = array();
// Create child process.
foreach ($processes as $name => $process) {
if ($process->isRunning()) {
$output = $process->getIncrementalOutput();
if (trim($output)) {
$poutput[$name] = explode(PHP_EOL, $output);
}
} else {
// Process exited.
$exitcodes[$name] = $process->getExitCode();
}
}
ksort($poutput);
// Get max depth of o/p before displaying.
$maxdepth = 0;
foreach ($poutput as $pout) {
$pdepth = count($pout);
$maxdepth = $pdepth >= $maxdepth ? $pdepth : $maxdepth;
}
// Iterate over each process to get line to print.
for ($i = 0; $i <= $maxdepth; $i++) {
$pline = "";
foreach ($tableheader as $name) {
$po = empty($poutput[$name][$i]) ? "" : substr($poutput[$name][$i], 0, $lengthofprocessline - 1);
$po = str_pad($po, $lengthofprocessline);
$pline .= "|". $po;
}
if (trim(str_replace("|", "", $pline))) {
echo $pline . PHP_EOL;
}
}
unset($poutput);
$poutput = null;
}
echo PHP_EOL;
return $exitcodes;
}
/**
* Print install output merging showing one run at a time.
* If any process fail then exit.
*
* @param array $processes list of processes.
* @param bool $showprefix show prefix.
* @return bool exitcode.
*/
function print_sequential_output($processes, $showprefix = true) {
$status = false;
foreach ($processes as $name => $process) {
$shownname = false;
while ($process->isRunning()) {
$op = $process->getIncrementalOutput();
if (trim($op)) {
// Show name of the run once for sequential.
if ($showprefix && !$shownname) {
echo '[' . $name . '] ';
$shownname = true;
}
echo $op;
}
}
// If any error then exit.
$exitcode = $process->getExitCode();
if ($exitcode != 0) {
exit($exitcode);
}
$status = $status || (bool)$exitcode;
}
return $status;
}

273
admin/tool/behat/cli/util_single_run.php Normal file
View File

@ -0,0 +1,273 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* CLI tool with utilities to manage Behat integration in Moodle
*
* All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as
* $CFG->dataroot and $CFG->prefix
*
* @package tool_behat
* @copyright 2012 David Monllaó
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
if (isset($_SERVER['REMOTE_ADDR'])) {
die(); // No access from web!.
}
// Basic functions.
require_once(__DIR__ . '/../../../../lib/clilib.php');
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
// CLI options.
list($options, $unrecognized) = cli_get_params(
array(
'help' => false,
'install' => false,
'parallel' => 0,
'run' => '',
'drop' => false,
'enable' => false,
'disable' => false,
'diag' => false,
'tags' => '',
'updatesteps' => false,
),
array(
'h' => 'help'
)
);
if ($options['install'] or $options['drop']) {
define('CACHE_DISABLE_ALL', true);
}
// Checking util_single_run.php CLI script usage.
$help = "
Behat utilities to manage the test environment
Options:
--install Installs the test environment for acceptance tests
--drop Drops the database tables and the dataroot contents
--enable Enables test environment and updates tests list
--disable Disables test environment
--diag Get behat test environment status code
--updatesteps Update feature step file.
-h, --help Print out this help
Example from Moodle root directory:
\$ php admin/tool/behat/cli/util_single_run.php --enable
More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
";
if (!empty($options['help'])) {
echo $help;
exit(0);
}
// Describe this script.
define('BEHAT_UTIL', true);
define('CLI_SCRIPT', true);
define('NO_OUTPUT_BUFFERING', true);
define('IGNORE_COMPONENT_CACHE', true);
// Set run value, to be used by setup for configuring proper CFG variables.
if ($options['run']) {
define('BEHAT_CURRENT_RUN', $options['run']);
}
// Only load CFG from config.php, stop ASAP in lib/setup.php.
define('ABORT_AFTER_CONFIG', true);
require_once(__DIR__ . '/../../../../config.php');
// Remove error handling overrides done in config.php.
$CFG->debug = (E_ALL | E_STRICT);
$CFG->debugdisplay = 1;
error_reporting($CFG->debug);
ini_set('display_errors', '1');
ini_set('log_errors', '1');
// Finish moodle init.
define('ABORT_AFTER_CONFIG_CANCEL', true);
require("$CFG->dirroot/lib/setup.php");
raise_memory_limit(MEMORY_HUGE);
require_once($CFG->libdir.'/adminlib.php');
require_once($CFG->libdir.'/upgradelib.php');
require_once($CFG->libdir.'/clilib.php');
require_once($CFG->libdir.'/installlib.php');
require_once($CFG->libdir.'/testing/classes/test_lock.php');
if ($unrecognized) {
$unrecognized = implode(PHP_EOL . " ", $unrecognized);
cli_error(get_string('cliunknowoption', 'admin', $unrecognized));
}
// Behat utilities.
require_once($CFG->libdir . '/behat/classes/util.php');
require_once($CFG->libdir . '/behat/classes/behat_command.php');
require_once($CFG->libdir . '/behat/classes/behat_config_manager.php');
// Ensure run option is <= parallel run installed.
if ($options['run']) {
if (!$options['parallel']) {
$options['parallel'] = behat_config_manager::get_parallel_test_runs();
}
if (empty($options['parallel']) || $options['run'] > $options['parallel']) {
echo "Parallel runs can't be more then ".$options['parallel'].PHP_EOL;
exit(1);
}
$CFG->behatrunprocess = $options['run'];
}
// Run command (only one per time).
if ($options['install']) {
behat_util::install_site();
// This is only displayed once for parallel install.
if (empty($options['run'])) {
mtrace("Acceptance tests site installed");
}
} else if ($options['drop']) {
// Ensure no tests are running.
test_lock::acquire('behat');
behat_util::drop_site();
// This is only displayed once for parallel install.
if (empty($options['run'])) {
mtrace("Acceptance tests site dropped");
}
} else if ($options['enable']) {
if (!empty($options['parallel'])) {
// Save parallel site info for enable and install options.
$filepath = behat_config_manager::get_parallel_test_file_path();
if (!file_put_contents($filepath, $options['parallel'])) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created');
}
}
// Enable test mode.
behat_util::start_test_mode();
// This is only displayed once for parallel install.
if (empty($options['run'])) {
$runtestscommand = behat_command::get_behat_command(true, !empty($options['run']));
$runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use: " . PHP_EOL .
$runtestscommand);
}
} else if ($options['disable']) {
behat_util::stop_test_mode();
// This is only displayed once for parallel install.
if (empty($options['run'])) {
mtrace("Acceptance tests environment disabled");
}
} else if ($options['diag']) {
$code = behat_util::get_behat_status();
exit($code);
} else if ($options['updatesteps']) {
if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) {
$behatstepfile = BEHAT_FEATURE_STEP_FILE;
} else {
echo "BEHAT_FEATURE_STEP_FILE is not set, please ensure you set this to writable file" . PHP_EOL;
exit(1);
}
// Rewrite config file to ensure we have all the features covered.
behat_config_manager::update_config_file();
// Run behat command to get steps in feature files.
$featurestepscmd = behat_command::get_behat_command(true);
$featurestepscmd .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
$featurestepscmd .= ' --dry-run --format=moodle_step_count';
$processes = cli_execute_parallel(array($featurestepscmd), __DIR__ . "/../../../../");
$status = print_update_step_output(array_pop($processes), $behatstepfile);
exit($status);
} else {
echo $help;
exit(1);
}
exit(0);
/**
* Print update progress as dots for updating feature file step list.
*
* @param Process $process process executing update step command.
* @param string $featurestepfile feature step file in which steps will be saved.
* @return int exitcode.
*/
function print_update_step_output($process, $featurestepfile) {
$printedlength = 0;
echo "Updating steps feature file for parallel behat runs" . PHP_EOL;
// Show progress while running command.
while ($process->isRunning()) {
usleep(10000);
$op = $process->getIncrementalOutput();
if (trim($op)) {
echo ".";
$printedlength++;
if ($printedlength > 70) {
$printedlength = 0;
echo PHP_EOL;
}
}
}
// If any error then exit.
$exitcode = $process->getExitCode();
// Output err.
if ($exitcode != 0) {
echo $process->getErrorOutput();
exit($exitcode);
}
// Extract features with step info and save it in file.
$featuresteps = $process->getOutput();
$featuresteps = explode(PHP_EOL, $featuresteps);
$realroot = realpath(__DIR__.'/../../../../').'/';
foreach ($featuresteps as $featurestep) {
if (trim($featurestep)) {
$step = explode("::", $featurestep);
$step[0] = str_replace($realroot, '', $step[0]);
$steps[$step[0]] = $step[1];
}
}
arsort($steps);
if (!@file_put_contents($featurestepfile, json_encode($steps, JSON_PRETTY_PRINT))) {
behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $featurestepfile . ' can not be created');
$exitcode = -1;
}
echo PHP_EOL. "Updated step count in " . $featurestepfile . PHP_EOL;
return $exitcode;
}

2
composer.json
View File

@ -8,6 +8,6 @@
"require-dev": {
"phpunit/phpunit": "3.7.*",
"phpunit/dbUnit": "1.2.*",
"moodlehq/behat-extension": "1.29.2"
"moodlehq/behat-extension": "1.29.3"
}
}

44
config-dist.php
View File

@ -724,6 +724,50 @@ $CFG->admin = 'admin';
// Example:
// $CFG->behat_faildump_path = '/my/path/to/save/failure/dumps';
//
// You can specify db, selenium wd_host etc. for behat parallel run by setting following variable.
// Example:
// $CFG->behat_parallel_run = array (
// array (
// 'dbtype' => 'mysqli',
// 'dblibrary' => 'native',
// 'dbhost' => 'localhost',
// 'dbname' => 'moodletest',
// 'dbuser' => 'moodle',
// 'dbpass' => 'moodle',
// 'behat_prefix' => 'mdl_',
// 'wd_host' => 'http://127.0.0.1:4444/wd/hub',
// 'behat_wwwroot' => 'http://127.0.0.1/moodle',
// 'behat_dataroot' => '/home/example/bht_moodledata'
// ),
// );
//
// To change name of behat parallel run site, define BEHAT_PARALLEL_SITE_NAME and parallel run sites will be suffixed
// with this value
// Example:
// define('BEHAT_PARALLEL_SITE_NAME', 'behatparallelsite');
//
// Command line output for parallel behat install is limited to 80 chars, if you are installing more then 4 sites and
// want to expand output to more then 80 chars, then define BEHAT_MAX_CMD_LINE_OUTPUT
// Example:
// define('BEHAT_MAX_CMD_LINE_OUTPUT', 120);
//
// Behat feature files will be distributed randomly between the processes by default. If you have timing file or want
// to create timing file then define BEHAT_FEATURE_TIMING_FILE with path to timing file. It will be updated for each
// run with latest time taken to execute feature.
// Example:
// define('BEHAT_FEATURE_TIMING_FILE', '/PATH_TO_TIMING_FILE/timing.json');
//
// If you don't have timing file and want some stable distribution of features, then you can use step counts to
// distribute the features. You can generate step file by executing php admin/tool/behat/cli/util.php --updatesteps
// this will update step file which is defined by BEHAT_FEATURE_STEP_FILE.
// Example:
// define('BEHAT_FEATURE_STEP_FILE', '/PATH_TO_FEATURE_STEP_COUNT_FILE/stepcount.json');
//
// Feature distribution for each process is displayed as histogram. you can disable it by setting
// BEHAT_DISABLE_HISTOGRAM
// Example:
// define('BEHAT_DISABLE_HISTOGRAM', true);
//
//=========================================================================
// 12. DEVELOPER DATA GENERATOR
//=========================================================================

35
lib/behat/classes/behat_command.php
View File

@ -44,12 +44,19 @@ class behat_command {
/**
* Ensures the behat dir exists in moodledata
* @param int $runprocess run process for which behat dir is returned.
* @return string Full path
*/
public static function get_behat_dir() {
public static function get_behat_dir($runprocess = 0) {
global $CFG;
$behatdir = $CFG->behat_dataroot . '/behat';
if (empty($runprocess)) {
$behatdir = $CFG->behat_dataroot . '/behat';
} else if (isset($CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'])) {
$behatdir = $CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'] . '/behat';;
} else {
$behatdir = $CFG->behat_dataroot . $runprocess . '/behat';
}
if (!is_dir($behatdir)) {
if (!mkdir($behatdir, $CFG->directorypermissions, true)) {
@ -73,23 +80,29 @@ class behat_command {
* normal cmd.exe (in Windows).
*
* @param bool $custombyterm If the provided command should depend on the terminal where it runs
* @param bool $parallelrun If parallel run is installed.
* @return string
*/
public final static function get_behat_command($custombyterm = false) {
public final static function get_behat_command($custombyterm = false, $parallerun = false) {
$separator = DIRECTORY_SEPARATOR;
$exec = 'behat';
if (!$parallerun) {
$exec = 'behat';
// Cygwin uses linux-style directory separators.
if ($custombyterm && testing_is_cygwin()) {
$separator = '/';
// Cygwin uses linux-style directory separators.
if ($custombyterm && testing_is_cygwin()) {
$separator = '/';
// MinGW can not execute .bat scripts.
if (!testing_is_mingw()) {
$exec = 'behat.bat';
// MinGW can not execute .bat scripts.
if (!testing_is_mingw()) {
$exec = 'behat.bat';
}
}
$command = 'vendor' . $separator . 'bin' . $separator . $exec;
} else {
$command = 'php admin' . $separator . 'tool' . $separator . 'behat' . $separator . 'cli' . $separator . 'run.php';
}
return 'vendor' . $separator . 'bin' . $separator . $exec;
return $command;
}
/**

254
lib/behat/classes/behat_config_manager.php
View File

@ -53,9 +53,10 @@ class behat_config_manager {
*
* @param string $component Restricts the obtained steps definitions to the specified component
* @param string $testsrunner If the config file will be used to run tests
* @param string $tags features files including tags.
* @return void
*/
public static function update_config_file($component = '', $testsrunner = true) {
public static function update_config_file($component = '', $testsrunner = true, $tags = '') {
global $CFG;
// Behat must have a separate behat.yml to have access to the whole set of features and steps definitions.
@ -79,7 +80,10 @@ class behat_config_manager {
$featurespaths[$uniquekey] = $path;
}
}
$features = array_values($featurespaths);
foreach ($featurespaths as $path) {
$additional = glob("$path/*.feature");
$features = array_merge($features, $additional);
}
}
// Optionally include features from additional directories.
@ -105,7 +109,7 @@ class behat_config_manager {
// Behat config file specifing the main context class,
// the required Behat extensions and Moodle test wwwroot.
$contents = self::get_config_file_contents($features, $stepsdefinitions);
$contents = self::get_config_file_contents(self::get_features_with_tags($features, $tags), $stepsdefinitions);
// Stores the file.
if (!file_put_contents($configfilepath, $contents)) {
@ -114,6 +118,40 @@ class behat_config_manager {
}
/**
* Search feature files for set of tags.
*
* @param array $features set of feature files.
* @param string $tags list of tags (currently support && only.)
* @return array filtered list of feature files with tags.
*/
public static function get_features_with_tags($features, $tags) {
if (empty($tags)) {
return $features;
}
$newfeaturelist = array();
$tagstosearch = explode('&&', $tags);
foreach ($features as $featurefile) {
$contents = file_get_contents($featurefile);
$includefeature = true;
foreach ($tagstosearch as $tag) {
// If negitive tag, then ensure it don't exist.
if (strpos($tag, '~') !== false) {
$tag = substr($tag, 1);
if ($contents && strpos($contents, $tag) !== false) {
$includefeature = false;
}
} else if ($contents && strpos($contents, $tag) === false) {
$includefeature = false;
}
}
if ($includefeature) {
$newfeaturelist[] = $featurefile;
}
}
return $newfeaturelist;
}
/**
* Gets the list of Moodle steps definitions
*
@ -169,12 +207,22 @@ class behat_config_manager {
/**
* Returns the behat config file path used by the behat cli command.
*
* @param int $runprocess Runprocess.
* @return string
*/
public static function get_behat_cli_config_filepath() {
public static function get_behat_cli_config_filepath($runprocess = 0) {
global $CFG;
$command = $CFG->behat_dataroot . DIRECTORY_SEPARATOR . 'behat' . DIRECTORY_SEPARATOR . 'behat.yml';
if ($runprocess) {
if (isset($CFG->behat_parallel_run[$runprocess - 1 ]['behat_dataroot'])) {
$command = $CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'];
} else {
$command = $CFG->behat_dataroot . $runprocess;
}
} else {
$command = $CFG->behat_dataroot;
}
$command .= DIRECTORY_SEPARATOR . 'behat' . DIRECTORY_SEPARATOR . 'behat.yml';
// Cygwin uses linux-style directory separators.
if (testing_is_cygwin()) {
@ -184,6 +232,99 @@ class behat_config_manager {
return $command;
}
/**
* Returns the path to the parallel run file which specifies if parallel test environment is enabled
* and how many parallel runs to execute.
*
* @param int $runprocess run process for which behat dir is returned.
* @return string
*/
public final static function get_parallel_test_file_path($runprocess = 0) {
return behat_command::get_behat_dir($runprocess) . '/parallel_environment_enabled.txt';
}
/**
* Returns number of parallel runs for which site is initialised.
*
* @param int $runprocess run process for which behat dir is returned.
* @return int
*/
public final static function get_parallel_test_runs($runprocess = 0) {
$parallelrun = 0;
// Get parallel run info from first file and last file.
$parallelrunconfigfile = self::get_parallel_test_file_path($runprocess);
if (file_exists($parallelrunconfigfile)) {
if ($parallel = file_get_contents($parallelrunconfigfile)) {
$parallelrun = (int) $parallel;
}
}
return $parallelrun;
}
/**
* Drops parallel site links.
*
* @return bool true on success else false.
*/
public final static function drop_parallel_site_links() {
global $CFG;
// Get parallel test runs from first run.
$parallelrun = self::get_parallel_test_runs(1);
if (empty($parallelrun)) {
return false;
}
// If parallel run then remove links and original file.
clearstatcache();
for ($i = 1; $i <= $parallelrun; $i++) {
// Don't delete links for specified sites, as they should be accessible.
if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) {
continue;
}
$link = $CFG->dirroot . '/' . BEHAT_PARALLEL_SITE_NAME . $i;
if (file_exists($link) && is_link($link)) {
@unlink($link);
}
}
return true;
}
/**
* Create parallel site links.
*
* @param int $fromrun first run
* @param int $torun last run.
* @return bool true for sucess, else false.
*/
public final static function create_parallel_site_links($fromrun, $torun) {
global $CFG;
// Create site symlink if necessary.
clearstatcache();
for ($i = $fromrun; $i <= $torun; $i++) {
// Don't create links for specified sites, as they should be accessible.
if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) {
continue;
}
$link = $CFG->dirroot.'/'.BEHAT_PARALLEL_SITE_NAME.$i;
clearstatcache();
if (file_exists($link)) {
if (!is_link($link) || !is_dir($link)) {
echo "File exists at link location ($link) but is not a link or directory!" . PHP_EOL;
return false;
}
} else if (!symlink($CFG->dirroot, $link)) {
// Try create link in case it's not already present.
echo "Unable to create behat site symlink ($link)" . PHP_EOL;
return false;
}
}
return true;
}
/**
* Behat config file specifing the main context class,
* the required Behat extensions and Moodle test wwwroot.
@ -198,12 +339,37 @@ class behat_config_manager {
// We require here when we are sure behat dependencies are available.
require_once($CFG->dirroot . '/vendor/autoload.php');
$selenium2wdhost = array('wd_host' => 'http://localhost:4444/wd/hub');
$parallelruns = self::get_parallel_test_runs();
// If parallel run, then only divide features.
if (!empty($CFG->behatrunprocess) && !empty($parallelruns)) {
// Attempt to split into weighted buckets using timing information, if available.
if ($alloc = self::profile_guided_allocate($features, max(1, $parallelruns), $CFG->behatrunprocess)) {
$features = $alloc;
} else {
// Divide the list of feature files amongst the parallel runners.
srand(crc32(floor(time() / 3600 / 24) . var_export($features, true)));
shuffle($features);
// Pull out the features for just this worker.
if (count($features)) {
$features = array_chunk($features, ceil(count($features) / max(1, $parallelruns)));
$features = $features[$CFG->behatrunprocess - 1];
}
}
// Set proper selenium2 wd_host if defined.
if (!empty($CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host'])) {
$selenium2wdhost = array('wd_host' => $CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host']);
}
}
// It is possible that it has no value as we don't require a full behat setup to list the step definitions.
if (empty($CFG->behat_wwwroot)) {
$CFG->behat_wwwroot = 'http://itwillnotbeused.com';
}
$basedir = $CFG->dirroot . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'behat';
$config = array(
'default' => array(
'paths' => array(
@ -217,12 +383,13 @@ class behat_config_manager {
'Behat\MinkExtension\Extension' => array(
'base_url' => $CFG->behat_wwwroot,
'goutte' => null,
'selenium2' => null
'selenium2' => $selenium2wdhost
),
'Moodle\BehatExtension\Extension' => array(
'formatters' => array(
'moodle_progress' => 'Moodle\BehatExtension\Formatter\MoodleProgressFormatter',
'moodle_list' => 'Moodle\BehatExtension\Formatter\MoodleListFormatter'
'moodle_list' => 'Moodle\BehatExtension\Formatter\MoodleListFormatter',
'moodle_step_count' => 'Moodle\BehatExtension\Formatter\MoodleStepCountFormatter'
),
'features' => $features,
'steps_definitions' => $stepsdefinitions
@ -242,6 +409,79 @@ class behat_config_manager {
return Symfony\Component\Yaml\Yaml::dump($config, 10, 2);
}
/**
* Attempt to split feature list into fairish buckets using timing information, if available.
* Simply add each one to lightest buckets until all files allocated.
* PGA = Profile Guided Allocation. I made it up just now.
* CAUTION: workers must agree on allocation, do not be random anywhere!
*
* @param array $features Behat feature files array
* @param int $nbuckets Number of buckets to divide into
* @param int $instance Index number of this instance
* @return array Feature files array, sorted into allocations
*/
protected static function profile_guided_allocate($features, $nbuckets, $instance) {
$behattimingfile = defined('BEHAT_FEATURE_TIMING_FILE') &&
@filesize(BEHAT_FEATURE_TIMING_FILE) ? BEHAT_FEATURE_TIMING_FILE : false;
if (!$behattimingfile || !$behattimingdata = @json_decode(file_get_contents($behattimingfile), true)) {
// No data available, fall back to relying on steps data.
$stepfile = "";
if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) {
$stepfile = BEHAT_FEATURE_STEP_FILE;
}
// We should never get this. But in case we can't do this then fall back on simple splitting.
if (empty($stepfile) || !$behattimingdata = @json_decode(file_get_contents($stepfile), true)) {
return false;
}
}
arsort($behattimingdata); // Ensure most expensive is first.
$realroot = realpath(__DIR__.'/../../../').'/';
$defaultweight = array_sum($behattimingdata) / count($behattimingdata);
$weights = array_fill(0, $nbuckets, 0);
$buckets = array_fill(0, $nbuckets, array());
$totalweight = 0;
// Re-key the features list to match timing data.
foreach ($features as $k => $file) {
$key = str_replace($realroot, '', $file);
$features[$key] = $file;
unset($features[$k]);
if (!isset($behattimingdata[$key])) {
$behattimingdata[$key] = $defaultweight;
}
}
// Sort features by known weights; largest ones should be allocated first.
$behattimingorder = array();
foreach ($features as $key => $file) {
$behattimingorder[$key] = $behattimingdata[$key];
}
arsort($behattimingorder);
// Finally, add each feature one by one to the lightest bucket.
foreach ($behattimingorder as $key => $weight) {
$file = $features[$key];
$lightbucket = array_search(min($weights), $weights);
$weights[$lightbucket] += $weight;
$buckets[$lightbucket][] = $file;
$totalweight += $weight;
}
if ($totalweight && !defined('BEHAT_DISABLE_HISTOGRAM') && $instance == $nbuckets) {
echo "Bucket weightings:\n";
foreach ($weights as $k => $weight) {
echo $k + 1 . ": " . str_repeat('*', 70 * $nbuckets * $weight / $totalweight) . PHP_EOL;
}
}
// Return the features for this worker.
return $buckets[$instance - 1];
}
/**
* Overrides default config with local config values
*

153
lib/behat/lib.php
View File

@ -41,6 +41,11 @@ define('BEHAT_EXITCODE_INSTALL', 254);
define('BEHAT_EXITCODE_COMPOSER', 255);
define('BEHAT_EXITCODE_INSTALLED', 256);
/**
* The behat test site fullname and shortname.
*/
define('BEHAT_PARALLEL_SITE_NAME', "behatrun");
/**
* Exits with an error code
*
@ -224,6 +229,7 @@ function behat_check_config_vars() {
behat_error(BEHAT_EXITCODE_CONFIG,
'Define $CFG->behat_dataroot in config.php');
}
clearstatcache();
if (!file_exists($CFG->behat_dataroot)) {
$permissions = isset($CFG->directorypermissions) ? $CFG->directorypermissions : 02777;
umask(0);
@ -272,6 +278,57 @@ function behat_is_test_site() {
return false;
}
/**
* Fix variables for parallel behat testing.
* - behat_wwwroot = behat_wwwroot{behatrunprocess}
* - behat_dataroot = behat_dataroot{behatrunprocess}
* - behat_prefix = behat_prefix.{behatrunprocess}_ (For oracle it will be firstletter of prefix and behatrunprocess)
**/
function behat_update_vars_for_process() {
global $CFG;
$allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'behat_prefix',
'behat_wwwroot', 'behat_dataroot');
$behatrunprocess = behat_get_run_process();
$CFG->behatrunprocess = $behatrunprocess;
if ($behatrunprocess) {
if (empty($CFG->behat_parallel_run[$behatrunprocess - 1]['behat_wwwroot'])) {
// Set www root for run process.
if (isset($CFG->behat_wwwroot) &&
!preg_match("#/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess . "\$#", $CFG->behat_wwwroot)) {
$CFG->behat_wwwroot .= "/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess;
}
}
if (empty($CFG->behat_parallel_run[$behatrunprocess - 1]['behat_dataroot'])) {
// Set behat_dataroot.
if (!preg_match("#" . $behatrunprocess . "\$#", $CFG->behat_dataroot)) {
$CFG->behat_dataroot .= $behatrunprocess;
}
}
// Set behat_prefix for db, just suffix run process number, to avoid max length exceed.
// For oracle only 2 letter prefix is possible.
// NOTE: This will not work for parallel process > 9.
if ($CFG->dbtype === 'oci') {
$CFG->behat_prefix = substr($CFG->behat_prefix, 0, 1);
$CFG->behat_prefix .= "{$behatrunprocess}";
} else {
$CFG->behat_prefix .= "{$behatrunprocess}_";
}
if (!empty($CFG->behat_parallel_run[$behatrunprocess - 1])) {
// Override allowed config vars.
foreach ($allowedconfigoverride as $config) {
if (isset($CFG->behat_parallel_run[$behatrunprocess - 1][$config])) {
$CFG->$config = $CFG->behat_parallel_run[$behatrunprocess - 1][$config];
}
}
}
}
}
/**
* Checks if the URL requested by the user matches the provided argument
*
@ -306,3 +363,99 @@ function behat_is_requested_url($url) {
return false;
}
/**
* Get behat run process from either $_SERVER or command config.
*
* @return bool|int false if single run, else run process number.
*/
function behat_get_run_process() {
global $argv, $CFG;
$behatrunprocess = false;
// Get behat run process, if set.
if (defined('BEHAT_CURRENT_RUN') && BEHAT_CURRENT_RUN) {
$behatrunprocess = BEHAT_CURRENT_RUN;
} else if (!empty($_SERVER['REMOTE_ADDR'])) {
// Try get it from config if present.
if (!empty($CFG->behat_parallel_run)) {
foreach ($CFG->behat_parallel_run as $run => $behatconfig) {
if (isset($behatconfig['behat_wwwroot']) && behat_is_requested_url($behatconfig['behat_wwwroot'])) {
$behatrunprocess = $run + 1; // We start process from 1.
break;
}
}
}
// Check if parallel site prefix is used.
if (empty($behatrunprocess) && preg_match('#/' . BEHAT_PARALLEL_SITE_NAME . '(.+?)/#', $_SERVER['REQUEST_URI'])) {
$dirrootrealpath = str_replace("\\", "/", realpath($CFG->dirroot));
$serverrealpath = str_replace("\\", "/", realpath($_SERVER['SCRIPT_FILENAME']));
$afterpath = str_replace($dirrootrealpath.'/', '', $serverrealpath);
if (!$behatrunprocess = preg_filter("#.*/" . BEHAT_PARALLEL_SITE_NAME . "(.+?)/$afterpath#", '$1',
$_SERVER['SCRIPT_FILENAME'])) {
throw new Exception("Unable to determine behat process [afterpath=" . $afterpath .
", scriptfilename=" . $_SERVER['SCRIPT_FILENAME'] . "]!");
}
}
} else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) {
if ($match = preg_filter('#--run=(.+)#', '$1', $argv)) {
$behatrunprocess = reset($match);
} else if ($k = array_search('--config', $argv)) {
$behatconfig = str_replace("\\", "/", $argv[$k + 1]);
// Try get it from config if present.
if (!empty($CFG->behat_parallel_run)) {
foreach ($CFG->behat_parallel_run as $run => $parallelconfig) {
if (!empty($parallelconfig['behat_dataroot']) &&
$parallelconfig['behat_dataroot'] . '/behat/behat.yml' == $behatconfig) {
$behatrunprocess = $run + 1; // We start process from 1.
break;
}
}
}
// Check if default behat datroot increment was done.
if (empty($behatrunprocess)) {
$behatdataroot = str_replace("\\", "/", $CFG->behat_dataroot);
$behatrunprocess = preg_filter("#^{$behatdataroot}" . "(.+?)[/|\\\]behat[/|\\\]behat\.yml#", '$1',
$behatconfig);
}
}
}
return $behatrunprocess;
}
/**
* Execute commands in parallel.
*
* @param array $cmds list of commands to be executed.
* @param string $cwd absolute path of working directory.
* @return array list of processes.
*/
function cli_execute_parallel($cmds, $cwd = null) {
require_once(__DIR__ . "/../../vendor/autoload.php");
$processes = array();
// Create child process.
foreach ($cmds as $name => $cmd) {
$process = new Symfony\Component\Process\Process($cmd);
$process->setWorkingDirectory($cwd);
$process->setTimeout(null);
$processes[$name] = $process;
$processes[$name]->start();
// If error creating process then exit.
if ($processes[$name]->getStatus() !== 'started') {
echo "Error starting process: $name";
foreach ($processes[$name] as $process) {
if ($process) {
$process->signal(SIGKILL);
}
}
exit(1);
}
}
return $processes;
}

1
lib/clilib.php
View File

@ -175,3 +175,4 @@ function cli_error($text, $errorcode=1) {
fwrite(STDERR, "\n");
die($errorcode);
}

11
lib/setup.php
View File

@ -80,7 +80,13 @@ if (defined('BEHAT_SITE_RUNNING')) {
// The behat is configured on this server, we need to find out if this is the behat test
// site based on the URL used for access.
require_once(__DIR__ . '/../lib/behat/lib.php');
// Update config variables for parallel behat runs.
behat_update_vars_for_process();
if (behat_is_test_site()) {
clearstatcache();
// Checking the integrity of the provided $CFG->behat_* vars and the
// selected wwwroot to prevent conflicts with production and phpunit environments.
behat_check_config_vars();
@ -89,10 +95,11 @@ if (defined('BEHAT_SITE_RUNNING')) {
if (!file_exists("$CFG->behat_dataroot/behattestdir.txt")) {
if ($dh = opendir($CFG->behat_dataroot)) {
while (($file = readdir($dh)) !== false) {
if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store') {
if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store' or is_numeric($file)) {
continue;
}
behat_error(BEHAT_EXITCODE_CONFIG, '$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot');
behat_error(BEHAT_EXITCODE_CONFIG, "$CFG->behat_dataroot directory is not empty, ensure this is the " .
"directory where you want to install behat test dataroot");
}
closedir($dh);
unset($dh);

2
lib/setuplib.php
View File

@ -1365,7 +1365,7 @@ function make_writable_directory($dir, $exceptiononerror = true) {
umask($CFG->umaskpermissions);
if (!file_exists($dir)) {
if (!mkdir($dir, $CFG->directorypermissions, true)) {
if (!@mkdir($dir, $CFG->directorypermissions, true)) {
clearstatcache();
// There might be a race condition when creating directory.
if (!is_dir($dir)) {

1
lib/testing/classes/test_lock.php
View File

@ -52,7 +52,6 @@ class test_lock {
*/
public static function acquire($framework) {
global $CFG;
$datarootpath = $CFG->{$framework . '_dataroot'} . '/' . $framework;
$lockfile = $datarootpath . '/lock';
if (!file_exists($datarootpath)) {

3
lib/testing/lib.php
View File

@ -156,6 +156,9 @@ function testing_error($errorcode, $text = '') {
// do not write to error stream because we need the error message in PHP exec result from web ui
echo($text."\n");
if (isset($_SERVER['REMOTE_ADDR'])) {
header('HTTP/1.1 500 Internal Server Error');
}
exit($errorcode);
}

77
lib/tests/behat/behat_hooks.php
View File

@ -31,6 +31,8 @@ require_once(__DIR__ . '/../../behat/behat_base.php');
use Behat\Behat\Event\SuiteEvent as SuiteEvent,
Behat\Behat\Event\ScenarioEvent as ScenarioEvent,
Behat\Behat\Event\FeatureEvent as FeatureEvent,
Behat\Behat\Event\OutlineExampleEvent as OutlineExampleEvent,
Behat\Behat\Event\StepEvent as StepEvent,
Behat\Mink\Exception\DriverException as DriverException,
WebDriver\Exception\NoSuchWindow as NoSuchWindow,
@ -84,17 +86,25 @@ class behat_hooks extends behat_base {
*/
protected static $faildumpdirname = false;
/**
* Keeps track of time taken by feature to execute.
*
* @var array list of feature timings
*/
protected static $timings = array();
/**
* Gives access to moodle codebase, ensures all is ready and sets up the test lock.
*
* Includes config.php to use moodle codebase with $CFG->behat_*
* instead of $CFG->prefix and $CFG->dataroot, called once per suite.
*
* @param SuiteEvent $event event before suite.
* @static
* @throws Exception
* @BeforeSuite
*/
public static function before_suite($event) {
public static function before_suite(SuiteEvent $event) {
global $CFG;
// Defined only when the behat CLI command is running, the moodle init setup process will
@ -153,9 +163,65 @@ class behat_hooks extends behat_base {
}
}
/**
* Gives access to moodle codebase, to keep track of feature start time.
*
* @param FeatureEvent $event event fired before feature.
* @BeforeFeature
*/
public static function before_feature(FeatureEvent $event) {
if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
return;
}
$file = $event->getFeature()->getFile();
self::$timings[$file] = microtime(true);
}
/**
* Gives access to moodle codebase, to keep track of feature end time.
*
* @param FeatureEvent $event event fired after feature.
* @AfterFeature
*/
public static function after_feature(FeatureEvent $event) {
if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
return;
}
$file = $event->getFeature()->getFile();
self::$timings[$file] = microtime(true) - self::$timings[$file];
// Probably didn't actually run this, don't output it.
if (self::$timings[$file] < 1) {
unset(self::$timings[$file]);
}
}
/**
* Gives access to moodle codebase, to keep track of suite timings.
*
* @param SuiteEvent $event event fired after suite.
* @AfterSuite
*/
public static function after_suite(SuiteEvent $event) {
if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
return;
}
$realroot = realpath(__DIR__.'/../../../').'/';
foreach (self::$timings as $k => $v) {
$new = str_replace($realroot, '', $k);
self::$timings[$new] = round($v, 1);
unset(self::$timings[$k]);
}
if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING_FILE), true)) {
self::$timings = array_merge($existing, self::$timings);
}
arsort(self::$timings);
@file_put_contents(BEHAT_FEATURE_TIMING_FILE, json_encode(self::$timings, JSON_PRETTY_PRINT));
}
/**
* Resets the test environment.
*
* @param OutlineExampleEvent|ScenarioEvent $event event fired before scenario.
* @throws coding_exception If here we are not using the test database it should be because of a coding error
* @BeforeScenario
*/
@ -244,9 +310,10 @@ class behat_hooks extends behat_base {
* default would be at framework level, which will stop the execution of
* the run.
*
* @param StepEvent $event event fired before step.
* @BeforeStep @javascript
*/
public function before_step_javascript($event) {
public function before_step_javascript(StepEvent $event) {
try {
$this->wait_for_pending_js();
@ -268,9 +335,10 @@ class behat_hooks extends behat_base {
* default would be at framework level, which will stop the execution of
* the run.
*
* @param StepEvent $event event fired after step.
* @AfterStep @javascript
*/
public function after_step_javascript($event) {
public function after_step_javascript(StepEvent $event) {
global $CFG;
// Save a screenshot if the step failed.
@ -303,9 +371,10 @@ class behat_hooks extends behat_base {
*
* This includes creating an HTML dump of the content if there was a failure.
*
* @param StepEvent $event event fired after step.
* @AfterStep
*/
public function after_step($event) {
public function after_step(StepEvent $event) {
global $CFG;
// Save the page content if the step failed.