mirror of
https://github.com/moodle/moodle.git
synced 2025-01-17 21:49:15 +01:00
MDL-39752 behat: Parallel execution support
This commit is contained in:
Tony Levi
Rajesh Taneja
parent
b90f98dade
commit
08e7f97ee4
@ -40,10 +40,41 @@ define('CACHE_DISABLE_ALL', true);
|
||||
require_once(__DIR__ . '/../../../../lib/clilib.php');
|
||||
require_once(__DIR__ . '/../../../../lib/behat/lib.php');
|
||||
|
||||
|
||||
list($options, $unrecognized) = cli_get_params(
|
||||
array(
|
||||
'parallel' => 0,
|
||||
'suffix' => '',
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']);
|
||||
$suffixarg = $options['suffix'] ? "--suffix={$options['suffix']} --parallel=$nproc" : '';
|
||||
|
||||
|
||||
if ($nproc && !$suffixarg) {
|
||||
foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) {
|
||||
if (file_exists($dir) && is_link($dir) && preg_match('#/behat\d+$#', $dir)) {
|
||||
unlink($dir);
|
||||
}
|
||||
}
|
||||
$cmds = array();
|
||||
for ($i = 1; $i <= $nproc; $i++) {
|
||||
$cmds[] = "php ".__FILE__." --suffix=$i --parallel=$nproc 2>&1";
|
||||
}
|
||||
// This is intensive compared to behat itself so halve the parallelism.
|
||||
foreach (array_chunk($cmds, max(1, floor($nproc/2)), true) as $chunk) {
|
||||
ns_parallel_popen($chunk, true);
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
// Changing the cwd to admin/tool/behat/cli.
|
||||
chdir(__DIR__);
|
||||
$output = null;
|
||||
exec("php util.php --diag", $output, $code);
|
||||
exec("php util.php --diag $suffixarg", $output, $code);
|
||||
if ($code == 0) {
|
||||
echo "Behat test environment already installed\n";
|
||||
|
||||
@ -53,7 +84,7 @@ if ($code == 0) {
|
||||
|
||||
// Behat and dependencies are installed and we need to install the test site.
|
||||
chdir(__DIR__);
|
||||
passthru("php util.php --install", $code);
|
||||
passthru("php util.php --install $suffixarg", $code);
|
||||
if ($code != 0) {
|
||||
exit($code);
|
||||
}
|
||||
@ -64,12 +95,12 @@ if ($code == 0) {
|
||||
|
||||
// Test site data is outdated.
|
||||
chdir(__DIR__);
|
||||
passthru("php util.php --drop", $code);
|
||||
passthru("php util.php --drop $suffixarg", $code);
|
||||
if ($code != 0) {
|
||||
exit($code);
|
||||
}
|
||||
|
||||
passthru("php util.php --install", $code);
|
||||
passthru("php util.php --install $suffixarg", $code);
|
||||
if ($code != 0) {
|
||||
exit($code);
|
||||
}
|
||||
@ -81,7 +112,7 @@ if ($code == 0) {
|
||||
|
||||
// Returning to admin/tool/behat/cli.
|
||||
chdir(__DIR__);
|
||||
passthru("php util.php --install", $code);
|
||||
passthru("php util.php --install $suffixarg", $code);
|
||||
if ($code != 0) {
|
||||
exit($code);
|
||||
}
|
||||
@ -93,7 +124,7 @@ if ($code == 0) {
|
||||
}
|
||||
|
||||
// Enable editing mode according to config.php vars.
|
||||
passthru("php util.php --enable", $code);
|
||||
passthru("php util.php --enable $suffixarg", $code);
|
||||
if ($code != 0) {
|
||||
exit($code);
|
||||
}
|
||||
|
||||
212
admin/tool/behat/cli/ns_parallel.php
Normal file
212
admin/tool/behat/cli/ns_parallel.php
Normal file
@ -0,0 +1,212 @@
|
||||
<?php
|
||||
// This file is part of Moodle - http://moodle.org/
|
||||
//
|
||||
// Moodle is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// Moodle is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/**
|
||||
* Wrapper to run previously set-up behat tests in parallel.
|
||||
*
|
||||
* @package tool_behat
|
||||
* @copyright 2014 NetSpot Pty Ltd
|
||||
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
|
||||
*/
|
||||
|
||||
|
||||
if (isset($_SERVER['REMOTE_ADDR'])) {
|
||||
die(); // No access from web!
|
||||
}
|
||||
|
||||
|
||||
define('BEHAT_UTIL', true);
|
||||
define('CLI_SCRIPT', true);
|
||||
define('ABORT_AFTER_CONFIG', true);
|
||||
define('NO_OUTPUT_BUFFERING', true);
|
||||
|
||||
error_reporting(E_ALL | E_STRICT);
|
||||
ini_set('display_errors', '1');
|
||||
ini_set('log_errors', '1');
|
||||
|
||||
|
||||
require_once __DIR__ .'/../../../../config.php';
|
||||
require_once __DIR__.'/../../../../lib/clilib.php';
|
||||
require_once __DIR__.'/../../../../lib/behat/lib.php';
|
||||
|
||||
|
||||
list($options, $unrecognised) = cli_get_params(
|
||||
array(
|
||||
'stop-on-failure' => 0,
|
||||
'parallel' => 0,
|
||||
'verbose' => false,
|
||||
'replace' => false,
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
if (empty($options['parallel']) && $dirs = glob("{$CFG->dirroot}/behat*")) {
|
||||
sort($dirs);
|
||||
if ($max = preg_filter('#.*behat(\d+)#', '$1', end($dirs))) {
|
||||
$options['parallel'] = $max;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$suffix = '';
|
||||
$time = microtime(true);
|
||||
$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']);
|
||||
array_walk($unrecognised, function (&$v) {
|
||||
if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) {
|
||||
$v = $x;
|
||||
} else if (!preg_match("#^-#", $v)) {
|
||||
$v = escapeshellarg($v);
|
||||
}
|
||||
});
|
||||
$extraopts = implode(' ', $unrecognised);
|
||||
|
||||
|
||||
if (empty($nproc)) {
|
||||
fwrite(STDERR, "Invalid or missing --parallel parameter, must be >= 1.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
$checkfail = array();
|
||||
$outputs = array();
|
||||
$handles = array();
|
||||
$pipe2i = array();
|
||||
$exits = array();
|
||||
$unused = null;
|
||||
$linelencnt = 0;
|
||||
$procs = array();
|
||||
|
||||
|
||||
for ($i = 1; $i <= $nproc; $i++) {
|
||||
$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts;
|
||||
$dirroot = dirname($CFG->behat_dataroot)."/behat$i";
|
||||
$cmd = "exec {$CFG->dirroot}/vendor/bin/behat --config $dirroot/behat/behat.yml $myopts";
|
||||
list($handle, $pipes) = ns_proc_open($cmd, true);
|
||||
@fclose($pipes[0]);
|
||||
unset($pipes[0]);
|
||||
$exits[$i] = 1;
|
||||
$handles[$i] = array($handle, $pipes[1], $pipes[2]);
|
||||
$procs[$i] = $handle;
|
||||
$checkfail[$i] = false;
|
||||
$outputs[$i] = array('');
|
||||
$pipe2i[(int) $pipes[1]] = $i;
|
||||
$pipe2i[(int) $pipes[2]] = $i;
|
||||
stream_set_blocking($pipes[1], 0);
|
||||
stream_set_blocking($pipes[2], 0);
|
||||
}
|
||||
|
||||
|
||||
while (!empty($procs)) {
|
||||
usleep(10000);
|
||||
|
||||
foreach ($handles as $i => $p) {
|
||||
if (!($status = @proc_get_status($p[0])) || !$status['running']) {
|
||||
if ($exits[$i] !== 0) {
|
||||
$exits[$i] = !empty($status) ? $status['exitcode'] : 1;
|
||||
}
|
||||
unset($procs[$i]);
|
||||
unset($handles[$i][0]);
|
||||
$last = array_pop($outputs[$i]);
|
||||
for ($l=2; $l>=1; $l--)
|
||||
while ($part = @fread($handles[$i][$l], 8192))
|
||||
$last .= $part;
|
||||
$outputs[$i] = array_merge($outputs[$i], explode("\n", $last));
|
||||
}
|
||||
}
|
||||
|
||||
$ready = array();
|
||||
foreach ($handles as $i => $set) {
|
||||
$ready[] = $set[1];
|
||||
$ready[] = $set[2];
|
||||
}
|
||||
|
||||
// Poll for any process with output or ended.
|
||||
if (!$result = @stream_select($ready, $unused, $unused, 1)) {
|
||||
// Nothing; try again.
|
||||
continue;
|
||||
}
|
||||
if (!$fh = reset($ready)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$i = $pipe2i[(int) $fh];
|
||||
$last = array_pop($outputs[$i]);
|
||||
$read = fread($fh, 4096);
|
||||
$newlines = explode("\n", $last.$read);
|
||||
$outputs[$i] = array_merge($outputs[$i], $newlines);
|
||||
|
||||
if (!$checkfail[$i]) {
|
||||
foreach ($newlines as $l => $line) {
|
||||
unset($newlines[$l]);
|
||||
if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) {
|
||||
$checkfail[$i] = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($progress = preg_filter('#^\s*([FUS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $newlines)) {
|
||||
if ($checkfail[$i] && preg_filter('#^\s*[S\.\-]*[FU][S\.\-]*(?:\s+\d+)?\s*$#', '$1', $progress)) {
|
||||
$exits[$i] = 1;
|
||||
if ($options['stop-on-failure']) {
|
||||
foreach ($handles as $l => $p) {
|
||||
$exits[$l] = $l != $i ? 0 : $exits[$i];
|
||||
@proc_terminate($p[0], SIGINT);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process has gone, assume this is the last output for it.
|
||||
if (empty($procs[$i])) {
|
||||
unset($handles[$i]);
|
||||
}
|
||||
if (empty($checkfail[$i]) || !($update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $read))) {
|
||||
continue;
|
||||
}
|
||||
while ($update) {
|
||||
$part = substr($update, 0, 70 - $linelencnt);
|
||||
$update = substr($update, strlen($part));
|
||||
$linelencnt += strlen($part);
|
||||
echo $part;
|
||||
if ($linelencnt >= 70) {
|
||||
echo "\n";
|
||||
$linelencnt = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
echo "\n\n";
|
||||
|
||||
|
||||
$exits = array_filter($exits, function ($v) {return $v !== 0;});
|
||||
|
||||
|
||||
if ($exits || $options['verbose']) {
|
||||
echo "Exit codes: ".implode(" ", $exits)."\n\n";
|
||||
foreach ($outputs as $i => $output) {
|
||||
unset($outputs[$i]);
|
||||
if (!end($output)) array_pop($output);
|
||||
$prefix = "[behat$i] ";
|
||||
array_walk($output, function (&$l) use ($prefix) {
|
||||
$l = $prefix.$l;
|
||||
});
|
||||
echo implode("\n", $output)."\n\n";
|
||||
}
|
||||
$failed = true;
|
||||
}
|
||||
|
||||
|
||||
$time = round(microtime(true) - $time, 1);
|
||||
echo "Finished in {$time}s\n";
|
||||
exit(!empty($failed) ? 1 : 0);
|
||||
@ -40,6 +40,8 @@ list($options, $unrecognized) = cli_get_params(
|
||||
array(
|
||||
'help' => false,
|
||||
'install' => false,
|
||||
'parallel' => 0,
|
||||
'suffix' => '',
|
||||
'drop' => false,
|
||||
'enable' => false,
|
||||
'disable' => false,
|
||||
@ -63,6 +65,7 @@ Options:
|
||||
--drop Drops the database tables and the dataroot contents
|
||||
--enable Enables test environment and updates tests list
|
||||
--disable Disables test environment
|
||||
--parallel Run operation for all parallel behat environments.
|
||||
--diag Get behat test environment status code
|
||||
|
||||
-h, --help Print out this help
|
||||
@ -78,11 +81,32 @@ if (!empty($options['help'])) {
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Describe this script.
|
||||
|
||||
if (!empty($options['parallel']) && empty($options['suffix'])) {
|
||||
foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) {
|
||||
if (file_exists($dir) && is_dir($dir)) {
|
||||
unlink($dir);
|
||||
}
|
||||
}
|
||||
$cmds = array();
|
||||
$extra = preg_filter('#(.*)\s*--parallel=\d+\s*(.*?)#', '$1 $2', implode(' ', array_slice($argv, 1)));
|
||||
for ($i = 1; $i <= $options['parallel']; $i++) {
|
||||
$cmds[] = "php ".__FILE__." $extra --suffix=$i 2>&1";
|
||||
}
|
||||
// This is intensive compared to behat itself so halve the parallelism.
|
||||
foreach (array_chunk($cmds, min(1, floor($options['parallel']/2)), true) as $chunk) {
|
||||
ns_parallel_popen($chunk, true);
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
// Checking $CFG->behat_* vars and values.
|
||||
define('BEHAT_UTIL', true);
|
||||
define('CLI_SCRIPT', true);
|
||||
define('NO_OUTPUT_BUFFERING', true);
|
||||
define('IGNORE_COMPONENT_CACHE', true);
|
||||
define('BEHAT_SUFFIX', $options['suffix']);
|
||||
|
||||
// Only load CFG from config.php, stop ASAP in lib/setup.php.
|
||||
define('ABORT_AFTER_CONFIG', true);
|
||||
|
||||
109
behat_pga_default.json
Normal file
109
behat_pga_default.json
Normal file
@ -0,0 +1,109 @@
|
||||
{
|
||||
"course\/tests\/behat\/course_controls.feature": 117.2,
|
||||
"backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": 108.2,
|
||||
"mod\/forum\/tests\/behat\/discussion_subscriptions.feature": 92.8,
|
||||
"course\/tests\/behat\/category_resort.feature": 75.4,
|
||||
"course\/tests\/behat\/course_resort.feature": 58.9,
|
||||
"admin\/tool\/behat\/tests\/behat\/data_generators.feature": 53.1,
|
||||
"course\/tests\/behat\/category_management.feature": 50.1,
|
||||
"blocks\/recent_activity\/tests\/behat\/structural_changes.feature": 48.3,
|
||||
"grade\/tests\/behat\/grade_override_letter.feature": 47.3,
|
||||
"blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": 42.7,
|
||||
"calendar\/tests\/behat\/calendar.feature": 42,
|
||||
"grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": 41.2,
|
||||
"mod\/forum\/tests\/behat\/forum_subscriptions.feature": 32.6,
|
||||
"course\/tests\/behat\/move_activities.feature": 30.8,
|
||||
"mod\/workshep\/tests\/behat\/workshep_assessment.feature": 29.3,
|
||||
"blocks\/glossary_random\/tests\/behat\/glossary_random.feature": 29,
|
||||
"mod\/workshop\/tests\/behat\/workshop_assessment.feature": 28.8,
|
||||
"blocks\/tests\/behat\/manage_blocks.feature": 28.6,
|
||||
"badges\/tests\/behat\/navrequirecap.feature": 28,
|
||||
"mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": 26.9,
|
||||
"mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": 26,
|
||||
"blocks\/html\/tests\/behat\/multiple_instances.feature": 26,
|
||||
"message\/tests\/behat\/display_history.feature": 24.7,
|
||||
"course\/tests\/behat\/category_change_visibility.feature": 24.2,
|
||||
"local\/uneditableblocks\/tests\/behat\/enable_uneditableblocks.feature": 23.1,
|
||||
"mod\/feedback\/tests\/behat\/defaultshortanswerlength.feature": 22.1,
|
||||
"backup\/util\/ui\/tests\/behat\/backup_courses.feature": 22,
|
||||
"course\/tests\/behat\/create_delete_course.feature": 21.9,
|
||||
"blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": 21.2,
|
||||
"course\/tests\/behat\/move_sections.feature": 21.1,
|
||||
"course\/tests\/behat\/course_category_management_listing.feature": 21,
|
||||
"local\/userpolicy\/tests\/behat\/fieldvisibility.feature": 20.9,
|
||||
"admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": 20.5,
|
||||
"mod\/wiki\/tests\/behat\/edit_tags.feature": 19.6,
|
||||
"mod\/attendance\/tests\/behat\/attendance_mod.feature": 19.5,
|
||||
"blocks\/participants\/tests\/behat\/block_participants_course.feature": 19.5,
|
||||
"blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": 17.7,
|
||||
"blocks\/tests\/behat\/configure_block_throughout_site.feature": 17.7,
|
||||
"grade\/report\/singleview\/tests\/behat\/singleview.feature": 17.6,
|
||||
"grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": 16.3,
|
||||
"mod\/wiki\/tests\/behat\/group_enhancements.feature": 16.2,
|
||||
"admin\/tests\/behat\/forcelogin_makefrontpagepublic.feature": 16,
|
||||
"message\/tests\/behat\/disablenotifications.feature": 16,
|
||||
"completion\/tests\/behat\/teacher_manual_completion.feature": 16,
|
||||
"mod\/glossary\/tests\/behat\/entries_require_approval.feature": 16,
|
||||
"backup\/util\/ui\/tests\/behat\/import_course.feature": 15.9,
|
||||
"admin\/tests\/behat\/set_admin_settings_value.feature": 15.8,
|
||||
"mod\/mediagallery\/tests\/behat\/separategroups.feature": 15.6,
|
||||
"blocks\/tests\/behat\/return_block_original_state.feature": 15.6,
|
||||
"admin\/tests\/behat\/custom_maxbytes.feature": 15.4,
|
||||
"mod\/oublog\/tests\/behat\/separate_individuals.feature": 15.4,
|
||||
"mod\/forum\/tests\/behat\/default_displaywordcount.feature": 15.4,
|
||||
"backup\/util\/ui\/tests\/behat\/restore_moodle2_course_numsections.feature": 14.9,
|
||||
"question\/tests\/behat\/question_defaultpenalty.feature": 14.8,
|
||||
"availability\/tests\/behat\/edit_availability.feature": 14.7,
|
||||
"mod\/data\/tests\/behat\/view_entries.feature": 14.6,
|
||||
"local\/catdelete\/tests\/behat\/catdelete.feature": 14.3,
|
||||
"group\/tests\/behat\/showusernameingroup.feature": 14.3,
|
||||
"mod\/forum\/tests\/behat\/edit_post_student.feature": 14.2,
|
||||
"admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": 14.1,
|
||||
"mod\/hsuforum\/tests\/behat\/edit_post_student.feature": 13.8,
|
||||
"mod\/quiz\/tests\/behat\/add_quiz.feature": 13.5,
|
||||
"grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": 13.5,
|
||||
"blocks\/autocreate_user\/tests\/behat\/add_introtext.feature": 13.5,
|
||||
"question\/type\/truefalse\/tests\/behat\/custompenalty.feature": 13.3,
|
||||
"blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": 13.1,
|
||||
"blocks\/html\/tests\/behat\/course_block.feature": 12.9,
|
||||
"mod\/quiz\/tests\/behat\/configsubnethide.feature": 12.8,
|
||||
"mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": 12.6,
|
||||
"blocks\/html\/tests\/behat\/configuring_html_block.feature": 12.3,
|
||||
"course\/tests\/behat\/course_change_visibility.feature": 12.2,
|
||||
"course\/tests\/behat\/modchooser_hidden.feature": 11.9,
|
||||
"blocks\/login\/tests\/behat\/login_block.feature": 11.7,
|
||||
"course\/tests\/behat\/paged_course_navigation.feature": 11.1,
|
||||
"course\/tests\/behat\/limitrolerenaming.feature": 11,
|
||||
"mod\/book\/tests\/behat\/create_chapters.feature": 11,
|
||||
"mod\/forum\/tests\/behat\/my_forum_posts.feature": 10.7,
|
||||
"auth\/tests\/behat\/login.feature": 10.5,
|
||||
"blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": 10.5,
|
||||
"mod\/data\/tests\/behat\/add_entries.feature": 10.1,
|
||||
"mod\/wiki\/tests\/behat\/preview_page.feature": 9.7,
|
||||
"mod\/oublog\/tests\/behat\/personalblog.feature": 9.7,
|
||||
"admin\/tests\/behat\/display_short_names.feature": 9.6,
|
||||
"mod\/forum\/tests\/behat\/separate_group_discussions.feature": 9.5,
|
||||
"course\/tests\/behat\/add_activities.feature": 9.4,
|
||||
"mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": 8.8,
|
||||
"blocks\/tests\/behat\/restrict_available_blocks.feature": 8.7,
|
||||
"blocks\/comments\/tests\/behat\/add_comment.feature": 8.7,
|
||||
"mod\/survey\/tests\/behat\/survey_types.feature": 8.5,
|
||||
"admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": 7.3,
|
||||
"local\/rolerenaming\/tests\/behat\/course_role_renaming.feature": 7.2,
|
||||
"course\/tests\/behat\/edit_settings.feature": 7.1,
|
||||
"group\/tests\/behat\/create_groups.feature": 6.8,
|
||||
"blocks\/tests\/behat\/add_blocks.feature": 6.5,
|
||||
"message\/tests\/behat\/manage_contacts.feature": 6.1,
|
||||
"course\/tests\/behat\/course_search.feature": 6,
|
||||
"course\/tests\/behat\/course_creation.feature": 5.8,
|
||||
"my\/tests\/behat\/reset_page.feature": 5.4,
|
||||
"my\/tests\/behat\/restrict_available_blocks.feature": 5.3,
|
||||
"mod\/oublog\/tests\/behat\/basic.feature": 5.2,
|
||||
"my\/tests\/behat\/add_blocks.feature": 4.6,
|
||||
"user\/tests\/behat\/reset_page.feature": 4,
|
||||
"message\/tests\/behat\/search_history.feature": 3.9,
|
||||
"blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": 3.3,
|
||||
"user\/tests\/behat\/add_blocks.feature": 2.8,
|
||||
"report\/usersessions\/tests\/behat\/usersessions_report.feature": 2.7,
|
||||
"admin\/tool\/behat\/tests\/behat\/test_environment.feature": 1
|
||||
}
|
||||
@ -79,7 +79,10 @@ class behat_config_manager {
|
||||
$featurespaths[$uniquekey] = $path;
|
||||
}
|
||||
}
|
||||
$features = array_values($featurespaths);
|
||||
foreach ($featurespaths as $path) {
|
||||
$additional = glob("$path/*.feature");
|
||||
$features = array_merge($features, $additional);
|
||||
}
|
||||
}
|
||||
|
||||
// Optionally include features from additional directories.
|
||||
@ -198,6 +201,29 @@ class behat_config_manager {
|
||||
// We require here when we are sure behat dependencies are available.
|
||||
require_once($CFG->dirroot . '/vendor/autoload.php');
|
||||
|
||||
$instance = 1;
|
||||
$parallel = 0;
|
||||
foreach ($_SERVER['argv'] as $arg) {
|
||||
if (strpos($arg, '--suffix=') === 0) {
|
||||
$instance = intval(substr($arg, strlen('--suffix=')));
|
||||
}
|
||||
if (empty($parallel)) {
|
||||
$parallel = preg_filter('#--parallel=(\d+)#', '$1', $arg);
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to split into weighted buckets using timing information, if available.
|
||||
if ($alloc = self::profile_guided_allocate($features, max(1, $parallel), $instance)) {
|
||||
$features = $alloc;
|
||||
} else {
|
||||
// Divide the list of feature files amongst the parallel runners.
|
||||
srand(crc32(floor(time() / 3600 / 24).var_export($features,true)));
|
||||
shuffle($features);
|
||||
// Pull out the features for just this worker.
|
||||
$features = array_chunk($features, ceil(count($features) / max(1, $parallel)));
|
||||
$features = $features[$instance-1];
|
||||
}
|
||||
|
||||
// It is possible that it has no value as we don't require a full behat setup to list the step definitions.
|
||||
if (empty($CFG->behat_wwwroot)) {
|
||||
$CFG->behat_wwwroot = 'http://itwillnotbeused.com';
|
||||
@ -242,6 +268,71 @@ class behat_config_manager {
|
||||
return Symfony\Component\Yaml\Yaml::dump($config, 10, 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to split feature list into fairish buckets using timing information, if available.
|
||||
* Simply add each one to lightest buckets until all files allocated.
|
||||
* PGA = Profile Guided Allocation. I made it up just now.
|
||||
* CAUTION: workers must agree on allocation, do not be random anywhere!
|
||||
*
|
||||
* @param array $features Behat feature files array
|
||||
* @param int $nbuckets Number of buckets to divide into
|
||||
* @param int $instance Index number of this instance
|
||||
* @return array Feature files array, sorted into allocations
|
||||
*/
|
||||
protected static function profile_guided_allocate($features, $nbuckets, $instance) {
|
||||
$pga = __DIR__.'/../../../behat_pga_default.json';
|
||||
$pga = defined('BEHAT_PGA_DATA') && file_exists(BEHAT_PGA_DATA) ? BEHAT_PGA_DATA : $pga;
|
||||
|
||||
if (defined('BEHAT_PGA_DISABLE') || !file_exists($pga) || !$pga = @json_decode(file_get_contents($pga), true)) {
|
||||
// No data available, fall back to relying on shuffle.
|
||||
return false;
|
||||
}
|
||||
|
||||
arsort($pga); // Ensure most expensive is first.
|
||||
|
||||
$realroot = realpath(__DIR__.'/../../../').'/';
|
||||
$defaultweight = array_sum($pga) / count($pga); // TODO: median is more ideal.
|
||||
$weights = array_fill(0, $nbuckets, 0);
|
||||
$buckets = array_fill(0, $nbuckets, array());
|
||||
$totalweight = 0;
|
||||
|
||||
// Re-key the features list to match pga data.
|
||||
foreach ($features as $k => $file) {
|
||||
$key = str_replace($realroot, '', $file);
|
||||
$features[$key] = $file;
|
||||
unset($features[$k]);
|
||||
if (!isset($pga[$key])) {
|
||||
$pga[$key] = $defaultweight;
|
||||
}
|
||||
}
|
||||
|
||||
// Sort features by known weights; largest ones should be allocated first.
|
||||
$pgaorder = array();
|
||||
foreach ($features as $key => $file) {
|
||||
$pgaorder[$key] = $pga[$key];
|
||||
}
|
||||
arsort($pgaorder);
|
||||
|
||||
// Finally, add each feature one by one to the lightest bucket.
|
||||
foreach ($pgaorder as $key => $weight) {
|
||||
$file = $features[$key];
|
||||
$light_bucket = array_search(min($weights), $weights);
|
||||
$weights[$light_bucket] += $weight;
|
||||
$buckets[$light_bucket][] = $file;
|
||||
$totalweight += $weight;
|
||||
}
|
||||
|
||||
if (!defined('BEHAT_PGA_DISABLE_HISTOGRAM') && $instance == $nbuckets) {
|
||||
echo "Bucket weightings:\n";
|
||||
foreach ($weights as $k => $weight) {
|
||||
echo "$k: ".str_repeat('*', 70 * $nbuckets * $weight / $totalweight)."\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Return the features for this worker.
|
||||
return $buckets[$instance-1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Overrides default config with local config values
|
||||
*
|
||||
|
||||
@ -213,6 +213,22 @@ class behat_util extends testing_util {
|
||||
// Updates all the Moodle features and steps definitions.
|
||||
behat_config_manager::update_config_file();
|
||||
|
||||
// Create suffix symlink if necessary.
|
||||
global $CFG;
|
||||
if ($CFG->behat_suffix) {
|
||||
$extra = preg_filter('#.*/(.+)$#', '$1', $CFG->behat_wwwroot);
|
||||
$link = $CFG->dirroot.'/'.$extra;
|
||||
if (file_exists($link)) {
|
||||
if (!is_link($link)) {
|
||||
throw new coding_exception("File exists at link location ($link) but is not a link!");
|
||||
}
|
||||
@unlink($link);
|
||||
}
|
||||
if (!symlink($CFG->dirroot, $link)) {
|
||||
throw new coding_exception("Unable to create behat suffix symlink ($link)");
|
||||
}
|
||||
}
|
||||
|
||||
if (self::is_test_mode_enabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -272,6 +272,34 @@ function behat_is_test_site() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add behat suffix to $CFG vars for parallel testing.
|
||||
**/
|
||||
function behat_add_suffix_to_vars($suffix = '') {
|
||||
global $CFG;
|
||||
|
||||
if (empty($suffix)) {
|
||||
if (!empty($CFG->behat_suffix)) {
|
||||
$suffix = $CFG->behat_suffix;
|
||||
|
||||
} else if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) {
|
||||
$suffix = BEHAT_SUFFIX;
|
||||
}
|
||||
}
|
||||
|
||||
$CFG->behat_suffix = $suffix;
|
||||
|
||||
if ($suffix) {
|
||||
if (isset($CFG->behat_wwwroot) && !preg_match("#/behat$suffix\$#", $CFG->behat_wwwroot)) {
|
||||
$CFG->behat_wwwroot .= "/behat{$suffix}";
|
||||
}
|
||||
if (!preg_match("#/behat{$suffix}\$#", $CFG->behat_dataroot)) {
|
||||
$CFG->behat_dataroot = dirname($CFG->behat_dataroot)."/behat{$suffix}";
|
||||
}
|
||||
$CFG->behat_prefix = "behat{$suffix}_";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the URL requested by the user matches the provided argument
|
||||
*
|
||||
|
||||
@ -175,3 +175,34 @@ function cli_error($text, $errorcode=1) {
|
||||
fwrite(STDERR, "\n");
|
||||
die($errorcode);
|
||||
}
|
||||
|
||||
|
||||
function ns_proc_open($cmd, $die = false) {
|
||||
$desc = array(
|
||||
0 => array('pipe', 'r'),
|
||||
1 => array('pipe', 'w'),
|
||||
2 => array('pipe', 'w'),
|
||||
);
|
||||
if (!($handle = proc_open($cmd, $desc, $pipes)) && $die) {
|
||||
throw new Exception('Error starting worker');
|
||||
}
|
||||
return array($handle, $pipes);
|
||||
}
|
||||
|
||||
|
||||
function ns_parallel_popen($cmds, $doexit = false) {
|
||||
$procs = array();
|
||||
foreach ($cmds as $k => $cmd) {
|
||||
$procs[] = popen($cmd, 'r');
|
||||
}
|
||||
$status = false;
|
||||
foreach ($procs as $p) {
|
||||
if (!$p) continue;
|
||||
while ($out = fgets($p)) echo $out;
|
||||
$status |= (bool) pclose($p);
|
||||
}
|
||||
if ($doexit && $status) {
|
||||
exit($status);
|
||||
}
|
||||
return $status;
|
||||
}
|
||||
|
||||
@ -77,9 +77,34 @@ if (defined('BEHAT_SITE_RUNNING')) {
|
||||
// We already switched to behat test site previously.
|
||||
|
||||
} else if (!empty($CFG->behat_wwwroot) or !empty($CFG->behat_dataroot) or !empty($CFG->behat_prefix)) {
|
||||
global $argv;
|
||||
$suffix = '';
|
||||
|
||||
if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) {
|
||||
$suffix = BEHAT_SUFFIX;
|
||||
|
||||
} else if (!empty($_SERVER['REMOTE_ADDR'])) {
|
||||
if (preg_match('#/behat(.+?)/#', $_SERVER['REQUEST_URI'])) {
|
||||
$afterpath = str_replace(realpath($CFG->dirroot).'/', '', realpath($_SERVER['SCRIPT_FILENAME']));
|
||||
if (!$suffix = preg_filter("#.*/behat(.+?)/$afterpath#", '$1', $_SERVER['SCRIPT_FILENAME'])) {
|
||||
throw new coding_exception("Unable to determine behat suffix [afterpath=$afterpath, scriptfilename={$_SERVER['SCRIPT_FILENAME']}]!");
|
||||
}
|
||||
}
|
||||
|
||||
} else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) {
|
||||
if ($match = preg_filter('#--suffix=(.+)#', '$1', $argv)) {
|
||||
$suffix = reset($match);
|
||||
}
|
||||
if ($k = array_search('--config', $argv)) {
|
||||
$behatconfig = $argv[$k+1];
|
||||
$suffix = preg_filter("#^{$CFG->behat_dataroot}(.+?)/behat/behat\.yml#", '$1', $behatconfig);
|
||||
}
|
||||
}
|
||||
|
||||
// The behat is configured on this server, we need to find out if this is the behat test
|
||||
// site based on the URL used for access.
|
||||
require_once(__DIR__ . '/../lib/behat/lib.php');
|
||||
behat_add_suffix_to_vars($suffix);
|
||||
if (behat_is_test_site()) {
|
||||
// Checking the integrity of the provided $CFG->behat_* vars and the
|
||||
// selected wwwroot to prevent conflicts with production and phpunit environments.
|
||||
@ -89,7 +114,7 @@ if (defined('BEHAT_SITE_RUNNING')) {
|
||||
if (!file_exists("$CFG->behat_dataroot/behattestdir.txt")) {
|
||||
if ($dh = opendir($CFG->behat_dataroot)) {
|
||||
while (($file = readdir($dh)) !== false) {
|
||||
if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store') {
|
||||
if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store' or is_numeric($file)) {
|
||||
continue;
|
||||
}
|
||||
behat_error(BEHAT_EXITCODE_CONFIG, '$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot');
|
||||
|
||||
@ -156,6 +156,9 @@ function testing_error($errorcode, $text = '') {
|
||||
|
||||
// do not write to error stream because we need the error message in PHP exec result from web ui
|
||||
echo($text."\n");
|
||||
if (isset($_SERVER['REMOTE_ADDR'])) {
|
||||
header('HTTP/1.1 500 Internal Server Error');
|
||||
}
|
||||
exit($errorcode);
|
||||
}
|
||||
|
||||
|
||||
@ -153,6 +153,43 @@ class behat_hooks extends behat_base {
|
||||
}
|
||||
}
|
||||
|
||||
protected static $timings = array();
|
||||
|
||||
/** @BeforeFeature */
|
||||
public static function before_feature($obj) {
|
||||
$file = $obj->getFeature()->getFile();
|
||||
self::$timings[$file] = microtime(true);
|
||||
}
|
||||
|
||||
/** @AfterFeature */
|
||||
public static function teardownFeature($obj) {
|
||||
$file = $obj->getFeature()->getFile();
|
||||
self::$timings[$file] = microtime(true) - self::$timings[$file];
|
||||
// Probably didn't actually run this, don't output it.
|
||||
if (self::$timings[$file] < 1) {
|
||||
unset(self::$timings[$file]);
|
||||
}
|
||||
}
|
||||
|
||||
/** @AfterSuite */
|
||||
public static function tearDown($obj) {
|
||||
global $CFG;
|
||||
if (!defined('BEHAT_FEATURE_TIMING')) {
|
||||
return;
|
||||
}
|
||||
$realroot = realpath(__DIR__.'/../../../').'/';
|
||||
foreach (self::$timings as $k => $v) {
|
||||
$new = str_replace($realroot, '', $k);
|
||||
self::$timings[$new] = round($v, 1);
|
||||
unset(self::$timings[$k]);
|
||||
}
|
||||
if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING), true)) {
|
||||
self::$timings = array_merge($existing, self::$timings);
|
||||
}
|
||||
arsort(self::$timings);
|
||||
@file_put_contents(BEHAT_FEATURE_TIMING, json_encode(self::$timings, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the test environment.
|
||||
*
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user