From 08e7f97ee499bb6abc125d0a25aa8abd8a40c4a4 Mon Sep 17 00:00:00 2001 From: Tony Levi Date: Sat, 31 May 2014 17:40:26 +0930 Subject: [PATCH 01/11] MDL-39752 behat: Parallel execution support --- admin/tool/behat/cli/init.php | 43 ++++- admin/tool/behat/cli/ns_parallel.php | 212 +++++++++++++++++++++ admin/tool/behat/cli/util.php | 26 ++- behat_pga_default.json | 109 +++++++++++ lib/behat/classes/behat_config_manager.php | 93 ++++++++- lib/behat/classes/util.php | 16 ++ lib/behat/lib.php | 28 +++ lib/clilib.php | 31 +++ lib/setup.php | 27 ++- lib/testing/lib.php | 3 + lib/tests/behat/behat_hooks.php | 37 ++++ 11 files changed, 616 insertions(+), 9 deletions(-) create mode 100644 admin/tool/behat/cli/ns_parallel.php create mode 100644 behat_pga_default.json diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index 5547b56eae3..97d613556d0 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -40,10 +40,41 @@ define('CACHE_DISABLE_ALL', true); require_once(__DIR__ . '/../../../../lib/clilib.php'); require_once(__DIR__ . '/../../../../lib/behat/lib.php'); + +list($options, $unrecognized) = cli_get_params( + array( + 'parallel' => 0, + 'suffix' => '', + ) +); + + +$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']); +$suffixarg = $options['suffix'] ? "--suffix={$options['suffix']} --parallel=$nproc" : ''; + + +if ($nproc && !$suffixarg) { + foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) { + if (file_exists($dir) && is_link($dir) && preg_match('#/behat\d+$#', $dir)) { + unlink($dir); + } + } + $cmds = array(); + for ($i = 1; $i <= $nproc; $i++) { + $cmds[] = "php ".__FILE__." --suffix=$i --parallel=$nproc 2>&1"; + } + // This is intensive compared to behat itself so halve the parallelism. + foreach (array_chunk($cmds, max(1, floor($nproc/2)), true) as $chunk) { + ns_parallel_popen($chunk, true); + } + exit(0); +} + + // Changing the cwd to admin/tool/behat/cli. chdir(__DIR__); $output = null; -exec("php util.php --diag", $output, $code); +exec("php util.php --diag $suffixarg", $output, $code); if ($code == 0) { echo "Behat test environment already installed\n"; @@ -53,7 +84,7 @@ if ($code == 0) { // Behat and dependencies are installed and we need to install the test site. chdir(__DIR__); - passthru("php util.php --install", $code); + passthru("php util.php --install $suffixarg", $code); if ($code != 0) { exit($code); } @@ -64,12 +95,12 @@ if ($code == 0) { // Test site data is outdated. chdir(__DIR__); - passthru("php util.php --drop", $code); + passthru("php util.php --drop $suffixarg", $code); if ($code != 0) { exit($code); } - passthru("php util.php --install", $code); + passthru("php util.php --install $suffixarg", $code); if ($code != 0) { exit($code); } @@ -81,7 +112,7 @@ if ($code == 0) { // Returning to admin/tool/behat/cli. chdir(__DIR__); - passthru("php util.php --install", $code); + passthru("php util.php --install $suffixarg", $code); if ($code != 0) { exit($code); } @@ -93,7 +124,7 @@ if ($code == 0) { } // Enable editing mode according to config.php vars. -passthru("php util.php --enable", $code); +passthru("php util.php --enable $suffixarg", $code); if ($code != 0) { exit($code); } diff --git a/admin/tool/behat/cli/ns_parallel.php b/admin/tool/behat/cli/ns_parallel.php new file mode 100644 index 00000000000..2f1ae7e7cfc --- /dev/null +++ b/admin/tool/behat/cli/ns_parallel.php @@ -0,0 +1,212 @@ +. + +/** + * Wrapper to run previously set-up behat tests in parallel. + * + * @package tool_behat + * @copyright 2014 NetSpot Pty Ltd + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + + +if (isset($_SERVER['REMOTE_ADDR'])) { + die(); // No access from web! +} + + +define('BEHAT_UTIL', true); +define('CLI_SCRIPT', true); +define('ABORT_AFTER_CONFIG', true); +define('NO_OUTPUT_BUFFERING', true); + +error_reporting(E_ALL | E_STRICT); +ini_set('display_errors', '1'); +ini_set('log_errors', '1'); + + +require_once __DIR__ .'/../../../../config.php'; +require_once __DIR__.'/../../../../lib/clilib.php'; +require_once __DIR__.'/../../../../lib/behat/lib.php'; + + +list($options, $unrecognised) = cli_get_params( + array( + 'stop-on-failure' => 0, + 'parallel' => 0, + 'verbose' => false, + 'replace' => false, + ) +); + + +if (empty($options['parallel']) && $dirs = glob("{$CFG->dirroot}/behat*")) { + sort($dirs); + if ($max = preg_filter('#.*behat(\d+)#', '$1', end($dirs))) { + $options['parallel'] = $max; + } +} + + +$suffix = ''; +$time = microtime(true); +$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']); +array_walk($unrecognised, function (&$v) { + if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) { + $v = $x; + } else if (!preg_match("#^-#", $v)) { + $v = escapeshellarg($v); + } +}); +$extraopts = implode(' ', $unrecognised); + + +if (empty($nproc)) { + fwrite(STDERR, "Invalid or missing --parallel parameter, must be >= 1.\n"); + exit(1); +} + + +$checkfail = array(); +$outputs = array(); +$handles = array(); +$pipe2i = array(); +$exits = array(); +$unused = null; +$linelencnt = 0; +$procs = array(); + + +for ($i = 1; $i <= $nproc; $i++) { + $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts; + $dirroot = dirname($CFG->behat_dataroot)."/behat$i"; + $cmd = "exec {$CFG->dirroot}/vendor/bin/behat --config $dirroot/behat/behat.yml $myopts"; + list($handle, $pipes) = ns_proc_open($cmd, true); + @fclose($pipes[0]); + unset($pipes[0]); + $exits[$i] = 1; + $handles[$i] = array($handle, $pipes[1], $pipes[2]); + $procs[$i] = $handle; + $checkfail[$i] = false; + $outputs[$i] = array(''); + $pipe2i[(int) $pipes[1]] = $i; + $pipe2i[(int) $pipes[2]] = $i; + stream_set_blocking($pipes[1], 0); + stream_set_blocking($pipes[2], 0); +} + + +while (!empty($procs)) { + usleep(10000); + + foreach ($handles as $i => $p) { + if (!($status = @proc_get_status($p[0])) || !$status['running']) { + if ($exits[$i] !== 0) { + $exits[$i] = !empty($status) ? $status['exitcode'] : 1; + } + unset($procs[$i]); + unset($handles[$i][0]); + $last = array_pop($outputs[$i]); + for ($l=2; $l>=1; $l--) + while ($part = @fread($handles[$i][$l], 8192)) + $last .= $part; + $outputs[$i] = array_merge($outputs[$i], explode("\n", $last)); + } + } + + $ready = array(); + foreach ($handles as $i => $set) { + $ready[] = $set[1]; + $ready[] = $set[2]; + } + + // Poll for any process with output or ended. + if (!$result = @stream_select($ready, $unused, $unused, 1)) { + // Nothing; try again. + continue; + } + if (!$fh = reset($ready)) { + continue; + } + + $i = $pipe2i[(int) $fh]; + $last = array_pop($outputs[$i]); + $read = fread($fh, 4096); + $newlines = explode("\n", $last.$read); + $outputs[$i] = array_merge($outputs[$i], $newlines); + + if (!$checkfail[$i]) { + foreach ($newlines as $l => $line) { + unset($newlines[$l]); + if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) { + $checkfail[$i] = true; + break; + } + } + } + if ($progress = preg_filter('#^\s*([FUS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $newlines)) { + if ($checkfail[$i] && preg_filter('#^\s*[S\.\-]*[FU][S\.\-]*(?:\s+\d+)?\s*$#', '$1', $progress)) { + $exits[$i] = 1; + if ($options['stop-on-failure']) { + foreach ($handles as $l => $p) { + $exits[$l] = $l != $i ? 0 : $exits[$i]; + @proc_terminate($p[0], SIGINT); + } + } + } + } + // Process has gone, assume this is the last output for it. + if (empty($procs[$i])) { + unset($handles[$i]); + } + if (empty($checkfail[$i]) || !($update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $read))) { + continue; + } + while ($update) { + $part = substr($update, 0, 70 - $linelencnt); + $update = substr($update, strlen($part)); + $linelencnt += strlen($part); + echo $part; + if ($linelencnt >= 70) { + echo "\n"; + $linelencnt = 0; + } + } +} +echo "\n\n"; + + +$exits = array_filter($exits, function ($v) {return $v !== 0;}); + + +if ($exits || $options['verbose']) { + echo "Exit codes: ".implode(" ", $exits)."\n\n"; + foreach ($outputs as $i => $output) { + unset($outputs[$i]); + if (!end($output)) array_pop($output); + $prefix = "[behat$i] "; + array_walk($output, function (&$l) use ($prefix) { + $l = $prefix.$l; + }); + echo implode("\n", $output)."\n\n"; + } + $failed = true; +} + + +$time = round(microtime(true) - $time, 1); +echo "Finished in {$time}s\n"; +exit(!empty($failed) ? 1 : 0); diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index d843bf2fb6b..b68992e3701 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -40,6 +40,8 @@ list($options, $unrecognized) = cli_get_params( array( 'help' => false, 'install' => false, + 'parallel' => 0, + 'suffix' => '', 'drop' => false, 'enable' => false, 'disable' => false, @@ -63,6 +65,7 @@ Options: --drop Drops the database tables and the dataroot contents --enable Enables test environment and updates tests list --disable Disables test environment +--parallel Run operation for all parallel behat environments. --diag Get behat test environment status code -h, --help Print out this help @@ -78,11 +81,32 @@ if (!empty($options['help'])) { exit(0); } -// Describe this script. + +if (!empty($options['parallel']) && empty($options['suffix'])) { + foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) { + if (file_exists($dir) && is_dir($dir)) { + unlink($dir); + } + } + $cmds = array(); + $extra = preg_filter('#(.*)\s*--parallel=\d+\s*(.*?)#', '$1 $2', implode(' ', array_slice($argv, 1))); + for ($i = 1; $i <= $options['parallel']; $i++) { + $cmds[] = "php ".__FILE__." $extra --suffix=$i 2>&1"; + } + // This is intensive compared to behat itself so halve the parallelism. + foreach (array_chunk($cmds, min(1, floor($options['parallel']/2)), true) as $chunk) { + ns_parallel_popen($chunk, true); + } + exit(0); +} + + +// Checking $CFG->behat_* vars and values. define('BEHAT_UTIL', true); define('CLI_SCRIPT', true); define('NO_OUTPUT_BUFFERING', true); define('IGNORE_COMPONENT_CACHE', true); +define('BEHAT_SUFFIX', $options['suffix']); // Only load CFG from config.php, stop ASAP in lib/setup.php. define('ABORT_AFTER_CONFIG', true); diff --git a/behat_pga_default.json b/behat_pga_default.json new file mode 100644 index 00000000000..dbce41d24e3 --- /dev/null +++ b/behat_pga_default.json @@ -0,0 +1,109 @@ +{ + "course\/tests\/behat\/course_controls.feature": 117.2, + "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": 108.2, + "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": 92.8, + "course\/tests\/behat\/category_resort.feature": 75.4, + "course\/tests\/behat\/course_resort.feature": 58.9, + "admin\/tool\/behat\/tests\/behat\/data_generators.feature": 53.1, + "course\/tests\/behat\/category_management.feature": 50.1, + "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": 48.3, + "grade\/tests\/behat\/grade_override_letter.feature": 47.3, + "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": 42.7, + "calendar\/tests\/behat\/calendar.feature": 42, + "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": 41.2, + "mod\/forum\/tests\/behat\/forum_subscriptions.feature": 32.6, + "course\/tests\/behat\/move_activities.feature": 30.8, + "mod\/workshep\/tests\/behat\/workshep_assessment.feature": 29.3, + "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": 29, + "mod\/workshop\/tests\/behat\/workshop_assessment.feature": 28.8, + "blocks\/tests\/behat\/manage_blocks.feature": 28.6, + "badges\/tests\/behat\/navrequirecap.feature": 28, + "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": 26.9, + "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": 26, + "blocks\/html\/tests\/behat\/multiple_instances.feature": 26, + "message\/tests\/behat\/display_history.feature": 24.7, + "course\/tests\/behat\/category_change_visibility.feature": 24.2, + "local\/uneditableblocks\/tests\/behat\/enable_uneditableblocks.feature": 23.1, + "mod\/feedback\/tests\/behat\/defaultshortanswerlength.feature": 22.1, + "backup\/util\/ui\/tests\/behat\/backup_courses.feature": 22, + "course\/tests\/behat\/create_delete_course.feature": 21.9, + "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": 21.2, + "course\/tests\/behat\/move_sections.feature": 21.1, + "course\/tests\/behat\/course_category_management_listing.feature": 21, + "local\/userpolicy\/tests\/behat\/fieldvisibility.feature": 20.9, + "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": 20.5, + "mod\/wiki\/tests\/behat\/edit_tags.feature": 19.6, + "mod\/attendance\/tests\/behat\/attendance_mod.feature": 19.5, + "blocks\/participants\/tests\/behat\/block_participants_course.feature": 19.5, + "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": 17.7, + "blocks\/tests\/behat\/configure_block_throughout_site.feature": 17.7, + "grade\/report\/singleview\/tests\/behat\/singleview.feature": 17.6, + "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": 16.3, + "mod\/wiki\/tests\/behat\/group_enhancements.feature": 16.2, + "admin\/tests\/behat\/forcelogin_makefrontpagepublic.feature": 16, + "message\/tests\/behat\/disablenotifications.feature": 16, + "completion\/tests\/behat\/teacher_manual_completion.feature": 16, + "mod\/glossary\/tests\/behat\/entries_require_approval.feature": 16, + "backup\/util\/ui\/tests\/behat\/import_course.feature": 15.9, + "admin\/tests\/behat\/set_admin_settings_value.feature": 15.8, + "mod\/mediagallery\/tests\/behat\/separategroups.feature": 15.6, + "blocks\/tests\/behat\/return_block_original_state.feature": 15.6, + "admin\/tests\/behat\/custom_maxbytes.feature": 15.4, + "mod\/oublog\/tests\/behat\/separate_individuals.feature": 15.4, + "mod\/forum\/tests\/behat\/default_displaywordcount.feature": 15.4, + "backup\/util\/ui\/tests\/behat\/restore_moodle2_course_numsections.feature": 14.9, + "question\/tests\/behat\/question_defaultpenalty.feature": 14.8, + "availability\/tests\/behat\/edit_availability.feature": 14.7, + "mod\/data\/tests\/behat\/view_entries.feature": 14.6, + "local\/catdelete\/tests\/behat\/catdelete.feature": 14.3, + "group\/tests\/behat\/showusernameingroup.feature": 14.3, + "mod\/forum\/tests\/behat\/edit_post_student.feature": 14.2, + "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": 14.1, + "mod\/hsuforum\/tests\/behat\/edit_post_student.feature": 13.8, + "mod\/quiz\/tests\/behat\/add_quiz.feature": 13.5, + "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": 13.5, + "blocks\/autocreate_user\/tests\/behat\/add_introtext.feature": 13.5, + "question\/type\/truefalse\/tests\/behat\/custompenalty.feature": 13.3, + "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": 13.1, + "blocks\/html\/tests\/behat\/course_block.feature": 12.9, + "mod\/quiz\/tests\/behat\/configsubnethide.feature": 12.8, + "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": 12.6, + "blocks\/html\/tests\/behat\/configuring_html_block.feature": 12.3, + "course\/tests\/behat\/course_change_visibility.feature": 12.2, + "course\/tests\/behat\/modchooser_hidden.feature": 11.9, + "blocks\/login\/tests\/behat\/login_block.feature": 11.7, + "course\/tests\/behat\/paged_course_navigation.feature": 11.1, + "course\/tests\/behat\/limitrolerenaming.feature": 11, + "mod\/book\/tests\/behat\/create_chapters.feature": 11, + "mod\/forum\/tests\/behat\/my_forum_posts.feature": 10.7, + "auth\/tests\/behat\/login.feature": 10.5, + "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": 10.5, + "mod\/data\/tests\/behat\/add_entries.feature": 10.1, + "mod\/wiki\/tests\/behat\/preview_page.feature": 9.7, + "mod\/oublog\/tests\/behat\/personalblog.feature": 9.7, + "admin\/tests\/behat\/display_short_names.feature": 9.6, + "mod\/forum\/tests\/behat\/separate_group_discussions.feature": 9.5, + "course\/tests\/behat\/add_activities.feature": 9.4, + "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": 8.8, + "blocks\/tests\/behat\/restrict_available_blocks.feature": 8.7, + "blocks\/comments\/tests\/behat\/add_comment.feature": 8.7, + "mod\/survey\/tests\/behat\/survey_types.feature": 8.5, + "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": 7.3, + "local\/rolerenaming\/tests\/behat\/course_role_renaming.feature": 7.2, + "course\/tests\/behat\/edit_settings.feature": 7.1, + "group\/tests\/behat\/create_groups.feature": 6.8, + "blocks\/tests\/behat\/add_blocks.feature": 6.5, + "message\/tests\/behat\/manage_contacts.feature": 6.1, + "course\/tests\/behat\/course_search.feature": 6, + "course\/tests\/behat\/course_creation.feature": 5.8, + "my\/tests\/behat\/reset_page.feature": 5.4, + "my\/tests\/behat\/restrict_available_blocks.feature": 5.3, + "mod\/oublog\/tests\/behat\/basic.feature": 5.2, + "my\/tests\/behat\/add_blocks.feature": 4.6, + "user\/tests\/behat\/reset_page.feature": 4, + "message\/tests\/behat\/search_history.feature": 3.9, + "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": 3.3, + "user\/tests\/behat\/add_blocks.feature": 2.8, + "report\/usersessions\/tests\/behat\/usersessions_report.feature": 2.7, + "admin\/tool\/behat\/tests\/behat\/test_environment.feature": 1 +} \ No newline at end of file diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index d6c852017de..b6f97154999 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -79,7 +79,10 @@ class behat_config_manager { $featurespaths[$uniquekey] = $path; } } - $features = array_values($featurespaths); + foreach ($featurespaths as $path) { + $additional = glob("$path/*.feature"); + $features = array_merge($features, $additional); + } } // Optionally include features from additional directories. @@ -198,6 +201,29 @@ class behat_config_manager { // We require here when we are sure behat dependencies are available. require_once($CFG->dirroot . '/vendor/autoload.php'); + $instance = 1; + $parallel = 0; + foreach ($_SERVER['argv'] as $arg) { + if (strpos($arg, '--suffix=') === 0) { + $instance = intval(substr($arg, strlen('--suffix='))); + } + if (empty($parallel)) { + $parallel = preg_filter('#--parallel=(\d+)#', '$1', $arg); + } + } + + // Attempt to split into weighted buckets using timing information, if available. + if ($alloc = self::profile_guided_allocate($features, max(1, $parallel), $instance)) { + $features = $alloc; + } else { + // Divide the list of feature files amongst the parallel runners. + srand(crc32(floor(time() / 3600 / 24).var_export($features,true))); + shuffle($features); + // Pull out the features for just this worker. + $features = array_chunk($features, ceil(count($features) / max(1, $parallel))); + $features = $features[$instance-1]; + } + // It is possible that it has no value as we don't require a full behat setup to list the step definitions. if (empty($CFG->behat_wwwroot)) { $CFG->behat_wwwroot = 'http://itwillnotbeused.com'; @@ -242,6 +268,71 @@ class behat_config_manager { return Symfony\Component\Yaml\Yaml::dump($config, 10, 2); } + /** + * Attempt to split feature list into fairish buckets using timing information, if available. + * Simply add each one to lightest buckets until all files allocated. + * PGA = Profile Guided Allocation. I made it up just now. + * CAUTION: workers must agree on allocation, do not be random anywhere! + * + * @param array $features Behat feature files array + * @param int $nbuckets Number of buckets to divide into + * @param int $instance Index number of this instance + * @return array Feature files array, sorted into allocations + */ + protected static function profile_guided_allocate($features, $nbuckets, $instance) { + $pga = __DIR__.'/../../../behat_pga_default.json'; + $pga = defined('BEHAT_PGA_DATA') && file_exists(BEHAT_PGA_DATA) ? BEHAT_PGA_DATA : $pga; + + if (defined('BEHAT_PGA_DISABLE') || !file_exists($pga) || !$pga = @json_decode(file_get_contents($pga), true)) { + // No data available, fall back to relying on shuffle. + return false; + } + + arsort($pga); // Ensure most expensive is first. + + $realroot = realpath(__DIR__.'/../../../').'/'; + $defaultweight = array_sum($pga) / count($pga); // TODO: median is more ideal. + $weights = array_fill(0, $nbuckets, 0); + $buckets = array_fill(0, $nbuckets, array()); + $totalweight = 0; + + // Re-key the features list to match pga data. + foreach ($features as $k => $file) { + $key = str_replace($realroot, '', $file); + $features[$key] = $file; + unset($features[$k]); + if (!isset($pga[$key])) { + $pga[$key] = $defaultweight; + } + } + + // Sort features by known weights; largest ones should be allocated first. + $pgaorder = array(); + foreach ($features as $key => $file) { + $pgaorder[$key] = $pga[$key]; + } + arsort($pgaorder); + + // Finally, add each feature one by one to the lightest bucket. + foreach ($pgaorder as $key => $weight) { + $file = $features[$key]; + $light_bucket = array_search(min($weights), $weights); + $weights[$light_bucket] += $weight; + $buckets[$light_bucket][] = $file; + $totalweight += $weight; + } + + if (!defined('BEHAT_PGA_DISABLE_HISTOGRAM') && $instance == $nbuckets) { + echo "Bucket weightings:\n"; + foreach ($weights as $k => $weight) { + echo "$k: ".str_repeat('*', 70 * $nbuckets * $weight / $totalweight)."\n"; + } + } + + // Return the features for this worker. + return $buckets[$instance-1]; + } + /** * Overrides default config with local config values * diff --git a/lib/behat/classes/util.php b/lib/behat/classes/util.php index 95f42fea62d..4a42c578653 100644 --- a/lib/behat/classes/util.php +++ b/lib/behat/classes/util.php @@ -213,6 +213,22 @@ class behat_util extends testing_util { // Updates all the Moodle features and steps definitions. behat_config_manager::update_config_file(); + // Create suffix symlink if necessary. + global $CFG; + if ($CFG->behat_suffix) { + $extra = preg_filter('#.*/(.+)$#', '$1', $CFG->behat_wwwroot); + $link = $CFG->dirroot.'/'.$extra; + if (file_exists($link)) { + if (!is_link($link)) { + throw new coding_exception("File exists at link location ($link) but is not a link!"); + } + @unlink($link); + } + if (!symlink($CFG->dirroot, $link)) { + throw new coding_exception("Unable to create behat suffix symlink ($link)"); + } + } + if (self::is_test_mode_enabled()) { return; } diff --git a/lib/behat/lib.php b/lib/behat/lib.php index 86718a92a8a..40612f4cd8e 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -272,6 +272,34 @@ function behat_is_test_site() { return false; } +/** + * Add behat suffix to $CFG vars for parallel testing. + **/ +function behat_add_suffix_to_vars($suffix = '') { + global $CFG; + + if (empty($suffix)) { + if (!empty($CFG->behat_suffix)) { + $suffix = $CFG->behat_suffix; + + } else if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) { + $suffix = BEHAT_SUFFIX; + } + } + + $CFG->behat_suffix = $suffix; + + if ($suffix) { + if (isset($CFG->behat_wwwroot) && !preg_match("#/behat$suffix\$#", $CFG->behat_wwwroot)) { + $CFG->behat_wwwroot .= "/behat{$suffix}"; + } + if (!preg_match("#/behat{$suffix}\$#", $CFG->behat_dataroot)) { + $CFG->behat_dataroot = dirname($CFG->behat_dataroot)."/behat{$suffix}"; + } + $CFG->behat_prefix = "behat{$suffix}_"; + } +} + /** * Checks if the URL requested by the user matches the provided argument * diff --git a/lib/clilib.php b/lib/clilib.php index 32272defc04..e4992528144 100644 --- a/lib/clilib.php +++ b/lib/clilib.php @@ -175,3 +175,34 @@ function cli_error($text, $errorcode=1) { fwrite(STDERR, "\n"); die($errorcode); } + + +function ns_proc_open($cmd, $die = false) { + $desc = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('pipe', 'w'), + ); + if (!($handle = proc_open($cmd, $desc, $pipes)) && $die) { + throw new Exception('Error starting worker'); + } + return array($handle, $pipes); +} + + +function ns_parallel_popen($cmds, $doexit = false) { + $procs = array(); + foreach ($cmds as $k => $cmd) { + $procs[] = popen($cmd, 'r'); + } + $status = false; + foreach ($procs as $p) { + if (!$p) continue; + while ($out = fgets($p)) echo $out; + $status |= (bool) pclose($p); + } + if ($doexit && $status) { + exit($status); + } + return $status; +} diff --git a/lib/setup.php b/lib/setup.php index d0a77601fdf..a0d870795d1 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -77,9 +77,34 @@ if (defined('BEHAT_SITE_RUNNING')) { // We already switched to behat test site previously. } else if (!empty($CFG->behat_wwwroot) or !empty($CFG->behat_dataroot) or !empty($CFG->behat_prefix)) { + global $argv; + $suffix = ''; + + if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) { + $suffix = BEHAT_SUFFIX; + + } else if (!empty($_SERVER['REMOTE_ADDR'])) { + if (preg_match('#/behat(.+?)/#', $_SERVER['REQUEST_URI'])) { + $afterpath = str_replace(realpath($CFG->dirroot).'/', '', realpath($_SERVER['SCRIPT_FILENAME'])); + if (!$suffix = preg_filter("#.*/behat(.+?)/$afterpath#", '$1', $_SERVER['SCRIPT_FILENAME'])) { + throw new coding_exception("Unable to determine behat suffix [afterpath=$afterpath, scriptfilename={$_SERVER['SCRIPT_FILENAME']}]!"); + } + } + + } else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) { + if ($match = preg_filter('#--suffix=(.+)#', '$1', $argv)) { + $suffix = reset($match); + } + if ($k = array_search('--config', $argv)) { + $behatconfig = $argv[$k+1]; + $suffix = preg_filter("#^{$CFG->behat_dataroot}(.+?)/behat/behat\.yml#", '$1', $behatconfig); + } + } + // The behat is configured on this server, we need to find out if this is the behat test // site based on the URL used for access. require_once(__DIR__ . '/../lib/behat/lib.php'); + behat_add_suffix_to_vars($suffix); if (behat_is_test_site()) { // Checking the integrity of the provided $CFG->behat_* vars and the // selected wwwroot to prevent conflicts with production and phpunit environments. @@ -89,7 +114,7 @@ if (defined('BEHAT_SITE_RUNNING')) { if (!file_exists("$CFG->behat_dataroot/behattestdir.txt")) { if ($dh = opendir($CFG->behat_dataroot)) { while (($file = readdir($dh)) !== false) { - if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store') { + if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store' or is_numeric($file)) { continue; } behat_error(BEHAT_EXITCODE_CONFIG, '$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot'); diff --git a/lib/testing/lib.php b/lib/testing/lib.php index c089c6a76cf..bbb93d3e293 100644 --- a/lib/testing/lib.php +++ b/lib/testing/lib.php @@ -156,6 +156,9 @@ function testing_error($errorcode, $text = '') { // do not write to error stream because we need the error message in PHP exec result from web ui echo($text."\n"); + if (isset($_SERVER['REMOTE_ADDR'])) { + header('HTTP/1.1 500 Internal Server Error'); + } exit($errorcode); } diff --git a/lib/tests/behat/behat_hooks.php b/lib/tests/behat/behat_hooks.php index 6e2653f9ed6..060998294f8 100644 --- a/lib/tests/behat/behat_hooks.php +++ b/lib/tests/behat/behat_hooks.php @@ -153,6 +153,43 @@ class behat_hooks extends behat_base { } } + protected static $timings = array(); + + /** @BeforeFeature */ + public static function before_feature($obj) { + $file = $obj->getFeature()->getFile(); + self::$timings[$file] = microtime(true); + } + + /** @AfterFeature */ + public static function teardownFeature($obj) { + $file = $obj->getFeature()->getFile(); + self::$timings[$file] = microtime(true) - self::$timings[$file]; + // Probably didn't actually run this, don't output it. + if (self::$timings[$file] < 1) { + unset(self::$timings[$file]); + } + } + + /** @AfterSuite */ + public static function tearDown($obj) { + global $CFG; + if (!defined('BEHAT_FEATURE_TIMING')) { + return; + } + $realroot = realpath(__DIR__.'/../../../').'/'; + foreach (self::$timings as $k => $v) { + $new = str_replace($realroot, '', $k); + self::$timings[$new] = round($v, 1); + unset(self::$timings[$k]); + } + if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING), true)) { + self::$timings = array_merge($existing, self::$timings); + } + arsort(self::$timings); + @file_put_contents(BEHAT_FEATURE_TIMING, json_encode(self::$timings, JSON_PRETTY_PRINT)); + } + /** * Resets the test environment. * From 3c71c15c2677e01ea490055af52e5af0dc9711cd Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Tue, 20 Jan 2015 13:56:29 +0800 Subject: [PATCH 02/11] MDL-39752 behat: Modified following for parallel run: 1. Create behat datadir within behat_dataroot not at same level 2. Define suffix for link and not use hard-coded values 3. Renamed ns_parallel to run.php 4. Rename variables to best understand them 5. Added support for each run to specify db, prefix, rerun and profile. 6. Showing number of steps in each line of parallel run. --- admin/tool/behat/cli/init.php | 83 ++-- .../behat/cli/{ns_parallel.php => run.php} | 180 ++++++-- admin/tool/behat/cli/util.php | 89 ++-- admin/tool/behat/cli/utilparallel.php | 195 +++++++++ behat_pga_default.json | 384 +++++++++++++----- config-dist.php | 15 + lib/behat/classes/behat_command.php | 31 +- lib/behat/classes/behat_config_manager.php | 189 +++++++-- lib/behat/classes/util.php | 16 - lib/behat/lib.php | 59 ++- lib/classes/process_manager.php | 211 ++++++++++ lib/clilib.php | 95 ++++- lib/setup.php | 42 +- lib/setuplib.php | 2 +- lib/testing/classes/test_lock.php | 1 - lib/tests/behat/behat_hooks.php | 65 ++- 16 files changed, 1327 insertions(+), 330 deletions(-) rename admin/tool/behat/cli/{ns_parallel.php => run.php} (50%) create mode 100644 admin/tool/behat/cli/utilparallel.php create mode 100644 lib/classes/process_manager.php diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index 97d613556d0..6afd936b899 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -40,92 +40,109 @@ define('CACHE_DISABLE_ALL', true); require_once(__DIR__ . '/../../../../lib/clilib.php'); require_once(__DIR__ . '/../../../../lib/behat/lib.php'); - list($options, $unrecognized) = cli_get_params( array( 'parallel' => 0, - 'suffix' => '', + 'maxruns' => false, + 'help' => false, + ), + array( + 'j' => 'parallel', + 'm' => 'maxruns', + 'h' => 'help', ) ); +// Checking run.php CLI script usage. +$help = " +Behat utilities to initialise behat tests -$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']); -$suffixarg = $options['suffix'] ? "--suffix={$options['suffix']} --parallel=$nproc" : ''; +Options: +-j, --parallel Number of parallel behat run to initialise +-m, --maxruns Max parallel processes to be executed at one time. +-h, --help Print out this help -if ($nproc && !$suffixarg) { - foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) { - if (file_exists($dir) && is_link($dir) && preg_match('#/behat\d+$#', $dir)) { - unlink($dir); - } - } - $cmds = array(); - for ($i = 1; $i <= $nproc; $i++) { - $cmds[] = "php ".__FILE__." --suffix=$i --parallel=$nproc 2>&1"; - } - // This is intensive compared to behat itself so halve the parallelism. - foreach (array_chunk($cmds, max(1, floor($nproc/2)), true) as $chunk) { - ns_parallel_popen($chunk, true); - } +Example from Moodle root directory: +\$ php admin/tool/behat/cli/init.php --parallel=2 + +More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests +"; + +if (!empty($options['help'])) { + echo $help; exit(0); } +// Check which util file to call. +$utilfile = 'util.php'; +$paralleloption = ""; +// If parallel run then use utilparallel. +if ($options['parallel']) { + $utilfile = 'utilparallel.php'; + $paralleloption = " --parallel=".$options['parallel']; +} // Changing the cwd to admin/tool/behat/cli. +$cwd = getcwd(); chdir(__DIR__); $output = null; -exec("php util.php --diag $suffixarg", $output, $code); + +exec("php $utilfile --diag $paralleloption", $output, $code); + +// Check if composer needs to be updated. +if (($code == BEHAT_EXITCODE_INSTALL) || $code == BEHAT_EXITCODE_REINSTALL || $code == BEHAT_EXITCODE_COMPOSER) { + testing_update_composer_dependencies(); +} + if ($code == 0) { echo "Behat test environment already installed\n"; } else if ($code == BEHAT_EXITCODE_INSTALL) { - - testing_update_composer_dependencies(); - // Behat and dependencies are installed and we need to install the test site. chdir(__DIR__); - passthru("php util.php --install $suffixarg", $code); + passthru("php $utilfile --install $paralleloption", $code); if ($code != 0) { + chdir($cwd); exit($code); } } else if ($code == BEHAT_EXITCODE_REINSTALL) { - - testing_update_composer_dependencies(); - // Test site data is outdated. chdir(__DIR__); - passthru("php util.php --drop $suffixarg", $code); + passthru("php $utilfile --drop $paralleloption", $code); if ($code != 0) { + chdir($cwd); exit($code); } - passthru("php util.php --install $suffixarg", $code); + passthru("php $utilfile --install $paralleloption", $code); if ($code != 0) { + chdir($cwd); exit($code); } } else if ($code == BEHAT_EXITCODE_COMPOSER) { // Missing Behat dependencies. - - testing_update_composer_dependencies(); - // Returning to admin/tool/behat/cli. chdir(__DIR__); - passthru("php util.php --install $suffixarg", $code); + passthru("php $utilfile --install $paralleloption", $code); if ($code != 0) { + chdir($cwd); exit($code); } } else { // Generic error, we just output it. echo implode("\n", $output)."\n"; + chdir($cwd); exit($code); } // Enable editing mode according to config.php vars. -passthru("php util.php --enable $suffixarg", $code); +passthru("php $utilfile --enable $paralleloption", $code); if ($code != 0) { + chdir($cwd); exit($code); } diff --git a/admin/tool/behat/cli/ns_parallel.php b/admin/tool/behat/cli/run.php similarity index 50% rename from admin/tool/behat/cli/ns_parallel.php rename to admin/tool/behat/cli/run.php index 2f1ae7e7cfc..4a3e1bca390 100644 --- a/admin/tool/behat/cli/ns_parallel.php +++ b/admin/tool/behat/cli/run.php @@ -22,48 +22,106 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ - if (isset($_SERVER['REMOTE_ADDR'])) { die(); // No access from web! } - define('BEHAT_UTIL', true); define('CLI_SCRIPT', true); define('ABORT_AFTER_CONFIG', true); +define('CACHE_DISABLE_ALL', true); define('NO_OUTPUT_BUFFERING', true); +require_once(__DIR__ .'/../../../../config.php'); +require_once(__DIR__.'/../../../../lib/clilib.php'); +require_once(__DIR__.'/../../../../lib/behat/lib.php'); +require_once(__DIR__.'/../../../../lib/behat/classes/behat_command.php'); +require_once(__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php'); + error_reporting(E_ALL | E_STRICT); ini_set('display_errors', '1'); ini_set('log_errors', '1'); - -require_once __DIR__ .'/../../../../config.php'; -require_once __DIR__.'/../../../../lib/clilib.php'; -require_once __DIR__.'/../../../../lib/behat/lib.php'; - - list($options, $unrecognised) = cli_get_params( array( 'stop-on-failure' => 0, - 'parallel' => 0, - 'verbose' => false, - 'replace' => false, + 'verbose' => false, + 'replace' => false, + 'help' => false, + 'tags' => '', + 'profile' => '', + ), + array( + 'h' => 'help', + 't' => 'tags', + 'p' => 'profile', ) ); +// Checking run.php CLI script usage. +$help = " +Behat utilities to run behat tests in parallel +Options: +-t, --tags{{color_green}} Tags to execute. +-p, --profile{{color_green}} Profile to execute. +--stop-on-failure{{color_green}} Stop on failure in any parallel run. +--verbose{{color_green}} Verbose output +--replace{{color_green}} Replace args string with run process number, useful for output. -if (empty($options['parallel']) && $dirs = glob("{$CFG->dirroot}/behat*")) { - sort($dirs); - if ($max = preg_filter('#.*behat(\d+)#', '$1', end($dirs))) { - $options['parallel'] = $max; +-h, --help{{color_green}} Print out this help + +Example from Moodle root directory: +\$ php admin/tool/behat/cli/run.php --parallel=2 + +More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests +"; + +if (!empty($options['help'])) { + echo $help; + exit(0); +} + +// Ensure we have parallel runs initialised and it's >= 1. +$parallelrun = behat_config_manager::get_parallel_test_runs(1); + +// Capture signals and ensure we clean symlinks. +pcntl_signal(SIGTERM, "signal_handler"); +pcntl_signal(SIGINT, "signal_handler"); +/** + * Signal handler for terminal exit. + * + * @param $signal + */ +function signal_handler($signal) { + switch ($signal) { + case SIGTERM: + case SIGKILL: + case SIGINT: + // Remove site symlink if necessary. + behat_config_manager::drop_parallel_site_links(); + exit(1); } } +// If empty parallelrun then just check with user if it's a run single behat test. +if (empty($parallelrun)) { + if (cli_input("This is not a parallel site, do you want to run single behat run? (Y/N)", 'n', array('y', 'n')) == 'y') { + $runtestscommand = behat_command::get_behat_command(); + $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); + exec("php $runtestscommand", $output, $code); + echo implode("\n", $output) . "\n"; + exit($code); + } else { + exit(1); + } +} + +// Create site symlink if necessary. +if (!behat_config_manager::create_parallel_site_links()) { + exit(1); +} -$suffix = ''; $time = microtime(true); -$nproc = (int) preg_filter('#.*(\d+).*#', '$1', $options['parallel']); array_walk($unrecognised, function (&$v) { if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1='\$2'", $v)) { $v = $x; @@ -73,13 +131,6 @@ array_walk($unrecognised, function (&$v) { }); $extraopts = implode(' ', $unrecognised); - -if (empty($nproc)) { - fwrite(STDERR, "Invalid or missing --parallel parameter, must be >= 1.\n"); - exit(1); -} - - $checkfail = array(); $outputs = array(); $handles = array(); @@ -88,13 +139,51 @@ $exits = array(); $unused = null; $linelencnt = 0; $procs = array(); +$behatdataroot = $CFG->behat_dataroot; +$tags = ''; +// Options parameters to be added to each run. +$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts; +if ($options['profile']) { + $profile = $options['profile']; + if (empty($CFG->behat_config[$profile]['filters']['tags'])) { + echo "Invaid profile passed: " . $profile; + exit(1); + } + $tags = $CFG->behat_config[$profile]['filters']['tags']; + $myopts .= '--profile=\'' . $profile . "'"; +} else if ($options['tags']) { + $tags = $options['tags']; + $myopts .= '--tags=' . $tags; +} -for ($i = 1; $i <= $nproc; $i++) { - $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts; - $dirroot = dirname($CFG->behat_dataroot)."/behat$i"; - $cmd = "exec {$CFG->dirroot}/vendor/bin/behat --config $dirroot/behat/behat.yml $myopts"; - list($handle, $pipes) = ns_proc_open($cmd, true); +// Update config file if tags defined. +if ($tags) { + // Hack to set proper dataroot and wwroot. + $behatdataroot = $CFG->behat_dataroot; + $behatwwwroot = $CFG->behat_wwwroot; + for ($i = 1; $i <= $parallelrun; $i++) { + $CFG->behatrunprocess = $i; + $CFG->behat_dataroot = $behatdataroot . $i; + $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i; + behat_config_manager::update_config_file('', true, $tags); + } + $CFG->behat_dataroot = $behatdataroot; + $CFG->behat_wwwroot = $behatwwwroot; + unset($CFG->behatrunprocess); +} + +for ($i = 1; $i <= $parallelrun; $i++) { + $CFG->behatrunprocess = $i; + $behatcommand = behat_command::get_behat_command(); + $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i); + + // Command to execute behat run. + $cmd = $behatcommand .' --config ' . $behatconfigpath . " " . $myopts; + + echo "[" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . "] ". $cmd . "\n"; + + list($handle, $pipes) = cli_execute($cmd, true); @fclose($pipes[0]); unset($pipes[0]); $exits[$i] = 1; @@ -108,7 +197,7 @@ for ($i = 1; $i <= $nproc; $i++) { stream_set_blocking($pipes[2], 0); } - +$progresscount = 0; while (!empty($procs)) { usleep(10000); @@ -120,9 +209,11 @@ while (!empty($procs)) { unset($procs[$i]); unset($handles[$i][0]); $last = array_pop($outputs[$i]); - for ($l=2; $l>=1; $l--) - while ($part = @fread($handles[$i][$l], 8192)) + for ($l = 2; $l >= 1; $l--) { + while ($part = @fread($handles[$i][$l], 8192)) { $last .= $part; + } + } $outputs[$i] = array_merge($outputs[$i], explode("\n", $last)); } } @@ -151,7 +242,8 @@ while (!empty($procs)) { if (!$checkfail[$i]) { foreach ($newlines as $l => $line) { unset($newlines[$l]); - if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) { + if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && + preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) { $checkfail[$i] = true; break; } @@ -181,23 +273,28 @@ while (!empty($procs)) { $linelencnt += strlen($part); echo $part; if ($linelencnt >= 70) { - echo "\n"; + $progresscount += 70; + echo " $progresscount\n"; $linelencnt = 0; } } } echo "\n\n"; - -$exits = array_filter($exits, function ($v) {return $v !== 0;}); - +$exits = array_filter($exits, + function ($v) { + return $v !== 0; + } +); if ($exits || $options['verbose']) { echo "Exit codes: ".implode(" ", $exits)."\n\n"; foreach ($outputs as $i => $output) { unset($outputs[$i]); - if (!end($output)) array_pop($output); - $prefix = "[behat$i] "; + if (!end($output)) { + array_pop($output); + } + $prefix = "[" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . "] "; array_walk($output, function (&$l) use ($prefix) { $l = $prefix.$l; }); @@ -206,7 +303,10 @@ if ($exits || $options['verbose']) { $failed = true; } - $time = round(microtime(true) - $time, 1); echo "Finished in {$time}s\n"; + +// Remove site symlink if necessary. +behat_config_manager::drop_parallel_site_links(); + exit(!empty($failed) ? 1 : 0); diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index b68992e3701..8915b3547bd 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -34,18 +34,18 @@ if (isset($_SERVER['REMOTE_ADDR'])) { require_once(__DIR__ . '/../../../../lib/clilib.php'); require_once(__DIR__ . '/../../../../lib/behat/lib.php'); - // CLI options. list($options, $unrecognized) = cli_get_params( array( 'help' => false, 'install' => false, 'parallel' => 0, - 'suffix' => '', + 'run' => '', 'drop' => false, 'enable' => false, 'disable' => false, - 'diag' => false + 'diag' => false, + 'tags' => '', ), array( 'h' => 'help' @@ -65,10 +65,9 @@ Options: --drop Drops the database tables and the dataroot contents --enable Enables test environment and updates tests list --disable Disables test environment ---parallel Run operation for all parallel behat environments. --diag Get behat test environment status code --h, --help Print out this help +-h, --help Print out this help Example from Moodle root directory: \$ php admin/tool/behat/cli/util.php --enable @@ -81,32 +80,16 @@ if (!empty($options['help'])) { exit(0); } - -if (!empty($options['parallel']) && empty($options['suffix'])) { - foreach ((array)glob(__DIR__."/../../../../behat*") as $dir) { - if (file_exists($dir) && is_dir($dir)) { - unlink($dir); - } - } - $cmds = array(); - $extra = preg_filter('#(.*)\s*--parallel=\d+\s*(.*?)#', '$1 $2', implode(' ', array_slice($argv, 1))); - for ($i = 1; $i <= $options['parallel']; $i++) { - $cmds[] = "php ".__FILE__." $extra --suffix=$i 2>&1"; - } - // This is intensive compared to behat itself so halve the parallelism. - foreach (array_chunk($cmds, min(1, floor($options['parallel']/2)), true) as $chunk) { - ns_parallel_popen($chunk, true); - } - exit(0); -} - - -// Checking $CFG->behat_* vars and values. +// Describe this script. define('BEHAT_UTIL', true); define('CLI_SCRIPT', true); define('NO_OUTPUT_BUFFERING', true); define('IGNORE_COMPONENT_CACHE', true); -define('BEHAT_SUFFIX', $options['suffix']); + +// Set run value, to be used by setup for configuring proper CFG variables. +if ($options['run']) { + define('BEHAT_CURRENT_RUN', $options['run']); +} // Only load CFG from config.php, stop ASAP in lib/setup.php. define('ABORT_AFTER_CONFIG', true); @@ -139,29 +122,69 @@ if ($unrecognized) { // Behat utilities. require_once($CFG->libdir . '/behat/classes/util.php'); require_once($CFG->libdir . '/behat/classes/behat_command.php'); +require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); + +// Ensure run option is <= parallel run installed. +if ($options['run']) { + if (!$options['parallel']) { + $options['parallel'] = behat_config_manager::get_parallel_test_runs(); + } + if (empty($options['parallel']) || $options['run'] > $options['parallel']) { + echo "Parallel runs can't be more then ".$options['parallel'].PHP_EOL; + exit(1); + } + $CFG->behatrunprocess = $options['run']; +} // Run command (only one per time). if ($options['install']) { behat_util::install_site(); - mtrace("Acceptance tests site installed"); + + // This is only displayed once for parallel install. + if (empty($options['run'])) { + mtrace("Acceptance tests site installed"); + } + } else if ($options['drop']) { // Ensure no tests are running. test_lock::acquire('behat'); behat_util::drop_site(); - mtrace("Acceptance tests site dropped"); + // This is only displayed once for parallel install. + if (empty($options['run'])) { + mtrace("Acceptance tests site dropped"); + } + } else if ($options['enable']) { behat_util::start_test_mode(); - $runtestscommand = behat_command::get_behat_command(true) . - ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); - mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:\n " . $runtestscommand); + + // This is only displayed once for parallel install. + if (empty($options['run'])) { + $runtestscommand = behat_command::get_behat_command(true, !empty($options['run'])); + + $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); + mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:\n " . $runtestscommand); + } else { + // Save parallel site info for enable and install options. + $filepath = behat_config_manager::get_parallel_test_file_path(); + if (!file_put_contents($filepath, $options['parallel'])) { + behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created'); + } + } + } else if ($options['disable']) { behat_util::stop_test_mode(); - mtrace("Acceptance tests environment disabled"); + // This is only displayed once for parallel install. + if (empty($options['run'])) { + mtrace("Acceptance tests environment disabled"); + } + } else if ($options['diag']) { $code = behat_util::get_behat_status(); exit($code); + } else { echo $help; + exit(1); } exit(0); diff --git a/admin/tool/behat/cli/utilparallel.php b/admin/tool/behat/cli/utilparallel.php new file mode 100644 index 00000000000..c47c5a5c90b --- /dev/null +++ b/admin/tool/behat/cli/utilparallel.php @@ -0,0 +1,195 @@ +. + +/** + * CLI tool with utilities to manage parallel Behat integration in Moodle + * + * All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as + * $CFG->dataroot and $CFG->prefix + * + * @package tool_behat + * @copyright 2015 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + + +if (isset($_SERVER['REMOTE_ADDR'])) { + die(); // No access from web!. +} + +define('BEHAT_UTIL', true); +define('CLI_SCRIPT', true); +define('NO_OUTPUT_BUFFERING', true); +define('IGNORE_COMPONENT_CACHE', true); + +require_once(__DIR__ . '/../../../../lib/clilib.php'); +require_once(__DIR__ . '/../../../../lib/behat/lib.php'); + +// CLI options. +list($options, $unrecognized) = cli_get_params( + array( + 'help' => false, + 'install' => false, + 'drop' => false, + 'enable' => false, + 'disable' => false, + 'diag' => false, + 'parallel' => 0, + 'maxruns' => false + ), + array( + 'h' => 'help', + 'j' => 'parallel', + 'm' => 'maxruns' + ) +); + +// Checking util.php CLI script usage. +$help = " +Behat utilities to manage the test environment + +Options: +--install Installs the test environment for acceptance tests +--drop Drops the database tables and the dataroot contents +--enable Enables test environment and updates tests list +--disable Disables test environment +--diag Get behat test environment status code +-j, --parallel Number of parallel behat run operation +-m, --maxruns Max parallel processes to be executed at one time. + +-h, --help Print out this help + +Example from Moodle root directory: +\$ php admin/tool/behat/cli/utilparallel.php --enable --parallel=4 + +More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests +"; + +if (!empty($options['help'])) { + echo $help; + exit(0); +} + +if (empty($options['parallel'])) { + echo $help; + exit(1); +} + +$status = 0; +$cmds = commands_to_execute($options); +$cwd = getcwd(); +chdir(__DIR__); + +// Start executing commands either sequential/parallel for options provided. +if ($options['diag'] || $options['drop'] || $options['enable'] || $options['disable']) { + $code = cli_execute_sequential($cmds, true); + // If any error then exit. + foreach ($code as $c) { + if ($c != 0) { + exit($c); + } + } +} else if ($options['install']) { + // This is intensive compared to behat itself so run them in chunk if $CFG->behat_max_parallel_init not set. + if ($options['maxruns']) { + foreach (array_chunk($cmds, maxruns, true) as $chunk) { + $chunkstatus = (bool)cli_execute_parallel($chunk, __DIR__, true, true); + $status = $chunkstatus || (bool) $status; + } + } else { + $status = (bool)cli_execute_parallel($cmds, __DIR__, true, true); + } +} else { + // We should never reach here. + echo $help; + exit(1); +} + +// Ensure we have success status to show following information. +if ($status) { + echo "Unknown failure $status".PHP_EOL; + exit((int)$status); +} + +// Only load CFG from config.php for 1st run amd stop ASAP in lib/setup.php. +define('ABORT_AFTER_CONFIG', true); +define('BEHAT_CURRENT_RUN', 1); +require_once(__DIR__ . '/../../../../config.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php'); + +// Remove first link from wwwroot, as it is set to first run. +$CFG->behat_wwwroot = str_replace('/'.BEHAT_PARALLEL_SITE_WWW_SUFFIX . '1', '', $CFG->behat_wwwroot); + +// Show command o/p (only one per time). +if ($options['install']) { + echo "Acceptance tests site installed for sites:".PHP_EOL; + // Display all sites which are installed/drop/diabled. + for ($i = 1; $i <= $options['parallel']; $i++ ) { + echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . PHP_EOL; + } +} else if ($options['drop']) { + echo "Acceptance tests site dropped for ".$options['parallel']." parallel sites".PHP_EOL; + +} else if ($options['enable']) { + echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:".PHP_EOL; + echo behat_command::get_behat_command(true, true); + echo PHP_EOL; + +} else if ($options['disable']) { + echo "Acceptance tests environment disabled for ".$options['parallel']." parallel sites".PHP_EOL; + +} else { + echo $help; +} + +chdir($cwd); +exit(0); + +/** + * Create commands to be executed for parallel run. + * + * @param array $options options provided by user. + * @return array commands to be executed. + */ +function commands_to_execute($options) { + $removeoptions = array('maxruns'); + $cmds = array(); + $extraoptions = $options; + $extra = ""; + + // Remove extra options not in util.php + foreach ($removeoptions as $ro) { + $extraoptions[$ro] = null; + unset($extraoptions[$ro]); + } + + foreach ($extraoptions as $option => $value) { + if ($options[$option]) { + $extra .= " --$option"; + if ($value) { + $extra .= "='$value'"; + } + } + } + + // Create commands which has to be executed for parallel site. + for ($i = 1; $i <= $options['parallel']; $i++) { + $prefix = BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i; + $cmds[$prefix] = "php util.php ".$extra." --run=".$i." 2>&1"; + } + return $cmds; +} \ No newline at end of file diff --git a/behat_pga_default.json b/behat_pga_default.json index dbce41d24e3..1f53315efc6 100644 --- a/behat_pga_default.json +++ b/behat_pga_default.json @@ -1,109 +1,283 @@ { - "course\/tests\/behat\/course_controls.feature": 117.2, - "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": 108.2, - "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": 92.8, - "course\/tests\/behat\/category_resort.feature": 75.4, - "course\/tests\/behat\/course_resort.feature": 58.9, - "admin\/tool\/behat\/tests\/behat\/data_generators.feature": 53.1, - "course\/tests\/behat\/category_management.feature": 50.1, - "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": 48.3, + "grade\/tests\/behat\/grade_aggregation.feature": 1675.3, + "grade\/tests\/behat\/grade_scales.feature": 909, + "grade\/tests\/behat\/grade_scales_aggregation.feature": 831.6, + "course\/tests\/behat\/course_category_management_listing.feature": 781.1, + "grade\/tests\/behat\/grade_single_item_scales.feature": 730.8, + "grade\/tests\/behat\/grade_calculated_weights.feature": 729.7, + "admin\/tool\/monitor\/tests\/behat\/subscription.feature": 628.5, + "course\/tests\/behat\/course_controls.feature": 579.5, + "admin\/tool\/monitor\/tests\/behat\/rule.feature": 387.4, + "mod\/forum\/tests\/behat\/discussion_navigation.feature": 382.5, + "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": 372, + "mod\/forum\/tests\/behat\/track_read_posts.feature": 367.2, + "mod\/forum\/tests\/behat\/discussion_display.feature": 357.4, + "badges\/tests\/behat\/award_badge.feature": 345, + "grade\/tests\/behat\/grade_natural_normalisation.feature": 330, + "grade\/tests\/behat\/grade_contribution_with_extra_credit.feature": 307.7, + "group\/tests\/behat\/update_groups.feature": 284.3, + "course\/tests\/behat\/course_resort.feature": 280.7, + "mod\/glossary\/tests\/behat\/search_entries.feature": 258.6, + "mod\/lesson\/tests\/behat\/link_to_gradebook.feature": 240.6, + "report\/outline\/tests\/behat\/outline.feature": 236.8, + "mod\/wiki\/tests\/behat\/wiki_search.feature": 227.9, + "grade\/tests\/behat\/grade_point_maximum.feature": 217.4, + "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": 215.9, + "grade\/tests\/behat\/grade_view.feature": 215.9, + "blocks\/navigation\/tests\/behat\/expand_courses_node.feature": 204.1, + "report\/participation\/tests\/behat\/filter_participation.feature": 197.9, + "report\/loglive\/tests\/behat\/loglive_report.feature": 196.9, + "course\/tests\/behat\/category_resort.feature": 196.9, + "mod\/workshop\/tests\/behat\/workshop_assessment.feature": 190.4, + "mod\/assign\/tests\/behat\/outcome_grading.feature": 186.5, + "mod\/forum\/tests\/behat\/edit_post_student.feature": 184, + "mod\/forum\/tests\/behat\/edit_post_teacher.feature": 182.7, + "mod\/assign\/tests\/behat\/quickgrading.feature": 178.5, + "completion\/tests\/behat\/restrict_section_availability.feature": 169.9, + "report\/outline\/tests\/behat\/user.feature": 167.4, + "availability\/tests\/behat\/edit_availability.feature": 166.8, + "course\/tests\/behat\/section_highlighting.feature": 162.5, + "mod\/assign\/tests\/behat\/prevent_submission_changes.feature": 161.1, + "mod\/wiki\/tests\/behat\/wiki_comments.feature": 155.8, + "mod\/assign\/feedback\/editpdf\/tests\/behat\/annotate_pdf.feature": 155.6, + "mod\/lesson\/tests\/behat\/lesson_practice.feature": 151.2, + "availability\/tests\/behat\/display_availability.feature": 150.6, + "mod\/assign\/tests\/behat\/grading_status.feature": 150.5, + "mod\/quiz\/tests\/behat\/editing_set_marks_with_attempts.feature": 148.9, + "cohort\/tests\/behat\/upload_cohorts.feature": 146.4, + "group\/tests\/behat\/delete_groups.feature": 143.8, + "group\/tests\/behat\/groups_import.feature": 143.6, + "mod\/lesson\/tests\/behat\/lesson_edit_pages.feature": 139.6, + "group\/tests\/behat\/auto_creation.feature": 136.6, + "course\/tests\/behat\/category_management.feature": 134.5, + "mod\/quiz\/tests\/behat\/editing_add.feature": 132.8, + "admin\/tool\/behat\/tests\/behat\/data_generators.feature": 131.7, + "mod\/choice\/tests\/behat\/publish_results.feature": 130.7, + "group\/tests\/behat\/create_groups.feature": 128.1, + "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": 126.7, + "availability\/condition\/profile\/tests\/behat\/availability_profile.feature": 124.6, + "grade\/tests\/behat\/grade_mingrade.feature": 123.8, + "mod\/lesson\/tests\/behat\/lesson_number_of_student_attempts.feature": 122.5, + "backup\/util\/ui\/tests\/behat\/backup_courses.feature": 122, + "mod\/lesson\/tests\/behat\/lesson_navigation.feature": 121.7, + "mod\/lesson\/tests\/behat\/lesson_with_clusters.feature": 119.9, + "blocks\/comments\/tests\/behat\/add_comment.feature": 118.8, + "mod\/forum\/tests\/behat\/separate_group_discussions.feature": 117.3, + "completion\/tests\/behat\/restrict_activity_by_date.feature": 117.2, + "lib\/editor\/tinymce\/tests\/behat\/edit_available_icons.feature": 113, + "mod\/glossary\/tests\/behat\/entries_always_editable.feature": 112.3, + "report\/log\/tests\/behat\/user_log.feature": 111.4, + "course\/tests\/behat\/activities_edit_completion.feature": 111.4, + "report\/log\/tests\/behat\/filter_log.feature": 108.2, + "mod\/assign\/tests\/behat\/group_submission.feature": 108.1, + "mod\/glossary\/tests\/behat\/categories.feature": 107.6, + "mod\/quiz\/tests\/behat\/editing_click_delete_icon.feature": 107.6, + "availability\/condition\/grade\/tests\/behat\/availability_grade.feature": 107.1, + "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": 106, + "course\/tests\/behat\/category_change_visibility.feature": 105.5, + "mod\/wiki\/tests\/behat\/collaborative_individual.feature": 104, + "course\/tests\/behat\/max_number_sections.feature": 104, + "course\/tests\/behat\/activities_visibility_icons.feature": 102.3, + "admin\/tool\/behat\/tests\/behat\/edit_permissions.feature": 100.4, + "mod\/lesson\/tests\/behat\/time_limit.feature": 98.7, + "mod\/quiz\/tests\/behat\/editing_set_marks_no_attempts.feature": 97, + "admin\/tests\/behat\/filter_users.feature": 94.4, + "mod\/forum\/tests\/behat\/add_forum.feature": 92.8, + "user\/tests\/behat\/delete_users.feature": 92, + "grade\/tests\/behat\/grade_UI_settings.feature": 91.9, + "completion\/tests\/behat\/restrict_activity_by_grade.feature": 91.9, + "enrol\/self\/tests\/behat\/self_enrolment.feature": 91.5, + "blocks\/navigation\/tests\/behat\/view_my_courses.feature": 91.3, + "question\/tests\/behat\/preview_question.feature": 90.9, + "availability\/condition\/group\/tests\/behat\/availability_group.feature": 89.3, + "blog\/tests\/behat\/comment.feature": 85.9, + "mod\/forum\/tests\/behat\/forum_subscriptions_availability.feature": 85.8, + "mod\/lesson\/tests\/behat\/lesson_edit_cluster.feature": 85.6, + "course\/tests\/behat\/section_visibility.feature": 84.9, + "question\/tests\/behat\/question_categories.feature": 84.6, + "question\/format\/xml\/tests\/behat\/import_export.feature": 83.5, + "question\/tests\/behat\/delete_questions.feature": 82.5, + "mod\/forum\/tests\/behat\/completion_condition_number_discussions.feature": 82.1, + "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": 82, + "question\/tests\/behat\/sort_questions.feature": 81.6, + "group\/tests\/behat\/id_uniqueness.feature": 80, + "admin\/tool\/filetypes\/tests\/behat\/add_filetypes.feature": 79.5, + "mod\/lesson\/tests\/behat\/date_availability.feature": 78.1, + "cohort\/tests\/behat\/add_cohort.feature": 76.8, + "grade\/report\/singleview\/tests\/behat\/singleview.feature": 76.5, + "mod\/wiki\/tests\/behat\/wiki_formats.feature": 75.8, + "course\/tests\/behat\/force_group_mode.feature": 75.5, + "blocks\/news_items\/tests\/behat\/display_news.feature": 75.1, + "course\/tests\/behat\/frontpage_display_modes.feature": 74.9, + "mod\/assign\/tests\/behat\/edit_previous_feedback.feature": 74.1, + "availability\/condition\/grouping\/tests\/behat\/availability_grouping.feature": 73.1, + "course\/tests\/behat\/restrict_available_activities.feature": 72.9, + "mod\/assign\/feedback\/editpdf\/tests\/behat\/group_annotations.feature": 72.2, + "mod\/lesson\/tests\/behat\/questions_images.feature": 72, + "mod\/lesson\/tests\/behat\/completion_condition_end_reached.feature": 72, + "mod\/forum\/tests\/behat\/single_forum_discussion.feature": 71, + "mod\/lesson\/tests\/behat\/lesson_review.feature": 70.5, + "mod\/quiz\/tests\/behat\/editing_repaginate.feature": 70.2, + "course\/tests\/behat\/paged_course_navigation.feature": 69.3, + "repository\/tests\/behat\/create_shortcut.feature": 69.1, + "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": 67, + "mod\/choice\/tests\/behat\/publish_results_anonymously.feature": 66.9, + "mod\/book\/tests\/behat\/show_hide_chapters.feature": 65.9, + "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": 65.1, + "cohort\/tests\/behat\/access_visible_cohorts.feature": 64.9, + "user\/tests\/behat\/view_full_profile.feature": 64.3, + "course\/tests\/behat\/course_creation.feature": 64, + "mod\/glossary\/tests\/behat\/print_friendly_version.feature": 64, + "grade\/export\/txt\/tests\/behat\/export.feature": 63, + "question\/tests\/behat\/copy_questions.feature": 62.8, + "lib\/editor\/atto\/plugins\/accessibilitychecker\/tests\/behat\/accessibilitychecker.feature": 62.1, + "mod\/quiz\/tests\/behat\/settings_form_fields_disableif.feature": 61.7, + "admin\/tool\/uploadcourse\/tests\/behat\/create.feature": 61.6, + "badges\/tests\/behat\/add_badge.feature": 61.4, + "mod\/assign\/tests\/behat\/allow_another_attempt.feature": 61.1, + "completion\/tests\/behat\/enable_manual_complete_mark.feature": 60.5, + "mod\/wiki\/tests\/behat\/page_history.feature": 59.9, + "cohort\/tests\/behat\/upload_cohort_users.feature": 58.8, + "mod\/data\/tests\/behat\/add_entries.feature": 58.2, + "mod\/feedback\/tests\/behat\/show_nonrespondents.feature": 58, + "mod\/lesson\/tests\/behat\/lesson_progress_bar.feature": 57.5, + "mod\/lesson\/tests\/behat\/lesson_essay_question.feature": 57.1, + "blocks\/tests\/behat\/restrict_available_blocks.feature": 56.8, + "repository\/tests\/behat\/overwrite_file.feature": 56.6, + "course\/tests\/behat\/add_activities.feature": 56.5, + "availability\/condition\/date\/tests\/behat\/availability_date.feature": 56.2, + "mod\/choice\/tests\/behat\/multiple_options.feature": 55.4, + "grade\/report\/history\/tests\/behat\/basic_functionality.feature": 55.3, + "enrol\/guest\/tests\/behat\/guest_access.feature": 55.1, + "mod\/assign\/tests\/behat\/comment_inline.feature": 54.9, + "blocks\/tests\/behat\/hidden_block_region.feature": 54.3, + "mod\/quiz\/tests\/behat\/add_quiz.feature": 53.5, + "cohort\/tests\/behat\/view_cohorts.feature": 52.8, + "admin\/tool\/availabilityconditions\/tests\/behat\/manage_conditions.feature": 52.8, + "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": 51.9, + "mod\/assign\/tests\/behat\/display_grade.feature": 51.6, + "mod\/lesson\/tests\/behat\/teacher_grade_essays.feature": 51.5, + "message\/tests\/behat\/send_message.feature": 51.1, + "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": 50.9, + "availability\/condition\/completion\/tests\/behat\/availability_completion.feature": 50.5, + "mod\/lesson\/tests\/behat\/password_protection.feature": 49.6, + "admin\/tool\/behat\/tests\/behat\/basic_actions.feature": 49.5, + "mod\/quiz\/tests\/behat\/editing_click_move_icon.feature": 48.4, + "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": 48.2, + "mod\/assign\/tests\/behat\/submission_comments.feature": 47.8, + "files\/tests\/behat\/course_files.feature": 47.5, + "admin\/tool\/behat\/tests\/behat\/list_steps.feature": 47.4, "grade\/tests\/behat\/grade_override_letter.feature": 47.3, - "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": 42.7, - "calendar\/tests\/behat\/calendar.feature": 42, - "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": 41.2, - "mod\/forum\/tests\/behat\/forum_subscriptions.feature": 32.6, - "course\/tests\/behat\/move_activities.feature": 30.8, - "mod\/workshep\/tests\/behat\/workshep_assessment.feature": 29.3, - "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": 29, - "mod\/workshop\/tests\/behat\/workshop_assessment.feature": 28.8, - "blocks\/tests\/behat\/manage_blocks.feature": 28.6, - "badges\/tests\/behat\/navrequirecap.feature": 28, - "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": 26.9, - "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": 26, - "blocks\/html\/tests\/behat\/multiple_instances.feature": 26, - "message\/tests\/behat\/display_history.feature": 24.7, - "course\/tests\/behat\/category_change_visibility.feature": 24.2, - "local\/uneditableblocks\/tests\/behat\/enable_uneditableblocks.feature": 23.1, - "mod\/feedback\/tests\/behat\/defaultshortanswerlength.feature": 22.1, - "backup\/util\/ui\/tests\/behat\/backup_courses.feature": 22, - "course\/tests\/behat\/create_delete_course.feature": 21.9, - "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": 21.2, - "course\/tests\/behat\/move_sections.feature": 21.1, - "course\/tests\/behat\/course_category_management_listing.feature": 21, - "local\/userpolicy\/tests\/behat\/fieldvisibility.feature": 20.9, - "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": 20.5, - "mod\/wiki\/tests\/behat\/edit_tags.feature": 19.6, - "mod\/attendance\/tests\/behat\/attendance_mod.feature": 19.5, - "blocks\/participants\/tests\/behat\/block_participants_course.feature": 19.5, - "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": 17.7, - "blocks\/tests\/behat\/configure_block_throughout_site.feature": 17.7, - "grade\/report\/singleview\/tests\/behat\/singleview.feature": 17.6, - "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": 16.3, - "mod\/wiki\/tests\/behat\/group_enhancements.feature": 16.2, - "admin\/tests\/behat\/forcelogin_makefrontpagepublic.feature": 16, - "message\/tests\/behat\/disablenotifications.feature": 16, - "completion\/tests\/behat\/teacher_manual_completion.feature": 16, - "mod\/glossary\/tests\/behat\/entries_require_approval.feature": 16, - "backup\/util\/ui\/tests\/behat\/import_course.feature": 15.9, - "admin\/tests\/behat\/set_admin_settings_value.feature": 15.8, - "mod\/mediagallery\/tests\/behat\/separategroups.feature": 15.6, - "blocks\/tests\/behat\/return_block_original_state.feature": 15.6, - "admin\/tests\/behat\/custom_maxbytes.feature": 15.4, - "mod\/oublog\/tests\/behat\/separate_individuals.feature": 15.4, - "mod\/forum\/tests\/behat\/default_displaywordcount.feature": 15.4, - "backup\/util\/ui\/tests\/behat\/restore_moodle2_course_numsections.feature": 14.9, - "question\/tests\/behat\/question_defaultpenalty.feature": 14.8, - "availability\/tests\/behat\/edit_availability.feature": 14.7, - "mod\/data\/tests\/behat\/view_entries.feature": 14.6, - "local\/catdelete\/tests\/behat\/catdelete.feature": 14.3, - "group\/tests\/behat\/showusernameingroup.feature": 14.3, - "mod\/forum\/tests\/behat\/edit_post_student.feature": 14.2, - "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": 14.1, - "mod\/hsuforum\/tests\/behat\/edit_post_student.feature": 13.8, - "mod\/quiz\/tests\/behat\/add_quiz.feature": 13.5, - "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": 13.5, - "blocks\/autocreate_user\/tests\/behat\/add_introtext.feature": 13.5, - "question\/type\/truefalse\/tests\/behat\/custompenalty.feature": 13.3, - "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": 13.1, - "blocks\/html\/tests\/behat\/course_block.feature": 12.9, - "mod\/quiz\/tests\/behat\/configsubnethide.feature": 12.8, - "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": 12.6, - "blocks\/html\/tests\/behat\/configuring_html_block.feature": 12.3, - "course\/tests\/behat\/course_change_visibility.feature": 12.2, - "course\/tests\/behat\/modchooser_hidden.feature": 11.9, - "blocks\/login\/tests\/behat\/login_block.feature": 11.7, - "course\/tests\/behat\/paged_course_navigation.feature": 11.1, - "course\/tests\/behat\/limitrolerenaming.feature": 11, - "mod\/book\/tests\/behat\/create_chapters.feature": 11, - "mod\/forum\/tests\/behat\/my_forum_posts.feature": 10.7, - "auth\/tests\/behat\/login.feature": 10.5, - "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": 10.5, - "mod\/data\/tests\/behat\/add_entries.feature": 10.1, - "mod\/wiki\/tests\/behat\/preview_page.feature": 9.7, - "mod\/oublog\/tests\/behat\/personalblog.feature": 9.7, - "admin\/tests\/behat\/display_short_names.feature": 9.6, - "mod\/forum\/tests\/behat\/separate_group_discussions.feature": 9.5, - "course\/tests\/behat\/add_activities.feature": 9.4, - "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": 8.8, - "blocks\/tests\/behat\/restrict_available_blocks.feature": 8.7, - "blocks\/comments\/tests\/behat\/add_comment.feature": 8.7, - "mod\/survey\/tests\/behat\/survey_types.feature": 8.5, - "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": 7.3, - "local\/rolerenaming\/tests\/behat\/course_role_renaming.feature": 7.2, - "course\/tests\/behat\/edit_settings.feature": 7.1, - "group\/tests\/behat\/create_groups.feature": 6.8, - "blocks\/tests\/behat\/add_blocks.feature": 6.5, - "message\/tests\/behat\/manage_contacts.feature": 6.1, - "course\/tests\/behat\/course_search.feature": 6, - "course\/tests\/behat\/course_creation.feature": 5.8, - "my\/tests\/behat\/reset_page.feature": 5.4, - "my\/tests\/behat\/restrict_available_blocks.feature": 5.3, - "mod\/oublog\/tests\/behat\/basic.feature": 5.2, + "mod\/assign\/tests\/behat\/file_submission.feature": 46.6, + "mod\/wiki\/tests\/behat\/preview_page.feature": 46.5, + "mod\/lesson\/tests\/behat\/lesson_informations_at_end.feature": 46.4, + "repository\/tests\/behat\/zip_and_unzip.feature": 46.1, + "backup\/util\/ui\/tests\/behat\/import_course.feature": 45.5, + "course\/tests\/behat\/category_role_assignment.feature": 44.6, + "user\/tests\/behat\/table_sorting.feature": 44.5, + "course\/format\/social\/tests\/behat\/social_adjust_discussion_count.feature": 44.3, + "mod\/choice\/tests\/behat\/change_response.feature": 44.1, + "mod\/book\/tests\/behat\/log_entries.feature": 43.8, + "lib\/editor\/atto\/plugins\/align\/tests\/behat\/align.feature": 42.8, + "mod\/assign\/tests\/behat\/online_submissions.feature": 42.7, + "mod\/assign\/tests\/behat\/filter_by_marker.feature": 41.7, + "lib\/editor\/atto\/plugins\/image\/tests\/behat\/image.feature": 41.3, + "mod\/assign\/tests\/behat\/grant_extension.feature": 41.2, + "mod\/label\/tests\/behat\/label_visibility.feature": 40.9, + "mod\/forum\/tests\/behat\/forum_subscriptions.feature": 40.2, + "course\/tests\/behat\/move_activities.feature": 40.2, + "message\/tests\/behat\/manage_contacts.feature": 39.1, + "my\/tests\/behat\/restrict_available_blocks.feature": 38.3, + "report\/eventlist\/tests\/behat\/mainsection.feature": 37.5, + "blocks\/comments\/tests\/behat\/delete_comment.feature": 37.2, + "calendar\/tests\/behat\/calendar.feature": 37, + "backup\/util\/ui\/tests\/behat\/duplicate_activities.feature": 37, + "question\/tests\/behat\/edit_questions.feature": 36.5, + "repository\/recent\/tests\/behat\/add_recent.feature": 36.2, + "mod\/lesson\/tests\/behat\/import_fillintheblank_question.feature": 35.7, + "course\/tests\/behat\/activities_group_icons.feature": 35.2, + "admin\/tests\/behat\/upload_users.feature": 34.3, + "course\/tests\/behat\/course_change_visibility.feature": 32.9, + "blocks\/tests\/behat\/return_block_original_state.feature": 32.5, + "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": 31.9, + "mod\/glossary\/tests\/behat\/prevent_duplicate_entries.feature": 31.4, + "admin\/tool\/behat\/tests\/behat\/manipulate_forms.feature": 31.1, + "course\/tests\/behat\/course_search.feature": 30.7, + "mod\/lesson\/tests\/behat\/import_images.feature": 30.7, + "lib\/editor\/atto\/plugins\/bold\/tests\/behat\/bold.feature": 30.5, + "mod\/choice\/tests\/behat\/limit_responses.feature": 30.4, + "course\/tests\/behat\/activities_edit_with_block_dock.feature": 30.4, + "lib\/editor\/atto\/plugins\/indent\/tests\/behat\/indent.feature": 30.1, + "lib\/editor\/atto\/plugins\/italic\/tests\/behat\/italic.feature": 29.7, + "lib\/editor\/atto\/plugins\/equation\/tests\/behat\/equation.feature": 29.6, + "course\/tests\/behat\/edit_settings.feature": 29.5, + "mod\/scorm\/tests\/behat\/add_scorm.feature": 29.1, + "repository\/tests\/behat\/delete_files.feature": 28.7, + "admin\/tool\/uploadcourse\/tests\/behat\/update.feature": 28.7, + "course\/tests\/behat\/activities_indentation.feature": 28.4, + "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": 28.1, + "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": 27.1, + "repository\/tests\/behat\/cancel_add_file.feature": 26, + "blocks\/html\/tests\/behat\/configuring_html_block.feature": 25.8, + "lib\/editor\/atto\/plugins\/table\/tests\/behat\/table.feature": 24.1, + "blocks\/tests\/behat\/add_blocks.feature": 24, + "course\/tests\/behat\/rename_roles.feature": 23.7, + "blocks\/tests\/behat\/manage_blocks.feature": 23.7, + "message\/tests\/behat\/block_users.feature": 23.7, + "course\/tests\/behat\/create_delete_course.feature": 23.6, + "message\/tests\/behat\/message_participants.feature": 23.6, + "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": 23.4, + "message\/tests\/behat\/display_history.feature": 23, + "mod\/choice\/tests\/behat\/add_choice.feature": 23, + "question\/format\/gift\/tests\/behat\/import_export.feature": 22.9, + "message\/tests\/behat\/search_history.feature": 22.8, + "repository\/tests\/behat\/create_folders.feature": 22.3, + "completion\/tests\/behat\/teacher_manual_completion.feature": 22.3, + "mod\/glossary\/tests\/behat\/entries_require_approval.feature": 21.7, + "mod\/wiki\/tests\/behat\/edit_tags.feature": 21, + "lib\/editor\/atto\/plugins\/link\/tests\/behat\/link.feature": 20.9, + "course\/tests\/behat\/move_sections.feature": 20, + "question\/format\/webct\/tests\/behat\/import.feature": 19.9, + "lib\/editor\/atto\/plugins\/accessibilityhelper\/tests\/behat\/accessibilityhelper.feature": 19.7, + "question\/format\/webct\/tests\/behat\/importcalculated.feature": 19.6, + "lib\/editor\/atto\/plugins\/media\/tests\/behat\/media.feature": 19.5, + "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": 18.2, + "blocks\/participants\/tests\/behat\/block_participants_course.feature": 17.4, + "admin\/tests\/behat\/set_admin_settings_value.feature": 16.8, + "lib\/editor\/atto\/plugins\/clear\/tests\/behat\/clear.feature": 16.7, + "admin\/tool\/monitor\/tests\/behat\/disabled.feature": 16.4, + "blocks\/html\/tests\/behat\/multiple_instances.feature": 15.6, + "grade\/export\/xml\/tests\/behat\/export.feature": 15.4, + "auth\/tests\/behat\/login.feature": 14.9, + "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": 14.8, + "lib\/editor\/atto\/plugins\/charmap\/tests\/behat\/charmap.feature": 14.4, + "lib\/editor\/atto\/plugins\/unorderedlist\/tests\/behat\/unorderedlist.feature": 14.4, + "lib\/editor\/atto\/plugins\/strike\/tests\/behat\/strike.feature": 14, + "blocks\/tests\/behat\/configure_block_throughout_site.feature": 14, + "lib\/editor\/atto\/plugins\/superscript\/tests\/behat\/superscript.feature": 13.9, + "lib\/editor\/atto\/plugins\/title\/tests\/behat\/title.feature": 13.8, + "lib\/editor\/atto\/plugins\/subscript\/tests\/behat\/subscript.feature": 13.8, + "lib\/editor\/atto\/plugins\/underline\/tests\/behat\/underline.feature": 13.7, + "lib\/editor\/atto\/plugins\/orderedlist\/tests\/behat\/orderedlist.feature": 13.3, + "user\/tests\/behat\/edituserpassword.feature": 12.7, + "mod\/data\/tests\/behat\/view_entries.feature": 12.4, + "repository\/upload\/tests\/behat\/upload_file.feature": 12.2, + "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": 9.4, + "lib\/editor\/atto\/plugins\/html\/tests\/behat\/html.feature": 9.2, + "mod\/forum\/tests\/behat\/my_forum_posts.feature": 9.1, + "mod\/book\/tests\/behat\/create_chapters.feature": 8.8, + "blocks\/html\/tests\/behat\/course_block.feature": 8.6, + "lib\/editor\/atto\/plugins\/collapse\/tests\/behat\/collapse.feature": 8.5, + "blocks\/login\/tests\/behat\/login_block.feature": 8.4, + "admin\/tests\/behat\/display_short_names.feature": 7.8, + "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": 7.1, + "mod\/survey\/tests\/behat\/survey_types.feature": 6.8, + "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": 6, + "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": 5.6, + "my\/tests\/behat\/reset_page.feature": 5.3, "my\/tests\/behat\/add_blocks.feature": 4.6, - "user\/tests\/behat\/reset_page.feature": 4, - "message\/tests\/behat\/search_history.feature": 3.9, - "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": 3.3, - "user\/tests\/behat\/add_blocks.feature": 2.8, - "report\/usersessions\/tests\/behat\/usersessions_report.feature": 2.7, - "admin\/tool\/behat\/tests\/behat\/test_environment.feature": 1 + "user\/tests\/behat\/reset_page.feature": 4.2, + "user\/tests\/behat\/add_blocks.feature": 4.1, + "report\/usersessions\/tests\/behat\/usersessions_report.feature": 2.8, + "admin\/tool\/behat\/tests\/behat\/test_environment.feature": 1.5 } \ No newline at end of file diff --git a/config-dist.php b/config-dist.php index fc594063539..648a4306eb2 100644 --- a/config-dist.php +++ b/config-dist.php @@ -724,6 +724,21 @@ $CFG->admin = 'admin'; // Example: // $CFG->behat_faildump_path = '/my/path/to/save/failure/dumps'; // +// You can specify db, selenium wd_host etc. for behat parallel run by setting following variable. +// Example: +// $CFG->behat_parallel_run = array ( +// array ( +// 'dbtype' => 'mysqli', +// 'dblibrary' => 'native', +// 'dbhost' => 'localhost', +// 'dbname' => 'moodletest', +// 'dbuser' => 'moodle', +// 'dbpass' => 'moodle', +// 'behat_prefix' => 'mdl_', +// 'wd_host' => 'http://127.0.0.1:4444/wd/hub' +// ), +// ); +// //========================================================================= // 12. DEVELOPER DATA GENERATOR //========================================================================= diff --git a/lib/behat/classes/behat_command.php b/lib/behat/classes/behat_command.php index a7060bd70d4..73e6aec02da 100644 --- a/lib/behat/classes/behat_command.php +++ b/lib/behat/classes/behat_command.php @@ -44,12 +44,15 @@ class behat_command { /** * Ensures the behat dir exists in moodledata + * @param int $runprocess run process for which behat dir is returned. * @return string Full path */ - public static function get_behat_dir() { + public static function get_behat_dir($runprocess = 0) { global $CFG; - $behatdir = $CFG->behat_dataroot . '/behat'; + $runprocess = empty($runprocess) ? "" : $runprocess; + + $behatdir = $CFG->behat_dataroot . $runprocess . '/behat'; if (!is_dir($behatdir)) { if (!mkdir($behatdir, $CFG->directorypermissions, true)) { @@ -73,23 +76,29 @@ class behat_command { * normal cmd.exe (in Windows). * * @param bool $custombyterm If the provided command should depend on the terminal where it runs + * @param bool $parallelrun If parallel run is installed. * @return string */ - public final static function get_behat_command($custombyterm = false) { + public final static function get_behat_command($custombyterm = false, $parallerun = false) { $separator = DIRECTORY_SEPARATOR; - $exec = 'behat'; + if (!$parallerun) { + $exec = 'behat'; - // Cygwin uses linux-style directory separators. - if ($custombyterm && testing_is_cygwin()) { - $separator = '/'; + // Cygwin uses linux-style directory separators. + if ($custombyterm && testing_is_cygwin()) { + $separator = '/'; - // MinGW can not execute .bat scripts. - if (!testing_is_mingw()) { - $exec = 'behat.bat'; + // MinGW can not execute .bat scripts. + if (!testing_is_mingw()) { + $exec = 'behat.bat'; + } } + $command = 'vendor' . $separator . 'bin' . $separator . $exec; + } else { + $command = 'php admin' . $separator . 'tool' . $separator . 'behat' . $separator . 'cli' . $separator . 'run.php'; } - return 'vendor' . $separator . 'bin' . $separator . $exec; + return $command; } /** diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index b6f97154999..c5cce499a01 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -53,9 +53,10 @@ class behat_config_manager { * * @param string $component Restricts the obtained steps definitions to the specified component * @param string $testsrunner If the config file will be used to run tests + * @param string $tags features files including tags. * @return void */ - public static function update_config_file($component = '', $testsrunner = true) { + public static function update_config_file($component = '', $testsrunner = true, $tags = '') { global $CFG; // Behat must have a separate behat.yml to have access to the whole set of features and steps definitions. @@ -108,7 +109,7 @@ class behat_config_manager { // Behat config file specifing the main context class, // the required Behat extensions and Moodle test wwwroot. - $contents = self::get_config_file_contents($features, $stepsdefinitions); + $contents = self::get_config_file_contents(self::get_fetaures_with_tags($features, $tags), $stepsdefinitions); // Stores the file. if (!file_put_contents($configfilepath, $contents)) { @@ -117,6 +118,40 @@ class behat_config_manager { } + /** + * Search feature files for set of tags. + * + * @param array $features set of feature files. + * @param string $tags list of tags (currently support && only.) + * @return array filtered list of feature files with tags. + */ + public static function get_fetaures_with_tags($features, $tags) { + if (empty($tags)) { + return $features; + } + $newfeaturelist = array(); + $tagstosearch = explode('&&', $tags); + foreach ($features as $featurefile) { + $contents = file_get_contents($featurefile); + $includefeature = true; + foreach ($tagstosearch as $tag) { + // If negitive tag, then ensure it don't exist. + if (strpos($tag, '~') !== false) { + $tag = substr($tag, 1); + if ($contents && strpos($contents, $tag) !== false) { + $includefeature = false; + } + } else if ($contents && strpos($contents, $tag) === false) { + $includefeature = false; + } + } + if ($includefeature) { + $newfeaturelist[] = $featurefile; + } + } + return $newfeaturelist; + } + /** * Gets the list of Moodle steps definitions * @@ -172,12 +207,18 @@ class behat_config_manager { /** * Returns the behat config file path used by the behat cli command. * + * @param int $runprocess Runprocess. * @return string */ - public static function get_behat_cli_config_filepath() { + public static function get_behat_cli_config_filepath($runprocess = 0) { global $CFG; - $command = $CFG->behat_dataroot . DIRECTORY_SEPARATOR . 'behat' . DIRECTORY_SEPARATOR . 'behat.yml'; + if ($runprocess) { + $command = $CFG->behat_dataroot . $runprocess; + } else { + $command = $CFG->behat_dataroot; + } + $command .= DIRECTORY_SEPARATOR . 'behat' . DIRECTORY_SEPARATOR . 'behat.yml'; // Cygwin uses linux-style directory separators. if (testing_is_cygwin()) { @@ -187,6 +228,92 @@ class behat_config_manager { return $command; } + /** + * Returns the path to the parallel run file which specifies if parallel test environment is enabled + * and how many parallel runs to execute. + * + * @param int $runprocess run process for which behat dir is returned. + * @return string + */ + public final static function get_parallel_test_file_path($runprocess = 0) { + return behat_command::get_behat_dir($runprocess) . '/parallel_environment_enabled.txt'; + } + + /** + * Returns number of parallel runs for which site is initialised. + * + * @param int $runprocess run process for which behat dir is returned. + * @return int + */ + public final static function get_parallel_test_runs($runprocess = 0) { + $parallelrun = 0; + // Get parallel run info from first file and last file. + $parallelrunconfigfile = self::get_parallel_test_file_path($runprocess); + if (file_exists($parallelrunconfigfile)) { + if ($parallel = file_get_contents($parallelrunconfigfile)) { + $parallelrun = (int) $parallel; + } + } + + return $parallelrun; + } + + /** + * Drops parallel site links. + * + * @return bool true on success else false. + */ + public final static function drop_parallel_site_links() { + global $CFG; + + // Get parallel test runs from first run. + $parallelrun = self::get_parallel_test_runs(1); + + if (empty($parallelrun)) { + return false; + } + + // If parallel run then remove links and original file. + clearstatcache(); + for ($i = 1; $i <= $parallelrun; $i++) { + $link = $CFG->dirroot . '/' . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i; + if (file_exists($link) && is_link($link)) { + @unlink($link); + } + } + return true; + } + + /** + * Create parallel site links. + * + * @return bool true for sucess, else false. + */ + public final static function create_parallel_site_links() { + global $CFG; + + // Get parallel test runs from first run. + $parallelrun = self::get_parallel_test_runs(1); + + // Create site symlink if necessary. + clearstatcache(); + for ($i = 1; $i <= $parallelrun; $i++) { + $link = $CFG->dirroot.'/'.BEHAT_PARALLEL_SITE_WWW_SUFFIX.$i; + clearstatcache(); + if (file_exists($link)) { + if (!is_link($link) || !is_dir($link)) { + echo "File exists at link location ($link) but is not a link or directory!\n"; + return false; + } + } else if (!symlink($CFG->dirroot, $link)) { + // Try create link in case it's not already present. + echo "Unable to create behat site symlink ($link)\n"; + return false; + } + } + return true; + } + /** * Behat config file specifing the main context class, * the required Behat extensions and Moodle test wwwroot. @@ -201,27 +328,26 @@ class behat_config_manager { // We require here when we are sure behat dependencies are available. require_once($CFG->dirroot . '/vendor/autoload.php'); - $instance = 1; - $parallel = 0; - foreach ($_SERVER['argv'] as $arg) { - if (strpos($arg, '--suffix=') === 0) { - $instance = intval(substr($arg, strlen('--suffix='))); - } - if (empty($parallel)) { - $parallel = preg_filter('#--parallel=(\d+)#', '$1', $arg); - } - } + $selenium2wdhost = array('wd_host' => 'http://localhost:4444/wd/hub'); - // Attempt to split into weighted buckets using timing information, if available. - if ($alloc = self::profile_guided_allocate($features, max(1, $parallel), $instance)) { - $features = $alloc; - } else { - // Divide the list of feature files amongst the parallel runners. - srand(crc32(floor(time() / 3600 / 24).var_export($features,true))); - shuffle($features); - // Pull out the features for just this worker. - $features = array_chunk($features, ceil(count($features) / max(1, $parallel))); - $features = $features[$instance-1]; + $parallelruns = self::get_parallel_test_runs(); + // If parallel run, then only divide features. + if (!empty($CFG->behatrunprocess) && !empty($parallelruns)) { + // Attempt to split into weighted buckets using timing information, if available. + if ($alloc = self::profile_guided_allocate($features, max(1, $parallelruns), $CFG->behatrunprocess)) { + $features = $alloc; + } else { + // Divide the list of feature files amongst the parallel runners. + srand(crc32(floor(time() / 3600 / 24) . var_export($features, true))); + shuffle($features); + // Pull out the features for just this worker. + $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); + $features = $features[$CFG->behatrunprocess - 1]; + } + // Set proper selenium2 wd_host if defined. + if (!empty($CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host'])) { + $selenium2wdhost = array('wd_host' => $CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host']); + } } // It is possible that it has no value as we don't require a full behat setup to list the step definitions. @@ -230,6 +356,7 @@ class behat_config_manager { } $basedir = $CFG->dirroot . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'behat'; + $config = array( 'default' => array( 'paths' => array( @@ -243,7 +370,7 @@ class behat_config_manager { 'Behat\MinkExtension\Extension' => array( 'base_url' => $CFG->behat_wwwroot, 'goutte' => null, - 'selenium2' => null + 'selenium2' => $selenium2wdhost ), 'Moodle\BehatExtension\Extension' => array( 'formatters' => array( @@ -273,7 +400,7 @@ class behat_config_manager { * Simply add each one to lightest buckets until all files allocated. * PGA = Profile Guided Allocation. I made it up just now. * CAUTION: workers must agree on allocation, do not be random anywhere! - * + * * @param array $features Behat feature files array * @param int $nbuckets Number of buckets to divide into * @param int $instance Index number of this instance @@ -296,7 +423,7 @@ class behat_config_manager { $buckets = array_fill(0, $nbuckets, array()); $totalweight = 0; - // Re-key the features list to match pga data. + // Re-key the features list to match timing data. foreach ($features as $k => $file) { $key = str_replace($realroot, '', $file); $features[$key] = $file; @@ -316,9 +443,9 @@ class behat_config_manager { // Finally, add each feature one by one to the lightest bucket. foreach ($pgaorder as $key => $weight) { $file = $features[$key]; - $light_bucket = array_search(min($weights), $weights); - $weights[$light_bucket] += $weight; - $buckets[$light_bucket][] = $file; + $lightbucket = array_search(min($weights), $weights); + $weights[$lightbucket] += $weight; + $buckets[$lightbucket][] = $file; $totalweight += $weight; } @@ -330,7 +457,7 @@ class behat_config_manager { } // Return the features for this worker. - return $buckets[$instance-1]; + return $buckets[$instance - 1]; } /** diff --git a/lib/behat/classes/util.php b/lib/behat/classes/util.php index 4a42c578653..95f42fea62d 100644 --- a/lib/behat/classes/util.php +++ b/lib/behat/classes/util.php @@ -213,22 +213,6 @@ class behat_util extends testing_util { // Updates all the Moodle features and steps definitions. behat_config_manager::update_config_file(); - // Create suffix symlink if necessary. - global $CFG; - if ($CFG->behat_suffix) { - $extra = preg_filter('#.*/(.+)$#', '$1', $CFG->behat_wwwroot); - $link = $CFG->dirroot.'/'.$extra; - if (file_exists($link)) { - if (!is_link($link)) { - throw new coding_exception("File exists at link location ($link) but is not a link!"); - } - @unlink($link); - } - if (!symlink($CFG->dirroot, $link)) { - throw new coding_exception("Unable to create behat suffix symlink ($link)"); - } - } - if (self::is_test_mode_enabled()) { return; } diff --git a/lib/behat/lib.php b/lib/behat/lib.php index 40612f4cd8e..ef305eee463 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -41,6 +41,11 @@ define('BEHAT_EXITCODE_INSTALL', 254); define('BEHAT_EXITCODE_COMPOSER', 255); define('BEHAT_EXITCODE_INSTALLED', 256); +/** + * The behat test site fullname and shortname. + */ +define('BEHAT_PARALLEL_SITE_WWW_SUFFIX', "behatrun"); + /** * Exits with an error code * @@ -224,6 +229,7 @@ function behat_check_config_vars() { behat_error(BEHAT_EXITCODE_CONFIG, 'Define $CFG->behat_dataroot in config.php'); } + clearstatcache(); if (!file_exists($CFG->behat_dataroot)) { $permissions = isset($CFG->directorypermissions) ? $CFG->directorypermissions : 02777; umask(0); @@ -273,30 +279,53 @@ function behat_is_test_site() { } /** - * Add behat suffix to $CFG vars for parallel testing. + * Fix variables for parallel behat testing. + * - behat_wwwroot = behat_wwwroot{behatrunprocess} + * - behat_dataroot = behat_dataroot{behatrunprocess} + * - behat_prefix = behat_prefix.{behatrunprocess}_ (For oracle it will be firstletter of prefix and behatrunprocess) + * + * @param string $behatrunprocess process index for which variables will be set. **/ -function behat_add_suffix_to_vars($suffix = '') { +function behat_update_vars_for_process($behatrunprocess = '') { global $CFG; - if (empty($suffix)) { - if (!empty($CFG->behat_suffix)) { - $suffix = $CFG->behat_suffix; + $allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass'); + $behatrunprocess = $CFG->behatrunprocess; - } else if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) { - $suffix = BEHAT_SUFFIX; + if ($behatrunprocess) { + // Set www root for run process. + if (isset($CFG->behat_wwwroot) && !preg_match("#/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $behatrunprocess . "\$#", + $CFG->behat_wwwroot)) { + $CFG->behat_wwwroot .= "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $behatrunprocess; } - } - $CFG->behat_suffix = $suffix; + // Set behat_dataroot. + if (!preg_match("#" . $behatrunprocess . "\$#", $CFG->behat_dataroot)) { + $CFG->behat_dataroot .= $behatrunprocess; + } - if ($suffix) { - if (isset($CFG->behat_wwwroot) && !preg_match("#/behat$suffix\$#", $CFG->behat_wwwroot)) { - $CFG->behat_wwwroot .= "/behat{$suffix}"; + // Set behat_prefix for db, just suffix run process number, to avoid max length exceed. + // For oracle only 2 letter prefix is possible. + // NOTE: This will not work for parallel process > 9. + if ($CFG->dbtype === 'oci') { + $CFG->behat_prefix = substr($CFG->behat_prefix, 0, 1); + $CFG->behat_prefix .= "{$behatrunprocess}"; + } else { + $CFG->behat_prefix .= "{$behatrunprocess}_"; } - if (!preg_match("#/behat{$suffix}\$#", $CFG->behat_dataroot)) { - $CFG->behat_dataroot = dirname($CFG->behat_dataroot)."/behat{$suffix}"; + + if (!empty($CFG->behat_parallel_run[$behatrunprocess - 1])) { + // Override allowed config vars. + foreach ($allowedconfigoverride as $config) { + if (isset($CFG->behat_parallel_run[$behatrunprocess - 1][$config])) { + $CFG->$config = $CFG->behat_parallel_run[$behatrunprocess - 1][$config]; + } + } + // Override behat prefix, if specified. + if (isset($CFG->behat_parallel_run[$behatrunprocess - 1]['behat_prefix'])) { + $CFG->behat_prefix = $CFG->behat_parallel_run[$behatrunprocess - 1]['behat_prefix']; + } } - $CFG->behat_prefix = "behat{$suffix}_"; } } diff --git a/lib/classes/process_manager.php b/lib/classes/process_manager.php new file mode 100644 index 00000000000..b49b9e5f56d --- /dev/null +++ b/lib/classes/process_manager.php @@ -0,0 +1,211 @@ +. + +/** + * Moodle implementation of process manager, to execute external commands. + * + * @package core + * @copyright 2015 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Moodle implementation of process manager, to execute external commands. + * + * @package core + * @copyright 2015 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class process_manager implements Countable { + + /** + * Standard in. + */ + const STDIN = 0; + + /** + * Standard out. + */ + const STDOUT = 1; + + /** + * Standard error. + */ + const STDERR = 2; + + /** + * Non blocking mode. + */ + const NON_BLOCKING = 0; + + /** + * Blocking mode. + */ + const BLOCKING = 1; + + /** @var array Descriptor used for process. */ + private static $DESCRIPTORSPEC = array( + self::STDIN => array('pipe', 'r'), + self::STDOUT => array('pipe', 'w'), + self::STDERR => array('pipe', 'w'), + ); + + /** @var array list of processes. */ + private $processes = array(); + + /** @var array list of stdin */ + private $stdins = array(); + + /** @var array list of stdout */ + private $stdouts = array(); + + /** @var array list of stderr. */ + private $stderrs = array(); + + /** + * Create new process and keep track of it. + * + * @param string $name name of the process, has to be unique for process identification. + * @param string $cmd command to execute. + * @param string $cwd absolute path of working directory for command to execute. + * @return false if failed to create process. + */ + public function create($name, $cmd, $cwd = NULL) { + $process = proc_open($cmd, self::$DESCRIPTORSPEC, $pipes, $cwd); + + if (false === is_resource($process)) { + throw new Exception('Error starting worker'); + } + + stream_set_blocking($pipes[self::STDOUT], self::NON_BLOCKING); + stream_set_blocking($pipes[self::STDERR], self::NON_BLOCKING); + + $this->processes[$name] = $process; + $this->stdins[$name] = $pipes[self::STDIN]; + $this->stdouts[$name] = $pipes[self::STDOUT]; + $this->stderrs[$name] = $pipes[self::STDERR]; + + return true; + } + + /** + * Keep listing to process and return status of it. + * + * @retrun array status, stdout and stderr. + */ + public function listen() { + $read = array(); + + foreach ($this->processes as $i => $p) { + // Update process info. + if (!($s = @proc_get_status($p)) || !$s['running']) { + $status[$i] = $this->detach($p); + } else { + $status[$i] = 0; + $read[] = $this->stdouts[$i]; + $read[] = $this->stderrs[$i]; + } + } + + if ($read) { + $changednum = stream_select($read, $write, $expect, 0); + } else { + return; + } + + if (false === $changednum) { + throw new \RuntimeException(); + } + + if (0 === $changednum) { + return; + } + + foreach ($read as $stream) { + $i = array_search($stream, $this->stdouts, true); + if (false === $i) { + $i = array_search($stream, $this->stderrs, true); + if (false === $i) { + continue; + } + } + + $stdout[$i] = stream_get_contents($this->stdouts[$i]); + $stderr[$i] = stream_get_contents($this->stderrs[$i]); + } + return (array($status, $stdout, $stderr)); + } + + /** + * Detach process. + * + * @param $process process to detatch. + * @return int status of process. + */ + public function detach($process) { + $i = array_search($process, $this->processes, true); + + if (false === $i) { + throw new \RuntimeException(); + } + + fclose($this->stdins[$i]); + fclose($this->stdouts[$i]); + fclose($this->stderrs[$i]); + $status = proc_close($this->processes[$i]); + + unset($this->processes[$i]); + unset($this->stdins[$i]); + unset($this->stdouts[$i]); + unset($this->stderrs[$i]); + + return $status; + } + + /** + * Detach all processes. + */ + public function detachall() { + foreach ($this->stdins as $stdin) { + fclose($stdin); + } + foreach ($this->stdouts as $stdout) { + fclose($stdout); + } + foreach ($this->stderrs as $stderr) { + fclose($stderr); + } + foreach ($this->processes as $processs) { + proc_close($processs); + } + } + + /** + * Return count of active processes. + * + * @return int count of active processes. + */ + public function count() { + return count($this->processes); + } + + /** + * Destructor. + */ + public function __destruct() { + $this->detachall(); + } +} diff --git a/lib/clilib.php b/lib/clilib.php index e4992528144..ebe0a08804f 100644 --- a/lib/clilib.php +++ b/lib/clilib.php @@ -176,8 +176,15 @@ function cli_error($text, $errorcode=1) { die($errorcode); } - -function ns_proc_open($cmd, $die = false) { +/** + * Executes cli command and return handle. + * + * @param string $cmd command to be executed. + * @param bool $die exit if command is not executed. + * @return array list of handles and pipe. + * @throws Exception if worker is not started, + */ +function cli_execute($cmd, $die = false) { $desc = array( 0 => array('pipe', 'r'), 1 => array('pipe', 'w'), @@ -189,20 +196,80 @@ function ns_proc_open($cmd, $die = false) { return array($handle, $pipes); } +/** + * Execute commands in parallel. + * + * @param array $cmds list of commands to be executed. + * @param string $cwd aabsolute path of working directory. + * @param bool $returnonfirstfail Will stop all process and return. + * + * @return bool status of all process. + */ +function cli_execute_parallel($cmds, $cwd = NULL, $returnonfirstfail = false, $addprefix = true) { + require_once(__DIR__ . '/classes/process_manager.php'); -function ns_parallel_popen($cmds, $doexit = false) { + $overallstatus = false; + $processmanager = new process_manager(); + + // Create child process. + foreach ($cmds as $name => $cmd) { + if (!$processmanager->create($name, $cmd, $cwd) && $returnonfirstfail) { + throw new Exception('Error starting worker'); + } + } + while (0 < count($processmanager)) { + usleep(10000); + list($status, $stdout, $stderr) = $processmanager->listen(); + + if (!empty($status)) { + foreach ($status as $name => $value) { + // Something went wrong. + if ((0 > $value)) { + throw new \RuntimeException(); + } + $overallstatus = $overallstatus || (bool)$value; + // Add prefix to process. + $prefix = ""; + if ($addprefix && ($value === 0)) { + $prefix = '[' . $name . '] '; + } + if (!empty($stdout[$name]) && trim($stdout[$name])) { + echo $prefix . $stdout[$name]; + } + if (!empty($stderr[$name]) && trim($stderr[$name])) { + echo $prefix . $stderr[$name]; + } + } + + // Return if fail found. + if ($returnonfirstfail && (bool)$value) { + unset($processmanager); + $processmanager = null; + echo PHP_EOL; + return $value; + } + } + } + + echo PHP_EOL; + return $overallstatus; +} + +/** + * Execute commands in sequence and return status code for each process. + * + * @param array $cmds commands to execute. + * @param bool $returnonfirstfail if true then returns on any fail. + * @return array status codes for each process. + */ +function cli_execute_sequential($cmds, $returnonfirstfail = false) { $procs = array(); foreach ($cmds as $k => $cmd) { - $procs[] = popen($cmd, 'r'); + $procs[$k] = popen($cmd, 'r'); + passthru($cmd, $procs[$k]); + if (($procs[$k] != 0) && $returnonfirstfail) { + return $procs; + } } - $status = false; - foreach ($procs as $p) { - if (!$p) continue; - while ($out = fgets($p)) echo $out; - $status |= (bool) pclose($p); - } - if ($doexit && $status) { - exit($status); - } - return $status; + return $procs; } diff --git a/lib/setup.php b/lib/setup.php index a0d870795d1..ef2b19dab0e 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -78,34 +78,46 @@ if (defined('BEHAT_SITE_RUNNING')) { } else if (!empty($CFG->behat_wwwroot) or !empty($CFG->behat_dataroot) or !empty($CFG->behat_prefix)) { global $argv; - $suffix = ''; + $behatrunprocess = false; - if (defined('BEHAT_SUFFIX') && BEHAT_SUFFIX) { - $suffix = BEHAT_SUFFIX; + require_once(__DIR__ . '/../lib/behat/lib.php'); + // Get behat run process, if set. + if (defined('BEHAT_CURRENT_RUN') && BEHAT_CURRENT_RUN) { + $behatrunprocess = BEHAT_CURRENT_RUN; } else if (!empty($_SERVER['REMOTE_ADDR'])) { - if (preg_match('#/behat(.+?)/#', $_SERVER['REQUEST_URI'])) { - $afterpath = str_replace(realpath($CFG->dirroot).'/', '', realpath($_SERVER['SCRIPT_FILENAME'])); - if (!$suffix = preg_filter("#.*/behat(.+?)/$afterpath#", '$1', $_SERVER['SCRIPT_FILENAME'])) { - throw new coding_exception("Unable to determine behat suffix [afterpath=$afterpath, scriptfilename={$_SERVER['SCRIPT_FILENAME']}]!"); + if (preg_match('#/' . BEHAT_PARALLEL_SITE_WWW_SUFFIX . '(.+?)/#', $_SERVER['REQUEST_URI'])) { + $dirrootrealpath = str_replace("\\", "/", realpath($CFG->dirroot)); + $serverrealpath = str_replace("\\", "/", realpath($_SERVER['SCRIPT_FILENAME'])); + $afterpath = str_replace($dirrootrealpath.'/', '', $serverrealpath); + if (!$behatrunprocess = preg_filter("#.*/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . "(.+?)/$afterpath#", '$1', + $_SERVER['SCRIPT_FILENAME'])) { + throw new Exception("Unable to determine behat process [afterpath=" . $afterpath . + ", scriptfilename=" . $_SERVER['SCRIPT_FILENAME'] . "]!"); } } - } else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) { - if ($match = preg_filter('#--suffix=(.+)#', '$1', $argv)) { - $suffix = reset($match); + if ($match = preg_filter('#--run=(.+)#', '$1', $argv)) { + $behatrunprocess = reset($match); } + if ($k = array_search('--config', $argv)) { - $behatconfig = $argv[$k+1]; - $suffix = preg_filter("#^{$CFG->behat_dataroot}(.+?)/behat/behat\.yml#", '$1', $behatconfig); + $behatconfig = str_replace("\\", "/", $argv[$k + 1]); + $behatdataroot = str_replace("\\", "/", $CFG->behat_dataroot); + $behatrunprocess = preg_filter("#^{$behatdataroot}" . + "(.+?)[/|\\\]behat[/|\\\]behat\.yml#", '$1', $behatconfig); } } + $CFG->behatrunprocess = $behatrunprocess; + // The behat is configured on this server, we need to find out if this is the behat test // site based on the URL used for access. - require_once(__DIR__ . '/../lib/behat/lib.php'); - behat_add_suffix_to_vars($suffix); + behat_update_vars_for_process($behatrunprocess); + if (behat_is_test_site()) { + clearstatcache(); + // Checking the integrity of the provided $CFG->behat_* vars and the // selected wwwroot to prevent conflicts with production and phpunit environments. behat_check_config_vars(); @@ -117,7 +129,7 @@ if (defined('BEHAT_SITE_RUNNING')) { if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store' or is_numeric($file)) { continue; } - behat_error(BEHAT_EXITCODE_CONFIG, '$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot'); + behat_error(BEHAT_EXITCODE_CONFIG, "$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot"); } closedir($dh); unset($dh); diff --git a/lib/setuplib.php b/lib/setuplib.php index 399308bd867..848df4a4779 100644 --- a/lib/setuplib.php +++ b/lib/setuplib.php @@ -1365,7 +1365,7 @@ function make_writable_directory($dir, $exceptiononerror = true) { umask($CFG->umaskpermissions); if (!file_exists($dir)) { - if (!mkdir($dir, $CFG->directorypermissions, true)) { + if (!@mkdir($dir, $CFG->directorypermissions, true)) { clearstatcache(); // There might be a race condition when creating directory. if (!is_dir($dir)) { diff --git a/lib/testing/classes/test_lock.php b/lib/testing/classes/test_lock.php index 710cdeabcf4..169241e2c47 100644 --- a/lib/testing/classes/test_lock.php +++ b/lib/testing/classes/test_lock.php @@ -52,7 +52,6 @@ class test_lock { */ public static function acquire($framework) { global $CFG; - $datarootpath = $CFG->{$framework . '_dataroot'} . '/' . $framework; $lockfile = $datarootpath . '/lock'; if (!file_exists($datarootpath)) { diff --git a/lib/tests/behat/behat_hooks.php b/lib/tests/behat/behat_hooks.php index 060998294f8..5d1deaa7da6 100644 --- a/lib/tests/behat/behat_hooks.php +++ b/lib/tests/behat/behat_hooks.php @@ -31,6 +31,8 @@ require_once(__DIR__ . '/../../behat/behat_base.php'); use Behat\Behat\Event\SuiteEvent as SuiteEvent, Behat\Behat\Event\ScenarioEvent as ScenarioEvent, + Behat\Behat\Event\FeatureEvent as FeatureEvent, + Behat\Behat\Event\OutlineExampleEvent as OutlineExampleEvent, Behat\Behat\Event\StepEvent as StepEvent, Behat\Mink\Exception\DriverException as DriverException, WebDriver\Exception\NoSuchWindow as NoSuchWindow, @@ -84,17 +86,25 @@ class behat_hooks extends behat_base { */ protected static $faildumpdirname = false; + /** + * Keeps track of time taken by feature to execute. + * + * @var array list of feature timings + */ + protected static $timings = array(); + /** * Gives access to moodle codebase, ensures all is ready and sets up the test lock. * * Includes config.php to use moodle codebase with $CFG->behat_* * instead of $CFG->prefix and $CFG->dataroot, called once per suite. * + * @param SuiteEvent $event event before suite. * @static * @throws Exception * @BeforeSuite */ - public static function before_suite($event) { + public static function before_suite(SuiteEvent $event) { global $CFG; // Defined only when the behat CLI command is running, the moodle init setup process will @@ -153,17 +163,33 @@ class behat_hooks extends behat_base { } } - protected static $timings = array(); - - /** @BeforeFeature */ - public static function before_feature($obj) { - $file = $obj->getFeature()->getFile(); + /** + * Gives access to moodle codebase, to keep track of feature start time. + * + * @param FeatureEvent $event event fired before feature. + * @static + * @BeforeFeature + */ + public static function before_feature(FeatureEvent $event) { + if (!defined('BEHAT_FEATURE_TIMING')) { + return; + } + $file = $event->getFeature()->getFile(); self::$timings[$file] = microtime(true); } - /** @AfterFeature */ - public static function teardownFeature($obj) { - $file = $obj->getFeature()->getFile(); + /** + * Gives access to moodle codebase, to keep track of feature end time. + * + * @param FeatureEvent $event event fired after feature. + * @static + * @AfterFeature + */ + public static function after_feature(FeatureEvent $event) { + if (!defined('BEHAT_FEATURE_TIMING')) { + return; + } + $file = $event->getFeature()->getFile(); self::$timings[$file] = microtime(true) - self::$timings[$file]; // Probably didn't actually run this, don't output it. if (self::$timings[$file] < 1) { @@ -171,9 +197,14 @@ class behat_hooks extends behat_base { } } - /** @AfterSuite */ - public static function tearDown($obj) { - global $CFG; + /** + * Gives access to moodle codebase, to keep track of suite timings. + * + * @param SuiteEvent $event event fired after suite. + * @static + * @AfterSuite + */ + public static function after_suite(SuiteEvent $event) { if (!defined('BEHAT_FEATURE_TIMING')) { return; } @@ -193,6 +224,7 @@ class behat_hooks extends behat_base { /** * Resets the test environment. * + * @param OutlineExampleEvent|ScenarioEvent $event event fired before scenario. * @throws coding_exception If here we are not using the test database it should be because of a coding error * @BeforeScenario */ @@ -281,9 +313,10 @@ class behat_hooks extends behat_base { * default would be at framework level, which will stop the execution of * the run. * + * @param StepEvent $event event fired before step. * @BeforeStep @javascript */ - public function before_step_javascript($event) { + public function before_step_javascript(StepEvent $event) { try { $this->wait_for_pending_js(); @@ -305,9 +338,10 @@ class behat_hooks extends behat_base { * default would be at framework level, which will stop the execution of * the run. * + * @param StepEvent $event event fired after step. * @AfterStep @javascript */ - public function after_step_javascript($event) { + public function after_step_javascript(StepEvent $event) { global $CFG; // Save a screenshot if the step failed. @@ -340,9 +374,10 @@ class behat_hooks extends behat_base { * * This includes creating an HTML dump of the content if there was a failure. * + * @param StepEvent $event event fired after step. * @AfterStep */ - public function after_step($event) { + public function after_step(StepEvent $event) { global $CFG; // Save the page content if the step failed. From 027212b0348fc52ec05a7a521553c77eb4b70397 Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Tue, 17 Feb 2015 14:19:00 +0800 Subject: [PATCH 03/11] MDL-39752 behat: Use symfony2/process --- admin/tool/behat/cli/init.php | 23 +- admin/tool/behat/cli/run.php | 354 +++++++++-------- admin/tool/behat/cli/util.php | 424 +++++++++++++++------ admin/tool/behat/cli/util_single_run.php | 271 +++++++++++++ admin/tool/behat/cli/utilparallel.php | 195 ---------- behat_features_step_count.json | 286 ++++++++++++++ behat_pga_default.json | 283 -------------- lib/behat/classes/behat_config_manager.php | 61 +-- lib/behat/lib.php | 47 ++- lib/classes/process_manager.php | 211 ---------- lib/clilib.php | 97 ----- lib/setup.php | 7 +- lib/tests/behat/behat_hooks.php | 13 +- 13 files changed, 1166 insertions(+), 1106 deletions(-) create mode 100644 admin/tool/behat/cli/util_single_run.php delete mode 100644 admin/tool/behat/cli/utilparallel.php create mode 100644 behat_features_step_count.json delete mode 100644 behat_pga_default.json delete mode 100644 lib/classes/process_manager.php diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index 6afd936b899..4b7917f1fe2 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -75,20 +75,30 @@ if (!empty($options['help'])) { } // Check which util file to call. -$utilfile = 'util.php'; +$utilfile = 'util_single_run.php'; $paralleloption = ""; // If parallel run then use utilparallel. if ($options['parallel']) { - $utilfile = 'utilparallel.php'; - $paralleloption = " --parallel=".$options['parallel']; + $utilfile = 'util.php'; + $paralleloption = "--parallel=" . $options['parallel']; + + // If maxruns then add that option. + if ($options['maxruns']) { + $paralleloption .= " --maxruns=" . $options['maxruns']; + } } // Changing the cwd to admin/tool/behat/cli. $cwd = getcwd(); -chdir(__DIR__); $output = null; -exec("php $utilfile --diag $paralleloption", $output, $code); +// If behat dependencies not downloaded then do it first, else symfony/process can't be used. +if ($options['parallel'] && !file_exists(__DIR__ . "/../../../../vendor/autoload.php")) { + $code = BEHAT_EXITCODE_COMPOSER; +} else { + chdir(__DIR__); + exec("php $utilfile --diag $paralleloption", $output, $code); +} // Check if composer needs to be updated. if (($code == BEHAT_EXITCODE_INSTALL) || $code == BEHAT_EXITCODE_REINSTALL || $code == BEHAT_EXITCODE_COMPOSER) { @@ -116,6 +126,7 @@ if ($code == 0) { exit($code); } + chdir(__DIR__); passthru("php $utilfile --install $paralleloption", $code); if ($code != 0) { chdir($cwd); @@ -140,8 +151,10 @@ if ($code == 0) { } // Enable editing mode according to config.php vars. +chdir(__DIR__); passthru("php $utilfile --enable $paralleloption", $code); if ($code != 0) { + echo "Error enabling site" . PHP_EOL; chdir($cwd); exit($code); } diff --git a/admin/tool/behat/cli/run.php b/admin/tool/behat/cli/run.php index 4a3e1bca390..92f4462869a 100644 --- a/admin/tool/behat/cli/run.php +++ b/admin/tool/behat/cli/run.php @@ -62,13 +62,13 @@ list($options, $unrecognised) = cli_get_params( $help = " Behat utilities to run behat tests in parallel Options: --t, --tags{{color_green}} Tags to execute. --p, --profile{{color_green}} Profile to execute. ---stop-on-failure{{color_green}} Stop on failure in any parallel run. ---verbose{{color_green}} Verbose output ---replace{{color_green}} Replace args string with run process number, useful for output. +-t, --tags Tags to execute. +-p, --profile Profile to execute. +--stop-on-failure Stop on failure in any parallel run. +--verbose Verbose output +--replace Replace args string with run process number, useful for output. --h, --help{{color_green}} Print out this help +-h, --help Print out this help Example from Moodle root directory: \$ php admin/tool/behat/cli/run.php --parallel=2 @@ -85,21 +85,11 @@ if (!empty($options['help'])) { $parallelrun = behat_config_manager::get_parallel_test_runs(1); // Capture signals and ensure we clean symlinks. -pcntl_signal(SIGTERM, "signal_handler"); -pcntl_signal(SIGINT, "signal_handler"); -/** - * Signal handler for terminal exit. - * - * @param $signal - */ -function signal_handler($signal) { - switch ($signal) { - case SIGTERM: - case SIGKILL: - case SIGINT: - // Remove site symlink if necessary. - behat_config_manager::drop_parallel_site_links(); - exit(1); +if (extension_loaded('pcntl')) { + $disabled = explode(',', ini_get('disable_functions')); + if (!in_array('pcntl_signal', $disabled)) { + pcntl_signal(SIGTERM, "signal_handler"); + pcntl_signal(SIGINT, "signal_handler"); } } @@ -109,7 +99,7 @@ if (empty($parallelrun)) { $runtestscommand = behat_command::get_behat_command(); $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); exec("php $runtestscommand", $output, $code); - echo implode("\n", $output) . "\n"; + echo implode(PHP_EOL, $output) . PHP_EOL; exit($code); } else { exit(1); @@ -118,6 +108,7 @@ if (empty($parallelrun)) { // Create site symlink if necessary. if (!behat_config_manager::create_parallel_site_links()) { + echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL; exit(1); } @@ -131,19 +122,8 @@ array_walk($unrecognised, function (&$v) { }); $extraopts = implode(' ', $unrecognised); -$checkfail = array(); -$outputs = array(); -$handles = array(); -$pipe2i = array(); -$exits = array(); -$unused = null; -$linelencnt = 0; -$procs = array(); -$behatdataroot = $CFG->behat_dataroot; - $tags = ''; -// Options parameters to be added to each run. -$myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts; + if ($options['profile']) { $profile = $options['profile']; if (empty($CFG->behat_config[$profile]['filters']['tags'])) { @@ -151,10 +131,10 @@ if ($options['profile']) { exit(1); } $tags = $CFG->behat_config[$profile]['filters']['tags']; - $myopts .= '--profile=\'' . $profile . "'"; + $extraopts .= '--profile=\'' . $profile . "'"; } else if ($options['tags']) { $tags = $options['tags']; - $myopts .= '--tags=' . $tags; + $extraopts .= '--tags="' . $tags . '"'; } // Update config file if tags defined. @@ -165,7 +145,7 @@ if ($tags) { for ($i = 1; $i <= $parallelrun; $i++) { $CFG->behatrunprocess = $i; $CFG->behat_dataroot = $behatdataroot . $i; - $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i; + $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i; behat_config_manager::update_config_file('', true, $tags); } $CFG->behat_dataroot = $behatdataroot; @@ -173,140 +153,196 @@ if ($tags) { unset($CFG->behatrunprocess); } +$cmds = array(); +echo "Running ${parallelrun} parallel behat sites:" . PHP_EOL; for ($i = 1; $i <= $parallelrun; $i++) { $CFG->behatrunprocess = $i; + + // Options parameters to be added to each run. + $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts; + $behatcommand = behat_command::get_behat_command(); $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i); // Command to execute behat run. - $cmd = $behatcommand .' --config ' . $behatconfigpath . " " . $myopts; - - echo "[" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . "] ". $cmd . "\n"; - - list($handle, $pipes) = cli_execute($cmd, true); - @fclose($pipes[0]); - unset($pipes[0]); - $exits[$i] = 1; - $handles[$i] = array($handle, $pipes[1], $pipes[2]); - $procs[$i] = $handle; - $checkfail[$i] = false; - $outputs[$i] = array(''); - $pipe2i[(int) $pipes[1]] = $i; - $pipe2i[(int) $pipes[2]] = $i; - stream_set_blocking($pipes[1], 0); - stream_set_blocking($pipes[2], 0); + $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts; + echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL; } -$progresscount = 0; -while (!empty($procs)) { - usleep(10000); - - foreach ($handles as $i => $p) { - if (!($status = @proc_get_status($p[0])) || !$status['running']) { - if ($exits[$i] !== 0) { - $exits[$i] = !empty($status) ? $status['exitcode'] : 1; - } - unset($procs[$i]); - unset($handles[$i][0]); - $last = array_pop($outputs[$i]); - for ($l = 2; $l >= 1; $l--) { - while ($part = @fread($handles[$i][$l], 8192)) { - $last .= $part; - } - } - $outputs[$i] = array_merge($outputs[$i], explode("\n", $last)); - } - } - - $ready = array(); - foreach ($handles as $i => $set) { - $ready[] = $set[1]; - $ready[] = $set[2]; - } - - // Poll for any process with output or ended. - if (!$result = @stream_select($ready, $unused, $unused, 1)) { - // Nothing; try again. - continue; - } - if (!$fh = reset($ready)) { - continue; - } - - $i = $pipe2i[(int) $fh]; - $last = array_pop($outputs[$i]); - $read = fread($fh, 4096); - $newlines = explode("\n", $last.$read); - $outputs[$i] = array_merge($outputs[$i], $newlines); - - if (!$checkfail[$i]) { - foreach ($newlines as $l => $line) { - unset($newlines[$l]); - if (preg_match('#^Started at [\d\-]+#', $line) || (strlen($line) > 3 && - preg_match('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', $line))) { - $checkfail[$i] = true; - break; - } - } - } - if ($progress = preg_filter('#^\s*([FUS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $newlines)) { - if ($checkfail[$i] && preg_filter('#^\s*[S\.\-]*[FU][S\.\-]*(?:\s+\d+)?\s*$#', '$1', $progress)) { - $exits[$i] = 1; - if ($options['stop-on-failure']) { - foreach ($handles as $l => $p) { - $exits[$l] = $l != $i ? 0 : $exits[$i]; - @proc_terminate($p[0], SIGINT); - } - } - } - } - // Process has gone, assume this is the last output for it. - if (empty($procs[$i])) { - unset($handles[$i]); - } - if (empty($checkfail[$i]) || !($update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $read))) { - continue; - } - while ($update) { - $part = substr($update, 0, 70 - $linelencnt); - $update = substr($update, strlen($part)); - $linelencnt += strlen($part); - echo $part; - if ($linelencnt >= 70) { - $progresscount += 70; - echo " $progresscount\n"; - $linelencnt = 0; - } - } -} -echo "\n\n"; - -$exits = array_filter($exits, - function ($v) { - return $v !== 0; - } -); - -if ($exits || $options['verbose']) { - echo "Exit codes: ".implode(" ", $exits)."\n\n"; - foreach ($outputs as $i => $output) { - unset($outputs[$i]); - if (!end($output)) { - array_pop($output); - } - $prefix = "[" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . "] "; - array_walk($output, function (&$l) use ($prefix) { - $l = $prefix.$l; - }); - echo implode("\n", $output)."\n\n"; - } - $failed = true; +if (empty($cmds)) { + echo "No commands to execute " . PHP_EOL; + exit(1); } +// Execute all commands. +$processes = cli_execute_parallel($cmds); +$stoponfail = empty($options['stop-on-failure']) ? false : true; + +// Print header. +print_process_start_info($processes); + +// Print combined run o/p from processes. +$exitcodes = print_combined_run_output($processes, $stoponfail); $time = round(microtime(true) - $time, 1); -echo "Finished in {$time}s\n"; +echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL; + + +// Print exit info from each run. +$status = false; +foreach ($exitcodes as $exitcode) { + $status = (bool)$status || (bool)$exitcode; +} + +// Show exit code from each process, if any process failed. +if ($status) { + echo "Exit codes: " . implode(" ", $exitcodes) . PHP_EOL; + echo "To re-run failed processes, you can use following commands:" . PHP_EOL; + foreach ($cmds as $name => $cmd) { + if (!empty($exitcodes[$name])) { + echo "[" . $name . "] " . $cmd . PHP_EOL; + } + } + echo PHP_EOL; +} + +// Run finished. Show exit code and output from individual process. +$verbose = empty($options['verbose']) ? false : true; +$verbose = $verbose || $status; +print_each_process_info($processes, $verbose); // Remove site symlink if necessary. behat_config_manager::drop_parallel_site_links(); -exit(!empty($failed) ? 1 : 0); +exit((int) $status); + +/** + * Signal handler for terminal exit. + * + * @param int $signal signal number. + */ +function signal_handler($signal) { + switch ($signal) { + case SIGTERM: + case SIGKILL: + case SIGINT: + // Remove site symlink if necessary. + behat_config_manager::drop_parallel_site_links(); + exit(1); + } +} + +/** + * Prints header from the first process. + * + * @param array $processes list of processes to loop though. + */ +function print_process_start_info($processes) { + $printed = false; + // Keep looping though processes, till we get first process o/p. + while (!$printed) { + usleep(10000); + foreach ($processes as $name => $process) { + // Exit if any process has stopped. + if (!$process->isRunning()) { + $printed = true; + break; + } + + $op = explode(PHP_EOL, $process->getOutput()); + if (count($op) >= 3) { + foreach ($op as $line) { + if (trim($line) && (strpos($line, '.') !== 0)) { + echo $line . PHP_EOL; + } + } + $printed = true; + } + } + } +} + +/** + * Loop though all processes and print combined o/p + * + * @param array $processes list of processes to loop though. + * @param bool $stoponfail Stop all processes and exit if failed. + * @return array list of exit codes from all processes. + */ +function print_combined_run_output($processes, $stoponfail = false) { + $exitcodes = array(); + $maxdotsonline = 70; + $remainingprintlen = $maxdotsonline; + $progresscount = 0; + while (count($exitcodes) != count($processes)) { + usleep(10000); + foreach ($processes as $name => $process) { + if ($process->isRunning()) { + $op = $process->getIncrementalOutput(); + if (trim($op)) { + $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op); + // Exit process if anything fails. + if ($stoponfail && (strpos($update, 'F') !== false)) { + $process->stop(0); + } + + $strlentoprint = strlen($update); + + // If not enough dots printed on line then just print. + if ($strlentoprint < $remainingprintlen) { + echo $update; + $remainingprintlen = $remainingprintlen - $strlentoprint; + } else if ($strlentoprint == $remainingprintlen) { + $progresscount += $maxdotsonline; + echo $update ." " . $progresscount . PHP_EOL; + $remainingprintlen = $maxdotsonline; + } else { + while ($part = substr($update, 0, $remainingprintlen) > 0) { + $progresscount += $maxdotsonline; + echo $part . " " . $progresscount . PHP_EOL; + $update = substr($update, $remainingprintlen); + $remainingprintlen = $maxdotsonline; + } + } + } + } else { + $exitcodes[$name] = $process->getExitCode(); + if ($stoponfail && ($exitcodes[$name] != 0)) { + foreach ($processes as $l => $p) { + $exitcodes[$l] = -1; + $process->stop(0); + } + } + } + } + } + + echo PHP_EOL; + return $exitcodes; +} + +/** + * Loop though all processes and print combined o/p + * + * @param array $processes list of processes to loop though. + * @param bool $verbose Show verbose output for each process. + */ +function print_each_process_info($processes, $verbose = false) { + foreach ($processes as $name => $process) { + echo "**************** [" . $name . "] ****************" . PHP_EOL; + if ($verbose) { + echo $process->getOutput(); + echo $process->getErrorOutput(); + } else { + $op = explode(PHP_EOL, $process->getOutput()); + foreach ($op as $line) { + // Don't print progress . + if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) && + (strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) && + (strpos($line, 'Browser specific fixes ') !== 0)) { + echo $line . PHP_EOL; + } + } + } + echo PHP_EOL; + } +} diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index 8915b3547bd..62007477b67 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -15,7 +15,7 @@ // along with Moodle. If not, see . /** - * CLI tool with utilities to manage Behat integration in Moodle + * CLI tool with utilities to manage parallel Behat integration in Moodle * * All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as * $CFG->dataroot and $CFG->prefix @@ -30,47 +30,52 @@ if (isset($_SERVER['REMOTE_ADDR'])) { die(); // No access from web!. } -// Basic functions. +define('BEHAT_UTIL', true); +define('CLI_SCRIPT', true); +define('NO_OUTPUT_BUFFERING', true); +define('IGNORE_COMPONENT_CACHE', true); + require_once(__DIR__ . '/../../../../lib/clilib.php'); require_once(__DIR__ . '/../../../../lib/behat/lib.php'); // CLI options. list($options, $unrecognized) = cli_get_params( array( - 'help' => false, - 'install' => false, - 'parallel' => 0, - 'run' => '', - 'drop' => false, - 'enable' => false, - 'disable' => false, - 'diag' => false, - 'tags' => '', + 'help' => false, + 'install' => false, + 'drop' => false, + 'enable' => false, + 'disable' => false, + 'diag' => false, + 'parallel' => 0, + 'maxruns' => false, + 'updatesteps' => false, ), array( - 'h' => 'help' + 'h' => 'help', + 'j' => 'parallel', + 'm' => 'maxruns' ) ); -if ($options['install'] or $options['drop']) { - define('CACHE_DISABLE_ALL', true); -} - // Checking util.php CLI script usage. $help = " Behat utilities to manage the test environment Options: ---install Installs the test environment for acceptance tests ---drop Drops the database tables and the dataroot contents ---enable Enables test environment and updates tests list ---disable Disables test environment ---diag Get behat test environment status code +--install Installs the test environment for acceptance tests +--drop Drops the database tables and the dataroot contents +--enable Enables test environment and updates tests list +--disable Disables test environment +--diag Get behat test environment status code +-j, --parallel Number of parallel behat run operation +-m, --maxruns Max parallel processes to be executed at one time. +--updatesteps Update feature step file. --h, --help Print out this help +-h, --help Print out this help Example from Moodle root directory: -\$ php admin/tool/behat/cli/util.php --enable +\$ php admin/tool/behat/cli/util.php --enable --parallel=4 More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests "; @@ -80,111 +85,304 @@ if (!empty($options['help'])) { exit(0); } -// Describe this script. -define('BEHAT_UTIL', true); -define('CLI_SCRIPT', true); -define('NO_OUTPUT_BUFFERING', true); -define('IGNORE_COMPONENT_CACHE', true); - -// Set run value, to be used by setup for configuring proper CFG variables. -if ($options['run']) { - define('BEHAT_CURRENT_RUN', $options['run']); -} - -// Only load CFG from config.php, stop ASAP in lib/setup.php. -define('ABORT_AFTER_CONFIG', true); -require_once(__DIR__ . '/../../../../config.php'); - -// Remove error handling overrides done in config.php. -$CFG->debug = (E_ALL | E_STRICT); -$CFG->debugdisplay = 1; -error_reporting($CFG->debug); -ini_set('display_errors', '1'); -ini_set('log_errors', '1'); - -// Finish moodle init. -define('ABORT_AFTER_CONFIG_CANCEL', true); -require("$CFG->dirroot/lib/setup.php"); - -raise_memory_limit(MEMORY_HUGE); - -require_once($CFG->libdir.'/adminlib.php'); -require_once($CFG->libdir.'/upgradelib.php'); -require_once($CFG->libdir.'/clilib.php'); -require_once($CFG->libdir.'/installlib.php'); -require_once($CFG->libdir.'/testing/classes/test_lock.php'); - -if ($unrecognized) { - $unrecognized = implode("\n ", $unrecognized); - cli_error(get_string('cliunknowoption', 'admin', $unrecognized)); -} - -// Behat utilities. -require_once($CFG->libdir . '/behat/classes/util.php'); -require_once($CFG->libdir . '/behat/classes/behat_command.php'); -require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); - -// Ensure run option is <= parallel run installed. -if ($options['run']) { - if (!$options['parallel']) { - $options['parallel'] = behat_config_manager::get_parallel_test_runs(); +$cwd = getcwd(); +// If not a parallel site then open single run. +if (empty($options['parallel'])) { + chdir(__DIR__); + // Check if behat is initialised, if not exit. + passthru("php util_single_run.php --diag", $status); + if ($status) { + exit ($status); } - if (empty($options['parallel']) || $options['run'] > $options['parallel']) { - echo "Parallel runs can't be more then ".$options['parallel'].PHP_EOL; - exit(1); - } - $CFG->behatrunprocess = $options['run']; + $cmd = commands_to_execute($options); + $processes = cli_execute_parallel(array($cmd), __DIR__); + $status = print_sequential_output($processes, false); + chdir($cwd); + exit($status); } -// Run command (only one per time). -if ($options['install']) { - behat_util::install_site(); +$status = false; +$cmds = commands_to_execute($options); - // This is only displayed once for parallel install. - if (empty($options['run'])) { - mtrace("Acceptance tests site installed"); +// Start executing commands either sequential/parallel for options provided. +if ($options['diag'] || $options['enable'] || $options['disable']) { + // Do it sequentially as it's fast and need to be displayed nicely. + foreach (array_chunk($cmds, 1, true) as $cmd) { + $processes = cli_execute_parallel($cmd, __DIR__); + print_sequential_output($processes); } } else if ($options['drop']) { - // Ensure no tests are running. - test_lock::acquire('behat'); - behat_util::drop_site(); - // This is only displayed once for parallel install. - if (empty($options['run'])) { - mtrace("Acceptance tests site dropped"); + $processes = cli_execute_parallel($cmds, __DIR__); + $exitcodes = print_combined_drop_output($processes); + foreach ($exitcodes as $exitcode) { + $status = (bool)$status || (bool)$exitcode; } -} else if ($options['enable']) { - behat_util::start_test_mode(); - - // This is only displayed once for parallel install. - if (empty($options['run'])) { - $runtestscommand = behat_command::get_behat_command(true, !empty($options['run'])); - - $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); - mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:\n " . $runtestscommand); - } else { - // Save parallel site info for enable and install options. - $filepath = behat_config_manager::get_parallel_test_file_path(); - if (!file_put_contents($filepath, $options['parallel'])) { - behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created'); +} else if ($options['install']) { + // This is intensive compared to behat itself so run them in chunk if option maxruns not set. + if ($options['maxruns']) { + foreach (array_chunk($cmds, $options['maxruns'], true) as $chunk) { + $processes = cli_execute_parallel($chunk, __DIR__); + $exitcodes = print_combined_install_output($processes); + foreach ($exitcodes as $name => $exitcode) { + if ($exitcode != 0) { + echo $processes[$name]->getErrorOutput(); + } + $status = (bool)$status || (bool)$exitcode; + } + } + } else { + $processes = cli_execute_parallel($cmds, __DIR__); + $exitcodes = print_combined_install_output($processes); + foreach ($exitcodes as $name => $exitcode) { + if ($exitcode != 0) { + echo $processes[$name]->getErrorOutput(); + } + $status = (bool)$status || (bool)$exitcode; } } -} else if ($options['disable']) { - behat_util::stop_test_mode(); - // This is only displayed once for parallel install. - if (empty($options['run'])) { - mtrace("Acceptance tests environment disabled"); - } - -} else if ($options['diag']) { - $code = behat_util::get_behat_status(); - exit($code); - } else { + // We should never reach here. echo $help; exit(1); } +// Ensure we have success status to show following information. +if ($status) { + echo "Unknown failure $status" . PHP_EOL; + exit((int)$status); +} + +// Only load CFG from config.php for 1st run amd stop ASAP in lib/setup.php. +define('ABORT_AFTER_CONFIG', true); +require_once(__DIR__ . '/../../../../config.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php'); + +// Show command o/p (only one per time). +if ($options['install']) { + echo "Acceptance tests site installed for sites:".PHP_EOL; + // Display all sites which are installed/drop/diabled. + for ($i = 1; $i <= $options['parallel']; $i++) { + echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL; + } +} else if ($options['drop']) { + echo "Acceptance tests site dropped for " . $options['parallel'] . " parallel sites" . PHP_EOL; + +} else if ($options['enable']) { + echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:" . PHP_EOL; + echo behat_command::get_behat_command(true, true); + echo PHP_EOL; + +} else if ($options['disable']) { + echo "Acceptance tests environment disabled for " . $options['parallel'] . " parallel sites" . PHP_EOL; + +} else { + echo $help; +} + +chdir($cwd); exit(0); + +/** + * Create commands to be executed for parallel run. + * + * @param array $options options provided by user. + * @return array commands to be executed. + */ +function commands_to_execute($options) { + $removeoptions = array('maxruns'); + $cmds = array(); + $extraoptions = $options; + $extra = ""; + + // Remove extra options not in util_single_run.php. + foreach ($removeoptions as $ro) { + $extraoptions[$ro] = null; + unset($extraoptions[$ro]); + } + + foreach ($extraoptions as $option => $value) { + if ($options[$option]) { + $extra .= " --$option"; + if ($value) { + $extra .= "=$value"; + } + } + } + + if (empty($options['parallel'])) { + $cmds = "php util_single_run.php " . $extra; + } else { + // Create commands which has to be executed for parallel site. + for ($i = 1; $i <= $options['parallel']; $i++) { + $prefix = BEHAT_PARALLEL_SITE_NAME . $i; + $cmds[$prefix] = "php util_single_run.php " . $extra . " --run=" . $i . " 2>&1"; + } + } + return $cmds; +} + +/** + * Print drop output merging each run. + * + * @param array $processes list of processes. + * @return array exit codes of each process. + */ +function print_combined_drop_output($processes) { + $exitcodes = array(); + $maxdotsonline = 70; + $remainingprintlen = $maxdotsonline; + $progresscount = 0; + echo "Dropping tables:" . PHP_EOL; + + while (count($exitcodes) != count($processes)) { + usleep(10000); + foreach ($processes as $name => $process) { + if ($process->isRunning()) { + $op = $process->getIncrementalOutput(); + if (trim($op)) { + $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op); + $strlentoprint = strlen($update); + + // If not enough dots printed on line then just print. + if ($strlentoprint < $remainingprintlen) { + echo $update; + $remainingprintlen = $remainingprintlen - $strlentoprint; + } else if ($strlentoprint == $remainingprintlen) { + $progresscount += $maxdotsonline; + echo $update . " " . $progresscount . PHP_EOL; + $remainingprintlen = $maxdotsonline; + } else { + while ($part = substr($update, 0, $remainingprintlen) > 0) { + $progresscount += $maxdotsonline; + echo $part . " " . $progresscount . PHP_EOL; + $update = substr($update, $remainingprintlen); + $remainingprintlen = $maxdotsonline; + } + } + } + } else { + // Process exited. + $process->clearOutput(); + $exitcodes[$name] = $process->getExitCode(); + } + } + } + + echo PHP_EOL; + return $exitcodes; +} + +/** + * Print install output merging each run. + * + * @param array $processes list of processes. + * @return array exit codes of each process. + */ +function print_combined_install_output($processes) { + $exitcodes = array(); + $line = array(); + + // Check what best we can do to accommodate all parallel run o/p on single line. + // Windows command line has length of 80 chars, so default we will try fit o/p in 80 chars. + if (defined('BEHAT_MAX_CMD_LINE_OUTPUT') && BEHAT_MAX_CMD_LINE_OUTPUT) { + $lengthofprocessline = (int)max(10, BEHAT_MAX_CMD_LINE_OUTPUT / count($processes)); + } else { + $lengthofprocessline = (int)max(10, 80 / count($processes)); + } + + echo "Installing behat site for " . count($processes) . " parallel behat run" . PHP_EOL; + + // Show process name in first row. + foreach ($processes as $name => $process) { + // If we don't have enough space to show full run name then show runX. + if ($lengthofprocessline < strlen($name + 2)) { + $name = substr($name, -5); + } + // One extra padding as we are adding | separator for rest of the data. + $line[$name] = str_pad('[' . $name . '] ', $lengthofprocessline + 1); + } + ksort($line); + $tableheader = array_keys($line); + echo implode("", $line) . PHP_EOL; + + // Now print o/p from each process. + while (count($exitcodes) != count($processes)) { + usleep(50000); + $poutput = array(); + // Create child process. + foreach ($processes as $name => $process) { + if ($process->isRunning()) { + $output = $process->getIncrementalOutput(); + if (trim($output)) { + $poutput[$name] = explode(PHP_EOL, $output); + } + } else { + // Process exited. + $exitcodes[$name] = $process->getExitCode(); + } + } + ksort($poutput); + + // Get max depth of o/p before displaying. + $maxdepth = 0; + foreach ($poutput as $pout) { + $pdepth = count($pout); + $maxdepth = $pdepth >= $maxdepth ? $pdepth : $maxdepth; + } + + // Iterate over each process to get line to print. + for ($i = 0; $i <= $maxdepth; $i++) { + $pline = ""; + foreach ($tableheader as $name) { + $po = empty($poutput[$name][$i]) ? "" : substr($poutput[$name][$i], 0, $lengthofprocessline - 1); + $po = str_pad($po, $lengthofprocessline); + $pline .= "|". $po; + } + if (trim(str_replace("|", "", $pline))) { + echo $pline . PHP_EOL; + } + } + unset($poutput); + $poutput = null; + + } + echo PHP_EOL; + return $exitcodes; +} + +/** + * Print install output merging showing one run at a time. + * If any process fail then exit. + * + * @param array $processes list of processes. + * @param bool $showprefix show prefix. + * @return bool exitcode. + */ +function print_sequential_output($processes, $showprefix = true) { + $status = false; + foreach ($processes as $name => $process) { + $shownname = false; + while ($process->isRunning()) { + $op = $process->getIncrementalOutput(); + if (trim($op)) { + // Show name of the run once for sequential. + if ($showprefix && !$shownname) { + echo '[' . $name . '] '; + $shownname = true; + } + echo $op; + } + } + // If any error then exit. + $exitcode = $process->getExitCode(); + if ($exitcode != 0) { + exit($exitcode); + } + $status = $status || (bool)$exitcode; + } + return $status; +} diff --git a/admin/tool/behat/cli/util_single_run.php b/admin/tool/behat/cli/util_single_run.php new file mode 100644 index 00000000000..c42a908ced6 --- /dev/null +++ b/admin/tool/behat/cli/util_single_run.php @@ -0,0 +1,271 @@ +. + +/** + * CLI tool with utilities to manage Behat integration in Moodle + * + * All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as + * $CFG->dataroot and $CFG->prefix + * + * @package tool_behat + * @copyright 2012 David Monllaó + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + + +if (isset($_SERVER['REMOTE_ADDR'])) { + die(); // No access from web!. +} + +// Basic functions. +require_once(__DIR__ . '/../../../../lib/clilib.php'); +require_once(__DIR__ . '/../../../../lib/behat/lib.php'); + +// CLI options. +list($options, $unrecognized) = cli_get_params( + array( + 'help' => false, + 'install' => false, + 'parallel' => 0, + 'run' => '', + 'drop' => false, + 'enable' => false, + 'disable' => false, + 'diag' => false, + 'tags' => '', + 'updatesteps' => false, + ), + array( + 'h' => 'help' + ) +); + +if ($options['install'] or $options['drop']) { + define('CACHE_DISABLE_ALL', true); +} + +// Checking util_single_run.php CLI script usage. +$help = " +Behat utilities to manage the test environment + +Options: +--install Installs the test environment for acceptance tests +--drop Drops the database tables and the dataroot contents +--enable Enables test environment and updates tests list +--disable Disables test environment +--diag Get behat test environment status code +--updatesteps Update feature step file. + +-h, --help Print out this help + +Example from Moodle root directory: +\$ php admin/tool/behat/cli/util_single_run.php --enable + +More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests +"; + +if (!empty($options['help'])) { + echo $help; + exit(0); +} + +// Describe this script. +define('BEHAT_UTIL', true); +define('CLI_SCRIPT', true); +define('NO_OUTPUT_BUFFERING', true); +define('IGNORE_COMPONENT_CACHE', true); + +// Set run value, to be used by setup for configuring proper CFG variables. +if ($options['run']) { + define('BEHAT_CURRENT_RUN', $options['run']); +} + +// Only load CFG from config.php, stop ASAP in lib/setup.php. +define('ABORT_AFTER_CONFIG', true); +require_once(__DIR__ . '/../../../../config.php'); + +// Remove error handling overrides done in config.php. +$CFG->debug = (E_ALL | E_STRICT); +$CFG->debugdisplay = 1; +error_reporting($CFG->debug); +ini_set('display_errors', '1'); +ini_set('log_errors', '1'); + +// Finish moodle init. +define('ABORT_AFTER_CONFIG_CANCEL', true); +require("$CFG->dirroot/lib/setup.php"); + +raise_memory_limit(MEMORY_HUGE); + +require_once($CFG->libdir.'/adminlib.php'); +require_once($CFG->libdir.'/upgradelib.php'); +require_once($CFG->libdir.'/clilib.php'); +require_once($CFG->libdir.'/installlib.php'); +require_once($CFG->libdir.'/testing/classes/test_lock.php'); + +if ($unrecognized) { + $unrecognized = implode(PHP_EOL . " ", $unrecognized); + cli_error(get_string('cliunknowoption', 'admin', $unrecognized)); +} + +// Behat utilities. +require_once($CFG->libdir . '/behat/classes/util.php'); +require_once($CFG->libdir . '/behat/classes/behat_command.php'); +require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); + +// Ensure run option is <= parallel run installed. +if ($options['run']) { + if (!$options['parallel']) { + $options['parallel'] = behat_config_manager::get_parallel_test_runs(); + } + if (empty($options['parallel']) || $options['run'] > $options['parallel']) { + echo "Parallel runs can't be more then ".$options['parallel'].PHP_EOL; + exit(1); + } + $CFG->behatrunprocess = $options['run']; +} + +// Run command (only one per time). +if ($options['install']) { + behat_util::install_site(); + + // This is only displayed once for parallel install. + if (empty($options['run'])) { + mtrace("Acceptance tests site installed"); + } + +} else if ($options['drop']) { + // Ensure no tests are running. + test_lock::acquire('behat'); + behat_util::drop_site(); + // This is only displayed once for parallel install. + if (empty($options['run'])) { + mtrace("Acceptance tests site dropped"); + } + +} else if ($options['enable']) { + if (!empty($options['parallel'])) { + // Save parallel site info for enable and install options. + $filepath = behat_config_manager::get_parallel_test_file_path(); + if (!file_put_contents($filepath, $options['parallel'])) { + behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created'); + } + } + + // Enable test mode. + behat_util::start_test_mode(); + + // This is only displayed once for parallel install. + if (empty($options['run'])) { + $runtestscommand = behat_command::get_behat_command(true, !empty($options['run'])); + + $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); + mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use: " . PHP_EOL . + $runtestscommand); + } + +} else if ($options['disable']) { + behat_util::stop_test_mode(); + // This is only displayed once for parallel install. + if (empty($options['run'])) { + mtrace("Acceptance tests environment disabled"); + } + +} else if ($options['diag']) { + $code = behat_util::get_behat_status(); + exit($code); + +} else if ($options['updatesteps']) { + $behatstepfile = __DIR__ . "/../../../../behat_features_step_count.json"; + if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) { + $behatstepfile = BEHAT_FEATURE_STEP_FILE; + } + + // Rewrite config file to ensure we have all the features covered. + behat_config_manager::update_config_file(); + + // Run behat command to get steps in feature files. + $featurestepscmd = behat_command::get_behat_command(true); + $featurestepscmd .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); + $featurestepscmd .= ' --dry-run --format=moodle_step_count'; + $processes = cli_execute_parallel(array($featurestepscmd), __DIR__ . "/../../../../"); + $status = print_update_step_output(array_pop($processes), $behatstepfile); + + exit($status); +} else { + echo $help; + exit(1); +} + +exit(0); + +/** + * Print update progress as dots for updating feature file step list. + * + * @param Process $process process executing update step command. + * @param string $featurestepfile feature step file in which steps will be saved. + * @return int exitcode. + */ +function print_update_step_output($process, $featurestepfile) { + $printedlength = 0; + + echo "Updating steps feature file for parallel behat runs" . PHP_EOL; + + // Show progress while running command. + while ($process->isRunning()) { + usleep(10000); + $op = $process->getIncrementalOutput(); + if (trim($op)) { + echo "."; + $printedlength++; + if ($printedlength > 70) { + $printedlength = 0; + echo PHP_EOL; + } + } + } + + // If any error then exit. + $exitcode = $process->getExitCode(); + // Output err. + if ($exitcode != 0) { + echo $process->getErrorOutput(); + exit($exitcode); + } + + // Extract features with step info and save it in file. + $featuresteps = $process->getOutput(); + $featuresteps = explode(PHP_EOL, $featuresteps); + + $realroot = realpath(__DIR__.'/../../../../').'/'; + foreach ($featuresteps as $featurestep) { + if (trim($featurestep)) { + $step = explode("::", $featurestep); + $step[0] = str_replace($realroot, '', $step[0]); + $steps[$step[0]] = $step[1]; + } + } + arsort($steps); + + if (!@file_put_contents($featurestepfile, json_encode($steps, JSON_PRETTY_PRINT))) { + behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $featurestepfile . ' can not be created'); + $exitcode = -1; + } + + echo PHP_EOL; + + return $exitcode; +} diff --git a/admin/tool/behat/cli/utilparallel.php b/admin/tool/behat/cli/utilparallel.php deleted file mode 100644 index c47c5a5c90b..00000000000 --- a/admin/tool/behat/cli/utilparallel.php +++ /dev/null @@ -1,195 +0,0 @@ -. - -/** - * CLI tool with utilities to manage parallel Behat integration in Moodle - * - * All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as - * $CFG->dataroot and $CFG->prefix - * - * @package tool_behat - * @copyright 2015 Rajesh Taneja - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ - - -if (isset($_SERVER['REMOTE_ADDR'])) { - die(); // No access from web!. -} - -define('BEHAT_UTIL', true); -define('CLI_SCRIPT', true); -define('NO_OUTPUT_BUFFERING', true); -define('IGNORE_COMPONENT_CACHE', true); - -require_once(__DIR__ . '/../../../../lib/clilib.php'); -require_once(__DIR__ . '/../../../../lib/behat/lib.php'); - -// CLI options. -list($options, $unrecognized) = cli_get_params( - array( - 'help' => false, - 'install' => false, - 'drop' => false, - 'enable' => false, - 'disable' => false, - 'diag' => false, - 'parallel' => 0, - 'maxruns' => false - ), - array( - 'h' => 'help', - 'j' => 'parallel', - 'm' => 'maxruns' - ) -); - -// Checking util.php CLI script usage. -$help = " -Behat utilities to manage the test environment - -Options: ---install Installs the test environment for acceptance tests ---drop Drops the database tables and the dataroot contents ---enable Enables test environment and updates tests list ---disable Disables test environment ---diag Get behat test environment status code --j, --parallel Number of parallel behat run operation --m, --maxruns Max parallel processes to be executed at one time. - --h, --help Print out this help - -Example from Moodle root directory: -\$ php admin/tool/behat/cli/utilparallel.php --enable --parallel=4 - -More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests -"; - -if (!empty($options['help'])) { - echo $help; - exit(0); -} - -if (empty($options['parallel'])) { - echo $help; - exit(1); -} - -$status = 0; -$cmds = commands_to_execute($options); -$cwd = getcwd(); -chdir(__DIR__); - -// Start executing commands either sequential/parallel for options provided. -if ($options['diag'] || $options['drop'] || $options['enable'] || $options['disable']) { - $code = cli_execute_sequential($cmds, true); - // If any error then exit. - foreach ($code as $c) { - if ($c != 0) { - exit($c); - } - } -} else if ($options['install']) { - // This is intensive compared to behat itself so run them in chunk if $CFG->behat_max_parallel_init not set. - if ($options['maxruns']) { - foreach (array_chunk($cmds, maxruns, true) as $chunk) { - $chunkstatus = (bool)cli_execute_parallel($chunk, __DIR__, true, true); - $status = $chunkstatus || (bool) $status; - } - } else { - $status = (bool)cli_execute_parallel($cmds, __DIR__, true, true); - } -} else { - // We should never reach here. - echo $help; - exit(1); -} - -// Ensure we have success status to show following information. -if ($status) { - echo "Unknown failure $status".PHP_EOL; - exit((int)$status); -} - -// Only load CFG from config.php for 1st run amd stop ASAP in lib/setup.php. -define('ABORT_AFTER_CONFIG', true); -define('BEHAT_CURRENT_RUN', 1); -require_once(__DIR__ . '/../../../../config.php'); -require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php'); -require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php'); - -// Remove first link from wwwroot, as it is set to first run. -$CFG->behat_wwwroot = str_replace('/'.BEHAT_PARALLEL_SITE_WWW_SUFFIX . '1', '', $CFG->behat_wwwroot); - -// Show command o/p (only one per time). -if ($options['install']) { - echo "Acceptance tests site installed for sites:".PHP_EOL; - // Display all sites which are installed/drop/diabled. - for ($i = 1; $i <= $options['parallel']; $i++ ) { - echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i . PHP_EOL; - } -} else if ($options['drop']) { - echo "Acceptance tests site dropped for ".$options['parallel']." parallel sites".PHP_EOL; - -} else if ($options['enable']) { - echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:".PHP_EOL; - echo behat_command::get_behat_command(true, true); - echo PHP_EOL; - -} else if ($options['disable']) { - echo "Acceptance tests environment disabled for ".$options['parallel']." parallel sites".PHP_EOL; - -} else { - echo $help; -} - -chdir($cwd); -exit(0); - -/** - * Create commands to be executed for parallel run. - * - * @param array $options options provided by user. - * @return array commands to be executed. - */ -function commands_to_execute($options) { - $removeoptions = array('maxruns'); - $cmds = array(); - $extraoptions = $options; - $extra = ""; - - // Remove extra options not in util.php - foreach ($removeoptions as $ro) { - $extraoptions[$ro] = null; - unset($extraoptions[$ro]); - } - - foreach ($extraoptions as $option => $value) { - if ($options[$option]) { - $extra .= " --$option"; - if ($value) { - $extra .= "='$value'"; - } - } - } - - // Create commands which has to be executed for parallel site. - for ($i = 1; $i <= $options['parallel']; $i++) { - $prefix = BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i; - $cmds[$prefix] = "php util.php ".$extra." --run=".$i." 2>&1"; - } - return $cmds; -} \ No newline at end of file diff --git a/behat_features_step_count.json b/behat_features_step_count.json new file mode 100644 index 00000000000..f0963e1cf23 --- /dev/null +++ b/behat_features_step_count.json @@ -0,0 +1,286 @@ +{ + "course\/tests\/behat\/course_category_management_listing.feature": "859", + "grade\/tests\/behat\/grade_aggregation.feature": "746", + "course\/tests\/behat\/course_controls.feature": "648", + "grade\/tests\/behat\/grade_scales.feature": "566", + "course\/tests\/behat\/course_resort.feature": "481", + "grade\/tests\/behat\/grade_single_item_scales.feature": "434", + "grade\/tests\/behat\/grade_calculated_weights.feature": "369", + "grade\/tests\/behat\/grade_scales_aggregation.feature": "349", + "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": "291", + "grade\/tests\/behat\/grade_natural_normalisation.feature": "288", + "admin\/tool\/monitor\/tests\/behat\/subscription.feature": "285", + "course\/tests\/behat\/category_resort.feature": "256", + "course\/tests\/behat\/category_change_visibility.feature": "234", + "course\/tests\/behat\/category_management.feature": "225", + "admin\/tool\/behat\/tests\/behat\/data_generators.feature": "218", + "admin\/tool\/monitor\/tests\/behat\/rule.feature": "213", + "mod\/quiz\/tests\/behat\/editing_add.feature": "207", + "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": "204", + "blocks\/navigation\/tests\/behat\/expand_courses_node.feature": "189", + "course\/format\/social\/tests\/behat\/social_adjust_discussion_count.feature": "184", + "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": "176", + "mod\/workshop\/tests\/behat\/workshop_assessment.feature": "162", + "badges\/tests\/behat\/award_badge.feature": "158", + "group\/tests\/behat\/update_groups.feature": "150", + "availability\/tests\/behat\/edit_availability.feature": "147", + "mod\/wiki\/tests\/behat\/wiki_search.feature": "140", + "mod\/forum\/tests\/behat\/track_read_posts.feature": "136", + "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": "132", + "mod\/assign\/tests\/behat\/quickgrading.feature": "132", + "grade\/tests\/behat\/grade_contribution_with_extra_credit.feature": "129", + "mod\/lesson\/tests\/behat\/link_to_gradebook.feature": "128", + "mod\/forum\/tests\/behat\/discussion_navigation.feature": "126", + "grade\/report\/singleview\/tests\/behat\/singleview.feature": "120", + "mod\/assign\/tests\/behat\/grading_status.feature": "116", + "grade\/tests\/behat\/grade_point_maximum.feature": "114", + "mod\/glossary\/tests\/behat\/search_entries.feature": "113", + "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": "110", + "mod\/quiz\/tests\/behat\/editing_repaginate.feature": "110", + "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": "108", + "mod\/assign\/tests\/behat\/outcome_grading.feature": "104", + "question\/tests\/behat\/preview_question.feature": "102", + "group\/tests\/behat\/groups_import.feature": "99", + "group\/tests\/behat\/auto_creation.feature": "98", + "report\/outline\/tests\/behat\/outline.feature": "98", + "mod\/assign\/tests\/behat\/prevent_submission_changes.feature": "97", + "blocks\/comments\/tests\/behat\/add_comment.feature": "96", + "course\/tests\/behat\/paged_course_navigation.feature": "96", + "cohort\/tests\/behat\/access_visible_cohorts.feature": "96", + "grade\/tests\/behat\/grade_view.feature": "94", + "grade\/tests\/behat\/grade_override_letter.feature": "94", + "availability\/tests\/behat\/display_availability.feature": "94", + "mod\/forum\/tests\/behat\/discussion_display.feature": "92", + "course\/format\/weeks\/tests\/behat\/edit_delete_sections.feature": "90", + "mod\/quiz\/tests\/behat\/editing_set_marks_with_attempts.feature": "89", + "mod\/lesson\/tests\/behat\/lesson_edit_pages.feature": "88", + "course\/format\/topics\/tests\/behat\/edit_delete_sections.feature": "87", + "report\/outline\/tests\/behat\/user.feature": "86", + "admin\/tool\/filetypes\/tests\/behat\/add_filetypes.feature": "86", + "mod\/lesson\/tests\/behat\/lesson_number_of_student_attempts.feature": "86", + "mod\/glossary\/tests\/behat\/categories.feature": "85", + "mod\/assign\/feedback\/editpdf\/tests\/behat\/annotate_pdf.feature": "84", + "mod\/wiki\/tests\/behat\/wiki_comments.feature": "83", + "report\/participation\/tests\/behat\/filter_participation.feature": "83", + "mod\/lesson\/tests\/behat\/lesson_navigation.feature": "82", + "question\/tests\/behat\/question_categories.feature": "81", + "group\/tests\/behat\/create_groups.feature": "80", + "course\/tests\/behat\/course_change_visibility.feature": "79", + "mod\/lesson\/tests\/behat\/lesson_with_clusters.feature": "79", + "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": "79", + "mod\/lesson\/tests\/behat\/lesson_practice.feature": "78", + "group\/tests\/behat\/delete_groups.feature": "76", + "cohort\/tests\/behat\/upload_cohorts.feature": "76", + "mod\/assign\/tests\/behat\/group_submission.feature": "76", + "question\/tests\/behat\/sort_questions.feature": "75", + "mod\/choice\/tests\/behat\/publish_results.feature": "73", + "mod\/quiz\/tests\/behat\/editing_click_delete_icon.feature": "71", + "report\/loglive\/tests\/behat\/loglive_report.feature": "71", + "blog\/tests\/behat\/comment.feature": "70", + "calendar\/tests\/behat\/calendar.feature": "70", + "completion\/tests\/behat\/restrict_section_availability.feature": "69", + "grade\/export\/txt\/tests\/behat\/export.feature": "69", + "blocks\/navigation\/tests\/behat\/view_my_courses.feature": "69", + "availability\/condition\/profile\/tests\/behat\/availability_profile.feature": "68", + "mod\/forum\/tests\/behat\/forum_subscriptions.feature": "68", + "mod\/forum\/tests\/behat\/edit_post_student.feature": "66", + "report\/log\/tests\/behat\/filter_log.feature": "65", + "availability\/condition\/grade\/tests\/behat\/availability_grade.feature": "64", + "question\/format\/xml\/tests\/behat\/import_export.feature": "64", + "mod\/quiz\/tests\/behat\/editing_click_move_icon.feature": "64", + "mod\/forum\/tests\/behat\/forum_subscriptions_availability.feature": "62", + "course\/tests\/behat\/create_delete_course.feature": "62", + "blocks\/tests\/behat\/manage_blocks.feature": "62", + "user\/tests\/behat\/delete_users.feature": "61", + "mod\/quiz\/tests\/behat\/settings_form_fields_disableif.feature": "60", + "mod\/wiki\/tests\/behat\/wiki_formats.feature": "59", + "grade\/tests\/behat\/grade_mingrade.feature": "59", + "mod\/lesson\/tests\/behat\/questions_images.feature": "59", + "course\/tests\/behat\/activities_edit_completion.feature": "59", + "mod\/forum\/tests\/behat\/edit_post_teacher.feature": "59", + "mod\/quiz\/tests\/behat\/editing_set_marks_no_attempts.feature": "59", + "mod\/lesson\/tests\/behat\/lesson_edit_cluster.feature": "58", + "mod\/wiki\/tests\/behat\/collaborative_individual.feature": "57", + "mod\/forum\/tests\/behat\/separate_group_discussions.feature": "57", + "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": "56", + "availability\/condition\/group\/tests\/behat\/availability_group.feature": "55", + "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": "55", + "lib\/editor\/atto\/plugins\/image\/tests\/behat\/image.feature": "55", + "course\/tests\/behat\/section_highlighting.feature": "54", + "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": "53", + "course\/tests\/behat\/move_activities.feature": "52", + "blocks\/tests\/behat\/hidden_block_region.feature": "52", + "course\/tests\/behat\/activities_visibility_icons.feature": "52", + "lib\/editor\/atto\/plugins\/accessibilitychecker\/tests\/behat\/accessibilitychecker.feature": "52", + "mod\/quiz\/tests\/behat\/add_quiz.feature": "51", + "mod\/lesson\/tests\/behat\/teacher_grade_essays.feature": "51", + "report\/eventlist\/tests\/behat\/mainsection.feature": "51", + "mod\/wiki\/tests\/behat\/edit_tags.feature": "51", + "report\/log\/tests\/behat\/user_log.feature": "50", + "grade\/tests\/behat\/grade_UI_settings.feature": "50", + "enrol\/self\/tests\/behat\/self_enrolment.feature": "50", + "course\/tests\/behat\/add_activities.feature": "49", + "mod\/lesson\/tests\/behat\/lesson_essay_question.feature": "49", + "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": "48", + "mod\/book\/tests\/behat\/log_entries.feature": "48", + "mod\/data\/tests\/behat\/view_entries.feature": "48", + "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": "48", + "availability\/condition\/grouping\/tests\/behat\/availability_grouping.feature": "47", + "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": "47", + "mod\/assign\/feedback\/editpdf\/tests\/behat\/group_annotations.feature": "46", + "lib\/editor\/tinymce\/tests\/behat\/edit_available_icons.feature": "46", + "mod\/book\/tests\/behat\/show_hide_chapters.feature": "46", + "grade\/report\/history\/tests\/behat\/basic_functionality.feature": "46", + "mod\/wiki\/tests\/behat\/page_history.feature": "45", + "mod\/assign\/tests\/behat\/edit_previous_feedback.feature": "45", + "question\/tests\/behat\/delete_questions.feature": "45", + "cohort\/tests\/behat\/add_cohort.feature": "45", + "admin\/tests\/behat\/filter_users.feature": "45", + "cohort\/tests\/behat\/view_cohorts.feature": "45", + "mod\/wiki\/tests\/behat\/preview_page.feature": "44", + "mod\/lesson\/tests\/behat\/lesson_progress_bar.feature": "44", + "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": "44", + "group\/tests\/behat\/id_uniqueness.feature": "44", + "repository\/tests\/behat\/create_shortcut.feature": "44", + "backup\/util\/ui\/tests\/behat\/backup_courses.feature": "44", + "mod\/lesson\/tests\/behat\/lesson_review.feature": "43", + "mod\/lesson\/tests\/behat\/date_availability.feature": "42", + "completion\/tests\/behat\/restrict_activity_by_date.feature": "41", + "mod\/glossary\/tests\/behat\/entries_require_approval.feature": "41", + "user\/tests\/behat\/view_full_profile.feature": "41", + "course\/tests\/behat\/move_sections.feature": "40", + "completion\/tests\/behat\/restrict_activity_by_grade.feature": "40", + "mod\/assign\/tests\/behat\/comment_inline.feature": "39", + "mod\/assign\/tests\/behat\/submission_comments.feature": "39", + "course\/tests\/behat\/section_visibility.feature": "39", + "message\/tests\/behat\/display_history.feature": "39", + "mod\/data\/tests\/behat\/add_entries.feature": "39", + "mod\/assign\/tests\/behat\/grant_extension.feature": "38", + "mod\/lesson\/tests\/behat\/completion_condition_end_reached.feature": "38", + "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": "38", + "question\/tests\/behat\/copy_questions.feature": "37", + "blocks\/participants\/tests\/behat\/block_participants_course.feature": "37", + "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": "37", + "mod\/book\/tests\/behat\/create_chapters.feature": "37", + "mod\/choice\/tests\/behat\/multiple_options.feature": "36", + "mod\/feedback\/tests\/behat\/show_nonrespondents.feature": "36", + "admin\/tool\/behat\/tests\/behat\/edit_permissions.feature": "36", + "course\/tests\/behat\/force_group_mode.feature": "36", + "cohort\/tests\/behat\/upload_cohort_users.feature": "35", + "mod\/assign\/tests\/behat\/allow_another_attempt.feature": "35", + "mod\/choice\/tests\/behat\/publish_results_anonymously.feature": "35", + "message\/tests\/behat\/manage_contacts.feature": "34", + "blocks\/html\/tests\/behat\/multiple_instances.feature": "34", + "message\/tests\/behat\/send_message.feature": "34", + "admin\/tool\/behat\/tests\/behat\/basic_actions.feature": "33", + "course\/tests\/behat\/frontpage_display_modes.feature": "33", + "mod\/choice\/tests\/behat\/change_response.feature": "32", + "enrol\/guest\/tests\/behat\/guest_access.feature": "32", + "admin\/tool\/uploadcourse\/tests\/behat\/create.feature": "32", + "blocks\/tests\/behat\/return_block_original_state.feature": "32", + "mod\/lesson\/tests\/behat\/lesson_informations_at_end.feature": "32", + "lib\/editor\/atto\/plugins\/align\/tests\/behat\/align.feature": "31", + "availability\/condition\/date\/tests\/behat\/availability_date.feature": "31", + "mod\/glossary\/tests\/behat\/print_friendly_version.feature": "30", + "mod\/assign\/tests\/behat\/display_grade.feature": "30", + "repository\/tests\/behat\/overwrite_file.feature": "30", + "course\/tests\/behat\/edit_settings.feature": "30", + "mod\/assign\/tests\/behat\/file_submission.feature": "30", + "blocks\/tests\/behat\/configure_block_throughout_site.feature": "30", + "course\/tests\/behat\/course_creation.feature": "29", + "admin\/tool\/availabilityconditions\/tests\/behat\/manage_conditions.feature": "29", + "completion\/tests\/behat\/teacher_manual_completion.feature": "29", + "lib\/editor\/atto\/plugins\/table\/tests\/behat\/table.feature": "29", + "question\/tests\/behat\/edit_questions.feature": "29", + "blocks\/html\/tests\/behat\/configuring_html_block.feature": "28", + "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": "28", + "admin\/tests\/behat\/upload_users.feature": "28", + "course\/tests\/behat\/max_number_sections.feature": "27", + "course\/tests\/behat\/restrict_available_activities.feature": "27", + "course\/tests\/behat\/activities_group_icons.feature": "27", + "mod\/lesson\/tests\/behat\/time_limit.feature": "27", + "mod\/assign\/tests\/behat\/filter_by_marker.feature": "26", + "user\/tests\/behat\/table_sorting.feature": "26", + "mod\/assign\/tests\/behat\/online_submissions.feature": "26", + "blocks\/tests\/behat\/restrict_available_blocks.feature": "26", + "course\/tests\/behat\/course_search.feature": "25", + "mod\/forum\/tests\/behat\/completion_condition_number_discussions.feature": "25", + "course\/tests\/behat\/rename_roles.feature": "25", + "badges\/tests\/behat\/add_badge.feature": "25", + "completion\/tests\/behat\/enable_manual_complete_mark.feature": "24", + "lib\/editor\/atto\/plugins\/equation\/tests\/behat\/equation.feature": "24", + "mod\/forum\/tests\/behat\/single_forum_discussion.feature": "24", + "availability\/condition\/completion\/tests\/behat\/availability_completion.feature": "24", + "blocks\/news_items\/tests\/behat\/display_news.feature": "24", + "mod\/lesson\/tests\/behat\/password_protection.feature": "24", + "mod\/lesson\/tests\/behat\/import_fillintheblank_question.feature": "24", + "mod\/scorm\/tests\/behat\/add_scorm.feature": "23", + "admin\/tool\/behat\/tests\/behat\/manipulate_forms.feature": "23", + "my\/tests\/behat\/restrict_available_blocks.feature": "23", + "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": "23", + "mod\/glossary\/tests\/behat\/entries_always_editable.feature": "21", + "auth\/tests\/behat\/login.feature": "21", + "lib\/editor\/atto\/plugins\/indent\/tests\/behat\/indent.feature": "21", + "blocks\/html\/tests\/behat\/course_block.feature": "21", + "lib\/editor\/atto\/plugins\/accessibilityhelper\/tests\/behat\/accessibilityhelper.feature": "21", + "lib\/editor\/atto\/plugins\/italic\/tests\/behat\/italic.feature": "20", + "mod\/lesson\/tests\/behat\/import_images.feature": "20", + "repository\/tests\/behat\/cancel_add_file.feature": "20", + "mod\/forum\/tests\/behat\/add_forum.feature": "20", + "lib\/editor\/atto\/plugins\/bold\/tests\/behat\/bold.feature": "20", + "backup\/util\/ui\/tests\/behat\/duplicate_activities.feature": "20", + "files\/tests\/behat\/course_files.feature": "20", + "lib\/editor\/atto\/plugins\/media\/tests\/behat\/media.feature": "20", + "course\/tests\/behat\/category_role_assignment.feature": "20", + "message\/tests\/behat\/message_participants.feature": "19", + "mod\/choice\/tests\/behat\/limit_responses.feature": "19", + "blocks\/comments\/tests\/behat\/delete_comment.feature": "19", + "course\/tests\/behat\/activities_indentation.feature": "19", + "admin\/tool\/uploadcourse\/tests\/behat\/update.feature": "18", + "repository\/tests\/behat\/zip_and_unzip.feature": "18", + "mod\/forum\/tests\/behat\/my_forum_posts.feature": "18", + "grade\/export\/xml\/tests\/behat\/export.feature": "18", + "message\/tests\/behat\/search_history.feature": "18", + "question\/format\/gift\/tests\/behat\/import_export.feature": "18", + "blocks\/login\/tests\/behat\/login_block.feature": "18", + "mod\/survey\/tests\/behat\/survey_types.feature": "18", + "admin\/tool\/behat\/tests\/behat\/list_steps.feature": "18", + "repository\/recent\/tests\/behat\/add_recent.feature": "17", + "blocks\/tests\/behat\/add_blocks.feature": "16", + "message\/tests\/behat\/block_users.feature": "16", + "admin\/tests\/behat\/set_admin_settings_value.feature": "16", + "lib\/editor\/atto\/plugins\/link\/tests\/behat\/link.feature": "16", + "blocks\/tests\/behat\/hide_blocks.feature": "16", + "backup\/util\/ui\/tests\/behat\/import_course.feature": "15", + "my\/tests\/behat\/reset_page.feature": "15", + "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": "15", + "mod\/label\/tests\/behat\/label_visibility.feature": "14", + "repository\/tests\/behat\/create_folders.feature": "14", + "question\/format\/webct\/tests\/behat\/importcalculated.feature": "14", + "question\/format\/webct\/tests\/behat\/import.feature": "14", + "mod\/glossary\/tests\/behat\/prevent_duplicate_entries.feature": "14", + "course\/tests\/behat\/activities_edit_with_block_dock.feature": "13", + "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": "13", + "mod\/choice\/tests\/behat\/add_choice.feature": "13", + "admin\/tests\/behat\/display_short_names.feature": "12", + "admin\/tool\/monitor\/tests\/behat\/disabled.feature": "12", + "repository\/tests\/behat\/delete_files.feature": "12", + "lib\/editor\/atto\/plugins\/clear\/tests\/behat\/clear.feature": "12", + "repository\/upload\/tests\/behat\/upload_file.feature": "11", + "my\/tests\/behat\/add_blocks.feature": "11", + "lib\/editor\/atto\/plugins\/charmap\/tests\/behat\/charmap.feature": "11", + "lib\/editor\/atto\/plugins\/subscript\/tests\/behat\/subscript.feature": "10", + "lib\/editor\/atto\/plugins\/strike\/tests\/behat\/strike.feature": "10", + "lib\/editor\/atto\/plugins\/superscript\/tests\/behat\/superscript.feature": "10", + "lib\/editor\/atto\/plugins\/title\/tests\/behat\/title.feature": "10", + "lib\/editor\/atto\/plugins\/underline\/tests\/behat\/underline.feature": "10", + "user\/tests\/behat\/reset_page.feature": "10", + "lib\/editor\/atto\/plugins\/unorderedlist\/tests\/behat\/unorderedlist.feature": "9", + "user\/tests\/behat\/edituserpassword.feature": "9", + "lib\/editor\/atto\/plugins\/orderedlist\/tests\/behat\/orderedlist.feature": "9", + "user\/tests\/behat\/add_blocks.feature": "8", + "lib\/editor\/atto\/plugins\/collapse\/tests\/behat\/collapse.feature": "6", + "lib\/editor\/atto\/plugins\/html\/tests\/behat\/html.feature": "6", + "report\/usersessions\/tests\/behat\/usersessions_report.feature": "3", + "admin\/tool\/behat\/tests\/behat\/test_environment.feature": "2" +} \ No newline at end of file diff --git a/behat_pga_default.json b/behat_pga_default.json deleted file mode 100644 index 1f53315efc6..00000000000 --- a/behat_pga_default.json +++ /dev/null @@ -1,283 +0,0 @@ -{ - "grade\/tests\/behat\/grade_aggregation.feature": 1675.3, - "grade\/tests\/behat\/grade_scales.feature": 909, - "grade\/tests\/behat\/grade_scales_aggregation.feature": 831.6, - "course\/tests\/behat\/course_category_management_listing.feature": 781.1, - "grade\/tests\/behat\/grade_single_item_scales.feature": 730.8, - "grade\/tests\/behat\/grade_calculated_weights.feature": 729.7, - "admin\/tool\/monitor\/tests\/behat\/subscription.feature": 628.5, - "course\/tests\/behat\/course_controls.feature": 579.5, - "admin\/tool\/monitor\/tests\/behat\/rule.feature": 387.4, - "mod\/forum\/tests\/behat\/discussion_navigation.feature": 382.5, - "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": 372, - "mod\/forum\/tests\/behat\/track_read_posts.feature": 367.2, - "mod\/forum\/tests\/behat\/discussion_display.feature": 357.4, - "badges\/tests\/behat\/award_badge.feature": 345, - "grade\/tests\/behat\/grade_natural_normalisation.feature": 330, - "grade\/tests\/behat\/grade_contribution_with_extra_credit.feature": 307.7, - "group\/tests\/behat\/update_groups.feature": 284.3, - "course\/tests\/behat\/course_resort.feature": 280.7, - "mod\/glossary\/tests\/behat\/search_entries.feature": 258.6, - "mod\/lesson\/tests\/behat\/link_to_gradebook.feature": 240.6, - "report\/outline\/tests\/behat\/outline.feature": 236.8, - "mod\/wiki\/tests\/behat\/wiki_search.feature": 227.9, - "grade\/tests\/behat\/grade_point_maximum.feature": 217.4, - "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": 215.9, - "grade\/tests\/behat\/grade_view.feature": 215.9, - "blocks\/navigation\/tests\/behat\/expand_courses_node.feature": 204.1, - "report\/participation\/tests\/behat\/filter_participation.feature": 197.9, - "report\/loglive\/tests\/behat\/loglive_report.feature": 196.9, - "course\/tests\/behat\/category_resort.feature": 196.9, - "mod\/workshop\/tests\/behat\/workshop_assessment.feature": 190.4, - "mod\/assign\/tests\/behat\/outcome_grading.feature": 186.5, - "mod\/forum\/tests\/behat\/edit_post_student.feature": 184, - "mod\/forum\/tests\/behat\/edit_post_teacher.feature": 182.7, - "mod\/assign\/tests\/behat\/quickgrading.feature": 178.5, - "completion\/tests\/behat\/restrict_section_availability.feature": 169.9, - "report\/outline\/tests\/behat\/user.feature": 167.4, - "availability\/tests\/behat\/edit_availability.feature": 166.8, - "course\/tests\/behat\/section_highlighting.feature": 162.5, - "mod\/assign\/tests\/behat\/prevent_submission_changes.feature": 161.1, - "mod\/wiki\/tests\/behat\/wiki_comments.feature": 155.8, - "mod\/assign\/feedback\/editpdf\/tests\/behat\/annotate_pdf.feature": 155.6, - "mod\/lesson\/tests\/behat\/lesson_practice.feature": 151.2, - "availability\/tests\/behat\/display_availability.feature": 150.6, - "mod\/assign\/tests\/behat\/grading_status.feature": 150.5, - "mod\/quiz\/tests\/behat\/editing_set_marks_with_attempts.feature": 148.9, - "cohort\/tests\/behat\/upload_cohorts.feature": 146.4, - "group\/tests\/behat\/delete_groups.feature": 143.8, - "group\/tests\/behat\/groups_import.feature": 143.6, - "mod\/lesson\/tests\/behat\/lesson_edit_pages.feature": 139.6, - "group\/tests\/behat\/auto_creation.feature": 136.6, - "course\/tests\/behat\/category_management.feature": 134.5, - "mod\/quiz\/tests\/behat\/editing_add.feature": 132.8, - "admin\/tool\/behat\/tests\/behat\/data_generators.feature": 131.7, - "mod\/choice\/tests\/behat\/publish_results.feature": 130.7, - "group\/tests\/behat\/create_groups.feature": 128.1, - "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": 126.7, - "availability\/condition\/profile\/tests\/behat\/availability_profile.feature": 124.6, - "grade\/tests\/behat\/grade_mingrade.feature": 123.8, - "mod\/lesson\/tests\/behat\/lesson_number_of_student_attempts.feature": 122.5, - "backup\/util\/ui\/tests\/behat\/backup_courses.feature": 122, - "mod\/lesson\/tests\/behat\/lesson_navigation.feature": 121.7, - "mod\/lesson\/tests\/behat\/lesson_with_clusters.feature": 119.9, - "blocks\/comments\/tests\/behat\/add_comment.feature": 118.8, - "mod\/forum\/tests\/behat\/separate_group_discussions.feature": 117.3, - "completion\/tests\/behat\/restrict_activity_by_date.feature": 117.2, - "lib\/editor\/tinymce\/tests\/behat\/edit_available_icons.feature": 113, - "mod\/glossary\/tests\/behat\/entries_always_editable.feature": 112.3, - "report\/log\/tests\/behat\/user_log.feature": 111.4, - "course\/tests\/behat\/activities_edit_completion.feature": 111.4, - "report\/log\/tests\/behat\/filter_log.feature": 108.2, - "mod\/assign\/tests\/behat\/group_submission.feature": 108.1, - "mod\/glossary\/tests\/behat\/categories.feature": 107.6, - "mod\/quiz\/tests\/behat\/editing_click_delete_icon.feature": 107.6, - "availability\/condition\/grade\/tests\/behat\/availability_grade.feature": 107.1, - "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": 106, - "course\/tests\/behat\/category_change_visibility.feature": 105.5, - "mod\/wiki\/tests\/behat\/collaborative_individual.feature": 104, - "course\/tests\/behat\/max_number_sections.feature": 104, - "course\/tests\/behat\/activities_visibility_icons.feature": 102.3, - "admin\/tool\/behat\/tests\/behat\/edit_permissions.feature": 100.4, - "mod\/lesson\/tests\/behat\/time_limit.feature": 98.7, - "mod\/quiz\/tests\/behat\/editing_set_marks_no_attempts.feature": 97, - "admin\/tests\/behat\/filter_users.feature": 94.4, - "mod\/forum\/tests\/behat\/add_forum.feature": 92.8, - "user\/tests\/behat\/delete_users.feature": 92, - "grade\/tests\/behat\/grade_UI_settings.feature": 91.9, - "completion\/tests\/behat\/restrict_activity_by_grade.feature": 91.9, - "enrol\/self\/tests\/behat\/self_enrolment.feature": 91.5, - "blocks\/navigation\/tests\/behat\/view_my_courses.feature": 91.3, - "question\/tests\/behat\/preview_question.feature": 90.9, - "availability\/condition\/group\/tests\/behat\/availability_group.feature": 89.3, - "blog\/tests\/behat\/comment.feature": 85.9, - "mod\/forum\/tests\/behat\/forum_subscriptions_availability.feature": 85.8, - "mod\/lesson\/tests\/behat\/lesson_edit_cluster.feature": 85.6, - "course\/tests\/behat\/section_visibility.feature": 84.9, - "question\/tests\/behat\/question_categories.feature": 84.6, - "question\/format\/xml\/tests\/behat\/import_export.feature": 83.5, - "question\/tests\/behat\/delete_questions.feature": 82.5, - "mod\/forum\/tests\/behat\/completion_condition_number_discussions.feature": 82.1, - "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": 82, - "question\/tests\/behat\/sort_questions.feature": 81.6, - "group\/tests\/behat\/id_uniqueness.feature": 80, - "admin\/tool\/filetypes\/tests\/behat\/add_filetypes.feature": 79.5, - "mod\/lesson\/tests\/behat\/date_availability.feature": 78.1, - "cohort\/tests\/behat\/add_cohort.feature": 76.8, - "grade\/report\/singleview\/tests\/behat\/singleview.feature": 76.5, - "mod\/wiki\/tests\/behat\/wiki_formats.feature": 75.8, - "course\/tests\/behat\/force_group_mode.feature": 75.5, - "blocks\/news_items\/tests\/behat\/display_news.feature": 75.1, - "course\/tests\/behat\/frontpage_display_modes.feature": 74.9, - "mod\/assign\/tests\/behat\/edit_previous_feedback.feature": 74.1, - "availability\/condition\/grouping\/tests\/behat\/availability_grouping.feature": 73.1, - "course\/tests\/behat\/restrict_available_activities.feature": 72.9, - "mod\/assign\/feedback\/editpdf\/tests\/behat\/group_annotations.feature": 72.2, - "mod\/lesson\/tests\/behat\/questions_images.feature": 72, - "mod\/lesson\/tests\/behat\/completion_condition_end_reached.feature": 72, - "mod\/forum\/tests\/behat\/single_forum_discussion.feature": 71, - "mod\/lesson\/tests\/behat\/lesson_review.feature": 70.5, - "mod\/quiz\/tests\/behat\/editing_repaginate.feature": 70.2, - "course\/tests\/behat\/paged_course_navigation.feature": 69.3, - "repository\/tests\/behat\/create_shortcut.feature": 69.1, - "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": 67, - "mod\/choice\/tests\/behat\/publish_results_anonymously.feature": 66.9, - "mod\/book\/tests\/behat\/show_hide_chapters.feature": 65.9, - "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": 65.1, - "cohort\/tests\/behat\/access_visible_cohorts.feature": 64.9, - "user\/tests\/behat\/view_full_profile.feature": 64.3, - "course\/tests\/behat\/course_creation.feature": 64, - "mod\/glossary\/tests\/behat\/print_friendly_version.feature": 64, - "grade\/export\/txt\/tests\/behat\/export.feature": 63, - "question\/tests\/behat\/copy_questions.feature": 62.8, - "lib\/editor\/atto\/plugins\/accessibilitychecker\/tests\/behat\/accessibilitychecker.feature": 62.1, - "mod\/quiz\/tests\/behat\/settings_form_fields_disableif.feature": 61.7, - "admin\/tool\/uploadcourse\/tests\/behat\/create.feature": 61.6, - "badges\/tests\/behat\/add_badge.feature": 61.4, - "mod\/assign\/tests\/behat\/allow_another_attempt.feature": 61.1, - "completion\/tests\/behat\/enable_manual_complete_mark.feature": 60.5, - "mod\/wiki\/tests\/behat\/page_history.feature": 59.9, - "cohort\/tests\/behat\/upload_cohort_users.feature": 58.8, - "mod\/data\/tests\/behat\/add_entries.feature": 58.2, - "mod\/feedback\/tests\/behat\/show_nonrespondents.feature": 58, - "mod\/lesson\/tests\/behat\/lesson_progress_bar.feature": 57.5, - "mod\/lesson\/tests\/behat\/lesson_essay_question.feature": 57.1, - "blocks\/tests\/behat\/restrict_available_blocks.feature": 56.8, - "repository\/tests\/behat\/overwrite_file.feature": 56.6, - "course\/tests\/behat\/add_activities.feature": 56.5, - "availability\/condition\/date\/tests\/behat\/availability_date.feature": 56.2, - "mod\/choice\/tests\/behat\/multiple_options.feature": 55.4, - "grade\/report\/history\/tests\/behat\/basic_functionality.feature": 55.3, - "enrol\/guest\/tests\/behat\/guest_access.feature": 55.1, - "mod\/assign\/tests\/behat\/comment_inline.feature": 54.9, - "blocks\/tests\/behat\/hidden_block_region.feature": 54.3, - "mod\/quiz\/tests\/behat\/add_quiz.feature": 53.5, - "cohort\/tests\/behat\/view_cohorts.feature": 52.8, - "admin\/tool\/availabilityconditions\/tests\/behat\/manage_conditions.feature": 52.8, - "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": 51.9, - "mod\/assign\/tests\/behat\/display_grade.feature": 51.6, - "mod\/lesson\/tests\/behat\/teacher_grade_essays.feature": 51.5, - "message\/tests\/behat\/send_message.feature": 51.1, - "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": 50.9, - "availability\/condition\/completion\/tests\/behat\/availability_completion.feature": 50.5, - "mod\/lesson\/tests\/behat\/password_protection.feature": 49.6, - "admin\/tool\/behat\/tests\/behat\/basic_actions.feature": 49.5, - "mod\/quiz\/tests\/behat\/editing_click_move_icon.feature": 48.4, - "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": 48.2, - "mod\/assign\/tests\/behat\/submission_comments.feature": 47.8, - "files\/tests\/behat\/course_files.feature": 47.5, - "admin\/tool\/behat\/tests\/behat\/list_steps.feature": 47.4, - "grade\/tests\/behat\/grade_override_letter.feature": 47.3, - "mod\/assign\/tests\/behat\/file_submission.feature": 46.6, - "mod\/wiki\/tests\/behat\/preview_page.feature": 46.5, - "mod\/lesson\/tests\/behat\/lesson_informations_at_end.feature": 46.4, - "repository\/tests\/behat\/zip_and_unzip.feature": 46.1, - "backup\/util\/ui\/tests\/behat\/import_course.feature": 45.5, - "course\/tests\/behat\/category_role_assignment.feature": 44.6, - "user\/tests\/behat\/table_sorting.feature": 44.5, - "course\/format\/social\/tests\/behat\/social_adjust_discussion_count.feature": 44.3, - "mod\/choice\/tests\/behat\/change_response.feature": 44.1, - "mod\/book\/tests\/behat\/log_entries.feature": 43.8, - "lib\/editor\/atto\/plugins\/align\/tests\/behat\/align.feature": 42.8, - "mod\/assign\/tests\/behat\/online_submissions.feature": 42.7, - "mod\/assign\/tests\/behat\/filter_by_marker.feature": 41.7, - "lib\/editor\/atto\/plugins\/image\/tests\/behat\/image.feature": 41.3, - "mod\/assign\/tests\/behat\/grant_extension.feature": 41.2, - "mod\/label\/tests\/behat\/label_visibility.feature": 40.9, - "mod\/forum\/tests\/behat\/forum_subscriptions.feature": 40.2, - "course\/tests\/behat\/move_activities.feature": 40.2, - "message\/tests\/behat\/manage_contacts.feature": 39.1, - "my\/tests\/behat\/restrict_available_blocks.feature": 38.3, - "report\/eventlist\/tests\/behat\/mainsection.feature": 37.5, - "blocks\/comments\/tests\/behat\/delete_comment.feature": 37.2, - "calendar\/tests\/behat\/calendar.feature": 37, - "backup\/util\/ui\/tests\/behat\/duplicate_activities.feature": 37, - "question\/tests\/behat\/edit_questions.feature": 36.5, - "repository\/recent\/tests\/behat\/add_recent.feature": 36.2, - "mod\/lesson\/tests\/behat\/import_fillintheblank_question.feature": 35.7, - "course\/tests\/behat\/activities_group_icons.feature": 35.2, - "admin\/tests\/behat\/upload_users.feature": 34.3, - "course\/tests\/behat\/course_change_visibility.feature": 32.9, - "blocks\/tests\/behat\/return_block_original_state.feature": 32.5, - "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": 31.9, - "mod\/glossary\/tests\/behat\/prevent_duplicate_entries.feature": 31.4, - "admin\/tool\/behat\/tests\/behat\/manipulate_forms.feature": 31.1, - "course\/tests\/behat\/course_search.feature": 30.7, - "mod\/lesson\/tests\/behat\/import_images.feature": 30.7, - "lib\/editor\/atto\/plugins\/bold\/tests\/behat\/bold.feature": 30.5, - "mod\/choice\/tests\/behat\/limit_responses.feature": 30.4, - "course\/tests\/behat\/activities_edit_with_block_dock.feature": 30.4, - "lib\/editor\/atto\/plugins\/indent\/tests\/behat\/indent.feature": 30.1, - "lib\/editor\/atto\/plugins\/italic\/tests\/behat\/italic.feature": 29.7, - "lib\/editor\/atto\/plugins\/equation\/tests\/behat\/equation.feature": 29.6, - "course\/tests\/behat\/edit_settings.feature": 29.5, - "mod\/scorm\/tests\/behat\/add_scorm.feature": 29.1, - "repository\/tests\/behat\/delete_files.feature": 28.7, - "admin\/tool\/uploadcourse\/tests\/behat\/update.feature": 28.7, - "course\/tests\/behat\/activities_indentation.feature": 28.4, - "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": 28.1, - "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": 27.1, - "repository\/tests\/behat\/cancel_add_file.feature": 26, - "blocks\/html\/tests\/behat\/configuring_html_block.feature": 25.8, - "lib\/editor\/atto\/plugins\/table\/tests\/behat\/table.feature": 24.1, - "blocks\/tests\/behat\/add_blocks.feature": 24, - "course\/tests\/behat\/rename_roles.feature": 23.7, - "blocks\/tests\/behat\/manage_blocks.feature": 23.7, - "message\/tests\/behat\/block_users.feature": 23.7, - "course\/tests\/behat\/create_delete_course.feature": 23.6, - "message\/tests\/behat\/message_participants.feature": 23.6, - "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": 23.4, - "message\/tests\/behat\/display_history.feature": 23, - "mod\/choice\/tests\/behat\/add_choice.feature": 23, - "question\/format\/gift\/tests\/behat\/import_export.feature": 22.9, - "message\/tests\/behat\/search_history.feature": 22.8, - "repository\/tests\/behat\/create_folders.feature": 22.3, - "completion\/tests\/behat\/teacher_manual_completion.feature": 22.3, - "mod\/glossary\/tests\/behat\/entries_require_approval.feature": 21.7, - "mod\/wiki\/tests\/behat\/edit_tags.feature": 21, - "lib\/editor\/atto\/plugins\/link\/tests\/behat\/link.feature": 20.9, - "course\/tests\/behat\/move_sections.feature": 20, - "question\/format\/webct\/tests\/behat\/import.feature": 19.9, - "lib\/editor\/atto\/plugins\/accessibilityhelper\/tests\/behat\/accessibilityhelper.feature": 19.7, - "question\/format\/webct\/tests\/behat\/importcalculated.feature": 19.6, - "lib\/editor\/atto\/plugins\/media\/tests\/behat\/media.feature": 19.5, - "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": 18.2, - "blocks\/participants\/tests\/behat\/block_participants_course.feature": 17.4, - "admin\/tests\/behat\/set_admin_settings_value.feature": 16.8, - "lib\/editor\/atto\/plugins\/clear\/tests\/behat\/clear.feature": 16.7, - "admin\/tool\/monitor\/tests\/behat\/disabled.feature": 16.4, - "blocks\/html\/tests\/behat\/multiple_instances.feature": 15.6, - "grade\/export\/xml\/tests\/behat\/export.feature": 15.4, - "auth\/tests\/behat\/login.feature": 14.9, - "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": 14.8, - "lib\/editor\/atto\/plugins\/charmap\/tests\/behat\/charmap.feature": 14.4, - "lib\/editor\/atto\/plugins\/unorderedlist\/tests\/behat\/unorderedlist.feature": 14.4, - "lib\/editor\/atto\/plugins\/strike\/tests\/behat\/strike.feature": 14, - "blocks\/tests\/behat\/configure_block_throughout_site.feature": 14, - "lib\/editor\/atto\/plugins\/superscript\/tests\/behat\/superscript.feature": 13.9, - "lib\/editor\/atto\/plugins\/title\/tests\/behat\/title.feature": 13.8, - "lib\/editor\/atto\/plugins\/subscript\/tests\/behat\/subscript.feature": 13.8, - "lib\/editor\/atto\/plugins\/underline\/tests\/behat\/underline.feature": 13.7, - "lib\/editor\/atto\/plugins\/orderedlist\/tests\/behat\/orderedlist.feature": 13.3, - "user\/tests\/behat\/edituserpassword.feature": 12.7, - "mod\/data\/tests\/behat\/view_entries.feature": 12.4, - "repository\/upload\/tests\/behat\/upload_file.feature": 12.2, - "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": 9.4, - "lib\/editor\/atto\/plugins\/html\/tests\/behat\/html.feature": 9.2, - "mod\/forum\/tests\/behat\/my_forum_posts.feature": 9.1, - "mod\/book\/tests\/behat\/create_chapters.feature": 8.8, - "blocks\/html\/tests\/behat\/course_block.feature": 8.6, - "lib\/editor\/atto\/plugins\/collapse\/tests\/behat\/collapse.feature": 8.5, - "blocks\/login\/tests\/behat\/login_block.feature": 8.4, - "admin\/tests\/behat\/display_short_names.feature": 7.8, - "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": 7.1, - "mod\/survey\/tests\/behat\/survey_types.feature": 6.8, - "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": 6, - "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": 5.6, - "my\/tests\/behat\/reset_page.feature": 5.3, - "my\/tests\/behat\/add_blocks.feature": 4.6, - "user\/tests\/behat\/reset_page.feature": 4.2, - "user\/tests\/behat\/add_blocks.feature": 4.1, - "report\/usersessions\/tests\/behat\/usersessions_report.feature": 2.8, - "admin\/tool\/behat\/tests\/behat\/test_environment.feature": 1.5 -} \ No newline at end of file diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index c5cce499a01..641789ca22d 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -109,7 +109,7 @@ class behat_config_manager { // Behat config file specifing the main context class, // the required Behat extensions and Moodle test wwwroot. - $contents = self::get_config_file_contents(self::get_fetaures_with_tags($features, $tags), $stepsdefinitions); + $contents = self::get_config_file_contents(self::get_features_with_tags($features, $tags), $stepsdefinitions); // Stores the file. if (!file_put_contents($configfilepath, $contents)) { @@ -125,7 +125,7 @@ class behat_config_manager { * @param string $tags list of tags (currently support && only.) * @return array filtered list of feature files with tags. */ - public static function get_fetaures_with_tags($features, $tags) { + public static function get_features_with_tags($features, $tags) { if (empty($tags)) { return $features; } @@ -276,7 +276,7 @@ class behat_config_manager { // If parallel run then remove links and original file. clearstatcache(); for ($i = 1; $i <= $parallelrun; $i++) { - $link = $CFG->dirroot . '/' . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $i; + $link = $CFG->dirroot . '/' . BEHAT_PARALLEL_SITE_NAME . $i; if (file_exists($link) && is_link($link)) { @unlink($link); } @@ -298,16 +298,16 @@ class behat_config_manager { // Create site symlink if necessary. clearstatcache(); for ($i = 1; $i <= $parallelrun; $i++) { - $link = $CFG->dirroot.'/'.BEHAT_PARALLEL_SITE_WWW_SUFFIX.$i; + $link = $CFG->dirroot.'/'.BEHAT_PARALLEL_SITE_NAME.$i; clearstatcache(); if (file_exists($link)) { if (!is_link($link) || !is_dir($link)) { - echo "File exists at link location ($link) but is not a link or directory!\n"; + echo "File exists at link location ($link) but is not a link or directory!" . PHP_EOL; return false; } } else if (!symlink($CFG->dirroot, $link)) { // Try create link in case it's not already present. - echo "Unable to create behat site symlink ($link)\n"; + echo "Unable to create behat site symlink ($link)" . PHP_EOL; return false; } } @@ -341,8 +341,10 @@ class behat_config_manager { srand(crc32(floor(time() / 3600 / 24) . var_export($features, true))); shuffle($features); // Pull out the features for just this worker. - $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); - $features = $features[$CFG->behatrunprocess - 1]; + if (count($features)) { + $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); + $features = $features[$CFG->behatrunprocess - 1]; + } } // Set proper selenium2 wd_host if defined. if (!empty($CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host'])) { @@ -375,7 +377,8 @@ class behat_config_manager { 'Moodle\BehatExtension\Extension' => array( 'formatters' => array( 'moodle_progress' => 'Moodle\BehatExtension\Formatter\MoodleProgressFormatter', - 'moodle_list' => 'Moodle\BehatExtension\Formatter\MoodleListFormatter' + 'moodle_list' => 'Moodle\BehatExtension\Formatter\MoodleListFormatter', + 'moodle_step_count' => 'Moodle\BehatExtension\Formatter\MoodleStepCountFormatter' ), 'features' => $features, 'steps_definitions' => $stepsdefinitions @@ -407,18 +410,28 @@ class behat_config_manager { * @return array Feature files array, sorted into allocations */ protected static function profile_guided_allocate($features, $nbuckets, $instance) { - $pga = __DIR__.'/../../../behat_pga_default.json'; - $pga = defined('BEHAT_PGA_DATA') && file_exists(BEHAT_PGA_DATA) ? BEHAT_PGA_DATA : $pga; - if (defined('BEHAT_PGA_DISABLE') || !file_exists($pga) || !$pga = @json_decode(file_get_contents($pga), true)) { - // No data available, fall back to relying on shuffle. - return false; + $behattimingfile = defined('BEHAT_FEATURE_TIMING_FILE') && + @filesize(BEHAT_FEATURE_TIMING_FILE) ? BEHAT_FEATURE_TIMING_FILE : false; + + if (!$behattimingfile || !$behattimingdata = @json_decode(file_get_contents($behattimingfile), true)) { + // No data available, fall back to relying on steps data. + $stepfile = __DIR__ . "/../../../behat_features_step_count.json"; + + if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) { + $stepfile = BEHAT_FEATURE_STEP_FILE; + } + // We should never get this. But in case we can't do this then fall back on simple splitting. + if (!$stepfile || !$behattimingdata = @json_decode(file_get_contents($stepfile), true)) { + echo "No step/timing information available, falling back on simple distribution" . PHP_EOL; + return false; + } } - arsort($pga); // Ensure most expensive is first. + arsort($behattimingdata); // Ensure most expensive is first. $realroot = realpath(__DIR__.'/../../../').'/'; - $defaultweight = array_sum($pga) / count($pga); // TODO: median is more ideal. + $defaultweight = array_sum($behattimingdata) / count($behattimingdata); $weights = array_fill(0, $nbuckets, 0); $buckets = array_fill(0, $nbuckets, array()); $totalweight = 0; @@ -428,20 +441,20 @@ class behat_config_manager { $key = str_replace($realroot, '', $file); $features[$key] = $file; unset($features[$k]); - if (!isset($pga[$key])) { - $pga[$key] = $defaultweight; + if (!isset($behattimingdata[$key])) { + $behattimingdata[$key] = $defaultweight; } } // Sort features by known weights; largest ones should be allocated first. - $pgaorder = array(); + $behattimingorder = array(); foreach ($features as $key => $file) { - $pgaorder[$key] = $pga[$key]; + $behattimingorder[$key] = $behattimingdata[$key]; } - arsort($pgaorder); + arsort($behattimingorder); // Finally, add each feature one by one to the lightest bucket. - foreach ($pgaorder as $key => $weight) { + foreach ($behattimingorder as $key => $weight) { $file = $features[$key]; $lightbucket = array_search(min($weights), $weights); $weights[$lightbucket] += $weight; @@ -449,10 +462,10 @@ class behat_config_manager { $totalweight += $weight; } - if (!defined('BEHAT_PGA_DISABLE_HISTOGRAM') && $instance == $nbuckets) { + if ($totalweight && !defined('BEHAT_DISABLE_HISTOGRAM') && $instance == $nbuckets) { echo "Bucket weightings:\n"; foreach ($weights as $k => $weight) { - echo "$k: ".str_repeat('*', 70 * $nbuckets * $weight / $totalweight)."\n"; + echo $k + 1 . ": " . str_repeat('*', 70 * $nbuckets * $weight / $totalweight) . PHP_EOL; } } diff --git a/lib/behat/lib.php b/lib/behat/lib.php index ef305eee463..e6f50ea4cca 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -44,7 +44,7 @@ define('BEHAT_EXITCODE_INSTALLED', 256); /** * The behat test site fullname and shortname. */ -define('BEHAT_PARALLEL_SITE_WWW_SUFFIX', "behatrun"); +define('BEHAT_PARALLEL_SITE_NAME', "behatrun"); /** * Exits with an error code @@ -289,14 +289,14 @@ function behat_is_test_site() { function behat_update_vars_for_process($behatrunprocess = '') { global $CFG; - $allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass'); + $allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'behat_prefix'); $behatrunprocess = $CFG->behatrunprocess; if ($behatrunprocess) { // Set www root for run process. - if (isset($CFG->behat_wwwroot) && !preg_match("#/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $behatrunprocess . "\$#", + if (isset($CFG->behat_wwwroot) && !preg_match("#/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess . "\$#", $CFG->behat_wwwroot)) { - $CFG->behat_wwwroot .= "/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . $behatrunprocess; + $CFG->behat_wwwroot .= "/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess; } // Set behat_dataroot. @@ -321,10 +321,6 @@ function behat_update_vars_for_process($behatrunprocess = '') { $CFG->$config = $CFG->behat_parallel_run[$behatrunprocess - 1][$config]; } } - // Override behat prefix, if specified. - if (isset($CFG->behat_parallel_run[$behatrunprocess - 1]['behat_prefix'])) { - $CFG->behat_prefix = $CFG->behat_parallel_run[$behatrunprocess - 1]['behat_prefix']; - } } } } @@ -363,3 +359,38 @@ function behat_is_requested_url($url) { return false; } + +/** + * Execute commands in parallel. + * + * @param array $cmds list of commands to be executed. + * @param string $cwd absolute path of working directory. + * @return array list of processes. + */ +function cli_execute_parallel($cmds, $cwd = null) { + require_once(__DIR__ . "/../../vendor/autoload.php"); + + $processes = array(); + + // Create child process. + foreach ($cmds as $name => $cmd) { + $process = new Symfony\Component\Process\Process($cmd); + + $process->setWorkingDirectory($cwd); + $process->setTimeout(null); + $processes[$name] = $process; + $processes[$name]->start(); + + // If error creating process then exit. + if ($processes[$name]->getStatus() !== 'started') { + echo "Error starting process: $name"; + foreach ($processes[$name] as $process) { + if ($process) { + $process->signal(SIGKILL); + } + } + exit(1); + } + } + return $processes; +} \ No newline at end of file diff --git a/lib/classes/process_manager.php b/lib/classes/process_manager.php deleted file mode 100644 index b49b9e5f56d..00000000000 --- a/lib/classes/process_manager.php +++ /dev/null @@ -1,211 +0,0 @@ -. - -/** - * Moodle implementation of process manager, to execute external commands. - * - * @package core - * @copyright 2015 Rajesh Taneja - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ - -/** - * Moodle implementation of process manager, to execute external commands. - * - * @package core - * @copyright 2015 Rajesh Taneja - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - */ -class process_manager implements Countable { - - /** - * Standard in. - */ - const STDIN = 0; - - /** - * Standard out. - */ - const STDOUT = 1; - - /** - * Standard error. - */ - const STDERR = 2; - - /** - * Non blocking mode. - */ - const NON_BLOCKING = 0; - - /** - * Blocking mode. - */ - const BLOCKING = 1; - - /** @var array Descriptor used for process. */ - private static $DESCRIPTORSPEC = array( - self::STDIN => array('pipe', 'r'), - self::STDOUT => array('pipe', 'w'), - self::STDERR => array('pipe', 'w'), - ); - - /** @var array list of processes. */ - private $processes = array(); - - /** @var array list of stdin */ - private $stdins = array(); - - /** @var array list of stdout */ - private $stdouts = array(); - - /** @var array list of stderr. */ - private $stderrs = array(); - - /** - * Create new process and keep track of it. - * - * @param string $name name of the process, has to be unique for process identification. - * @param string $cmd command to execute. - * @param string $cwd absolute path of working directory for command to execute. - * @return false if failed to create process. - */ - public function create($name, $cmd, $cwd = NULL) { - $process = proc_open($cmd, self::$DESCRIPTORSPEC, $pipes, $cwd); - - if (false === is_resource($process)) { - throw new Exception('Error starting worker'); - } - - stream_set_blocking($pipes[self::STDOUT], self::NON_BLOCKING); - stream_set_blocking($pipes[self::STDERR], self::NON_BLOCKING); - - $this->processes[$name] = $process; - $this->stdins[$name] = $pipes[self::STDIN]; - $this->stdouts[$name] = $pipes[self::STDOUT]; - $this->stderrs[$name] = $pipes[self::STDERR]; - - return true; - } - - /** - * Keep listing to process and return status of it. - * - * @retrun array status, stdout and stderr. - */ - public function listen() { - $read = array(); - - foreach ($this->processes as $i => $p) { - // Update process info. - if (!($s = @proc_get_status($p)) || !$s['running']) { - $status[$i] = $this->detach($p); - } else { - $status[$i] = 0; - $read[] = $this->stdouts[$i]; - $read[] = $this->stderrs[$i]; - } - } - - if ($read) { - $changednum = stream_select($read, $write, $expect, 0); - } else { - return; - } - - if (false === $changednum) { - throw new \RuntimeException(); - } - - if (0 === $changednum) { - return; - } - - foreach ($read as $stream) { - $i = array_search($stream, $this->stdouts, true); - if (false === $i) { - $i = array_search($stream, $this->stderrs, true); - if (false === $i) { - continue; - } - } - - $stdout[$i] = stream_get_contents($this->stdouts[$i]); - $stderr[$i] = stream_get_contents($this->stderrs[$i]); - } - return (array($status, $stdout, $stderr)); - } - - /** - * Detach process. - * - * @param $process process to detatch. - * @return int status of process. - */ - public function detach($process) { - $i = array_search($process, $this->processes, true); - - if (false === $i) { - throw new \RuntimeException(); - } - - fclose($this->stdins[$i]); - fclose($this->stdouts[$i]); - fclose($this->stderrs[$i]); - $status = proc_close($this->processes[$i]); - - unset($this->processes[$i]); - unset($this->stdins[$i]); - unset($this->stdouts[$i]); - unset($this->stderrs[$i]); - - return $status; - } - - /** - * Detach all processes. - */ - public function detachall() { - foreach ($this->stdins as $stdin) { - fclose($stdin); - } - foreach ($this->stdouts as $stdout) { - fclose($stdout); - } - foreach ($this->stderrs as $stderr) { - fclose($stderr); - } - foreach ($this->processes as $processs) { - proc_close($processs); - } - } - - /** - * Return count of active processes. - * - * @return int count of active processes. - */ - public function count() { - return count($this->processes); - } - - /** - * Destructor. - */ - public function __destruct() { - $this->detachall(); - } -} diff --git a/lib/clilib.php b/lib/clilib.php index ebe0a08804f..be869e331dc 100644 --- a/lib/clilib.php +++ b/lib/clilib.php @@ -176,100 +176,3 @@ function cli_error($text, $errorcode=1) { die($errorcode); } -/** - * Executes cli command and return handle. - * - * @param string $cmd command to be executed. - * @param bool $die exit if command is not executed. - * @return array list of handles and pipe. - * @throws Exception if worker is not started, - */ -function cli_execute($cmd, $die = false) { - $desc = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('pipe', 'w'), - ); - if (!($handle = proc_open($cmd, $desc, $pipes)) && $die) { - throw new Exception('Error starting worker'); - } - return array($handle, $pipes); -} - -/** - * Execute commands in parallel. - * - * @param array $cmds list of commands to be executed. - * @param string $cwd aabsolute path of working directory. - * @param bool $returnonfirstfail Will stop all process and return. - * - * @return bool status of all process. - */ -function cli_execute_parallel($cmds, $cwd = NULL, $returnonfirstfail = false, $addprefix = true) { - require_once(__DIR__ . '/classes/process_manager.php'); - - $overallstatus = false; - $processmanager = new process_manager(); - - // Create child process. - foreach ($cmds as $name => $cmd) { - if (!$processmanager->create($name, $cmd, $cwd) && $returnonfirstfail) { - throw new Exception('Error starting worker'); - } - } - while (0 < count($processmanager)) { - usleep(10000); - list($status, $stdout, $stderr) = $processmanager->listen(); - - if (!empty($status)) { - foreach ($status as $name => $value) { - // Something went wrong. - if ((0 > $value)) { - throw new \RuntimeException(); - } - $overallstatus = $overallstatus || (bool)$value; - // Add prefix to process. - $prefix = ""; - if ($addprefix && ($value === 0)) { - $prefix = '[' . $name . '] '; - } - if (!empty($stdout[$name]) && trim($stdout[$name])) { - echo $prefix . $stdout[$name]; - } - if (!empty($stderr[$name]) && trim($stderr[$name])) { - echo $prefix . $stderr[$name]; - } - } - - // Return if fail found. - if ($returnonfirstfail && (bool)$value) { - unset($processmanager); - $processmanager = null; - echo PHP_EOL; - return $value; - } - } - } - - echo PHP_EOL; - return $overallstatus; -} - -/** - * Execute commands in sequence and return status code for each process. - * - * @param array $cmds commands to execute. - * @param bool $returnonfirstfail if true then returns on any fail. - * @return array status codes for each process. - */ -function cli_execute_sequential($cmds, $returnonfirstfail = false) { - $procs = array(); - foreach ($cmds as $k => $cmd) { - $procs[$k] = popen($cmd, 'r'); - passthru($cmd, $procs[$k]); - if (($procs[$k] != 0) && $returnonfirstfail) { - return $procs; - } - } - return $procs; -} diff --git a/lib/setup.php b/lib/setup.php index ef2b19dab0e..9d0d41ea57d 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -86,11 +86,11 @@ if (defined('BEHAT_SITE_RUNNING')) { if (defined('BEHAT_CURRENT_RUN') && BEHAT_CURRENT_RUN) { $behatrunprocess = BEHAT_CURRENT_RUN; } else if (!empty($_SERVER['REMOTE_ADDR'])) { - if (preg_match('#/' . BEHAT_PARALLEL_SITE_WWW_SUFFIX . '(.+?)/#', $_SERVER['REQUEST_URI'])) { + if (preg_match('#/' . BEHAT_PARALLEL_SITE_NAME . '(.+?)/#', $_SERVER['REQUEST_URI'])) { $dirrootrealpath = str_replace("\\", "/", realpath($CFG->dirroot)); $serverrealpath = str_replace("\\", "/", realpath($_SERVER['SCRIPT_FILENAME'])); $afterpath = str_replace($dirrootrealpath.'/', '', $serverrealpath); - if (!$behatrunprocess = preg_filter("#.*/" . BEHAT_PARALLEL_SITE_WWW_SUFFIX . "(.+?)/$afterpath#", '$1', + if (!$behatrunprocess = preg_filter("#.*/" . BEHAT_PARALLEL_SITE_NAME . "(.+?)/$afterpath#", '$1', $_SERVER['SCRIPT_FILENAME'])) { throw new Exception("Unable to determine behat process [afterpath=" . $afterpath . ", scriptfilename=" . $_SERVER['SCRIPT_FILENAME'] . "]!"); @@ -129,7 +129,8 @@ if (defined('BEHAT_SITE_RUNNING')) { if ($file === 'behat' or $file === '.' or $file === '..' or $file === '.DS_Store' or is_numeric($file)) { continue; } - behat_error(BEHAT_EXITCODE_CONFIG, "$CFG->behat_dataroot directory is not empty, ensure this is the directory where you want to install behat test dataroot"); + behat_error(BEHAT_EXITCODE_CONFIG, "$CFG->behat_dataroot directory is not empty, ensure this is the " . + "directory where you want to install behat test dataroot"); } closedir($dh); unset($dh); diff --git a/lib/tests/behat/behat_hooks.php b/lib/tests/behat/behat_hooks.php index 5d1deaa7da6..5df77bbaac2 100644 --- a/lib/tests/behat/behat_hooks.php +++ b/lib/tests/behat/behat_hooks.php @@ -167,11 +167,10 @@ class behat_hooks extends behat_base { * Gives access to moodle codebase, to keep track of feature start time. * * @param FeatureEvent $event event fired before feature. - * @static * @BeforeFeature */ public static function before_feature(FeatureEvent $event) { - if (!defined('BEHAT_FEATURE_TIMING')) { + if (!defined('BEHAT_FEATURE_TIMING_FILE')) { return; } $file = $event->getFeature()->getFile(); @@ -182,11 +181,10 @@ class behat_hooks extends behat_base { * Gives access to moodle codebase, to keep track of feature end time. * * @param FeatureEvent $event event fired after feature. - * @static * @AfterFeature */ public static function after_feature(FeatureEvent $event) { - if (!defined('BEHAT_FEATURE_TIMING')) { + if (!defined('BEHAT_FEATURE_TIMING_FILE')) { return; } $file = $event->getFeature()->getFile(); @@ -201,11 +199,10 @@ class behat_hooks extends behat_base { * Gives access to moodle codebase, to keep track of suite timings. * * @param SuiteEvent $event event fired after suite. - * @static * @AfterSuite */ public static function after_suite(SuiteEvent $event) { - if (!defined('BEHAT_FEATURE_TIMING')) { + if (!defined('BEHAT_FEATURE_TIMING_FILE')) { return; } $realroot = realpath(__DIR__.'/../../../').'/'; @@ -214,11 +211,11 @@ class behat_hooks extends behat_base { self::$timings[$new] = round($v, 1); unset(self::$timings[$k]); } - if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING), true)) { + if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING_FILE), true)) { self::$timings = array_merge($existing, self::$timings); } arsort(self::$timings); - @file_put_contents(BEHAT_FEATURE_TIMING, json_encode(self::$timings, JSON_PRETTY_PRINT)); + @file_put_contents(BEHAT_FEATURE_TIMING_FILE, json_encode(self::$timings, JSON_PRETTY_PRINT)); } /** From ef95c163e4bae69672d42607f8a8f644e53fc3d7 Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Mon, 23 Feb 2015 16:52:20 +0800 Subject: [PATCH 04/11] MDL-39752 behat: Don't package default step count with moodle --- admin/tool/behat/cli/util_single_run.php | 6 +- behat_features_step_count.json | 286 --------------------- lib/behat/classes/behat_config_manager.php | 6 +- 3 files changed, 6 insertions(+), 292 deletions(-) delete mode 100644 behat_features_step_count.json diff --git a/admin/tool/behat/cli/util_single_run.php b/admin/tool/behat/cli/util_single_run.php index c42a908ced6..fa90359f174 100644 --- a/admin/tool/behat/cli/util_single_run.php +++ b/admin/tool/behat/cli/util_single_run.php @@ -189,9 +189,11 @@ if ($options['install']) { exit($code); } else if ($options['updatesteps']) { - $behatstepfile = __DIR__ . "/../../../../behat_features_step_count.json"; if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) { $behatstepfile = BEHAT_FEATURE_STEP_FILE; + } else { + echo "BEHAT_FEATURE_STEP_FILE is not set, please ensure you set this to writable file" . PHP_EOL; + exit(1); } // Rewrite config file to ensure we have all the features covered. @@ -265,7 +267,7 @@ function print_update_step_output($process, $featurestepfile) { $exitcode = -1; } - echo PHP_EOL; + echo PHP_EOL. "Updated step count in " . $featurestepfile . PHP_EOL; return $exitcode; } diff --git a/behat_features_step_count.json b/behat_features_step_count.json deleted file mode 100644 index f0963e1cf23..00000000000 --- a/behat_features_step_count.json +++ /dev/null @@ -1,286 +0,0 @@ -{ - "course\/tests\/behat\/course_category_management_listing.feature": "859", - "grade\/tests\/behat\/grade_aggregation.feature": "746", - "course\/tests\/behat\/course_controls.feature": "648", - "grade\/tests\/behat\/grade_scales.feature": "566", - "course\/tests\/behat\/course_resort.feature": "481", - "grade\/tests\/behat\/grade_single_item_scales.feature": "434", - "grade\/tests\/behat\/grade_calculated_weights.feature": "369", - "grade\/tests\/behat\/grade_scales_aggregation.feature": "349", - "mod\/forum\/tests\/behat\/discussion_subscriptions.feature": "291", - "grade\/tests\/behat\/grade_natural_normalisation.feature": "288", - "admin\/tool\/monitor\/tests\/behat\/subscription.feature": "285", - "course\/tests\/behat\/category_resort.feature": "256", - "course\/tests\/behat\/category_change_visibility.feature": "234", - "course\/tests\/behat\/category_management.feature": "225", - "admin\/tool\/behat\/tests\/behat\/data_generators.feature": "218", - "admin\/tool\/monitor\/tests\/behat\/rule.feature": "213", - "mod\/quiz\/tests\/behat\/editing_add.feature": "207", - "grade\/grading\/form\/rubric\/tests\/behat\/edit_rubric.feature": "204", - "blocks\/navigation\/tests\/behat\/expand_courses_node.feature": "189", - "course\/format\/social\/tests\/behat\/social_adjust_discussion_count.feature": "184", - "admin\/tool\/behat\/tests\/behat\/get_and_set_fields.feature": "176", - "mod\/workshop\/tests\/behat\/workshop_assessment.feature": "162", - "badges\/tests\/behat\/award_badge.feature": "158", - "group\/tests\/behat\/update_groups.feature": "150", - "availability\/tests\/behat\/edit_availability.feature": "147", - "mod\/wiki\/tests\/behat\/wiki_search.feature": "140", - "mod\/forum\/tests\/behat\/track_read_posts.feature": "136", - "blocks\/recent_activity\/tests\/behat\/structural_changes.feature": "132", - "mod\/assign\/tests\/behat\/quickgrading.feature": "132", - "grade\/tests\/behat\/grade_contribution_with_extra_credit.feature": "129", - "mod\/lesson\/tests\/behat\/link_to_gradebook.feature": "128", - "mod\/forum\/tests\/behat\/discussion_navigation.feature": "126", - "grade\/report\/singleview\/tests\/behat\/singleview.feature": "120", - "mod\/assign\/tests\/behat\/grading_status.feature": "116", - "grade\/tests\/behat\/grade_point_maximum.feature": "114", - "mod\/glossary\/tests\/behat\/search_entries.feature": "113", - "backup\/util\/ui\/tests\/behat\/restore_moodle2_courses.feature": "110", - "mod\/quiz\/tests\/behat\/editing_repaginate.feature": "110", - "blocks\/activity_modules\/tests\/behat\/block_activity_modules.feature": "108", - "mod\/assign\/tests\/behat\/outcome_grading.feature": "104", - "question\/tests\/behat\/preview_question.feature": "102", - "group\/tests\/behat\/groups_import.feature": "99", - "group\/tests\/behat\/auto_creation.feature": "98", - "report\/outline\/tests\/behat\/outline.feature": "98", - "mod\/assign\/tests\/behat\/prevent_submission_changes.feature": "97", - "blocks\/comments\/tests\/behat\/add_comment.feature": "96", - "course\/tests\/behat\/paged_course_navigation.feature": "96", - "cohort\/tests\/behat\/access_visible_cohorts.feature": "96", - "grade\/tests\/behat\/grade_view.feature": "94", - "grade\/tests\/behat\/grade_override_letter.feature": "94", - "availability\/tests\/behat\/display_availability.feature": "94", - "mod\/forum\/tests\/behat\/discussion_display.feature": "92", - "course\/format\/weeks\/tests\/behat\/edit_delete_sections.feature": "90", - "mod\/quiz\/tests\/behat\/editing_set_marks_with_attempts.feature": "89", - "mod\/lesson\/tests\/behat\/lesson_edit_pages.feature": "88", - "course\/format\/topics\/tests\/behat\/edit_delete_sections.feature": "87", - "report\/outline\/tests\/behat\/user.feature": "86", - "admin\/tool\/filetypes\/tests\/behat\/add_filetypes.feature": "86", - "mod\/lesson\/tests\/behat\/lesson_number_of_student_attempts.feature": "86", - "mod\/glossary\/tests\/behat\/categories.feature": "85", - "mod\/assign\/feedback\/editpdf\/tests\/behat\/annotate_pdf.feature": "84", - "mod\/wiki\/tests\/behat\/wiki_comments.feature": "83", - "report\/participation\/tests\/behat\/filter_participation.feature": "83", - "mod\/lesson\/tests\/behat\/lesson_navigation.feature": "82", - "question\/tests\/behat\/question_categories.feature": "81", - "group\/tests\/behat\/create_groups.feature": "80", - "course\/tests\/behat\/course_change_visibility.feature": "79", - "mod\/lesson\/tests\/behat\/lesson_with_clusters.feature": "79", - "blocks\/glossary_random\/tests\/behat\/glossary_random.feature": "79", - "mod\/lesson\/tests\/behat\/lesson_practice.feature": "78", - "group\/tests\/behat\/delete_groups.feature": "76", - "cohort\/tests\/behat\/upload_cohorts.feature": "76", - "mod\/assign\/tests\/behat\/group_submission.feature": "76", - "question\/tests\/behat\/sort_questions.feature": "75", - "mod\/choice\/tests\/behat\/publish_results.feature": "73", - "mod\/quiz\/tests\/behat\/editing_click_delete_icon.feature": "71", - "report\/loglive\/tests\/behat\/loglive_report.feature": "71", - "blog\/tests\/behat\/comment.feature": "70", - "calendar\/tests\/behat\/calendar.feature": "70", - "completion\/tests\/behat\/restrict_section_availability.feature": "69", - "grade\/export\/txt\/tests\/behat\/export.feature": "69", - "blocks\/navigation\/tests\/behat\/view_my_courses.feature": "69", - "availability\/condition\/profile\/tests\/behat\/availability_profile.feature": "68", - "mod\/forum\/tests\/behat\/forum_subscriptions.feature": "68", - "mod\/forum\/tests\/behat\/edit_post_student.feature": "66", - "report\/log\/tests\/behat\/filter_log.feature": "65", - "availability\/condition\/grade\/tests\/behat\/availability_grade.feature": "64", - "question\/format\/xml\/tests\/behat\/import_export.feature": "64", - "mod\/quiz\/tests\/behat\/editing_click_move_icon.feature": "64", - "mod\/forum\/tests\/behat\/forum_subscriptions_availability.feature": "62", - "course\/tests\/behat\/create_delete_course.feature": "62", - "blocks\/tests\/behat\/manage_blocks.feature": "62", - "user\/tests\/behat\/delete_users.feature": "61", - "mod\/quiz\/tests\/behat\/settings_form_fields_disableif.feature": "60", - "mod\/wiki\/tests\/behat\/wiki_formats.feature": "59", - "grade\/tests\/behat\/grade_mingrade.feature": "59", - "mod\/lesson\/tests\/behat\/questions_images.feature": "59", - "course\/tests\/behat\/activities_edit_completion.feature": "59", - "mod\/forum\/tests\/behat\/edit_post_teacher.feature": "59", - "mod\/quiz\/tests\/behat\/editing_set_marks_no_attempts.feature": "59", - "mod\/lesson\/tests\/behat\/lesson_edit_cluster.feature": "58", - "mod\/wiki\/tests\/behat\/collaborative_individual.feature": "57", - "mod\/forum\/tests\/behat\/separate_group_discussions.feature": "57", - "mod\/quiz\/tests\/behat\/completion_condition_passing_grade.feature": "56", - "availability\/condition\/group\/tests\/behat\/availability_group.feature": "55", - "mod\/quiz\/tests\/behat\/completion_condition_attempts_used.feature": "55", - "lib\/editor\/atto\/plugins\/image\/tests\/behat\/image.feature": "55", - "course\/tests\/behat\/section_highlighting.feature": "54", - "mod\/forum\/tests\/behat\/separate_group_single_group_discussions.feature": "53", - "course\/tests\/behat\/move_activities.feature": "52", - "blocks\/tests\/behat\/hidden_block_region.feature": "52", - "course\/tests\/behat\/activities_visibility_icons.feature": "52", - "lib\/editor\/atto\/plugins\/accessibilitychecker\/tests\/behat\/accessibilitychecker.feature": "52", - "mod\/quiz\/tests\/behat\/add_quiz.feature": "51", - "mod\/lesson\/tests\/behat\/teacher_grade_essays.feature": "51", - "report\/eventlist\/tests\/behat\/mainsection.feature": "51", - "mod\/wiki\/tests\/behat\/edit_tags.feature": "51", - "report\/log\/tests\/behat\/user_log.feature": "50", - "grade\/tests\/behat\/grade_UI_settings.feature": "50", - "enrol\/self\/tests\/behat\/self_enrolment.feature": "50", - "course\/tests\/behat\/add_activities.feature": "49", - "mod\/lesson\/tests\/behat\/lesson_essay_question.feature": "49", - "grade\/grading\/form\/rubric\/tests\/behat\/reuse_own_rubrics.feature": "48", - "mod\/book\/tests\/behat\/log_entries.feature": "48", - "mod\/data\/tests\/behat\/view_entries.feature": "48", - "grade\/grading\/form\/rubric\/tests\/behat\/publish_rubric_templates.feature": "48", - "availability\/condition\/grouping\/tests\/behat\/availability_grouping.feature": "47", - "admin\/tool\/behat\/tests\/behat\/nasty_strings.feature": "47", - "mod\/assign\/feedback\/editpdf\/tests\/behat\/group_annotations.feature": "46", - "lib\/editor\/tinymce\/tests\/behat\/edit_available_icons.feature": "46", - "mod\/book\/tests\/behat\/show_hide_chapters.feature": "46", - "grade\/report\/history\/tests\/behat\/basic_functionality.feature": "46", - "mod\/wiki\/tests\/behat\/page_history.feature": "45", - "mod\/assign\/tests\/behat\/edit_previous_feedback.feature": "45", - "question\/tests\/behat\/delete_questions.feature": "45", - "cohort\/tests\/behat\/add_cohort.feature": "45", - "admin\/tests\/behat\/filter_users.feature": "45", - "cohort\/tests\/behat\/view_cohorts.feature": "45", - "mod\/wiki\/tests\/behat\/preview_page.feature": "44", - "mod\/lesson\/tests\/behat\/lesson_progress_bar.feature": "44", - "admin\/tool\/langimport\/tests\/behat\/manage_langpacks.feature": "44", - "group\/tests\/behat\/id_uniqueness.feature": "44", - "repository\/tests\/behat\/create_shortcut.feature": "44", - "backup\/util\/ui\/tests\/behat\/backup_courses.feature": "44", - "mod\/lesson\/tests\/behat\/lesson_review.feature": "43", - "mod\/lesson\/tests\/behat\/date_availability.feature": "42", - "completion\/tests\/behat\/restrict_activity_by_date.feature": "41", - "mod\/glossary\/tests\/behat\/entries_require_approval.feature": "41", - "user\/tests\/behat\/view_full_profile.feature": "41", - "course\/tests\/behat\/move_sections.feature": "40", - "completion\/tests\/behat\/restrict_activity_by_grade.feature": "40", - "mod\/assign\/tests\/behat\/comment_inline.feature": "39", - "mod\/assign\/tests\/behat\/submission_comments.feature": "39", - "course\/tests\/behat\/section_visibility.feature": "39", - "message\/tests\/behat\/display_history.feature": "39", - "mod\/data\/tests\/behat\/add_entries.feature": "39", - "mod\/assign\/tests\/behat\/grant_extension.feature": "38", - "mod\/lesson\/tests\/behat\/completion_condition_end_reached.feature": "38", - "blocks\/course_summary\/tests\/behat\/block_course_summary_course.feature": "38", - "question\/tests\/behat\/copy_questions.feature": "37", - "blocks\/participants\/tests\/behat\/block_participants_course.feature": "37", - "blocks\/navigation\/tests\/behat\/expand_my_courses_setting.feature": "37", - "mod\/book\/tests\/behat\/create_chapters.feature": "37", - "mod\/choice\/tests\/behat\/multiple_options.feature": "36", - "mod\/feedback\/tests\/behat\/show_nonrespondents.feature": "36", - "admin\/tool\/behat\/tests\/behat\/edit_permissions.feature": "36", - "course\/tests\/behat\/force_group_mode.feature": "36", - "cohort\/tests\/behat\/upload_cohort_users.feature": "35", - "mod\/assign\/tests\/behat\/allow_another_attempt.feature": "35", - "mod\/choice\/tests\/behat\/publish_results_anonymously.feature": "35", - "message\/tests\/behat\/manage_contacts.feature": "34", - "blocks\/html\/tests\/behat\/multiple_instances.feature": "34", - "message\/tests\/behat\/send_message.feature": "34", - "admin\/tool\/behat\/tests\/behat\/basic_actions.feature": "33", - "course\/tests\/behat\/frontpage_display_modes.feature": "33", - "mod\/choice\/tests\/behat\/change_response.feature": "32", - "enrol\/guest\/tests\/behat\/guest_access.feature": "32", - "admin\/tool\/uploadcourse\/tests\/behat\/create.feature": "32", - "blocks\/tests\/behat\/return_block_original_state.feature": "32", - "mod\/lesson\/tests\/behat\/lesson_informations_at_end.feature": "32", - "lib\/editor\/atto\/plugins\/align\/tests\/behat\/align.feature": "31", - "availability\/condition\/date\/tests\/behat\/availability_date.feature": "31", - "mod\/glossary\/tests\/behat\/print_friendly_version.feature": "30", - "mod\/assign\/tests\/behat\/display_grade.feature": "30", - "repository\/tests\/behat\/overwrite_file.feature": "30", - "course\/tests\/behat\/edit_settings.feature": "30", - "mod\/assign\/tests\/behat\/file_submission.feature": "30", - "blocks\/tests\/behat\/configure_block_throughout_site.feature": "30", - "course\/tests\/behat\/course_creation.feature": "29", - "admin\/tool\/availabilityconditions\/tests\/behat\/manage_conditions.feature": "29", - "completion\/tests\/behat\/teacher_manual_completion.feature": "29", - "lib\/editor\/atto\/plugins\/table\/tests\/behat\/table.feature": "29", - "question\/tests\/behat\/edit_questions.feature": "29", - "blocks\/html\/tests\/behat\/configuring_html_block.feature": "28", - "blocks\/course_summary\/tests\/behat\/block_course_summary_frontpage.feature": "28", - "admin\/tests\/behat\/upload_users.feature": "28", - "course\/tests\/behat\/max_number_sections.feature": "27", - "course\/tests\/behat\/restrict_available_activities.feature": "27", - "course\/tests\/behat\/activities_group_icons.feature": "27", - "mod\/lesson\/tests\/behat\/time_limit.feature": "27", - "mod\/assign\/tests\/behat\/filter_by_marker.feature": "26", - "user\/tests\/behat\/table_sorting.feature": "26", - "mod\/assign\/tests\/behat\/online_submissions.feature": "26", - "blocks\/tests\/behat\/restrict_available_blocks.feature": "26", - "course\/tests\/behat\/course_search.feature": "25", - "mod\/forum\/tests\/behat\/completion_condition_number_discussions.feature": "25", - "course\/tests\/behat\/rename_roles.feature": "25", - "badges\/tests\/behat\/add_badge.feature": "25", - "completion\/tests\/behat\/enable_manual_complete_mark.feature": "24", - "lib\/editor\/atto\/plugins\/equation\/tests\/behat\/equation.feature": "24", - "mod\/forum\/tests\/behat\/single_forum_discussion.feature": "24", - "availability\/condition\/completion\/tests\/behat\/availability_completion.feature": "24", - "blocks\/news_items\/tests\/behat\/display_news.feature": "24", - "mod\/lesson\/tests\/behat\/password_protection.feature": "24", - "mod\/lesson\/tests\/behat\/import_fillintheblank_question.feature": "24", - "mod\/scorm\/tests\/behat\/add_scorm.feature": "23", - "admin\/tool\/behat\/tests\/behat\/manipulate_forms.feature": "23", - "my\/tests\/behat\/restrict_available_blocks.feature": "23", - "mod\/forum\/tests\/behat\/forum_subscriptions_management.feature": "23", - "mod\/glossary\/tests\/behat\/entries_always_editable.feature": "21", - "auth\/tests\/behat\/login.feature": "21", - "lib\/editor\/atto\/plugins\/indent\/tests\/behat\/indent.feature": "21", - "blocks\/html\/tests\/behat\/course_block.feature": "21", - "lib\/editor\/atto\/plugins\/accessibilityhelper\/tests\/behat\/accessibilityhelper.feature": "21", - "lib\/editor\/atto\/plugins\/italic\/tests\/behat\/italic.feature": "20", - "mod\/lesson\/tests\/behat\/import_images.feature": "20", - "repository\/tests\/behat\/cancel_add_file.feature": "20", - "mod\/forum\/tests\/behat\/add_forum.feature": "20", - "lib\/editor\/atto\/plugins\/bold\/tests\/behat\/bold.feature": "20", - "backup\/util\/ui\/tests\/behat\/duplicate_activities.feature": "20", - "files\/tests\/behat\/course_files.feature": "20", - "lib\/editor\/atto\/plugins\/media\/tests\/behat\/media.feature": "20", - "course\/tests\/behat\/category_role_assignment.feature": "20", - "message\/tests\/behat\/message_participants.feature": "19", - "mod\/choice\/tests\/behat\/limit_responses.feature": "19", - "blocks\/comments\/tests\/behat\/delete_comment.feature": "19", - "course\/tests\/behat\/activities_indentation.feature": "19", - "admin\/tool\/uploadcourse\/tests\/behat\/update.feature": "18", - "repository\/tests\/behat\/zip_and_unzip.feature": "18", - "mod\/forum\/tests\/behat\/my_forum_posts.feature": "18", - "grade\/export\/xml\/tests\/behat\/export.feature": "18", - "message\/tests\/behat\/search_history.feature": "18", - "question\/format\/gift\/tests\/behat\/import_export.feature": "18", - "blocks\/login\/tests\/behat\/login_block.feature": "18", - "mod\/survey\/tests\/behat\/survey_types.feature": "18", - "admin\/tool\/behat\/tests\/behat\/list_steps.feature": "18", - "repository\/recent\/tests\/behat\/add_recent.feature": "17", - "blocks\/tests\/behat\/add_blocks.feature": "16", - "message\/tests\/behat\/block_users.feature": "16", - "admin\/tests\/behat\/set_admin_settings_value.feature": "16", - "lib\/editor\/atto\/plugins\/link\/tests\/behat\/link.feature": "16", - "blocks\/tests\/behat\/hide_blocks.feature": "16", - "backup\/util\/ui\/tests\/behat\/import_course.feature": "15", - "my\/tests\/behat\/reset_page.feature": "15", - "blocks\/participants\/tests\/behat\/block_participants_frontpage.feature": "15", - "mod\/label\/tests\/behat\/label_visibility.feature": "14", - "repository\/tests\/behat\/create_folders.feature": "14", - "question\/format\/webct\/tests\/behat\/importcalculated.feature": "14", - "question\/format\/webct\/tests\/behat\/import.feature": "14", - "mod\/glossary\/tests\/behat\/prevent_duplicate_entries.feature": "14", - "course\/tests\/behat\/activities_edit_with_block_dock.feature": "13", - "blocks\/glossary_random\/tests\/behat\/glossary_random_frontpage.feature": "13", - "mod\/choice\/tests\/behat\/add_choice.feature": "13", - "admin\/tests\/behat\/display_short_names.feature": "12", - "admin\/tool\/monitor\/tests\/behat\/disabled.feature": "12", - "repository\/tests\/behat\/delete_files.feature": "12", - "lib\/editor\/atto\/plugins\/clear\/tests\/behat\/clear.feature": "12", - "repository\/upload\/tests\/behat\/upload_file.feature": "11", - "my\/tests\/behat\/add_blocks.feature": "11", - "lib\/editor\/atto\/plugins\/charmap\/tests\/behat\/charmap.feature": "11", - "lib\/editor\/atto\/plugins\/subscript\/tests\/behat\/subscript.feature": "10", - "lib\/editor\/atto\/plugins\/strike\/tests\/behat\/strike.feature": "10", - "lib\/editor\/atto\/plugins\/superscript\/tests\/behat\/superscript.feature": "10", - "lib\/editor\/atto\/plugins\/title\/tests\/behat\/title.feature": "10", - "lib\/editor\/atto\/plugins\/underline\/tests\/behat\/underline.feature": "10", - "user\/tests\/behat\/reset_page.feature": "10", - "lib\/editor\/atto\/plugins\/unorderedlist\/tests\/behat\/unorderedlist.feature": "9", - "user\/tests\/behat\/edituserpassword.feature": "9", - "lib\/editor\/atto\/plugins\/orderedlist\/tests\/behat\/orderedlist.feature": "9", - "user\/tests\/behat\/add_blocks.feature": "8", - "lib\/editor\/atto\/plugins\/collapse\/tests\/behat\/collapse.feature": "6", - "lib\/editor\/atto\/plugins\/html\/tests\/behat\/html.feature": "6", - "report\/usersessions\/tests\/behat\/usersessions_report.feature": "3", - "admin\/tool\/behat\/tests\/behat\/test_environment.feature": "2" -} \ No newline at end of file diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index 641789ca22d..61c7f856c32 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -416,14 +416,12 @@ class behat_config_manager { if (!$behattimingfile || !$behattimingdata = @json_decode(file_get_contents($behattimingfile), true)) { // No data available, fall back to relying on steps data. - $stepfile = __DIR__ . "/../../../behat_features_step_count.json"; - + $stepfile = ""; if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) { $stepfile = BEHAT_FEATURE_STEP_FILE; } // We should never get this. But in case we can't do this then fall back on simple splitting. - if (!$stepfile || !$behattimingdata = @json_decode(file_get_contents($stepfile), true)) { - echo "No step/timing information available, falling back on simple distribution" . PHP_EOL; + if (empty($stepfile) || !$behattimingdata = @json_decode(file_get_contents($stepfile), true)) { return false; } } From 1bff616611d82c9d07e9a78e6f20ad2d39a2f1cf Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Thu, 26 Feb 2015 13:40:29 +0800 Subject: [PATCH 05/11] MDL-39752 behat: Added Fromrun and torun for running on multiple vms https://circleci.com --- admin/tool/behat/cli/init.php | 16 ++++++++++------ admin/tool/behat/cli/run.php | 19 ++++++++++++++----- admin/tool/behat/cli/util.php | 14 +++++++++++--- lib/behat/classes/behat_config_manager.php | 9 ++++----- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index 4b7917f1fe2..52f783f517c 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -44,7 +44,9 @@ list($options, $unrecognized) = cli_get_params( array( 'parallel' => 0, 'maxruns' => false, - 'help' => false, + 'help' => false, + 'fromrun' => 1, + 'torun' => 0, ), array( 'j' => 'parallel', @@ -60,6 +62,8 @@ Behat utilities to initialise behat tests Options: -j, --parallel Number of parallel behat run to initialise -m, --maxruns Max parallel processes to be executed at one time. +--fromrun Execute run starting from (Used for parallel runs on different vms) +--torun Execute run till (Used for parallel runs on different vms) -h, --help Print out this help @@ -80,11 +84,11 @@ $paralleloption = ""; // If parallel run then use utilparallel. if ($options['parallel']) { $utilfile = 'util.php'; - $paralleloption = "--parallel=" . $options['parallel']; - - // If maxruns then add that option. - if ($options['maxruns']) { - $paralleloption .= " --maxruns=" . $options['maxruns']; + $paralleloption = ""; + foreach ($options as $option => $value) { + if ($value) { + $paralleloption .= " --$option=\"$value\""; + } } } diff --git a/admin/tool/behat/cli/run.php b/admin/tool/behat/cli/run.php index 92f4462869a..3433e46fcae 100644 --- a/admin/tool/behat/cli/run.php +++ b/admin/tool/behat/cli/run.php @@ -50,6 +50,8 @@ list($options, $unrecognised) = cli_get_params( 'help' => false, 'tags' => '', 'profile' => '', + 'fromrun' => 1, + 'torun' => 0, ), array( 'h' => 'help', @@ -67,6 +69,8 @@ Options: --stop-on-failure Stop on failure in any parallel run. --verbose Verbose output --replace Replace args string with run process number, useful for output. +--fromrun Execute run starting from (Used for parallel runs on different vms) +--torun Execute run till (Used for parallel runs on different vms) -h, --help Print out this help @@ -81,8 +85,12 @@ if (!empty($options['help'])) { exit(0); } -// Ensure we have parallel runs initialised and it's >= 1. -$parallelrun = behat_config_manager::get_parallel_test_runs(1); +$parallelrun = behat_config_manager::get_parallel_test_runs($options['fromrun']); + +// Default torun is maximum parallel runs. +if (empty($options['torun'])) { + $options['torun'] = $parallelrun; +} // Capture signals and ensure we clean symlinks. if (extension_loaded('pcntl')) { @@ -107,7 +115,7 @@ if (empty($parallelrun)) { } // Create site symlink if necessary. -if (!behat_config_manager::create_parallel_site_links()) { +if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) { echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL; exit(1); } @@ -154,8 +162,9 @@ if ($tags) { } $cmds = array(); -echo "Running ${parallelrun} parallel behat sites:" . PHP_EOL; -for ($i = 1; $i <= $parallelrun; $i++) { +echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL; + +for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { $CFG->behatrunprocess = $i; // Options parameters to be added to each run. diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index 62007477b67..25196685f4a 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -50,6 +50,8 @@ list($options, $unrecognized) = cli_get_params( 'parallel' => 0, 'maxruns' => false, 'updatesteps' => false, + 'fromrun' => 1, + 'torun' => 0, ), array( 'h' => 'help', @@ -101,6 +103,11 @@ if (empty($options['parallel'])) { exit($status); } +// Default torun is maximum parallel runs. +if (empty($options['torun'])) { + $options['torun'] = $options['parallel']; +} + $status = false; $cmds = commands_to_execute($options); @@ -164,8 +171,9 @@ require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php' // Show command o/p (only one per time). if ($options['install']) { echo "Acceptance tests site installed for sites:".PHP_EOL; + // Display all sites which are installed/drop/diabled. - for ($i = 1; $i <= $options['parallel']; $i++) { + for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL; } } else if ($options['drop']) { @@ -193,7 +201,7 @@ exit(0); * @return array commands to be executed. */ function commands_to_execute($options) { - $removeoptions = array('maxruns'); + $removeoptions = array('maxruns', 'fromrun', 'torun'); $cmds = array(); $extraoptions = $options; $extra = ""; @@ -217,7 +225,7 @@ function commands_to_execute($options) { $cmds = "php util_single_run.php " . $extra; } else { // Create commands which has to be executed for parallel site. - for ($i = 1; $i <= $options['parallel']; $i++) { + for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { $prefix = BEHAT_PARALLEL_SITE_NAME . $i; $cmds[$prefix] = "php util_single_run.php " . $extra . " --run=" . $i . " 2>&1"; } diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index 61c7f856c32..8d840d7a7c6 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -287,17 +287,16 @@ class behat_config_manager { /** * Create parallel site links. * + * @param int $fromrun first run + * @param int $torun last run. * @return bool true for sucess, else false. */ - public final static function create_parallel_site_links() { + public final static function create_parallel_site_links($fromrun, $torun) { global $CFG; - // Get parallel test runs from first run. - $parallelrun = self::get_parallel_test_runs(1); - // Create site symlink if necessary. clearstatcache(); - for ($i = 1; $i <= $parallelrun; $i++) { + for ($i = $fromrun; $i <= $torun; $i++) { $link = $CFG->dirroot.'/'.BEHAT_PARALLEL_SITE_NAME.$i; clearstatcache(); if (file_exists($link)) { From a2754d791bd02f842003df5b856ea89cf55bee8b Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Tue, 3 Mar 2015 12:04:54 +0800 Subject: [PATCH 06/11] MDL-39752 behat: Moved parallel run checks from setup to behat/lib.php --- lib/behat/lib.php | 46 ++++++++++++++++++++++++++++++++++++++++++---- lib/setup.php | 39 ++++----------------------------------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/lib/behat/lib.php b/lib/behat/lib.php index e6f50ea4cca..ecc439fc1e0 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -283,14 +283,13 @@ function behat_is_test_site() { * - behat_wwwroot = behat_wwwroot{behatrunprocess} * - behat_dataroot = behat_dataroot{behatrunprocess} * - behat_prefix = behat_prefix.{behatrunprocess}_ (For oracle it will be firstletter of prefix and behatrunprocess) - * - * @param string $behatrunprocess process index for which variables will be set. **/ -function behat_update_vars_for_process($behatrunprocess = '') { +function behat_update_vars_for_process() { global $CFG; $allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'behat_prefix'); - $behatrunprocess = $CFG->behatrunprocess; + $behatrunprocess = behat_get_run_process(); + $CFG->behatrunprocess = $behatrunprocess; if ($behatrunprocess) { // Set www root for run process. @@ -360,6 +359,45 @@ function behat_is_requested_url($url) { return false; } +/** + * Get behat run process from either $_SERVER or command config. + * + * @return bool|int false if single run, else run process number. + */ +function behat_get_run_process() { + global $argv, $CFG; + $behatrunprocess = false; + + // Get behat run process, if set. + if (defined('BEHAT_CURRENT_RUN') && BEHAT_CURRENT_RUN) { + $behatrunprocess = BEHAT_CURRENT_RUN; + } else if (!empty($_SERVER['REMOTE_ADDR'])) { + if (preg_match('#/' . BEHAT_PARALLEL_SITE_NAME . '(.+?)/#', $_SERVER['REQUEST_URI'])) { + $dirrootrealpath = str_replace("\\", "/", realpath($CFG->dirroot)); + $serverrealpath = str_replace("\\", "/", realpath($_SERVER['SCRIPT_FILENAME'])); + $afterpath = str_replace($dirrootrealpath.'/', '', $serverrealpath); + if (!$behatrunprocess = preg_filter("#.*/" . BEHAT_PARALLEL_SITE_NAME . "(.+?)/$afterpath#", '$1', + $_SERVER['SCRIPT_FILENAME'])) { + throw new Exception("Unable to determine behat process [afterpath=" . $afterpath . + ", scriptfilename=" . $_SERVER['SCRIPT_FILENAME'] . "]!"); + } + } + } else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) { + if ($match = preg_filter('#--run=(.+)#', '$1', $argv)) { + $behatrunprocess = reset($match); + } + + if ($k = array_search('--config', $argv)) { + $behatconfig = str_replace("\\", "/", $argv[$k + 1]); + $behatdataroot = str_replace("\\", "/", $CFG->behat_dataroot); + $behatrunprocess = preg_filter("#^{$behatdataroot}" . + "(.+?)[/|\\\]behat[/|\\\]behat\.yml#", '$1', $behatconfig); + } + } + + return $behatrunprocess; +} + /** * Execute commands in parallel. * diff --git a/lib/setup.php b/lib/setup.php index 9d0d41ea57d..6b093d3f7e0 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -77,43 +77,12 @@ if (defined('BEHAT_SITE_RUNNING')) { // We already switched to behat test site previously. } else if (!empty($CFG->behat_wwwroot) or !empty($CFG->behat_dataroot) or !empty($CFG->behat_prefix)) { - global $argv; - $behatrunprocess = false; - - require_once(__DIR__ . '/../lib/behat/lib.php'); - - // Get behat run process, if set. - if (defined('BEHAT_CURRENT_RUN') && BEHAT_CURRENT_RUN) { - $behatrunprocess = BEHAT_CURRENT_RUN; - } else if (!empty($_SERVER['REMOTE_ADDR'])) { - if (preg_match('#/' . BEHAT_PARALLEL_SITE_NAME . '(.+?)/#', $_SERVER['REQUEST_URI'])) { - $dirrootrealpath = str_replace("\\", "/", realpath($CFG->dirroot)); - $serverrealpath = str_replace("\\", "/", realpath($_SERVER['SCRIPT_FILENAME'])); - $afterpath = str_replace($dirrootrealpath.'/', '', $serverrealpath); - if (!$behatrunprocess = preg_filter("#.*/" . BEHAT_PARALLEL_SITE_NAME . "(.+?)/$afterpath#", '$1', - $_SERVER['SCRIPT_FILENAME'])) { - throw new Exception("Unable to determine behat process [afterpath=" . $afterpath . - ", scriptfilename=" . $_SERVER['SCRIPT_FILENAME'] . "]!"); - } - } - } else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) { - if ($match = preg_filter('#--run=(.+)#', '$1', $argv)) { - $behatrunprocess = reset($match); - } - - if ($k = array_search('--config', $argv)) { - $behatconfig = str_replace("\\", "/", $argv[$k + 1]); - $behatdataroot = str_replace("\\", "/", $CFG->behat_dataroot); - $behatrunprocess = preg_filter("#^{$behatdataroot}" . - "(.+?)[/|\\\]behat[/|\\\]behat\.yml#", '$1', $behatconfig); - } - } - - $CFG->behatrunprocess = $behatrunprocess; - // The behat is configured on this server, we need to find out if this is the behat test // site based on the URL used for access. - behat_update_vars_for_process($behatrunprocess); + require_once(__DIR__ . '/../lib/behat/lib.php'); + + // Update config variables for parallel behat runs. + behat_update_vars_for_process(); if (behat_is_test_site()) { clearstatcache(); From b32b2284fec35c14316afdcf1f2db8e343618b50 Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Wed, 4 Mar 2015 16:29:03 +0800 Subject: [PATCH 07/11] MDL-39752 behat: Show output of failed install process --- admin/tool/behat/cli/util.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index 25196685f4a..8e359fbd63c 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -134,7 +134,11 @@ if ($options['diag'] || $options['enable'] || $options['disable']) { $exitcodes = print_combined_install_output($processes); foreach ($exitcodes as $name => $exitcode) { if ($exitcode != 0) { + echo "Failed process [[$name]]" . PHP_EOL; + echo $processes[$name]->getOutput(); + echo PHP_EOL; echo $processes[$name]->getErrorOutput(); + echo PHP_EOL . PHP_EOL; } $status = (bool)$status || (bool)$exitcode; } @@ -144,7 +148,11 @@ if ($options['diag'] || $options['enable'] || $options['disable']) { $exitcodes = print_combined_install_output($processes); foreach ($exitcodes as $name => $exitcode) { if ($exitcode != 0) { + echo "Failed process [[$name]]" . PHP_EOL; + echo $processes[$name]->getOutput(); + echo PHP_EOL; echo $processes[$name]->getErrorOutput(); + echo PHP_EOL . PHP_EOL; } $status = (bool)$status || (bool)$exitcode; } From 52acb7c9aaba149b8c7eccd12339c2652f8dc61f Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Thu, 5 Mar 2015 12:09:25 +0800 Subject: [PATCH 08/11] MDL-39752 behat: Added information about defines in config-dist --- config-dist.php | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/config-dist.php b/config-dist.php index 648a4306eb2..7f6c974f165 100644 --- a/config-dist.php +++ b/config-dist.php @@ -739,6 +739,33 @@ $CFG->admin = 'admin'; // ), // ); // +// To change name of behat parallel run site, define BEHAT_PARALLEL_SITE_NAME and parallel run sites will be suffixed +// with this value +// Example: +// define('BEHAT_PARALLEL_SITE_NAME', 'behatparallelsite'); +// +// Command line output for parallel behat install is limited to 80 chars, if you are installing more then 4 sites and +// want to expand output to more then 80 chars, then define BEHAT_MAX_CMD_LINE_OUTPUT +// Example: +// define('BEHAT_MAX_CMD_LINE_OUTPUT', 120); +// +// Behat feature files will be distributed randomly between the processes by default. If you have timing file or want +// to create timing file then define BEHAT_FEATURE_TIMING_FILE with path to timing file. It will be updated for each +// run with latest time taken to execute feature. +// Example: +// define('BEHAT_FEATURE_TIMING_FILE', '/PATH_TO_TIMING_FILE/timing.json'); +// +// If you don't have timing file and want some stable distribution of features, then you can use step counts to +// distribute the features. You can generate step file by executing php admin/tool/behat/cli/util.php --updatesteps +// this will update step file which is defined by BEHAT_FEATURE_STEP_FILE. +// Example: +// define('BEHAT_FEATURE_STEP_FILE', '/PATH_TO_FEATURE_STEP_COUNT_FILE/stepcount.json'); +// +// Feature distribution for each process is displayed as histogram. you can disable it by setting +// BEHAT_DISABLE_HISTOGRAM +// Example: +// define('BEHAT_DISABLE_HISTOGRAM', true); +// //========================================================================= // 12. DEVELOPER DATA GENERATOR //========================================================================= From b1c36d966ea43ff6163266cf173f0dd971430a69 Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Thu, 5 Mar 2015 12:44:06 +0800 Subject: [PATCH 09/11] MDL-39752 behat: for drop option don't have to pass parallel option --- admin/tool/behat/cli/util.php | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index 8e359fbd63c..9e2198d819f 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -34,9 +34,13 @@ define('BEHAT_UTIL', true); define('CLI_SCRIPT', true); define('NO_OUTPUT_BUFFERING', true); define('IGNORE_COMPONENT_CACHE', true); +define('ABORT_AFTER_CONFIG', true); +require_once(__DIR__ . '/../../../../config.php'); require_once(__DIR__ . '/../../../../lib/clilib.php'); require_once(__DIR__ . '/../../../../lib/behat/lib.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php'); +require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php'); // CLI options. list($options, $unrecognized) = cli_get_params( @@ -88,6 +92,13 @@ if (!empty($options['help'])) { } $cwd = getcwd(); + +// For drop option check if parallel site. +if ((empty($options['parallel'])) && $options['drop']) { + // Get parallel run info from first run. + $options['parallel'] = behat_config_manager::get_parallel_test_runs($options['fromrun']); +} + // If not a parallel site then open single run. if (empty($options['parallel'])) { chdir(__DIR__); @@ -170,12 +181,6 @@ if ($status) { exit((int)$status); } -// Only load CFG from config.php for 1st run amd stop ASAP in lib/setup.php. -define('ABORT_AFTER_CONFIG', true); -require_once(__DIR__ . '/../../../../config.php'); -require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php'); -require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php'); - // Show command o/p (only one per time). if ($options['install']) { echo "Acceptance tests site installed for sites:".PHP_EOL; From 5b9e6b5526583c2ad0a5f6c4a103f730aa03fae7 Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Fri, 6 Mar 2015 10:00:37 +0800 Subject: [PATCH 10/11] MDL-39752 behat: Added support for behat_wwwroot and behat_dataroot --- admin/tool/behat/cli/run.php | 8 ++- admin/tool/behat/cli/util.php | 7 ++- config-dist.php | 4 +- lib/behat/classes/behat_command.php | 10 ++-- lib/behat/classes/behat_config_manager.php | 14 +++++- lib/behat/lib.php | 57 ++++++++++++++++------ 6 files changed, 77 insertions(+), 23 deletions(-) diff --git a/admin/tool/behat/cli/run.php b/admin/tool/behat/cli/run.php index 3433e46fcae..c83259f1d8b 100644 --- a/admin/tool/behat/cli/run.php +++ b/admin/tool/behat/cli/run.php @@ -147,13 +147,17 @@ if ($options['profile']) { // Update config file if tags defined. if ($tags) { - // Hack to set proper dataroot and wwroot. + // Hack to set proper dataroot and wwwroot. $behatdataroot = $CFG->behat_dataroot; $behatwwwroot = $CFG->behat_wwwroot; for ($i = 1; $i <= $parallelrun; $i++) { $CFG->behatrunprocess = $i; $CFG->behat_dataroot = $behatdataroot . $i; - $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i; + if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) { + $CFG->behat_wwwroot = $CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot']; + } else { + $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i; + } behat_config_manager::update_config_file('', true, $tags); } $CFG->behat_dataroot = $behatdataroot; diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index 9e2198d819f..1bdb72dd833 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -187,7 +187,12 @@ if ($options['install']) { // Display all sites which are installed/drop/diabled. for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { - echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL; + if (empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) { + echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL; + } else { + echo $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'] . PHP_EOL; + } + } } else if ($options['drop']) { echo "Acceptance tests site dropped for " . $options['parallel'] . " parallel sites" . PHP_EOL; diff --git a/config-dist.php b/config-dist.php index 7f6c974f165..4973d520faf 100644 --- a/config-dist.php +++ b/config-dist.php @@ -735,7 +735,9 @@ $CFG->admin = 'admin'; // 'dbuser' => 'moodle', // 'dbpass' => 'moodle', // 'behat_prefix' => 'mdl_', -// 'wd_host' => 'http://127.0.0.1:4444/wd/hub' +// 'wd_host' => 'http://127.0.0.1:4444/wd/hub', +// 'behat_wwwroot' => 'http://127.0.0.1/moodle', +// 'behat_dataroot' => '/home/example/bht_moodledata' // ), // ); // diff --git a/lib/behat/classes/behat_command.php b/lib/behat/classes/behat_command.php index 73e6aec02da..ce6c744a731 100644 --- a/lib/behat/classes/behat_command.php +++ b/lib/behat/classes/behat_command.php @@ -50,9 +50,13 @@ class behat_command { public static function get_behat_dir($runprocess = 0) { global $CFG; - $runprocess = empty($runprocess) ? "" : $runprocess; - - $behatdir = $CFG->behat_dataroot . $runprocess . '/behat'; + if (empty($runprocess)) { + $behatdir = $CFG->behat_dataroot . '/behat'; + } else if (isset($CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'])) { + $behatdir = $CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot'] . '/behat';; + } else { + $behatdir = $CFG->behat_dataroot . $runprocess . '/behat'; + } if (!is_dir($behatdir)) { if (!mkdir($behatdir, $CFG->directorypermissions, true)) { diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index 8d840d7a7c6..f3b8680378d 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -214,7 +214,11 @@ class behat_config_manager { global $CFG; if ($runprocess) { - $command = $CFG->behat_dataroot . $runprocess; + if (isset($CFG->behat_parallel_run[$runprocess - 1 ]['behat_dataroot'])) { + $command = $CFG->behat_parallel_run[$runprocess - 1]['behat_dataroot']; + } else { + $command = $CFG->behat_dataroot . $runprocess; + } } else { $command = $CFG->behat_dataroot; } @@ -276,6 +280,10 @@ class behat_config_manager { // If parallel run then remove links and original file. clearstatcache(); for ($i = 1; $i <= $parallelrun; $i++) { + // Don't delete links for specified sites, as they should be accessible. + if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) { + continue; + } $link = $CFG->dirroot . '/' . BEHAT_PARALLEL_SITE_NAME . $i; if (file_exists($link) && is_link($link)) { @unlink($link); @@ -297,6 +305,10 @@ class behat_config_manager { // Create site symlink if necessary. clearstatcache(); for ($i = $fromrun; $i <= $torun; $i++) { + // Don't create links for specified sites, as they should be accessible. + if (!empty($CFG->behat_parallel_run['behat_wwwroot'][$i - 1]['behat_wwwroot'])) { + continue; + } $link = $CFG->dirroot.'/'.BEHAT_PARALLEL_SITE_NAME.$i; clearstatcache(); if (file_exists($link)) { diff --git a/lib/behat/lib.php b/lib/behat/lib.php index ecc439fc1e0..7a422ed9c24 100644 --- a/lib/behat/lib.php +++ b/lib/behat/lib.php @@ -287,20 +287,25 @@ function behat_is_test_site() { function behat_update_vars_for_process() { global $CFG; - $allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'behat_prefix'); + $allowedconfigoverride = array('dbtype', 'dblibrary', 'dbhost', 'dbname', 'dbuser', 'dbpass', 'behat_prefix', + 'behat_wwwroot', 'behat_dataroot'); $behatrunprocess = behat_get_run_process(); $CFG->behatrunprocess = $behatrunprocess; if ($behatrunprocess) { - // Set www root for run process. - if (isset($CFG->behat_wwwroot) && !preg_match("#/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess . "\$#", - $CFG->behat_wwwroot)) { - $CFG->behat_wwwroot .= "/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess; + if (empty($CFG->behat_parallel_run[$behatrunprocess - 1]['behat_wwwroot'])) { + // Set www root for run process. + if (isset($CFG->behat_wwwroot) && + !preg_match("#/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess . "\$#", $CFG->behat_wwwroot)) { + $CFG->behat_wwwroot .= "/" . BEHAT_PARALLEL_SITE_NAME . $behatrunprocess; + } } - // Set behat_dataroot. - if (!preg_match("#" . $behatrunprocess . "\$#", $CFG->behat_dataroot)) { - $CFG->behat_dataroot .= $behatrunprocess; + if (empty($CFG->behat_parallel_run[$behatrunprocess - 1]['behat_dataroot'])) { + // Set behat_dataroot. + if (!preg_match("#" . $behatrunprocess . "\$#", $CFG->behat_dataroot)) { + $CFG->behat_dataroot .= $behatrunprocess; + } } // Set behat_prefix for db, just suffix run process number, to avoid max length exceed. @@ -372,7 +377,17 @@ function behat_get_run_process() { if (defined('BEHAT_CURRENT_RUN') && BEHAT_CURRENT_RUN) { $behatrunprocess = BEHAT_CURRENT_RUN; } else if (!empty($_SERVER['REMOTE_ADDR'])) { - if (preg_match('#/' . BEHAT_PARALLEL_SITE_NAME . '(.+?)/#', $_SERVER['REQUEST_URI'])) { + // Try get it from config if present. + if (!empty($CFG->behat_parallel_run)) { + foreach ($CFG->behat_parallel_run as $run => $behatconfig) { + if (isset($behatconfig['behat_wwwroot']) && behat_is_requested_url($behatconfig['behat_wwwroot'])) { + $behatrunprocess = $run + 1; // We start process from 1. + break; + } + } + } + // Check if parallel site prefix is used. + if (empty($behatrunprocess) && preg_match('#/' . BEHAT_PARALLEL_SITE_NAME . '(.+?)/#', $_SERVER['REQUEST_URI'])) { $dirrootrealpath = str_replace("\\", "/", realpath($CFG->dirroot)); $serverrealpath = str_replace("\\", "/", realpath($_SERVER['SCRIPT_FILENAME'])); $afterpath = str_replace($dirrootrealpath.'/', '', $serverrealpath); @@ -385,13 +400,25 @@ function behat_get_run_process() { } else if (defined('BEHAT_TEST') || defined('BEHAT_UTIL')) { if ($match = preg_filter('#--run=(.+)#', '$1', $argv)) { $behatrunprocess = reset($match); - } - - if ($k = array_search('--config', $argv)) { + } else if ($k = array_search('--config', $argv)) { $behatconfig = str_replace("\\", "/", $argv[$k + 1]); - $behatdataroot = str_replace("\\", "/", $CFG->behat_dataroot); - $behatrunprocess = preg_filter("#^{$behatdataroot}" . - "(.+?)[/|\\\]behat[/|\\\]behat\.yml#", '$1', $behatconfig); + // Try get it from config if present. + if (!empty($CFG->behat_parallel_run)) { + foreach ($CFG->behat_parallel_run as $run => $parallelconfig) { + if (!empty($parallelconfig['behat_dataroot']) && + $parallelconfig['behat_dataroot'] . '/behat/behat.yml' == $behatconfig) { + + $behatrunprocess = $run + 1; // We start process from 1. + break; + } + } + } + // Check if default behat datroot increment was done. + if (empty($behatrunprocess)) { + $behatdataroot = str_replace("\\", "/", $CFG->behat_dataroot); + $behatrunprocess = preg_filter("#^{$behatdataroot}" . "(.+?)[/|\\\]behat[/|\\\]behat\.yml#", '$1', + $behatconfig); + } } } From 003dc31998f29fbfaef61a65f01e9a31a782767b Mon Sep 17 00:00:00 2001 From: Rajesh Taneja Date: Mon, 9 Mar 2015 07:11:01 +0800 Subject: [PATCH 11/11] MDL-39752 behat: Updated composer to use step count formatter --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index af77f59195e..60864a892e3 100644 --- a/composer.json +++ b/composer.json @@ -8,6 +8,6 @@ "require-dev": { "phpunit/phpunit": "3.7.*", "phpunit/dbUnit": "1.2.*", - "moodlehq/behat-extension": "1.29.2" + "moodlehq/behat-extension": "1.29.3" } }