MDL-46514 behat: Catch php errors in shutdown hook

If error is not displayed in web browser then
it will be caught in shutdown hook, so we can
show it to user

lib/behat/behat_base.php

@@ -764,6 +764,17 @@ class behat_base extends Behat\MinkExtension\Context\RawMinkContext {
             // Joined xpath expression. Most of the time there will be no exceptions, so this pre-check
             // is faster than to send the 4 xpath queries for each step.
             if (!$this->getSession()->getDriver()->find($joinedxpath)) {
+                // Check if we have recorded any errors in driver process.
+                $phperrors = behat_get_shutdown_process_errors();
+                if (!empty($phperrors)) {
+                    foreach ($phperrors as $error) {
+                        $errnostring = behat_get_error_string($error['type']);
+                        $msgs[] = $errnostring . ": " .$error['message'] . " at " . $error['file'] . ": " . $error['line'];
+                    }
+                    $msg = "PHP errors found:\n" . implode("\n", $msgs);
+                    throw new \Exception(htmlentities($msg));
+                }
+
                 return;
             }
 
@@ -776,8 +787,20 @@ class behat_base extends Behat\MinkExtension\Context\RawMinkContext {
                 if (empty($errorinfoboxes)) {
                     $errorinfoboxes = $this->getSession()->getPage()->findAll('css', 'div.notifytiny');
                 }
-                $errorinfo = $this->get_debug_text($errorinfoboxes[0]->getHtml()) . "\n" .
-                    $this->get_debug_text($errorinfoboxes[1]->getHtml());
+
+                // If errorinfoboxes is empty, try find ajax/JS exception in dialogue.
+                if (empty($errorinfoboxes)) {
+                    $errorinfoboxes = $this->getSession()->getPage()->findAll('css', 'div.moodle-exception-message');
+
+                    // If ajax/JS exception.
+                    if ($errorinfoboxes) {
+                        $errorinfo = $this->get_debug_text($errorinfoboxes[0]->getHtml());
+                    }
+
+                } else {
+                    $errorinfo = $this->get_debug_text($errorinfoboxes[0]->getHtml()) . "\n" .
+                        $this->get_debug_text($errorinfoboxes[1]->getHtml());
+                }
 
                 $msg = "Moodle exception: " . $errormsg->getText() . "\n" . $errorinfo;
                 throw new \Exception(html_entity_decode($msg));

lib/behat/lib.php

@@ -84,6 +84,36 @@ function behat_error($errorcode, $text = '') {
     testing_error($errorcode, $text);
 }
 
+/**
+ * Return logical error string.
+ *
+ * @param int $errtype php error type.
+ * @return string string which will be returned.
+ */
+function behat_get_error_string($errtype) {
+    switch ($errtype) {
+        case E_USER_ERROR:
+            $errnostr = 'Fatal error';
+            break;
+        case E_WARNING:
+        case E_USER_WARNING:
+            $errnostr = 'Warning';
+            break;
+        case E_NOTICE:
+        case E_USER_NOTICE:
+        case E_STRICT:
+            $errnostr = 'Notice';
+            break;
+        case E_RECOVERABLE_ERROR:
+            $errnostr = 'Catchable';
+            break;
+        default:
+            $errnostr = 'Unknown error type';
+    }
+
+    return $errnostr;
+}
+
 /**
  * PHP errors handler to use when running behat tests.
  *
@@ -121,35 +151,57 @@ function behat_error_handler($errno, $errstr, $errfile, $errline, $errcontext) {
     // Using the default one in case there is a fatal catchable error.
     default_error_handler($errno, $errstr, $errfile, $errline, $errcontext);
 
-    switch ($errno) {
-        case E_USER_ERROR:
-            $errnostr = 'Fatal error';
-            break;
-        case E_WARNING:
-        case E_USER_WARNING:
-            $errnostr = 'Warning';
-            break;
-        case E_NOTICE:
-        case E_USER_NOTICE:
-        case E_STRICT:
-            $errnostr = 'Notice';
-            break;
-        case E_RECOVERABLE_ERROR:
-            $errnostr = 'Catchable';
-            break;
-        default:
-            $errnostr = 'Unknown error type';
-    }
+    $errnostr = behat_get_error_string($errno);
 
-    // Wrapping the output.
-    echo '<div class="phpdebugmessage" data-rel="phpdebugmessage">' . PHP_EOL;
-    echo "$errnostr: $errstr in $errfile on line $errline" . PHP_EOL;
-    echo '</div>';
+    // If ajax script then throw exception, so the calling api catch it and show it on web page.
+    if (defined('AJAX_SCRIPT')) {
+        throw new Exception("$errnostr: $errstr in $errfile on line $errline");
+    } else {
+        // Wrapping the output.
+        echo '<div class="phpdebugmessage" data-rel="phpdebugmessage">' . PHP_EOL;
+        echo "$errnostr: $errstr in $errfile on line $errline" . PHP_EOL;
+        echo '</div>';
+    }
 
     // Also use the internal error handler so we keep the usual behaviour.
     return false;
 }
 
+/**
+ * Before shutdown save last error entries, so we can fail the test.
+ */
+function behat_shutdown_function() {
+    global $DB;
+
+    // If any error found, then save it.
+    if ($error = error_get_last()) {
+        // Ignore E_WARNING, as they might come via ( @ )suppression and might lead to false failure.
+        if (isset($error['type']) && !($error['type'] & E_WARNING)) {
+            $errors = behat_get_shutdown_process_errors();
+
+            $errors[] = $error;
+            $errorstosave = json_encode($errors);
+
+            set_config('process_errors', $errorstosave, 'tool_behat');
+        }
+    }
+}
+
+/**
+ * Return php errors save which were save during shutdown.
+ *
+ * @return array
+ */
+function behat_get_shutdown_process_errors() {
+    $phperrors = get_config('tool_behat', 'process_errors');
+
+    if (!empty($phperrors)) {
+        return json_decode($phperrors, true);
+    } else {
+        return array();
+    }
+}
+
 /**
  * Restrict the config.php settings allowed.
  *

lib/setup.php

@@ -1038,6 +1038,12 @@ if (isset($CFG->maintenance_later) and $CFG->maintenance_later <= time()) {
     }
 }
 
+// Add behat_shutdown_function to shutdown manager, so we can capture php errors,
+// but not necessary for behat CLI command as it's being captured by behat process.
+if (defined('BEHAT_SITE_RUNNING') && !defined('BEHAT_TEST')) {
+    core_shutdown_manager::register_function('behat_shutdown_function');
+}
+
 // note: we can not block non utf-8 installations here, because empty mysql database
 // might be converted to utf-8 in admin/index.php during installation