- Optimize acl_getf_global a bit

- a little performance improvement of the IP regular expressions
- convert post_text/subject collation to utf8_unicode_ci if a user wants to use mysql_fulltext to allow case insensitivity [Bug #6272]
- mysql_fulltext should alter all necessary columns at once to speed up the process
- validate URLs against RFC3986
- fixed some weirdness in make_clickable
I hope I didn't break any URLs with this commit, if I did then report it to the bugtracker please!


git-svn-id: file:///svn/phpbb/trunk@6774 89ea8834-ac86-4346-8a33-228a782c2dd0

phpBB/develop/ip_regex.php

@@ -1,37 +0,0 @@
-<?php
-$dec_octet = '(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])';
-$h16 = '[\dA-F]{1,4}';
-$ipv4 = "(?:$dec_octet\.){3}$dec_octet";
-$ls32 = "(?:$h16:$h16|$ipv4)";
-
-$ipv6_construct = array(
-	array(false,	'',		'{6}',	$ls32),
-	array(false,	'::',	'{5}',	$ls32),
-	array('',		':',	'{4}',	$ls32),
-	array('{1,2}',	':',	'{3}',	$ls32),
-	array('{1,3}',	':',	'{2}',	$ls32),
-	array('{1,4}',	':',	'',		$ls32),
-	array('{1,5}',	':',	false,	$ls32),
-	array('{1,6}',	':',	false,	$h16),
-	array('{1,7}',	':',	false,	'')
-);
-
-$ipv6 = '(?:';
-foreach ($ipv6_construct as $ip_type)
-{
-	$ipv6 .= '(?:';
-	if ($ip_type[0] !== false)
-	{
-		$ipv6 .= "(?:$h16:)" . $ip_type[0];
-	}
-	$ipv6 .= $ip_type[1];
-	if ($ip_type[2] !== false)
-	{
-		$ipv6 .= "(?:$h16:)" . $ip_type[2];
-	}
-	$ipv6 .= $ip_type[3] . ')|';
-}
-$ipv6 = substr($ipv6, 0, -1) . ')';
-
-echo 'IPv4: ' . $ipv4 . "<br />\nIPv6: " . $ipv6;
-?>

phpBB/develop/regex.php

@@ -0,0 +1,74 @@
+<?php
+
+// IP regular expressions
+
+$dec_octet = '(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])';
+$h16 = '[\dA-F]{1,4}';
+$ipv4 = "(?:$dec_octet\.){3}$dec_octet";
+$ls32 = "(?:$h16:$h16|$ipv4)";
+
+$ipv6_construct = array(
+	array(false,	'',		'{6}',	$ls32),
+	array(false,	'::',	'{5}',	$ls32),
+	array('',		':',	'{4}',	$ls32),
+	array('{1,2}',	':',	'{3}',	$ls32),
+	array('{1,3}',	':',	'{2}',	$ls32),
+	array('{1,4}',	':',	'',		$ls32),
+	array('{1,5}',	':',	false,	$ls32),
+	array('{1,6}',	':',	false,	$h16),
+	array('{1,7}',	':',	false,	'')
+);
+
+$ipv6 = '(?:';
+foreach ($ipv6_construct as $ip_type)
+{
+	$ipv6 .= '(?:';
+	if ($ip_type[0] !== false)
+	{
+		$ipv6 .= "(?:$h16:)" . $ip_type[0];
+	}
+	$ipv6 .= $ip_type[1];
+	if ($ip_type[2] !== false)
+	{
+		$ipv6 .= "(?:$h16:)" . $ip_type[2];
+	}
+	$ipv6 .= $ip_type[3] . ')|';
+}
+$ipv6 = substr($ipv6, 0, -1) . ')';
+
+echo 'IPv4: ' . $ipv4 . "<br />\nIPv6: " . $ipv6 . "<br />\n";
+
+// URL regular expressions
+
+$pct_encoded = "%[\dA-F]{2}";
+$unreserved = 'a-z0-9\-._~';
+$sub_delims = '!$&\'()*+,;=';
+$pchar = "(?:[$unreserved$sub_delims:@|]|$pct_encoded)"; // rfc: no "|"
+
+$scheme = '[a-z][a-z\d+\-.]*';
+$reg_name = "(?:[$unreserved$sub_delims|]|$pct_encoded)+"; // rfc: * instead of + and no "|"
+$authority = "(?:(?:[\w\-.~!$&'()*+,;=:]|$pct_encoded)*@){0,1}(?:$reg_name|$ipv4|\[$ipv6\])[:]?\d*";
+$userinfo = "(?:(?:[$unreserved$sub_delims:]|$pct_encoded))*";
+$ipv4_simple = '[0-9.]+';
+$ipv6_simple = '\[[a-z0-9.:]+\]';
+$host = "(?:$reg_name|$ipv4_simple|$ipv6_simple)";
+$port = '\d*';
+$authority = "(?:$userinfo@)?$host(?::$port)?";
+$segment = "$pchar*";
+$path_abempty = "(?:/$segment)*";
+$hier_part = "/{2}$authority$path_abempty";
+$query = "(?:[$unreserved$sub_delims:@/?|]|$pct_encoded)*"; // pchar | "/" | "?", rfc: no "|"
+$fragment = $query;
+
+$url =  "$scheme:$hier_part(?:\?$query)?(?:\#$fragment)?";
+echo 'URL: ' . $url . "<br />\n";
+
+// no scheme, shortened authority, but host has to start with www.
+$www_url =  "www\.$reg_name(?::$port)?$path_abempty(?:\?$query)?(?:\#$fragment)?";
+echo 'www.URL: ' . $www_url . "<br />\n";
+
+// no schema and no authority
+$relative_url = "$segment$path_abempty(?:\?$query)?(?:\#$fragment)?";
+echo 'relative URL: ' . $relative_url . "<br />\n";
+
+?>

phpBB/includes/auth.php

@@ -221,16 +221,18 @@ class auth
 	*/
 	function acl_getf_global($opt)
 	{
-		$allowed = false;
-
 		if (is_array($opt))
 		{
+			// evaluates to true as soon as acl_getf_global is true for one option
 			foreach ($opt as $check_option)
 			{
-				$allowed |= $this->acl_getf_global($check_option);
+				if ($this->acl_getf_global($check_option))
+				{
+					return true;
+				}
 			}
 
-			return $allowed;
+			return false;
 		}
 
 		if (isset($this->acl_options['local'][$opt]))
@@ -243,20 +245,19 @@ class auth
 					continue;
 				}
 
-				$allowed = (!isset($this->cache[$f][$opt])) ? $this->acl_get($opt, $f) : $this->cache[$f][$opt];
+				// as soon as the user has any permission we're done so return true
-
+				if ((!isset($this->cache[$f][$opt])) ? $this->acl_get($opt, $f) : $this->cache[$f][$opt])
-				if ($allowed)
 				{
-					break;
+					return true;
 				}
 			}
 		}
 		else if (isset($this->acl_options['global'][$opt]))
 		{
-			$allowed = $this->acl_get($opt);
+			return $this->acl_get($opt);
 		}
 
-		return $allowed;
+		return false;
 	}
 
 	/**

phpBB/includes/functions.php

@@ -2379,15 +2379,16 @@ function make_clickable($text, $server_url = false)
 		// Be sure to not let the matches cross over. ;)
 
 		// relative urls for this board
-		$magic_url_match[] = '#(^|[\n\t (])(' . preg_quote($server_url, '#') . ')/(([^[ \t\n\r<"\'\)&]+|&(?!lt;|quot;))*)#ie';
+		$magic_url_match[] = '#(^|[\n\t (])(' . preg_quote($server_url, '#') . ')/(' . get_preg_expression('relative_url') . ')#ie';
-		$magic_url_replace[] = "'\$1<!-- l --><a href=\"\$2/' . preg_replace('/(&amp;|\?)sid=[0-9a-f]{32}/', '\\1', '\$3') . '\">' . preg_replace('/(&amp;|\?)sid=[0-9a-f]{32}/', '\\1', '\$3') . '</a><!-- l -->'";
+		$magic_url_replace[] = "'\$1<!-- l --><a href=\"\$2/' . preg_replace('/(&amp;|\?)sid=[0-9a-f]{32}/', '\\\\1', '\$3') . '\">' . preg_replace('/(&amp;|\?)sid=[0-9a-f]{32}/', '\\\\1', '\$3') . '</a><!-- l -->'";
 
 		// matches a xxxx://aaaaa.bbb.cccc. ...
-		$magic_url_match[] = '#(^|[\n\t (])([\w]+:/{2}.*?([^[ \t\n\r<"\'\)&]+|&(?!lt;|quot;))*)#ie';
+		//$magic_url_match[] = '#(^|[\n\t (])(' . get_preg_expression('url') . ')([[ \t\n\r<"\'\)]|&(?!lt;|quot;))*#ie';
+		$magic_url_match[] = '#(^|[\n\t (])(' . get_preg_expression('url') . ')#ie';
 		$magic_url_replace[] = "'\$1<!-- m --><a href=\"\$2\">' . ((strlen('\$2') > 55) ? substr(str_replace('&amp;', '&', '\$2'), 0, 39) . ' ... ' . substr(str_replace('&amp;', '&', '\$2'), -10) : '\$2') . '</a><!-- m -->'";
 
 		// matches a "www.xxxx.yyyy[/zzzz]" kinda lazy URL thing
-		$magic_url_match[] = '#(^|[\n\t (])(w{3}\.[\w\-]+\.[\w\-.\~]+(?:[^[ \t\n\r<"\'\)&]+|&(?!lt;|quot;))*)#ie';
+		$magic_url_match[] = '#(^|[\n\t (])(' . get_preg_expression('www_url') . ')#ie';
 		$magic_url_replace[] = "'\$1<!-- w --><a href=\"http://\$2\">' . ((strlen('\$2') > 55) ? substr(str_replace('&amp;', '&', '\$2'), 0, 39) . ' ... ' . substr(str_replace('&amp;', '&', '\$2'), -10) : '\$2') . '</a><!-- w -->'";
 
 		// matches an email@domain type address at the start of a line, or after a space or after what might be a BBCode.
@@ -2725,8 +2726,8 @@ function get_backtrace()
 
 /**
 * This function returns a regular expression pattern for commonly used expressions
-* Use with / as delimiter for email mode
+* Use with / as delimiter for email mode and # for url modes
-* mode can be: email|bbcode_htm
+* mode can be: email|bbcode_htm|url|www_url|relative_url
 */
 function get_preg_expression($mode)
 {
@@ -2745,6 +2746,19 @@ function get_preg_expression($mode)
 				'#<.*?>#s',
 			);
 		break;
+
+		case 'url':
+			// generated with regex generation file in the develop folder
+			return "[a-z][a-z\d+\-.]*:/{2}(?:(?:(?:[a-z0-9\-._~!$&'()*+,;=:]|%[\dA-F]{2}))*@)?(?:(?:[a-z0-9\-._~!$&'()*+,;=|]|%[\dA-F]{2})+|[0-9.]+|\[[a-z0-9.:]+\])(?::\d*)?(?:/(?:[a-z0-9\-._~!$&'()*+,;=:@|]|%[\dA-F]{2})*)*(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?|]|%[\dA-F]{2})*)?(?:\#(?:[a-z0-9\-._~!$&'()*+,;=:@/?|]|%[\dA-F]{2})*)?";
+		break;
+
+		case 'www_url':
+			return "www\.(?:[a-z0-9\-._~!$&'()*+,;=|]|%[\dA-F]{2})+(?::\d*)?(?:/(?:[a-z0-9\-._~!$&'()*+,;=:@|]|%[\dA-F]{2})*)*(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?|]|%[\dA-F]{2})*)?(?:\#(?:[a-z0-9\-._~!$&'()*+,;=:@/?|]|%[\dA-F]{2})*)?";
+		break;
+
+		case 'relative_url':
+			return "(?:[a-z0-9\-._~!$&'()*+,;=:@|]|%[\dA-F]{2})*(?:/(?:[a-z0-9\-._~!$&'()*+,;=:@|]|%[\dA-F]{2})*)*(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?|]|%[\dA-F]{2})*)?(?:\#(?:[a-z0-9\-._~!$&'()*+,;=:@/?|]|%[\dA-F]{2})*)?";
+		break;
 	}
 
 	return '';

phpBB/includes/message_parser.php

@@ -783,6 +783,9 @@ class bbcode_firstpass extends bbcode
 
 	/**
 	* Validate url
+	*
+	* @param string $var1 optional url parameter for url bbcode: [url(=$var1)]$var2[/url]
+	* @param string $var2 url bbcode content: [url(=$var1)]$var2[/url] 
 	*/
 	function validate_url($var1, $var2)
 	{
@@ -792,38 +795,35 @@ class bbcode_firstpass extends bbcode
 		$var2 = str_replace("\r\n", "\n", str_replace('\"', '"', trim($var2)));
 
 		$url = ($var1) ? $var1 : $var2;
-		$valid = false;
 
 		if (!$url || ($var1 && !$var2))
 		{
 			return '';
 		}
 
-		// Before we check anything, we make sure certain characters are not included
+		$valid = false;
-		if (!preg_match('#[\t\n\r<"\']#', $url))
+
-		{
+		$url = str_replace(' ', '%20', $url);
+
 		// Checking urls
-			if (preg_match('#' . preg_quote(generate_board_url(), '#') . '/([^ \t\n\r<"\']+)#i', $url) ||
+		if (preg_match('#^' . get_preg_expression('url') . '$#i', $url) ||
-				preg_match('#([\w]+?://.*?[^ \t\n\r<"\']*)#i', $url) ||
+			preg_match('#^' . get_preg_expression('www_url') . '$#i', $url) ||
-				preg_match('#(www\.[\w\-]+\.[\w\-.\~]+(?:/[^ \t\n\r<"\']*)?)#i', $url))
+			preg_match('#^' . preg_quote(generate_board_url(), '#') . get_preg_expression('relative_url') . '$#i', $url))
 		{
 			$valid = true;
 		}
-		}
 
 		if ($valid)
 		{
-			// Do we want to transform some characters?
-			$url = str_replace(' ', '%20', $url);
-
 			$this->parsed_items['url']++;
 
-			if (!preg_match('#^[\w]+?://.*?#i', $url))
+			// if there is no scheme, then add http schema
+			if (!preg_match('#^[a-z][a-z\d+\-.]*:/{2}#i', $url))
 			{
 				$url = 'http://' . $url;
 			}
 
-			// We take our test url and stick on the first bit of text we get to check if we are really at the domain. If so, lets go!
+			// Is this a link to somewhere inside this board? If so then remove the session id from the url
 			if (strpos($url, generate_board_url()) !== false && strpos($url, 'sid=') !== false)
 			{
 				$url = preg_replace('/(&amp;|\?)sid=[0-9a-f]{32}/', '\1', $url);

phpBB/includes/search/fulltext_mysql.php

@@ -639,14 +639,29 @@ class fulltext_mysql extends search_backend
 			$this->get_stats();
 		}
 
+		$alter = array();
+
 		if (!isset($this->stats['post_subject']))
 		{
-			$db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ADD FULLTEXT (post_subject)');
+			if (version_compare($db->mysql_version, '4.1.3', '>='))
+			{
+				$alter[] = 'MODIFY post_subject varchar(100) COLLATE utf8_unicode_ci DEFAULT \'\' NOT NULL';
+			}
+			$alter[] = 'ADD FULLTEXT (post_subject)';
 		}
 
 		if (!isset($this->stats['post_text']))
 		{
-			$db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ADD FULLTEXT (post_text)');
+			if (version_compare($db->mysql_version, '4.1.3', '>='))
+			{
+				$alter[] = 'MODIFY post_text mediumtext COLLATE utf8_unicode_ci NOT NULL';
+			}
+			$alter[] = 'ADD FULLTEXT (post_text)';
+		}
+
+		if (sizeof($alter))
+		{
+			$db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter));
 		}
 
 		$db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE);
@@ -672,14 +687,21 @@ class fulltext_mysql extends search_backend
 			$this->get_stats();
 		}
 
+		$alter = array();
+
 		if (isset($this->stats['post_subject']))
 		{
-			$db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' DROP INDEX post_subject');
+			$alter[] = 'DROP INDEX post_subject';
 		}
 
 		if (isset($this->stats['post_text']))
 		{
-			$db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' DROP INDEX post_text');
+			$alter[] = 'DROP INDEX post_text';
+		}
+
+		if (sizeof($alter))
+		{
+			$db->sql_query('ALTER TABLE ' . POSTS_TABLE . ' ' . implode(', ', $alter));
 		}
 
 		$db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE);

phpBB/includes/session.php

@@ -158,8 +158,8 @@ class session
 			// Whoa these look impressive!
 			// The code to generate the following two regular expressions which match valid IPv4/IPv6 addresses
 			// can be found in the develop directory
-			$ipv4 = '#^(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])$#';
+			$ipv4 = '#^(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])$#';
-			$ipv6 = '#^(?:(?:(?:[\dA-F]{1,4}:){6}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:::(?:[\dA-F]{1,4}:){5}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:):(?:[\dA-F]{1,4}:){4}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,2}:(?:[\dA-F]{1,4}:){3}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,3}:(?:[\dA-F]{1,4}:){2}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,4}:(?:[\dA-F]{1,4}:)(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,5}:(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d?\d|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,6}:[\dA-F]{1,4})|(?:(?:[\dA-F]{1,4}:){1,7}:))$#';
+			$ipv6 = '#^(?:(?:(?:[\dA-F]{1,4}:){6}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:::(?:[\dA-F]{1,4}:){5}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:):(?:[\dA-F]{1,4}:){4}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,2}:(?:[\dA-F]{1,4}:){3}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,3}:(?:[\dA-F]{1,4}:){2}(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,4}:(?:[\dA-F]{1,4}:)(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,5}:(?:[\dA-F]{1,4}:[\dA-F]{1,4}|(?:(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])))|(?:(?:[\dA-F]{1,4}:){1,6}:[\dA-F]{1,4})|(?:(?:[\dA-F]{1,4}:){1,7}:))$#i';
 
 			// split the list of IPs
 			$ips = explode(', ', $this->forwarded_for);
@@ -313,7 +313,7 @@ class session
 					// Added logging temporarly to help debug bugs...
 					if (defined('DEBUG_EXTRA'))
 					{
-						add_log('critical', 'LOG_IP_BROWSER_FORWARDED_CHECK', $u_ip, $s_ip, $u_browser, $s_browser, $u_forwarded, $s_forwarded);
+						add_log('critical', 'LOG_IP_BROWSER_FORWARDED_CHECK', $u_ip, $s_ip, $u_browser, $s_browser, $u_forwarded_for, $s_forwarded_for);
 					}
 				}
 			}