mirror of
https://github.com/phpbb/phpbb.git
synced 2025-08-13 12:14:06 +02:00
- search result extract shouldn't end in the middle of a multibyte character [Bug #11863]
- missing localisation for an imageset shouldn't create lots of "imageset refreshed" log messages [Bug #12027] - explain that themes which need parsing cannot be stored on the filesystem [Bug #11134] - normalize usernames (we really need to make sure we normalize everything) - improved utf8_clean_string, more complete list of homographs and NFKC normalization, also the resulting string is now trimmed - corrected searching subforums explanation [Bug #12209] git-svn-id: file:///svn/phpbb/trunk@7890 89ea8834-ac86-4346-8a33-228a782c2dd0
This commit is contained in:
Nils Adermann
211
phpBB/develop/generate_utf_confusables.php
Normal file
211
phpBB/develop/generate_utf_confusables.php
Normal file
@@ -0,0 +1,211 @@
|
||||
<?php
|
||||
/**
|
||||
*
|
||||
* @package phpBB3
|
||||
* @version $Id$
|
||||
* @copyright (c) 2005 phpBB Group
|
||||
* @license http://opensource.org/licenses/gpl-license.php GNU Public License
|
||||
*
|
||||
*/
|
||||
|
||||
if (php_sapi_name() != 'cli')
|
||||
{
|
||||
die("This program must be run from the command line.\n");
|
||||
}
|
||||
|
||||
//
|
||||
// Security message:
|
||||
//
|
||||
// This script is potentially dangerous.
|
||||
// Remove or comment the next line (die(".... ) to enable this script.
|
||||
// Do NOT FORGET to either remove this script or disable it after you have used it.
|
||||
//
|
||||
die("Please read the first lines of this script for instructions on how to enable it");
|
||||
|
||||
set_time_limit(0);
|
||||
|
||||
define('IN_PHPBB', true);
|
||||
$phpbb_root_path = '../';
|
||||
$phpEx = substr(strrchr(__FILE__, '.'), 1);
|
||||
|
||||
echo "Checking for required files\n";
|
||||
download('http://unicode.org/reports/tr39/data/confusables.txt');
|
||||
echo "\n";
|
||||
|
||||
|
||||
/**
|
||||
* Load the CaseFolding table
|
||||
*/
|
||||
echo "Loading confusables\n";
|
||||
$unidata = file_get_contents('confusables.txt');
|
||||
|
||||
|
||||
function utf8_chr($cp)
|
||||
{
|
||||
if ($cp > 0xFFFF)
|
||||
{
|
||||
return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
|
||||
}
|
||||
else if ($cp > 0x7FF)
|
||||
{
|
||||
return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
|
||||
}
|
||||
else if ($cp > 0x7F)
|
||||
{
|
||||
return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
|
||||
}
|
||||
else
|
||||
{
|
||||
return chr($cp);
|
||||
}
|
||||
}
|
||||
|
||||
preg_match_all('/^([0-9A-F]+) ;\s((?:[0-9A-F]+ )*);/im', $unidata, $array, PREG_SET_ORDER);
|
||||
|
||||
// some that we defined ourselves
|
||||
$uniarray = array(
|
||||
"\xC2\xA1" => "\x69", // EXCLAMATION MARK, INVERTED => LATIN SMALL LETTER I
|
||||
"\xC7\x83" => "\x21", // LATIN LETTER RETROFLEX CLICK => EXCLAMATION MARK
|
||||
"\xCE\xB1" => "\x61", // GREEK SMALL LETTER ALPHA => LATIN SMALL LETTER A
|
||||
"\xE1\x9A\x80" => "\x20", // OGHAM SPACE MARK
|
||||
|
||||
"\xC2\xAD" => '', // HYPHEN, SOFT => empty string
|
||||
"\xDB\x9D" => '', // ARABIC END OF AYAH
|
||||
"\xDC\x8F" => '', // SYRIAC ABBREVIATION MARK
|
||||
"\xE1\xA0\x86" => '', // MONGOLIAN TODO SOFT HYPHEN
|
||||
"\xE1\xA0\x8E" => '', // MONGOLIAN VOWEL SEPARATOR
|
||||
"\xE2\x80\x8B" => '', // ZERO WIDTH SPACE
|
||||
"\xE2\x80\x8C" => '', // ZERO WIDTH NON-JOINER
|
||||
"\xE2\x80\x8D" => '', // ZERO WIDTH JOINER
|
||||
"\xE2\x80\xA8" => '', // LINE SEPARATOR
|
||||
"\xE2\x80\xA9" => '', // PARAGRAPH SEPARATOR
|
||||
"\xE2\x81\xA0" => '', // WORD JOINER
|
||||
"\xE2\x81\xA1" => '', // FUNCTION APPLICATION
|
||||
"\xE2\x81\xA2" => '', // INVISIBLE TIMES
|
||||
"\xE2\x81\xA3" => '', // INVISIBLE SEPARATOR
|
||||
"\xE2\x81\xAA" => '', // [CONTROL CHARACTERS]
|
||||
"\xE2\x81\xAB" => '', // [CONTROL CHARACTERS]
|
||||
"\xE2\x81\xAC" => '', // [CONTROL CHARACTERS]
|
||||
"\xE2\x81\xAD" => '', // [CONTROL CHARACTERS]
|
||||
"\xE2\x81\xAE" => '', // [CONTROL CHARACTERS]
|
||||
"\xE2\x81\xAF" => '', // [CONTROL CHARACTERS]
|
||||
"\xEF\xBB\xBF" => '', // ZERO WIDTH NO-BREAK SPACE
|
||||
"\xEF\xBF\xB9" => '', // [CONTROL CHARACTERS]
|
||||
"\xEF\xBF\xBA" => '', // [CONTROL CHARACTERS]
|
||||
"\xEF\xBF\xBB" => '', // [CONTROL CHARACTERS]
|
||||
"\xEF\xBF\xBC" => '', // [CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB3" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB4" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB5" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB6" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB7" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB8" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xB9" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
"\xF0\x9D\x85\xBA" => '', // [MUSICAL CONTROL CHARACTERS]
|
||||
);
|
||||
|
||||
$copy = $uniarray;
|
||||
|
||||
foreach ($array as $value)
|
||||
{
|
||||
if (isset($copy[utf8_chr(hexdec((string)$value[1]))]))
|
||||
{
|
||||
$num = '';
|
||||
$string = utf8_chr(hexdec((string)$value[1]));
|
||||
for ($i = 0; $i < strlen($string); $i++)
|
||||
{
|
||||
$num .= '\x' . str_pad(base_convert(ord($string[$i]), 10, 16), 2, '0', STR_PAD_LEFT);
|
||||
}
|
||||
echo $num . "\n";
|
||||
if ($uniarray[$string] != implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2]))))))
|
||||
{
|
||||
echo " --> $string\n";
|
||||
echo " --> " . implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2]))))) . "\n";
|
||||
}
|
||||
}
|
||||
$uniarray[utf8_chr(hexdec((string)$value[1]))] = implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2])))));
|
||||
}
|
||||
|
||||
echo "Writing to confusables.$phpEx\n";
|
||||
|
||||
$fp = fopen($phpbb_root_path . 'includes/utf/data/confusables.' . $phpEx, 'wb');
|
||||
fwrite($fp, '<?php return ' . my_var_export($uniarray) . ';');
|
||||
fclose($fp);
|
||||
|
||||
/**
|
||||
* Return a parsable string representation of a variable
|
||||
*
|
||||
* This is function is limited to array/strings/integers
|
||||
*
|
||||
* @param mixed $var Variable
|
||||
* @return string PHP code representing the variable
|
||||
*/
|
||||
function my_var_export($var)
|
||||
{
|
||||
if (is_array($var))
|
||||
{
|
||||
$lines = array();
|
||||
|
||||
foreach ($var as $k => $v)
|
||||
{
|
||||
$lines[] = my_var_export($k) . '=>' . my_var_export($v);
|
||||
}
|
||||
|
||||
return 'array(' . implode(',', $lines) . ')';
|
||||
}
|
||||
else if (is_string($var))
|
||||
{
|
||||
return "'" . str_replace(array('\\', "'"), array('\\\\', "\\'"), $var) . "'";
|
||||
}
|
||||
else
|
||||
{
|
||||
return $var;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file to the develop/ dir
|
||||
*
|
||||
* @param string $url URL of the file to download
|
||||
* @return void
|
||||
*/
|
||||
function download($url)
|
||||
{
|
||||
global $phpbb_root_path;
|
||||
|
||||
if (file_exists($phpbb_root_path . 'develop/' . basename($url)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
echo 'Downloading from ', $url, ' ';
|
||||
|
||||
if (!$fpr = fopen($url, 'rb'))
|
||||
{
|
||||
die("Can't download from $url\nPlease download it yourself and put it in the develop/ dir, kthxbai");
|
||||
}
|
||||
|
||||
if (!$fpw = fopen($phpbb_root_path . 'develop/' . basename($url), 'wb'))
|
||||
{
|
||||
die("Can't open develop/" . basename($url) . " for output... please check your permissions or something");
|
||||
}
|
||||
|
||||
$i = 0;
|
||||
$chunk = 32768;
|
||||
$done = '';
|
||||
|
||||
while (!feof($fpr))
|
||||
{
|
||||
$i += fwrite($fpw, fread($fpr, $chunk));
|
||||
echo str_repeat("\x08", strlen($done));
|
||||
|
||||
$done = ($i >> 10) . ' KiB';
|
||||
echo $done;
|
||||
}
|
||||
fclose($fpr);
|
||||
fclose($fpw);
|
||||
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
?>
|
||||
120
phpBB/develop/unicode_testing.php
Normal file
120
phpBB/develop/unicode_testing.php
Normal file
@@ -0,0 +1,120 @@
|
||||
<?php
|
||||
//
|
||||
// This file provides some useful functions for debugging the unicode/UTF-8 library
|
||||
// It requires utf_tools.php to be loaded
|
||||
//
|
||||
die("Please read the first lines of this script for instructions on how to enable it");
|
||||
|
||||
if (!headers_sent())
|
||||
{
|
||||
header('Content-type: text/html; charset=UTF-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts unicode escape sequences (\u0123) into UTF-8 characters
|
||||
*
|
||||
* @param string A unicode sequence
|
||||
* @return string UTF-8 representation of the given unicode sequence
|
||||
*/
|
||||
function unicode_to_utf8($string)
|
||||
{
|
||||
$utf8 = '';
|
||||
$chars = array();
|
||||
for ($i = 0; $i < strlen($string); $i++)
|
||||
{
|
||||
if (isset($string[$i + 5]) && substr($string, $i, 2) == '\\u' && ctype_xdigit(substr($string, $i + 2, 4)))
|
||||
{
|
||||
$utf8 .= utf8_from_unicode(array(base_convert(substr($string, $i + 2, 4), 16, 10)));
|
||||
$i += 5;
|
||||
}
|
||||
else
|
||||
{
|
||||
$utf8 .= $string[$i];
|
||||
}
|
||||
}
|
||||
return $utf8;
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes an array of ints representing the Unicode characters and returns
|
||||
* a UTF-8 string.
|
||||
*
|
||||
* @param array $array array of unicode code points representing a string
|
||||
* @return string UTF-8 character string
|
||||
*/
|
||||
function utf8_from_unicode($array)
|
||||
{
|
||||
$str = '';
|
||||
foreach ($array as $value)
|
||||
{
|
||||
$str .= utf8_chr($value);
|
||||
}
|
||||
return $str;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a UTF-8 string to unicode code points
|
||||
*
|
||||
* @param string $text UTF-8 string
|
||||
* @return string Unicode code points
|
||||
*/
|
||||
function utf8_to_unicode($text)
|
||||
{
|
||||
return preg_replace_callback(
|
||||
'#[\\xC2-\\xF4][\\x80-\\xBF]?[\\x80-\\xBF]?[\\x80-\\xBF]#',
|
||||
'utf8_to_unicode_callback',
|
||||
preg_replace_callback(
|
||||
'#[\\x00-\\x7f]#',
|
||||
'utf8_to_unicode_callback',
|
||||
$text
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a UTF-8 char and replaces it with its unicode escape sequence. Attention, $m is an array
|
||||
*
|
||||
* @param array $m 0-based numerically indexed array passed by preg_replace_callback()
|
||||
* @return string A unicode escape sequence
|
||||
*/
|
||||
function utf8_to_unicode_callback($m)
|
||||
{
|
||||
return '\u' . str_pad(base_convert(utf8_ord($m[0]), 10, 16), 4, '0', STR_PAD_LEFT) . '';
|
||||
}
|
||||
|
||||
/**
|
||||
* A wrapper function for the normalizer which takes care of including the class if required and modifies the passed strings
|
||||
* to be in NFKC
|
||||
*
|
||||
* @param mixed $strings a string or an array of strings to normalize
|
||||
* @return mixed the normalized content, preserving array keys if array given.
|
||||
*/
|
||||
function utf8_normalize_nfkc($strings)
|
||||
{
|
||||
if (empty($strings))
|
||||
{
|
||||
return $strings;
|
||||
}
|
||||
|
||||
if (!class_exists('utf_normalizer'))
|
||||
{
|
||||
global $phpbb_root_path, $phpEx;
|
||||
include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
|
||||
}
|
||||
|
||||
if (!is_array($strings))
|
||||
{
|
||||
utf_normalizer::nfkc($strings);
|
||||
}
|
||||
else if (is_array($strings))
|
||||
{
|
||||
foreach ($strings as $key => $string)
|
||||
{
|
||||
utf_normalizer::nfkc($strings[$key]);
|
||||
}
|
||||
}
|
||||
|
||||
return $strings;
|
||||
}
|
||||
|
||||
?>
|
||||
Reference in New Issue
Block a user