mirror of
https://github.com/phpbb/phpbb.git
synced 2025-06-04 21:44:57 +02:00
Added: that script will run the UTF normalizer through a serie of conformance tests designed by the Unicode consortium for that purpose. Only the first part of the tests has been implemented yet
TODO: test for invariants git-svn-id: file:///svn/phpbb/trunk@6158 89ea8834-ac86-4346-8a33-228a782c2dd0
This commit is contained in:
Ludovic Arnaud
parent
17cad8f11f
commit
9ab98d0d79
305
phpBB/develop/utf_normalizer_test.php
Normal file
305
phpBB/develop/utf_normalizer_test.php
Normal file
@ -0,0 +1,305 @@
|
||||
<?php
|
||||
/**
|
||||
*
|
||||
* @package phpBB3
|
||||
* @version $Id$
|
||||
* @copyright (c) 2005 phpBB Group
|
||||
* @license http://opensource.org/licenses/gpl-license.php GNU Public License
|
||||
*
|
||||
*/
|
||||
|
||||
if (php_sapi_name() != 'cli')
|
||||
{
|
||||
die("This program must be run from the command line.\n");
|
||||
}
|
||||
|
||||
set_time_limit(0);
|
||||
error_reporting(E_ALL);
|
||||
|
||||
define('IN_PHPBB', true);
|
||||
$phpbb_root_path = '../';
|
||||
$phpEx = substr(strrchr(__FILE__, '.'), 1);
|
||||
|
||||
|
||||
/**
|
||||
* Let's download some files we need
|
||||
*/
|
||||
download('http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt');
|
||||
download('http://www.unicode.org/Public/UNIDATA/UnicodeData.txt');
|
||||
|
||||
/**
|
||||
* Those are the tests we run
|
||||
*/
|
||||
$test_suite = array(
|
||||
/**
|
||||
* NFC
|
||||
* c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
* c4 == NFC(c4) == NFC(c5)
|
||||
*/
|
||||
'NFC' => array(
|
||||
'c2' => array('c1', 'c2', 'c3'),
|
||||
'c4' => array('c4', 'c5')
|
||||
),
|
||||
|
||||
/**
|
||||
* NFD
|
||||
* c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
* c5 == NFD(c4) == NFD(c5)
|
||||
*/
|
||||
'NFD' => array(
|
||||
'c3' => array('c1', 'c2', 'c3'),
|
||||
'c5' => array('c4', 'c5')
|
||||
),
|
||||
|
||||
/**
|
||||
* NFKC
|
||||
* c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
*/
|
||||
'NFKC' => array(
|
||||
'c4' => array('c1', 'c2', 'c3', 'c4', 'c5')
|
||||
),
|
||||
|
||||
/**
|
||||
* NFKD
|
||||
* c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
*/
|
||||
'NFKD' => array(
|
||||
'c5' => array('c1', 'c2', 'c3', 'c4', 'c5')
|
||||
)
|
||||
);
|
||||
|
||||
require_once($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
|
||||
|
||||
$i = $n = 0;
|
||||
$failed = FALSE;
|
||||
|
||||
$fp = fopen($phpbb_root_path . 'develop/NormalizationTest.txt', 'rb');
|
||||
while (!feof($fp))
|
||||
{
|
||||
$line = fgets($fp);
|
||||
++$n;
|
||||
|
||||
if ($line[0] == '@')
|
||||
{
|
||||
if ($i)
|
||||
{
|
||||
echo "done\n";
|
||||
}
|
||||
|
||||
$i = 0;
|
||||
echo "\n", substr($line, 1), "\n\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!strpos(' 0123456789ABCDEF', $line[0]))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (++$i % 100 == 0)
|
||||
{
|
||||
echo $i, ' ';
|
||||
}
|
||||
|
||||
list($c1, $c2, $c3, $c4, $c5) = explode(';', $line);
|
||||
|
||||
foreach ($test_suite as $form => $serie)
|
||||
{
|
||||
foreach ($serie as $expected => $tests)
|
||||
{
|
||||
$hex_expected = ${$expected};
|
||||
$utf_expected = hexseq_to_utf($hex_expected);
|
||||
|
||||
foreach ($tests as $test)
|
||||
{
|
||||
$utf_result = call_user_func(array('utf_normalizer', $form), $utf_expected);
|
||||
|
||||
if (strcmp($utf_expected, $utf_result))
|
||||
{
|
||||
$failed = TRUE;
|
||||
$hex_result = utf_to_hexseq($utf_result);
|
||||
|
||||
echo "FAILED $expected == $form($test) ($hex_expected != $hex_result)\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($failed)
|
||||
{
|
||||
die("\nFailed at line $n\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
die("\n\nALL TESTS PASSED SUCCESSFULLY\n");
|
||||
|
||||
/**
|
||||
* Download a file to the develop/ dir
|
||||
*
|
||||
* @param string $url URL of the file to download
|
||||
* @return void
|
||||
*/
|
||||
function download($url)
|
||||
{
|
||||
global $phpbb_root_path;
|
||||
|
||||
if (file_exists($phpbb_root_path . 'develop/' . basename($url)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
echo 'Downloading from ', $url, ' ';
|
||||
|
||||
if (!$fpr = fopen($url, 'rb'))
|
||||
{
|
||||
die("Can't download from $url\nPlease download it yourself and put it in the develop/ dir, kthxbai");
|
||||
}
|
||||
|
||||
if (!$fpw = fopen($phpbb_root_path . 'develop/' . basename($url), 'wb'))
|
||||
{
|
||||
die("Can't open develop/" . basename($url) . " for output... please check your permissions or something");
|
||||
}
|
||||
|
||||
$i = 0;
|
||||
$chunk = 32768;
|
||||
$done = '';
|
||||
|
||||
while (!feof($fpr))
|
||||
{
|
||||
$i += fwrite($fpw, fread($fpr, $chunk));
|
||||
echo str_repeat("\x08", strlen($done));
|
||||
|
||||
$done = ($i >> 10) . ' KiB';
|
||||
echo $done;
|
||||
}
|
||||
fclose($fpr);
|
||||
fclose($fpw);
|
||||
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a UTF string to a sequence of codepoints in hexadecimal
|
||||
*
|
||||
* @param string $utf UTF string
|
||||
* @return integer Unicode codepoints in hex
|
||||
*/
|
||||
function utf_to_hexseq($str)
|
||||
{
|
||||
$pos = 0;
|
||||
$len = strlen($str);
|
||||
$ret = array();
|
||||
|
||||
while ($pos < $len)
|
||||
{
|
||||
$c = $str[$pos];
|
||||
switch ($c & "\xF0")
|
||||
{
|
||||
case "\xC0":
|
||||
case "\xD0":
|
||||
$utf_char = substr($str, $pos, 2);
|
||||
$pos += 2;
|
||||
break;
|
||||
|
||||
case "\xE0":
|
||||
$utf_char = substr($str, $pos, 3);
|
||||
$pos += 3;
|
||||
break;
|
||||
|
||||
case "\xF0":
|
||||
$utf_char = substr($str, $pos, 4);
|
||||
$pos += 4;
|
||||
break;
|
||||
|
||||
default:
|
||||
$utf_char = $c;
|
||||
++$pos;
|
||||
}
|
||||
|
||||
$hex = dechex(utf_to_cp($utf_char));
|
||||
|
||||
if (!isset($hex[3]))
|
||||
{
|
||||
$hex = substr('000' . $hex, -4);
|
||||
}
|
||||
|
||||
$ret[] = $hex;
|
||||
}
|
||||
|
||||
return strtr(implode(' ', $ret), 'abcdef', 'ABCDEF');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a UTF-8 char to its codepoint
|
||||
*
|
||||
* @param string $utf_char UTF-8 char
|
||||
* @return integer Unicode codepoint
|
||||
*/
|
||||
function utf_to_cp($utf_char)
|
||||
{
|
||||
switch (strlen($utf_char))
|
||||
{
|
||||
case 1:
|
||||
return ord($utf_char);
|
||||
|
||||
case 2:
|
||||
return ((ord($utf_char[0]) & 0x1F) << 6) | (ord($utf_char[1]) & 0x3F);
|
||||
|
||||
case 3:
|
||||
return ((ord($utf_char[0]) & 0x0F) << 12) | ((ord($utf_char[1]) & 0x3F) << 6) | (ord($utf_char[2]) & 0x3F);
|
||||
|
||||
case 4:
|
||||
return ((ord($utf_char[0]) & 0x07) << 18) | ((ord($utf_char[1]) & 0x3F) << 12) | ((ord($utf_char[2]) & 0x3F) << 6) | (ord($utf_char[3]) & 0x3F);
|
||||
|
||||
default:
|
||||
die('UTF-8 chars can only be 1-4 bytes long');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a UTF string formed from a sequence of codepoints in hexadecimal
|
||||
*
|
||||
* @param string $seq Sequence of codepoints, separated with a space
|
||||
* @return string UTF-8 string
|
||||
*/
|
||||
function hexseq_to_utf($seq)
|
||||
{
|
||||
return implode('', array_map('hex_to_utf', explode(' ', $seq)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a codepoint in hexadecimal to a UTF-8 char
|
||||
*
|
||||
* @param string $hex Codepoint, in hexadecimal
|
||||
* @return string UTF-8 char
|
||||
*/
|
||||
function hex_to_utf($hex)
|
||||
{
|
||||
return cp_to_utf(hexdec($hex));
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a codepoint to a UTF-8 char
|
||||
*
|
||||
* @param integer $cp Unicode codepoint
|
||||
* @return string UTF-8 string
|
||||
*/
|
||||
function cp_to_utf($cp)
|
||||
{
|
||||
if ($cp > 0xFFFF)
|
||||
{
|
||||
return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
|
||||
}
|
||||
elseif ($cp > 0x7FF)
|
||||
{
|
||||
return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
|
||||
}
|
||||
elseif ($cp > 0x7F)
|
||||
{
|
||||
return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
|
||||
}
|
||||
else
|
||||
{
|
||||
return chr($cp);
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user