1
0
mirror of https://github.com/phpbb/phpbb.git synced 2025-07-31 14:00:31 +02:00

[feature/patchwork-utf8] Normalize with intl, use patchwork/utf8 as fallback

This commit is contained in:
Nicolas Grekas
2013-11-20 13:47:31 +01:00
parent 1601b61ef9
commit 5a7caf6508
20 changed files with 110 additions and 2883 deletions

View File

@@ -32,262 +32,11 @@ $phpbb_root_path = '../';
$phpEx = substr(strrchr(__FILE__, '.'), 1);
echo "Checking for required files\n";
download('http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt');
download('http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt');
download('http://www.unicode.org/Public/UNIDATA/UnicodeData.txt');
echo "\n";
require_once($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
$file_contents = array();
/**
* Generate some Hangul/Jamo stuff
*/
echo "\nGenerating Hangul and Jamo tables\n";
for ($i = 0; $i < UNICODE_HANGUL_LCOUNT; ++$i)
{
$utf_char = cp_to_utf(UNICODE_HANGUL_LBASE + $i);
$file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_VCOUNT * UNICODE_HANGUL_TCOUNT + UNICODE_HANGUL_SBASE;
$file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_L;
}
for ($i = 0; $i < UNICODE_HANGUL_VCOUNT; ++$i)
{
$utf_char = cp_to_utf(UNICODE_HANGUL_VBASE + $i);
$file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_TCOUNT;
$file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_V;
}
for ($i = 0; $i < UNICODE_HANGUL_TCOUNT; ++$i)
{
$utf_char = cp_to_utf(UNICODE_HANGUL_TBASE + $i);
$file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i;
$file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_T;
}
/**
* Load the CompositionExclusions table
*/
echo "Loading CompositionExclusion\n";
$fp = fopen('CompositionExclusions.txt', 'rt');
$exclude = array();
while (!feof($fp))
{
$line = fgets($fp, 1024);
if (!strpos(' 0123456789ABCDEFabcdef', $line[0]))
{
continue;
}
$cp = strtok($line, ' ');
if ($pos = strpos($cp, '..'))
{
$start = hexdec(substr($cp, 0, $pos));
$end = hexdec(substr($cp, $pos + 2));
for ($i = $start; $i < $end; ++$i)
{
$exclude[$i] = 1;
}
}
else
{
$exclude[hexdec($cp)] = 1;
}
}
fclose($fp);
/**
* Load QuickCheck tables
*/
echo "Generating QuickCheck tables\n";
$fp = fopen('DerivedNormalizationProps.txt', 'rt');
while (!feof($fp))
{
$line = fgets($fp, 1024);
if (!strpos(' 0123456789ABCDEFabcdef', $line[0]))
{
continue;
}
$p = array_map('trim', explode(';', strtok($line, '#')));
/**
* Capture only NFC_QC, NFKC_QC
*/
if (!preg_match('#^NFK?C_QC$#', $p[1]))
{
continue;
}
if ($pos = strpos($p[0], '..'))
{
$start = hexdec(substr($p[0], 0, $pos));
$end = hexdec(substr($p[0], $pos + 2));
}
else
{
$start = $end = hexdec($p[0]);
}
if ($start >= UTF8_HANGUL_FIRST && $end <= UTF8_HANGUL_LAST)
{
/**
* We do not store Hangul syllables in the array
*/
continue;
}
if ($p[2] == 'M')
{
$val = UNICODE_QC_MAYBE;
}
else
{
$val = UNICODE_QC_NO;
}
if ($p[1] == 'NFKC_QC')
{
$file = 'utf_nfkc_qc';
}
else
{
$file = 'utf_nfc_qc';
}
for ($i = $start; $i <= $end; ++$i)
{
/**
* The vars have the same name as the file: $utf_nfc_qc is in utf_nfc_qc.php
*/
$file_contents[$file][$file][cp_to_utf($i)] = $val;
}
}
fclose($fp);
/**
* Do mappings
*/
echo "Loading Unicode decomposition mappings\n";
$fp = fopen($phpbb_root_path . 'develop/UnicodeData.txt', 'rt');
$map = array();
while (!feof($fp))
{
$p = explode(';', fgets($fp, 1024));
$cp = hexdec($p[0]);
if (!empty($p[3]))
{
/**
* Store combining class > 0
*/
$file_contents['utf_normalizer_common']['utf_combining_class'][cp_to_utf($cp)] = (int) $p[3];
}
if (!isset($p[5]) || !preg_match_all('#[0-9A-F]+#', strip_tags($p[5]), $m))
{
continue;
}
if (strpos($p[5], '>'))
{
$map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0]));
}
else
{
$map['NFD'][$cp] = $map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0]));
}
}
fclose($fp);
/**
* Build the canonical composition table
*/
echo "Generating the Canonical Composition table\n";
foreach ($map['NFD'] as $cp => $decomp_seq)
{
if (!strpos($decomp_seq, ' ') || isset($exclude[$cp]))
{
/**
* Singletons are excluded from canonical composition
*/
continue;
}
$utf_seq = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq)));
if (!isset($file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq]))
{
$file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq] = cp_to_utf($cp);
}
}
/**
* Decompose the NF[K]D mappings recursively and prepare the file contents
*/
echo "Generating the Canonical and Compatibility Decomposition tables\n\n";
foreach ($map as $type => $decomp_map)
{
foreach ($decomp_map as $cp => $decomp_seq)
{
$decomp_map[$cp] = decompose($decomp_map, $decomp_seq);
}
unset($decomp_seq);
if ($type == 'NFKD')
{
$file = 'utf_compatibility_decomp';
$var = 'utf_compatibility_decomp';
}
else
{
$file = 'utf_canonical_decomp';
$var = 'utf_canonical_decomp';
}
/**
* Generate the corresponding file
*/
foreach ($decomp_map as $cp => $decomp_seq)
{
$file_contents[$file][$var][cp_to_utf($cp)] = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq)));
}
}
/**
* Generate and/or alter the files
*/
foreach ($file_contents as $file => $contents)
{
/**
* Generate a new file
*/
echo "Writing to $file.$phpEx\n";
if (!$fp = fopen($phpbb_root_path . 'includes/utf/data/' . $file . '.' . $phpEx, 'wb'))
{
trigger_error('Cannot open ' . $file . ' for write');
}
fwrite($fp, '<?php');
foreach ($contents as $var => $val)
{
fwrite($fp, "\n\$GLOBALS[" . my_var_export($var) . ']=' . my_var_export($val) . ";");
}
fclose($fp);
}
echo "\n*** UTF-8 normalization tables done\n\n";
/**
* Now we'll generate the files needed by the search indexer
* Generate the files needed by the search indexer
*/
echo "Generating search indexer tables\n";
@@ -424,32 +173,6 @@ die("\nAll done!\n");
// Internal functions //
////////////////////////////////////////////////////////////////////////////////
/**
* Decompose a sequence recusively
*
* @param array $decomp_map Decomposition mapping, passed by reference
* @param string $decomp_seq Decomposition sequence as decimal codepoints separated with a space
* @return string Decomposition sequence, fully decomposed
*/
function decompose(&$decomp_map, $decomp_seq)
{
$ret = array();
foreach (explode(' ', $decomp_seq) as $cp)
{
if (isset($decomp_map[$cp]))
{
$ret[] = decompose($decomp_map, $decomp_map[$cp]);
}
else
{
$ret[] = $cp;
}
}
return implode(' ', $ret);
}
/**
* Return a parsable string representation of a variable
*
@@ -537,17 +260,6 @@ function hex_to_utf($hex)
return cp_to_utf(hexdec($hex));
}
/**
* Return a UTF string formed from a sequence of codepoints in hexadecimal
*
* @param string $seq Sequence of codepoints, separated with a space
* @return string UTF-8 string
*/
function hexseq_to_utf($seq)
{
return implode('', array_map('hex_to_utf', explode(' ', $seq)));
}
/**
* Convert a codepoint to a UTF-8 char
*