/src/typo3_src-4.2.0alpha1/t3lib/class.t3lib_cs.php

<?php
/***************************************************************
*  Copyright notice
*
*  (c) 2003-2007 Kasper Skaarhoj (kasperYYYY@typo3.com)
*  All rights reserved
*
*  This script is part of the Typo3 project. The Typo3 project is
*  free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2 of the License, or
*  (at your option) any later version.
*
*  The GNU General Public License can be found at
*  http://www.gnu.org/copyleft/gpl.html.
*
*  This script is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU General Public License for more details.
*
*  This copyright notice MUST APPEAR in all copies of the script!
***************************************************************/
class t3lib_cs {
        var $noCharByteVal=63;          // ASCII Value for chars with no equivalent.

                // This is the array where parsed conversion tables are stored (cached)
        var $parsedCharsets=array();

                // An array where case folding data will be stored (cached)
        var $caseFolding=array();

                // An array where charset-to-ASCII mappings are stored (cached)
        var $toASCII=array();

                // This tells the converter which charsets has two bytes per char:
        var $twoByteSets=array(
                'ucs-2'=>1,     // 2-byte Unicode
        );

                // This tells the converter which charsets has four bytes per char:
        var $fourByteSets=array(
                'ucs-4'=>1,     // 4-byte Unicode
                'utf-32'=>1,    // 4-byte Unicode (limited to the 21-bits of UTF-16)
        );

                // This tells the converter which charsets use a scheme like the Extended Unix Code:
        var $eucBasedSets=array(
                'gb2312'=>1,            // Chinese, simplified.
                'big5'=>1,              // Chinese, traditional.
                'euc-kr'=>1,            // Korean
                'shift_jis'=>1,         // Japanese - WARNING: Shift-JIS includes half-width katakana single-bytes characters above 0x80!
        );

                // see  http://developer.apple.com/documentation/macos8/TextIntlSvcs/TextEncodingConversionManager/TEC1.5/TEC.b0.html
                // http://czyborra.com/charsets/iso8859.html
        var $synonyms=array(
                'us' => 'ascii',
                'us-ascii'=> 'ascii',
                'cp819' => 'iso-8859-1',
                'ibm819' => 'iso-8859-1',
                'iso-ir-100' => 'iso-8859-1',
                'iso-ir-109' => 'iso-8859-2',
                'iso-ir-148' => 'iso-8859-9',
                'iso-ir-199' => 'iso-8859-14',
                'iso-ir-203' => 'iso-8859-15',
                'csisolatin1' => 'iso-8859-1',
                'csisolatin2' => 'iso-8859-2',
                'csisolatin3' => 'iso-8859-3',
                'csisolatin5' => 'iso-8859-9',
                'csisolatin8' => 'iso-8859-14',
                'csisolatin9' => 'iso-8859-15',
                'csisolatingreek' => 'iso-8859-7',
                'iso-celtic' => 'iso-8859-14',
                'latin1' => 'iso-8859-1',
                'latin2' => 'iso-8859-2',
                'latin3' => 'iso-8859-3',
                'latin5' => 'iso-8859-9',
                'latin6' => 'iso-8859-10',
                'latin8' => 'iso-8859-14',
                'latin9' => 'iso-8859-15',
                'l1' => 'iso-8859-1',
                'l2' => 'iso-8859-2',
                'l3' => 'iso-8859-3',
                'l5' => 'iso-8859-9',
                'l6' => 'iso-8859-10',
                'l8' => 'iso-8859-14',
                'l9' => 'iso-8859-15',
                'cyrillic' => 'iso-8859-5',
                'arabic' => 'iso-8859-6',
                'tis-620' => 'iso-8859-11',
                'win874' => 'windows-874',
                'win1250' => 'windows-1250',
                'win1251' => 'windows-1251',
                'win1252' => 'windows-1252',
                'win1253' => 'windows-1253',
                'win1254' => 'windows-1254',
                'win1255' => 'windows-1255',
                'win1256' => 'windows-1256',
                'win1257' => 'windows-1257',
                'win1258' => 'windows-1258',
                'cp1250' => 'windows-1250',
                'cp1251' => 'windows-1251',
                'cp1252' => 'windows-1252',
                'ms-ee' => 'windows-1250',
                'ms-ansi' => 'windows-1252',
                'ms-greek' => 'windows-1253',
                'ms-turk' => 'windows-1254',
                'winbaltrim' => 'windows-1257',
                'koi-8ru' => 'koi-8r',
                'koi8r' => 'koi-8r',
                'cp878' => 'koi-8r',
                'mac' => 'macroman',
                'macintosh' => 'macroman',
                'euc-cn' => 'gb2312',
                'x-euc-cn' => 'gb2312',
                'euccn' => 'gb2312',
                'cp936' => 'gb2312',
                'big-5' => 'big5',
                'cp950' => 'big5',
                'eucjp' => 'euc-jp',
                'sjis' => 'shift_jis',
                'shift-jis' => 'shift_jis',
                'cp932' => 'shift_jis',
                'cp949' => 'euc-kr',
                'utf7' => 'utf-7',
                'utf8' => 'utf-8',
                'utf16' => 'utf-16',
                'utf32' => 'utf-32',
                'utf8' => 'utf-8',
                'ucs2' => 'ucs-2',
                'ucs4' => 'ucs-4',
        );

                // mapping of iso-639:2 language codes to script names
        var $lang_to_script=array(
                        // iso-639:2 language codes, see:
                        //  http://www.w3.org/WAI/ER/IG/ert/iso639.htm
                        //  http://www.loc.gov/standards/iso639-2/langcodes.html
                        //  http://www.unicode.org/onlinedat/languages.html
                'ar' => 'arabic',
                'bg' => 'cyrillic',             // Bulgarian
                'bs' => 'east_european',        // Bosnian
                'cs' => 'east_european',        // Czech
                'da' => 'west_european',        // Danish
                'de' => 'west_european',        // German
                'es' => 'west_european',        // Spanish
                'et' => 'estonian',
                'eo' => 'unicode',              // Esperanto
                'eu' => 'west_european',        // Basque
                'fa' => 'arabic',       // Persian
                'fi' => 'west_european',        // Finish
                'fo' => 'west_european',        // Faroese
                'fr' => 'west_european',        // French
                'gr' => 'greek',
                'ge' => 'unicode',              // Georgian
                'he' => 'hebrew',               // Hebrew (since 1998)
                'hi' => 'unicode',              // Hindi
                'hr' => 'east_european',        // Croatian
                'hu' => 'east_european',        // Hungarian
                'iw' => 'hebrew',               // Hebrew (til 1998)
                'is' => 'west_european',        // Icelandic
                'it' => 'west_european',        // Italian
                'ja' => 'japanese',
                'kl' => 'west_european',        // Greenlandic
                'ko' => 'korean',
                'lt' => 'lithuanian',
                'lv' => 'west_european',        // Latvian/Lettish
                'nl' => 'west_european',        // Dutch
                'no' => 'west_european',        // Norwegian
                'pl' => 'east_european',        // Polish
                'pt' => 'west_european',        // Portuguese
                'ro' => 'east_european',        // Romanian
                'ru' => 'cyrillic',             // Russian
                'sk' => 'east_european',        // Slovak
                'sl' => 'east_european',        // Slovenian
                'sr' => 'cyrillic',             // Serbian
                'sv' => 'west_european',        // Swedish
                'sq' => 'albanian',             // Albanian
                'th' => 'thai',
                'uk' => 'cyrillic',             // Ukranian
                'vi' => 'vietnamese',
                'zh' => 'chinese',
                        // MS language codes, see http://msdn.microsoft.com/library/default.asp?url=/library/en-us/vclib/html/_crt_language_strings.asp
                        // http://msdn.microsoft.com/library/default.asp?url=/library/en-us/wceinternational5/html/wce50conLanguageIdentifiersandLocales.asp
                'ara' => 'arabic',
                'bgr' => 'cyrillic',            // Bulgarian
                'cat' => 'west_european',       // Catalan
                'chs' => 'simpl_chinese',
                'cht' => 'trad_chinese',
                'csy' => 'east_european',       // Czech
                'dan' => 'west_european',       // Danisch
                'deu' => 'west_european',       // German
                'dea' => 'west_european',       // German (Austrian)
                'des' => 'west_european',       // German (Swiss)
                'ena' => 'west_european',       // English (Australian)
                'enc' => 'west_european',       // English (Canadian)
                'eng' => 'west_european',       // English
                'enz' => 'west_european',       // English (New Zealand)
                'enu' => 'west_european',       // English (United States)
                'euq' => 'west_european',       // Basque
                'fos' => 'west_european',       // Faroese
                'far' => 'arabic',      // Persian
                'fin' => 'west_european',       // Finish
                'fra' => 'west_european',       // French
                'frb' => 'west_european',       // French (Belgian)
                'frc' => 'west_european',       // French (Canadian)
                'frs' => 'west_european',       // French (Swiss)
                'geo' => 'unicode',                     // Georgian
                'ell' => 'greek',
                'heb' => 'hebrew',
                'hin' => 'unicode',     // Hindi
                'hun' => 'east_european',       // Hungarian
                'isl' => 'west_euorpean',       // Icelandic
                'ita' => 'west_european',       // Italian
                'its' => 'west_european',       // Italian (Swiss)
                'jpn' => 'japanese',
                'kor' => 'korean',
                'lth' => 'lithuanian',
                'lvi' => 'west_european',       // Latvian/Lettish
                'msl' => 'west_european',       // Malay
                'nlb' => 'west_european',       // Dutch (Belgian)
                'nld' => 'west_european',       // Dutch
                'nor' => 'west_european',       // Norwegian (bokmal)
                'non' => 'west_european',       // Norwegian (nynorsk)
                'plk' => 'east_european',       // Polish
                'ptg' => 'west_european',       // Portuguese
                'ptb' => 'west_european',       // Portuguese (Brazil)
                'rom' => 'east_european',       // Romanian
                'rus' => 'cyrillic',            // Russian
                'slv' => 'east_european',       // Slovenian
                'sky' => 'east_european',       // Slovak
                'srl' => 'east_european',       // Serbian (Latin)
                'srb' => 'cyrillic',            // Serbian (Cyrillic)
                'esp' => 'west_european',       // Spanish (trad. sort)
                'esm' => 'west_european',       // Spanish (Mexican)
                'esn' => 'west_european',       // Spanish (internat. sort)
                'sve' => 'west_european',       // Swedish
                'sqi' => 'albanian',            // Albanian
                'tha' => 'thai',
                'trk' => 'turkish',
                'ukr' => 'cyrillic',    // Ukrainian
                        // English language names
                'albanian' => 'albanian',
                'arabic' => 'arabic',
                'basque' => 'west_european',
                'bosnian' => 'east_european',
                'bulgarian' => 'east_european',
                'catalan' => 'west_european',
                'croatian' => 'east_european',
                'czech' => 'east_european',
                'danish' => 'west_european',
                'dutch' => 'west_european',
                'english' => 'west_european',
                'esperanto' => 'unicode',
                'estonian' => 'estonian',
                'faroese' => 'west_european',
                'farsi' => 'arabic',
                'finnish' => 'west_european',
                'french' => 'west_european',
                'galician' => 'west_european',
                'georgian' => 'unicode',
                'german' => 'west_european',
                'greek' => 'greek',
                'greenlandic' => 'west_european',
                'hebrew' => 'hebrew',
                'hindi' => 'unicode',
                'hungarian' => 'east_european',
                'icelandic' => 'west_european',
                'italian' => 'west_european',
                'latvian' => 'west_european',
                'lettish' => 'west_european',
                'lithuanian' => 'lithuanian',
                'malay' => 'west_european',
                'norwegian' => 'west_european',
                'persian' => 'arabic',
                'polish' => 'east_european',
                'portuguese' => 'west_european',
                'russian' => 'cyrillic',
                'romanian' => 'east_european',
                'serbian' => 'cyrillic',
                'slovak' => 'east_european',
                'slovenian' => 'east_european',
                'spanish' => 'west_european',
                'svedish' => 'west_european',
                'that' => 'thai',
                'turkish' => 'turkish',
                'ukrainian' => 'cyrillic',
        );

                // mapping of language (family) names to charsets on Unix
        var $script_to_charset_unix=array(
                'west_european' => 'iso-8859-1',
                'estonian' => 'iso-8859-1',
                'east_european' => 'iso-8859-2',
                'baltic' => 'iso-8859-4',
                'cyrillic' => 'iso-8859-5',
                'arabic' => 'iso-8859-6',
                'greek' => 'iso-8859-7',
                'hebrew' => 'iso-8859-8',
                'turkish' => 'iso-8859-9',
                'thai' => 'iso-8859-11', // = TIS-620
                'lithuanian' => 'iso-8859-13',
                'chinese' => 'gb2312', // = euc-cn
                'japanese' => 'euc-jp',
                'korean' => 'euc-kr',
                'simpl_chinese' => 'gb2312',
                'trad_chinese' => 'big5',
                'vietnamese' => '',
                'unicode' => 'utf-8',
                'albanian' => 'utf-8'
        );

                // mapping of language (family) names to charsets on Windows
        var $script_to_charset_windows=array(
                'east_european' => 'windows-1250',
                'cyrillic' => 'windows-1251',
                'west_european' => 'windows-1252',
                'greek' => 'windows-1253',
                'turkish' => 'windows-1254',
                'hebrew' => 'windows-1255',
                'arabic' => 'windows-1256',
                'baltic' => 'windows-1257',
                'estonian' => 'windows-1257',
                'lithuanian' => 'windows-1257',
                'vietnamese' => 'windows-1258',
                'thai' => 'cp874',
                'korean' => 'cp949',
                'chinese' => 'gb2312',
                'japanese' => 'shift_jis',
                'simpl_chinese' => 'gb2312',
                'trad_chinese' => 'big5',
                'albanian' => 'windows-1250',
                'unicode' => 'utf-8'
        );

                // mapping of locale names to charsets
        var $locale_to_charset=array(
                'japanese.euc' => 'euc-jp',
                'ja_jp.ujis' => 'euc-jp',
                'korean.euc' => 'euc-kr',
                'sr@Latn' => 'iso-8859-2',
                'zh_cn' => 'gb2312',
                'zh_hk' => 'big5',
                'zh_tw' => 'big5',
        );

                // TYPO3 specific: Array with the system charsets used for each system language in TYPO3:
                // Empty values means "iso-8859-1"
        var $charSetArray = array(
                'dk' => '',
                'de' => '',
                'no' => '',
                'it' => '',
                'fr' => '',
                'es' => '',
                'nl' => '',
                'cz' => 'windows-1250',
                'pl' => 'iso-8859-2',
                'si' => 'windows-1250',
                'fi' => '',
                'tr' => 'iso-8859-9',
                'se' => '',
                'pt' => '',
                'ru' => 'windows-1251',
                'ro' => 'iso-8859-2',
                'ch' => 'gb2312',
                'sk' => 'windows-1250',
                'lt' => 'windows-1257',
                'is' => 'utf-8',
                'hr' => 'windows-1250',
                'hu' => 'iso-8859-2',
                'gl' => '',
                'th' => 'iso-8859-11',
                'gr' => 'iso-8859-7',
                'hk' => 'big5',
                'eu' => '',
                'bg' => 'windows-1251',
                'br' => '',
                'et' => 'iso-8859-4',
                'ar' => 'iso-8859-6',
                'he' => 'utf-8',
                'ua' => 'windows-1251',
                'jp' => 'shift_jis',
                'lv' => 'utf-8',
                'vn' => 'utf-8',
                'ca' => 'iso-8859-15',
                'ba' => 'iso-8859-2',
                'kr' => 'euc-kr',
                'eo' => 'utf-8',
                'my' => '',
                'hi' => 'utf-8',
                'fo' => 'utf-8',
                'fa' => 'utf-8',
                'sr' => 'utf-8',
                'sq' => 'utf-8',
                'ge' => 'utf-8'
        );

                // TYPO3 specific: Array with the iso names used for each system language in TYPO3:
                // Missing keys means: same as Typo3
        var $isoArray = array(
                'ba' => 'bs',
                'br' => 'pt_BR',
                'ch' => 'zh_CN',
                'cz' => 'cs',
                'dk' => 'da',
                'si' => 'sl',
                'se' => 'sv',
                'gl' => 'kl',
                'gr' => 'el',
                'hk' => 'zh_HK',
                'kr' => 'ko',
                'ua' => 'uk',
                'jp' => 'ja',
                'vn' => 'vi',
        );

        function parse_charset($charset)        {
                $charset = trim(strtolower($charset));
                if (isset($this->synonyms[$charset]))   $charset = $this->synonyms[$charset];

                return $charset;
        }

        function get_locale_charset($locale)    {
                $locale = strtolower($locale);

                        // exact locale specific charset?
                if (isset($this->locale_to_charset[$locale]))   return $this->locale_to_charset[$locale];

                        // get modifier
                list($locale,$modifier) = explode('@',$locale);

                        // locale contains charset: use it
                list($locale,$charset) = explode('.',$locale);
                if ($charset)   return $this->parse_charset($charset);

                        // modifier is 'euro' (after charset check, because of xx.utf-8@euro)
                if ($modifier == 'euro')        return 'iso-8859-15';

                        // get language
                list($language,$country) = explode('_',$locale);
                if (isset($this->lang_to_script[$language]))    $script = $this->lang_to_script[$language];

                if (TYPO3_OS == 'WIN')  {
                        $cs = $this->script_to_charset_windows[$script] ? $this->script_to_charset_windows[$script] : 'window-1252';
                } else {
                        $cs = $this->script_to_charset_unix[$script] ? $this->script_to_charset_unix[$script] : 'iso-8859-1';
                }

                return $cs;
        }









        /********************************************
         *
         * Charset Conversion functions
         *
         ********************************************/

        function conv($str,$fromCS,$toCS,$useEntityForNoChar=0) {
                if ($fromCS==$toCS)     return $str;

                        // PHP-libs don't support fallback to SGML entities, but UTF-8 handles everything
                if ($toCS=='utf-8' || !$useEntityForNoChar)     {
                        switch($GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'])       {
                        case 'mbstring':
                                $conv_str = mb_convert_encoding($str,$toCS,$fromCS);
                                if (false !== $conv_str)        return $conv_str; // returns false for unsupported charsets
                                break;

                        case 'iconv':
                                $conv_str = iconv($fromCS,$toCS.'//TRANSLIT',$str);
                                if (false !== $conv_str)        return $conv_str;
                                break;

                        case 'recode':
                                $conv_str = recode_string($fromCS.'..'.$toCS,$str);
                                if (false !== $conv_str)        return $conv_str;
                                break;
                        }
                        // fallback to TYPO3 conversion
                }

                if ($fromCS!='utf-8')   $str=$this->utf8_encode($str,$fromCS);
                if ($toCS!='utf-8')     $str=$this->utf8_decode($str,$toCS,$useEntityForNoChar);
                return $str;
        }

        function convArray(&$array,$fromCS,$toCS,$useEntityForNoChar=0) {
                foreach($array as $key => $value)       {
                        if (is_array($array[$key]))     {
                                $this->convArray($array[$key],$fromCS,$toCS,$useEntityForNoChar);
                        } else {
                                $array[$key] = $this->conv($array[$key],$fromCS,$toCS,$useEntityForNoChar);
                        }
                }
        }

        function utf8_encode($str,$charset)     {

                if ($charset === 'utf-8')       return $str;

                        // Charset is case-insensitive.
                if ($this->initCharset($charset))       {       // Parse conv. table if not already...
                        $strLen = strlen($str);
                        $outStr='';

                        for ($a=0;$a<$strLen;$a++)      {       // Traverse each char in string.
                                $chr=substr($str,$a,1);
                                $ord=ord($chr);
                                if (isset($this->twoByteSets[$charset]))        {       // If the charset has two bytes per char
                                        $ord2 = ord($str{$a+1});
                                        $ord = $ord<<8 | $ord2; // assume big endian

                                        if (isset($this->parsedCharsets[$charset]['local'][$ord]))      {       // If the local char-number was found in parsed conv. table then we use that, otherwise 127 (no char?)
                                                $outStr.=$this->parsedCharsets[$charset]['local'][$ord];
                                        } else $outStr.=chr($this->noCharByteVal);      // No char exists
                                        $a++;
                                } elseif ($ord>127)     {       // If char has value over 127 it's a multibyte char in UTF-8
                                        if (isset($this->eucBasedSets[$charset]))       {       // EUC uses two-bytes above 127; we get both and advance pointer and make $ord a 16bit int.
                                                if ($charset != 'shift_jis' || ($ord < 0xA0 || $ord > 0xDF))    {       // Shift-JIS: chars between 160 and 223 are single byte
                                                        $a++;
                                                        $ord2=ord(substr($str,$a,1));
                                                        $ord = $ord*256+$ord2;
                                                }
                                        }

                                        if (isset($this->parsedCharsets[$charset]['local'][$ord]))      {       // If the local char-number was found in parsed conv. table then we use that, otherwise 127 (no char?)
                                                $outStr.= $this->parsedCharsets[$charset]['local'][$ord];
                                        } else $outStr.= chr($this->noCharByteVal);     // No char exists
                                } else $outStr.= $chr;  // ... otherwise it's just ASCII 0-127 and one byte. Transparent
                        }
                        return $outStr;
                }
        }

        function utf8_decode($str,$charset,$useEntityForNoChar=0)       {

                        // Charset is case-insensitive.
                if ($this->initCharset($charset))       {       // Parse conv. table if not already...
                        $strLen = strlen($str);
                        $outStr='';
                        $buf='';
                        for ($a=0,$i=0;$a<$strLen;$a++,$i++)    {       // Traverse each char in UTF-8 string.
                                $chr=substr($str,$a,1);
                                $ord=ord($chr);
                                if ($ord>127)   {       // This means multibyte! (first byte!)
                                        if ($ord & 64)  {       // Since the first byte must have the 7th bit set we check that. Otherwise we might be in the middle of a byte sequence.

                                                $buf=$chr;      // Add first byte
                                                for ($b=0;$b<8;$b++)    {       // for each byte in multibyte string...
                                                        $ord = $ord << 1;       // Shift it left and ...
                                                        if ($ord & 128) {       // ... and with 8th bit - if that is set, then there are still bytes in sequence.
                                                                $a++;   // Increase pointer...
                                                                $buf.=substr($str,$a,1);        // ... and add the next char.
                                                        } else break;
                                                }

                                                if (isset($this->parsedCharsets[$charset]['utf8'][$buf]))       {       // If the UTF-8 char-sequence is found then...
                                                        $mByte = $this->parsedCharsets[$charset]['utf8'][$buf]; // The local number
                                                        if ($mByte>255) {       // If the local number is greater than 255 we will need to split the byte (16bit word assumed) in two chars.
                                                                $outStr.= chr(($mByte >> 8) & 255).chr($mByte & 255);
                                                        } else $outStr.= chr($mByte);
                                                } elseif ($useEntityForNoChar) {        // Create num entity:
                                                        $outStr.='&#'.$this->utf8CharToUnumber($buf,1).';';
                                                } else $outStr.=chr($this->noCharByteVal);      // No char exists
                                        } else $outStr.=chr($this->noCharByteVal);      // No char exists (MIDDLE of MB sequence!)
                                } else $outStr.=$chr;   // ... otherwise it's just ASCII 0-127 and one byte. Transparent
                        }
                        return $outStr;
                }
        }

        function utf8_to_entities($str) {
                $strLen = strlen($str);
                $outStr='';
                $buf='';
                for ($a=0;$a<$strLen;$a++)      {       // Traverse each char in UTF-8 string.
                        $chr=substr($str,$a,1);
                        $ord=ord($chr);
                        if ($ord>127)   {       // This means multibyte! (first byte!)
                                if ($ord & 64)  {       // Since the first byte must have the 7th bit set we check that. Otherwise we might be in the middle of a byte sequence.
                                        $buf=$chr;      // Add first byte
                                        for ($b=0;$b<8;$b++)    {       // for each byte in multibyte string...
                                                $ord = $ord << 1;       // Shift it left and ...
                                                if ($ord & 128) {       // ... and with 8th bit - if that is set, then there are still bytes in sequence.
                                                        $a++;   // Increase pointer...
                                                        $buf.=substr($str,$a,1);        // ... and add the next char.
                                                } else break;
                                        }

                                        $outStr.='&#'.$this->utf8CharToUnumber($buf,1).';';
                                } else $outStr.=chr($this->noCharByteVal);      // No char exists (MIDDLE of MB sequence!)
                        } else $outStr.=$chr;   // ... otherwise it's just ASCII 0-127 and one byte. Transparent
                }

                return $outStr;
        }

        function entities_to_utf8($str,$alsoStdHtmlEnt=0)       {
                if ($alsoStdHtmlEnt)    {
                        $trans_tbl = array_flip(get_html_translation_table(HTML_ENTITIES));             // Getting them in iso-8859-1 - but thats ok since this is observed below.
                }

                $token = md5(microtime());
                $parts = explode($token,ereg_replace('(&([#[:alnum:]]*);)',$token.'\2'.$token,$str));
                foreach($parts as $k => $v)     {
                        if ($k%2)       {
                                if (substr($v,0,1)=='#')        {       // Dec or hex entities:
                                        if (substr($v,1,1)=='x')        {
                                                $parts[$k] = $this->UnumberToChar(hexdec(substr($v,2)));
                                        } else {
                                                $parts[$k] = $this->UnumberToChar(substr($v,1));
                                        }
                                } elseif ($alsoStdHtmlEnt && $trans_tbl['&'.$v.';']) {  // Other entities:
                                        $parts[$k] = $this->utf8_encode($trans_tbl['&'.$v.';'],'iso-8859-1');
                                } else {        // No conversion:
                                        $parts[$k] ='&'.$v.';';
                                }
                        }
                }

                return implode('',$parts);
        }

        function utf8_to_numberarray($str,$convEntities=0,$retChar=0)   {
                        // If entities must be registered as well...:
                if ($convEntities)      {
                        $str = $this->entities_to_utf8($str,1);
                }
                        // Do conversion:
                $strLen = strlen($str);
                $outArr=array();
                $buf='';
                for ($a=0;$a<$strLen;$a++)      {       // Traverse each char in UTF-8 string.
                        $chr=substr($str,$a,1);
                        $ord=ord($chr);
                        if ($ord>127)   {       // This means multibyte! (first byte!)
                                if ($ord & 64)  {       // Since the first byte must have the 7th bit set we check that. Otherwise we might be in the middle of a byte sequence.
                                        $buf=$chr;      // Add first byte
                                        for ($b=0;$b<8;$b++)    {       // for each byte in multibyte string...
                                                $ord = $ord << 1;       // Shift it left and ...
                                                if ($ord & 128) {       // ... and with 8th bit - if that is set, then there are still bytes in sequence.
                                                        $a++;   // Increase pointer...
                                                        $buf.=substr($str,$a,1);        // ... and add the next char.
                                                } else break;
                                        }

                                        $outArr[]=$retChar?$buf:$this->utf8CharToUnumber($buf);
                                } else $outArr[]=$retChar?chr($this->noCharByteVal):$this->noCharByteVal;       // No char exists (MIDDLE of MB sequence!)
                        } else $outArr[]=$retChar?chr($ord):$ord;       // ... otherwise it's just ASCII 0-127 and one byte. Transparent
                }

                return $outArr;
        }

        function UnumberToChar($cbyte)  {
                $str='';

                if ($cbyte < 0x80) {
                        $str.=chr($cbyte);
                } else if ($cbyte < 0x800) {
                        $str.=chr(0xC0 | ($cbyte >> 6));
                        $str.=chr(0x80 | ($cbyte & 0x3F));
                } else if ($cbyte < 0x10000) {
                        $str.=chr(0xE0 | ($cbyte >> 12));
                        $str.=chr(0x80 | (($cbyte >> 6) & 0x3F));
                        $str.=chr(0x80 | ($cbyte & 0x3F));
                } else if ($cbyte < 0x200000) {
                        $str.=chr(0xF0 | ($cbyte >> 18));
                        $str.=chr(0x80 | (($cbyte >> 12) & 0x3F));
                        $str.=chr(0x80 | (($cbyte >> 6) & 0x3F));
                        $str.=chr(0x80 | ($cbyte & 0x3F));
                } else if ($cbyte < 0x4000000) {
                        $str.=chr(0xF8 | ($cbyte >> 24));
                        $str.=chr(0x80 | (($cbyte >> 18) & 0x3F));
                        $str.=chr(0x80 | (($cbyte >> 12) & 0x3F));
                        $str.=chr(0x80 | (($cbyte >> 6) & 0x3F));
                        $str.=chr(0x80 | ($cbyte & 0x3F));
                } else if ($cbyte < 0x80000000) {
                        $str.=chr(0xFC | ($cbyte >> 30));
                        $str.=chr(0x80 | (($cbyte >> 24) & 0x3F));
                        $str.=chr(0x80 | (($cbyte >> 18) & 0x3F));
                        $str.=chr(0x80 | (($cbyte >> 12) & 0x3F));
                        $str.=chr(0x80 | (($cbyte >> 6) & 0x3F));
                        $str.=chr(0x80 | ($cbyte & 0x3F));
                } else { // Cannot express a 32-bit character in UTF-8
                        $str .= chr($this->noCharByteVal);
                }
                return $str;
        }

        function utf8CharToUnumber($str,$hex=0) {
                $ord=ord(substr($str,0,1));     // First char

                if (($ord & 192) == 192)        {       // This verifyes that it IS a multi byte string
                        $binBuf='';
                        for ($b=0;$b<8;$b++)    {       // for each byte in multibyte string...
                                $ord = $ord << 1;       // Shift it left and ...
                                if ($ord & 128) {       // ... and with 8th bit - if that is set, then there are still bytes in sequence.
                                        $binBuf.=substr('00000000'.decbin(ord(substr($str,$b+1,1))),-6);
                                } else break;
                        }
                        $binBuf=substr('00000000'.decbin(ord(substr($str,0,1))),-(6-$b)).$binBuf;

                        $int = bindec($binBuf);
                } else $int = $ord;

                return $hex ? 'x'.dechex($int) : $int;
        }









        /********************************************
         *
         * Init functions
         *
         ********************************************/

        function initCharset($charset)  {
                        // Only process if the charset is not yet loaded:
                if (!is_array($this->parsedCharsets[$charset])) {

                                // Conversion table filename:
                        $charsetConvTableFile = PATH_t3lib.'csconvtbl/'.$charset.'.tbl';

                                // If the conversion table is found:
                        if ($charset && t3lib_div::validPathStr($charsetConvTableFile) && @is_file($charsetConvTableFile))      {
                                        // Cache file for charsets:
                                        // Caching brought parsing time for gb2312 down from 2400 ms to 150 ms. For other charsets we are talking 11 ms down to zero.
                                $cacheFile = t3lib_div::getFileAbsFileName('typo3temp/cs/charset_'.$charset.'.tbl');
                                if ($cacheFile && @is_file($cacheFile)) {
                                        $this->parsedCharsets[$charset]=unserialize(t3lib_div::getUrl($cacheFile));
                                } else {
                                                // Parse conversion table into lines:
                                        $lines=t3lib_div::trimExplode(chr(10),t3lib_div::getUrl($charsetConvTableFile),1);
                                                // Initialize the internal variable holding the conv. table:
                                        $this->parsedCharsets[$charset]=array('local'=>array(),'utf8'=>array());
                                                // traverse the lines:
                                        $detectedType='';
                                        foreach($lines as $value)       {
                                                if (trim($value) && substr($value,0,1)!='#')    {       // Comment line or blanks are ignored.

                                                                // Detect type if not done yet: (Done on first real line)
                                                                // The "whitespaced" type is on the syntax      "0x0A   0x000A  #LINE FEED"     while   "ms-token" is like              "B9 = U+00B9 : SUPERSCRIPT ONE"
                                                        if (!$detectedType)             $detectedType = ereg('[[:space:]]*0x([[:alnum:]]*)[[:space:]]+0x([[:alnum:]]*)[[:space:]]+',$value) ? 'whitespaced' : 'ms-token';

                                                        if ($detectedType=='ms-token')  {
                                                                list($hexbyte,$utf8) = split('=|:',$value,3);
                                                        } elseif ($detectedType=='whitespaced') {
                                                                $regA=array();
                                                                ereg('[[:space:]]*0x([[:alnum:]]*)[[:space:]]+0x([[:alnum:]]*)[[:space:]]+',$value,$regA);
                                                                $hexbyte = $regA[1];
                                                                $utf8 = 'U+'.$regA[2];
                                                        }
                                                        $decval = hexdec(trim($hexbyte));
                                                        if ($decval>127)        {
                                                                $utf8decval = hexdec(substr(trim($utf8),2));
                                                                $this->parsedCharsets[$charset]['local'][$decval]=$this->UnumberToChar($utf8decval);
                                                                $this->parsedCharsets[$charset]['utf8'][$this->parsedCharsets[$charset]['local'][$decval]]=$decval;
                                                        }
                                                }
                                        }
                                        if ($cacheFile) {
                                                t3lib_div::writeFileToTypo3tempDir($cacheFile,serialize($this->parsedCharsets[$charset]));
                                        }
                                }
                                return 2;
                        } else return false;
                } else return 1;
        }

        function initUnicodeData($mode=null)    {
                        // cache files
                $cacheFileCase = t3lib_div::getFileAbsFileName('typo3temp/cs/cscase_utf-8.tbl');
                $cacheFileASCII = t3lib_div::getFileAbsFileName('typo3temp/cs/csascii_utf-8.tbl');

                        // Only process if the tables are not yet loaded
                switch($mode)   {
                        case 'case':
                                if (is_array($this->caseFolding['utf-8']))      return 1;

                                        // Use cached version if possible
                                if ($cacheFileCase && @is_file($cacheFileCase)) {
                                        $this->caseFolding['utf-8'] = unserialize(t3lib_div::getUrl($cacheFileCase));
                                        return 2;
                                }
                                break;

                        case 'ascii':
                                if (is_array($this->toASCII['utf-8']))  return 1;

                                        // Use cached version if possible
                                if ($cacheFileASCII && @is_file($cacheFileASCII))       {
                                        $this->toASCII['utf-8'] = unserialize(t3lib_div::getUrl($cacheFileASCII));
                                        return 2;
                                }
                                break;
                }

                        // process main Unicode data file
                $unicodeDataFile = PATH_t3lib.'unidata/UnicodeData.txt';
                if (!(t3lib_div::validPathStr($unicodeDataFile) && @is_file($unicodeDataFile))) return false;

                $fh = fopen($unicodeDataFile,'rb');
                if (!$fh)       return false;

                        // key = utf8 char (single codepoint), value = utf8 string (codepoint sequence)
                        // note: we use the UTF-8 characters here and not the Unicode numbers to avoid conversion roundtrip in utf8_strtolower/-upper)
                $this->caseFolding['utf-8'] = array();
                $utf8CaseFolding =& $this->caseFolding['utf-8']; // a shorthand
                $utf8CaseFolding['toUpper'] = array();
                $utf8CaseFolding['toLower'] = array();
                $utf8CaseFolding['toTitle'] = array();

                $decomposition = array();       // array of temp. decompositions
                $mark = array();                // array of chars that are marks (eg. composing accents)
                $number = array();              // array of chars that are numbers (eg. digits)
                $omit = array();                // array of chars to be omitted (eg. Russian hard sign)

                while (!feof($fh))      {
                        $line = fgets($fh,4096);
                                // has a lot of info
                        list($char,$name,$cat,,,$decomp,,,$num,,,,$upper,$lower,$title,) = split(';', rtrim($line));

                        $ord = hexdec($char);
                        if ($ord > 0xFFFF)      break;  // only process the BMP

                        $utf8_char = $this->UnumberToChar($ord);

                        if ($upper)     $utf8CaseFolding['toUpper'][$utf8_char] = $this->UnumberToChar(hexdec($upper));
                        if ($lower)     $utf8CaseFolding['toLower'][$utf8_char] = $this->UnumberToChar(hexdec($lower));
                                // store "title" only when different from "upper" (only a few)
                        if ($title && $title != $upper) $utf8CaseFolding['toTitle'][$utf8_char] = $this->UnumberToChar(hexdec($title));

                        switch ($cat{0})        {
                                case 'M':       // mark (accent, umlaut, ...)
                                        $mark["U+$char"] = 1;
                                        break;

                                case 'N':       // numeric value
                                        if ($ord > 0x80 && $num != '')  $number["U+$char"] = $num;
                        }

                                // accented Latin letters without "official" decomposition
                        $match = array();
                        if (ereg('^LATIN (SMALL|CAPITAL) LETTER ([A-Z]) WITH',$name,$match) && !$decomp)        {
                                $c = ord($match[2]);
                                if ($match[1] == 'SMALL')       $c += 32;

                                $decomposition["U+$char"] = array(dechex($c));
                                continue;
                        }

                        $match = array();
                        if (ereg('(<.*>)? *(.+)',$decomp,$match))       {
                                switch($match[1])       {
                                        case '<circle>':        // add parenthesis as circle replacement, eg (1)
                                                $match[2] = '0028 '.$match[2].' 0029';
                                                break;

                                        case '<square>':        // add square brackets as square replacement, eg [1]
                                                $match[2] = '005B '.$match[2].' 005D';
                                                break;

                                        case '<compat>':        // ignore multi char decompositions that start with a space
                                                if (ereg('^0020 ',$match[2]))   continue 2;
                                                break;

                                                // ignore Arabic and vertical layout presentation decomposition
                                        case '<initial>':
                                        case '<medial>':
                                        case '<final>':
                                        case '<isolated>':
                                        case '<vertical>':
                                                continue 2;
                                }
                                $decomposition["U+$char"] = split(' ',$match[2]);
                        }
                }
                fclose($fh);

                        // process additional Unicode data for casing (allow folded characters to expand into a sequence)
                $specialCasingFile = PATH_t3lib.'unidata/SpecialCasing.txt';
                if (t3lib_div::validPathStr($specialCasingFile) && @is_file($specialCasingFile))        {
                        $fh = fopen($specialCasingFile,'rb');
                        if ($fh)        {
                                while (!feof($fh))      {
                                        $line = fgets($fh,4096);
                                        if ($line{0} != '#' && trim($line) != '')       {

                                                list($char,$lower,$title,$upper,$cond) = t3lib_div::trimExplode(';', $line);
                                                if ($cond == '' || $cond{0} == '#')     {
                                                        $utf8_char = $this->UnumberToChar(hexdec($char));
                                                        if ($char != $lower)    {
                                                                $arr = split(' ',$lower);
                                                                for ($i=0; isset($arr[$i]); $i++)       $arr[$i] = $this->UnumberToChar(hexdec($arr[$i]));
                                                                $utf8CaseFolding['toLower'][$utf8_char] = implode('',$arr);
                                                        }
                                                        if ($char != $title && $title != $upper)        {
                                                                $arr = split(' ',$title);
                                                                for ($i=0; isset($arr[$i]); $i++)       $arr[$i] = $this->UnumberToChar(hexdec($arr[$i]));
                                                                $utf8CaseFolding['toTitle'][$utf8_char] = implode('',$arr);
                                                        }
                                                        if ($char != $upper)    {
                                                                        $arr = split(' ',$upper);
                                                                for ($i=0; isset($arr[$i]); $i++)       $arr[$i] = $this->UnumberToChar(hexdec($arr[$i]));
                                                                $utf8CaseFolding['toUpper'][$utf8_char] = implode('',$arr);
                                                        }
                                                }
                                        }
                                }
                                fclose($fh);
                        }
                }

                        // process custom decompositions
                $customTranslitFile = PATH_t3lib.'unidata/Translit.txt';
                if (t3lib_div::validPathStr($customTranslitFile) && @is_file($customTranslitFile))      {
                        $fh = fopen($customTranslitFile,'rb');
                        if ($fh)        {
                                while (!feof($fh))      {
                                        $line = fgets($fh,4096);
                                        if ($line{0} != '#' && trim($line) != '')       {
                                                list($char,$translit) = t3lib_div::trimExplode(';', $line);
                                                if (!$translit) $omit["U+$char"] = 1;
                                                $decomposition["U+$char"] = split(' ', $translit);

                                        }
                                }
                                fclose($fh);
                        }
                }

                        // decompose and remove marks; inspired by unac (Loic Dachary <loic@senga.org>)
                foreach($decomposition as $from => $to) {
                        $code_decomp = array();

                        while ($code_value = array_shift($to))  {
                                if (isset($decomposition["U+$code_value"]))     {       // do recursive decomposition
                                        foreach(array_reverse($decomposition["U+$code_value"]) as $cv)  {
                                                array_unshift($to, $cv);
                                        }
                                } elseif (!isset($mark["U+$code_value"])) {     // remove mark
                                        array_push($code_decomp, $code_value);
                                }
                        }
                        if (count($code_decomp) || isset($omit[$from])) {
                                $decomposition[$from] = $code_decomp;
                        } else {
                                unset($decomposition[$from]);
                        }
                }

                        // create ascii only mapping
                $this->toASCII['utf-8'] = array();
                $ascii =& $this->toASCII['utf-8'];

                foreach($decomposition as $from => $to) {
                        $code_decomp = array();
                        while ($code_value = array_shift($to))  {
                                $ord = hexdec($code_value);
                                if ($ord > 127)
                                        continue 2;     // skip decompositions containing non-ASCII chars
                                else
                                        array_push($code_decomp,chr($ord));
                        }
                        $ascii[$this->UnumberToChar(hexdec($from))] = join('',$code_decomp);
                }

                        // add numeric decompositions
                foreach($number as $from => $to)        {
                        $utf8_char = $this->UnumberToChar(hexdec($from));
                        if (!isset($ascii[$utf8_char])) {
                                $ascii[$utf8_char] = $to;
                        }
                }

                if ($cacheFileCase)     {
                                t3lib_div::writeFileToTypo3tempDir($cacheFileCase,serialize($utf8CaseFolding));
                }

                if ($cacheFileASCII)    {
                                t3lib_div::writeFileToTypo3tempDir($cacheFileASCII,serialize($ascii));
                }

                return 3;
        }

        function initCaseFolding($charset)      {
                        // Only process if the case table is not yet loaded:
                if (is_array($this->caseFolding[$charset]))     return 1;

                        // Use cached version if possible
                $cacheFile = t3lib_div::getFileAbsFileName('typo3temp/cs/cscase_'.$charset.'.tbl');
                if ($cacheFile && @is_file($cacheFile)) {
                        $this->caseFolding[$charset] = unserialize(t3lib_div::getUrl($cacheFile));
                        return 2;
                }

                        // init UTF-8 conversion for this charset
                if (!$this->initCharset($charset))      {
                        return false;
                }

                        // UTF-8 case folding is used as the base conversion table
                if (!$this->initUnicodeData('case'))    {
                        return false;
                }

                $nochar = chr($this->noCharByteVal);
                foreach ($this->parsedCharsets[$charset]['local'] as $ci => $utf8)      {
                                // reconvert to charset (don't use chr() of numeric value, might be muli-byte)
                        $c = $this->utf8_decode($utf8, $charset);

                                // $cc = $this->conv($this->caseFolding['utf-8']['toUpper'][$utf8], 'utf-8', $charset);
                        $cc = $this->utf8_decode($this->caseFolding['utf-8']['toUpper'][$utf8], $charset);
                        if ($cc != '' && $cc != $nochar)        $this->caseFolding[$charset]['toUpper'][$c] = $cc;

                                // $cc = $this->conv($this->caseFolding['utf-8']['toLower'][$utf8], 'utf-8', $charset);
                        $cc = $this->utf8_decode($this->caseFolding['utf-8']['toLower'][$utf8], $charset);
                        if ($cc != '' && $cc != $nochar)        $this->caseFolding[$charset]['toLower'][$c] = $cc;

                                // $cc = $this->conv($this->caseFolding['utf-8']['toTitle'][$utf8], 'utf-8', $charset);
                        $cc = $this->utf8_decode($this->caseFolding['utf-8']['toTitle'][$utf8], $charset);
                        if ($cc != '' && $cc != $nochar)        $this->caseFolding[$charset]['toTitle'][$c] = $cc;
                }

                        // add the ASCII case table
                for ($i=ord('a'); $i<=ord('z'); $i++)   {
                        $this->caseFolding[$charset]['toUpper'][chr($i)] = chr($i-32);
                }
                for ($i=ord('A'); $i<=ord('Z'); $i++)   {
                        $this->caseFolding[$charset]['toLower'][chr($i)] = chr($i+32);
                }

                if ($cacheFile) {
                                t3lib_div::writeFileToTypo3tempDir($cacheFile,serialize($this->caseFolding[$charset]));
                }

                return 3;
        }

        function initToASCII($charset)  {
                        // Only process if the case table is not yet loaded:
                if (is_array($this->toASCII[$charset])) return 1;

                        // Use cached version if possible
                $cacheFile = t3lib_div::getFileAbsFileName('typo3temp/cs/csascii_'.$charset.'.tbl');
                if ($cacheFile && @is_file($cacheFile)) {
                        $this->toASCII[$charset] = unserialize(t3lib_div::getUrl($cacheFile));
                        return 2;
                }

                        // init UTF-8 conversion for this charset
                if (!$this->initCharset($charset))      {
                        return false;
                }

                        // UTF-8/ASCII transliteration is used as the base conversion table
                if (!$this->initUnicodeData('ascii'))   {
                        return false;
                }

                $nochar = chr($this->noCharByteVal);
                foreach ($this->parsedCharsets[$charset]['local'] as $ci => $utf8)      {
                                // reconvert to charset (don't use chr() of numeric value, might be muli-byte)
                        $c = $this->utf8_decode($utf8, $charset);

                        if (isset($this->toASCII['utf-8'][$utf8]))      {
                                $this->toASCII[$charset][$c] = $this->toASCII['utf-8'][$utf8];
                        }
                }

                if ($cacheFile) {
                                t3lib_div::writeFileToTypo3tempDir($cacheFile,serialize($this->toASCII[$charset]));
                }

                return 3;
        }
















        /********************************************
         *
         * String operation functions
         *
         ********************************************/

        function substr($charset,$string,$start,$len=null)      {
                if ($len===0)   return '';

                if ($GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] == 'mbstring') {
                                // cannot omit $len, when specifying charset
                        if ($len==null) {
                                $enc = mb_internal_encoding();  // save internal encoding
                                mb_internal_encoding($charset);
                                $str = mb_substr($string,$start);
                                mb_internal_encoding($enc);     // restore internal encoding

                                return $str;
                        }
                        else {
                                return mb_substr($string,$start,$len,$charset);
                        }
                } elseif ($GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] == 'iconv')      {
                                // cannot omit $len, when specifying charset
                        if ($len==null) {
                                $enc = iconv_get_encoding('internal_encoding'); // save internal encoding
                                iconv_set_encoding('internal_encoding',$charset);
                                $str = iconv_substr($string,$start);
                                iconv_set_encoding('internal_encoding',$enc);   // restore internal encoding

                                return $str;
                        }
                        else {
                                return iconv_substr($string,$start,$len,$charset);
                        }
                } elseif ($charset == 'utf-8')  {
                        return $this->utf8_substr($string,$start,$len);
                } elseif ($this->eucBasedSets[$charset])        {
                        return $this->euc_substr($string,$start,$charset,$len);
                } elseif ($this->twoByteSets[$charset]) {
                        return substr($string,$start*2,$len*2);
                } elseif ($this->fourByteSets[$charset])        {
                        return substr($string,$start*4,$len*4);
                }

                // treat everything else as single-byte encoding
                return $len === NULL ? substr($string,$start) : substr($string,$start,$len);
        }

        function strlen($charset,$string)       {
                if ($GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] == 'mbstring') {
                        return mb_strlen($string,$charset);
                } elseif ($GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] == 'iconv')      {
                        return iconv_strlen($string,$charset);
                } elseif ($charset == 'utf-8')  {
                        return $this->utf8_strlen($string);
                } elseif ($this->eucBasedSets[$charset])        {
                        return $this->euc_strlen($string,$charset);
                } elseif ($this->twoByteSets[$charset]) {
                        return strlen($string)/2;
                } elseif ($this->fourByteSets[$charset])        {
                        return strlen($string)/4;
                }
                // treat everything else as single-byte encoding
                return strlen($string);
        }

        function crop($charset,$string,$len,$crop='')   {
                if (intval($len) == 0)  return $string;

                if ($charset == 'utf-8')        {
                        $i = $this->utf8_char2byte_pos($string,$len);
                } elseif ($this->eucBasedSets[$charset])        {
                        $i = $this->euc_char2byte_pos($string,$len,$charset);
                } else {
                        if ($len > 0)   {
                                $i = $len;
                        } else {
                                $i = strlen($string)+$len;
                                if ($i<=0)      $i = false;
                        }
                }

                if ($i === false)       {       // $len outside actual string length
                        return $string;
                } else  {