DedeBIZ
/
DedeV6
Watch 2
Star 0
Fork 0
Code Pull Requests 0 Releases 25 Activity
国内流行的内容管理系统（CMS）多端全媒体解决方案 https://www.dedebiz.com
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
939 Commits
3 Branches
1.8GB
Tree: 2f7009c758
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2f7009c758'
${ noResults }
DedeV6/src/system/libraries/splitword.class.php

1048 lines
37KB
Raw Blame History 

  1. <?php

  2. if (!defined('DEDEINC')) exit('dedebiz');

  3. /**

  4.  * Unicode编码词典的php分词器

  5.  *

  6.  *  1、只适用于php5，必要函数 iconv

  7.  *  2、本程序是使用RMM逆向匹配算法进行分词的，词库需要特别编译，本类里提供了 MakeDict() 方法

  8.  *  3、简单操作流程：SetSource -> StartAnalysis -> Get***Result

  9.  *  4、对主词典使用特殊格式进行编码, 不需要载入词典到内存操作

  10.  *

  11.  * @version        $Id: splitword.class.php 2 11:45 2011-2-14 itplato $

  12.  * @package        DedeBIZ.Libraries

  13.  * @copyright      Copyright (c) 2022, DedeBIZ.COM

  14.  * @license        https://www.dedebiz.com/license

  15.  * @link           https://www.dedebiz.com

  16.  */

  17. //常量定义

  18. define('_SP_', chr(0xFF).chr(0xFE)); 

  19. define('UCS2', 'ucs-2be');

  20. class SplitWord

  21. {

  22.     //hash算法选项

  23.     var $mask_value = 0xFFFF;

  24.     //输入和输出的字符编码（只允许 utf-8、gbk/gb2312/gb18030、big5 三种类型）  

  25.     var $sourceCharSet = 'utf-8';

  26.     var $targetCharSet = 'utf-8';

  27.     //生成的分词结果数据类型 1 为全部， 2为 词典词汇及单个中日韩简繁字符及英文， 3 为词典词汇及英文

  28.     var $resultType = 1;

  29.     //句子长度小于这个数值时不拆分，notSplitLen = n(个汉字) * 2 + 1

  30.     var $notSplitLen = 5;

  31.     //把英文单词全部转小写

  32.     var $toLower = FALSE;

  33.     //使用最大切分模式对二元词进行消岐

  34.     var $differMax = FALSE;

  35.     //尝试合并单字

  36.     var $unitWord = TRUE;

  37.     //初始化类时直接加载词典

  38.     var $loadInit = TRUE;

  39.     //使用热门词优先模式进行消岐

  40.     var $differFreq = FALSE;

  41.     //被转换为unicode的源字符串

  42.     var $sourceString = '';

  43.     //附加词典

  44.     var $addonDic = array();

  45.     var $addonDicFile = 'data/words_addons.dic';

  46.     //主词典 

  47.     var $dicStr = '';

  48.     var $mainDic = array();

  49.     var $mainDicHand = FALSE;

  50.     var $mainDicInfos = array();

  51.     var $mainDicFile = 'data/base_dic_full.dic';

  52.     //是否直接载入词典（选是载入速度较慢，但解析较快；选否载入较快，但解析较慢，需要时才会载入特定的词条）

  53.     var $mainDicFileZip = 'data/base_dic_full.zip';

  54.     var $isLoadAll = FALSE;

  55.     var $isUnpacked = FALSE;

  56.     //主词典词语最大长度 x / 2

  57.     var $dicWordMax = 14;

  58.     //粗分后的数组（通常是截取句子等用途）

  59.     var $simpleResult = array();

  60.     //最终结果(用空格分开的词汇列表)

  61.     var $finallyResult = '';

  62.     //是否已经载入词典

  63.     var $isLoadDic = FALSE;

  64.     //系统识别或合并的新词

  65.     var $newWords = array();

  66.     var $foundWordStr = '';

  67.     //词库载入时间

  68.     var $loadTime = 0;

  69.     /**

  70.      * 构造函数

  71.      * @param $source_charset

  72.      * @param $target_charset

  73.      * @param $load_alldic 

  74.      * @param $source

  75.      *

  76.      * @return void

  77.      */

  78.     function __construct($source_charset='utf-8', $target_charset='utf-8', $load_all=TRUE, $source='')

  79.     {

  80.         $this->SetSource( $source, $source_charset, $target_charset );

  81.         $this->isLoadAll = $load_all;

  82.         if(file_exists(DEDEINC.'/'.$this->mainDicFile)) $this->isUnpacked = TRUE;

  83.         if($this->loadInit) $this->LoadDict();

  84.     }

  85.     function SplitWord($source_charset='utf-8', $target_charset='utf-8', $load_all=TRUE, $source='')

  86.     {

  87.         $this->__construct($source_charset, $target_charset, $load_all, $source);

  88.     }

  89.     /**

  90.     * 析构函数

  91.     */

  92.     function __destruct()

  93.     {

  94.         if( $this->mainDicHand !== FALSE )

  95.         {

  96.             @fclose( $this->mainDicHand );

  97.         }

  98.     }

  99.     /**

  100.      * 根据字符串计算key索引

  101.      * @param $key

  102.      * @return short int

  103.      */

  104.     function _get_index( $key )

  105.     {

  106.         $l = strlen($key);

  107.         $h = 0x238f13af;

  108.         while ($l--)

  109.         {

  110.             $h += ($h << 5);

  111.             $h ^= ord($key[$l]);

  112.             $h &= 0x7fffffff;

  113.         }

  114.         return ($h % $this->mask_value);

  115.     }

  116.     /**

  117.      * 从文件获得词

  118.      * @param $key

  119.      * @param $type (类型 word 或 key_groups)

  120.      * @return short int

  121.      */

  122.     function GetWordInfos( $key, $type='word' )

  123.     {

  124.         if( !$this->mainDicHand )

  125.         {

  126.             $this->mainDicHand = fopen($this->mainDicFile, 'r');

  127.         }

  128.         $p = 0;

  129.         $keynum = $this->_get_index( $key );

  130.         if( isset($this->mainDicInfos[ $keynum ]) )

  131.         {

  132.             $data = $this->mainDicInfos[ $keynum ];

  133.         } else {

  134.             //rewind( $this->mainDicHand );

  135.             $move_pos = $keynum * 8;

  136.             fseek($this->mainDicHand, $move_pos, SEEK_SET);

  137.             $dat = fread($this->mainDicHand, 8);

  138.             $arr = unpack('I1s/n1l/n1c', $dat);

  139.             if( $arr['l'] == 0 )

  140.             {

  141.                 return FALSE;

  142.             }

  143.             fseek($this->mainDicHand, $arr['s'], SEEK_SET);

  144.             $data = @unserialize(fread($this->mainDicHand, $arr['l']));

  145.             $this->mainDicInfos[ $keynum ] = $data;

  146.        }

  147.        if( !is_array($data) || !isset($data[$key]) ) 

  148.        {

  149.            return FALSE;

  150.        }

  151.        return ($type=='word' ? $data[$key] : $data);

  152.     }

  153.     /**

  154.      * 设置源字符串

  155.      * @param $source

  156.      * @param $source_charset

  157.      * @param $target_charset

  158.      *

  159.      * @return bool

  160.      */

  161.     function SetSource( $source, $source_charset='utf-8', $target_charset='utf-8' )

  162.     {

  163.         $this->sourceCharSet = strtolower($source_charset);

  164.         $this->targetCharSet = strtolower($target_charset);

  165.         $this->simpleResult = array();

  166.         $this->finallyResult = array();

  167.         $this->finallyIndex = array();

  168.         if( $source != '' )

  169.         {

  170.             $rs = TRUE;

  171.             if( preg_match("/^utf/", $source_charset) ) {

  172.                 $this->sourceString = @iconv('utf-8', UCS2, $source);

  173.             }

  174.             else if( preg_match("/^gb/", $source_charset) ) {

  175.                 $this->sourceString = @iconv('utf-8', UCS2, iconv('gb18030', 'utf-8', $source));

  176.             }

  177.             else if( preg_match("/^big/", $source_charset) ) {

  178.                 $this->sourceString = @iconv('utf-8', UCS2, iconv('big5', 'utf-8', $source));

  179.             } else {

  180.                 $rs = FALSE;

  181.             }

  182.         } else {

  183.            $rs = FALSE;

  184.         }

  185.         return $rs;

  186.     }

  187.     /**

  188.      * 设置结果类型(只在获取finallyResult才有效)

  189.      * @param $rstype 1 为全部， 2去除特殊符号

  190.      *

  191.      * @return void

  192.      */

  193.     function SetResultType( $rstype )

  194.     {

  195.         $this->resultType = $rstype;

  196.     }

  197.     /**

  198.      * 载入词典

  199.      *

  200.      * @return void

  201.      */

  202.     function LoadDict( $maindic='' )

  203.     {

  204. 		$this->addonDicFile = DEDEINC.'/'.$this->addonDicFile;

  205. 		$this->mainDicFile = DEDEINC.'/'.$this->mainDicFile;

  206. 		$this->mainDicFileZip = DEDEINC.'/'.$this->mainDicFileZip;

  207.         $startt = microtime(TRUE);

  208.         //正常读取文件

  209.         $dicAddon = $this->addonDicFile;

  210.         if($maindic=='' || !file_exists($maindic) )

  211.         {

  212.             $dicWords = $this->mainDicFile ;

  213.         } else {

  214.             $dicWords = $maindic;

  215.             $this->mainDicFile = $maindic;

  216.         }

  217.         //加载主词典（只打开）

  218.         if($this->isUnpacked){

  219.         	$this->mainDicHand = fopen($dicWords, 'r');

  220.         } else {

  221.         	$this->InportDict($this->mainDicFileZip);

  222.         }

  223.         //载入副词典

  224.         $hw = '';

  225.         $ds = file($dicAddon);

  226.         foreach($ds as $d)

  227.         {

  228.             $d = trim($d);

  229.             if($d=='') continue;

  230.             $estr = substr($d, 1, 1);

  231.             if( $estr==':' ) {

  232.                 $hw = substr($d, 0, 1);

  233.             } else {

  234.                 $spstr = _SP_;

  235.                 $spstr = iconv(UCS2, 'utf-8', $spstr);

  236.                 $ws = explode(',', $d);

  237.                 $wall = iconv('utf-8', UCS2, join($spstr, $ws));

  238.                 $ws = explode(_SP_, $wall);

  239.                 foreach($ws as $estr)

  240.                 {

  241.                     $this->addonDic[$hw][$estr] = strlen($estr);

  242.                 }

  243.             }

  244.         }

  245.         $this->loadTime = microtime(TRUE) - $startt;

  246.         $this->isLoadDic = TRUE;

  247.     }

  248.     /**

  249.     * 检测某个词是否存在

  250.     */

  251.     function IsWord( $word )

  252.     {

  253.          $winfos = $this->GetWordInfos( $word );

  254.          return ($winfos !== FALSE);

  255.     }

  256.     /**

  257.      * 获得某个词的词性及词频信息

  258.      * @parem $word unicode编码的词

  259.      * @return void

  260.      */

  261.     function GetWordProperty($word)

  262.     {

  263.         if( strlen($word)<4 )

  264.         {

  265.             return '/s';

  266.         }

  267.         $infos = $this->GetWordInfos($word);

  268.         return isset($infos[1]) ? "/{$infos[1]}{$infos[0]}" : "/s";

  269.     }

  270.     /**

  271.      * 指定某词的词性信息（通常是新词）

  272.      * @parem $word unicode编码的词

  273.      * @parem $infos array('c' => 词频, 'm' => 词性);

  274.      * @return void;

  275.      */

  276.     function SetWordInfos($word, $infos)

  277.     {

  278.         if( strlen($word)<4 )

  279.         {

  280.             return ;

  281.         }

  282.         if( isset($this->mainDicInfos[$word]) )

  283.         {

  284.             $this->newWords[$word]++;

  285.             $this->mainDicInfos[$word]['c']++;

  286.         } else {

  287.             $this->newWords[$word] = 1;

  288.             $this->mainDicInfos[$word] = $infos;

  289.         }

  290.     }

  291.     /**

  292.      * 开始执行分析

  293.      * @parem bool optimize 是否对结果进行优化

  294.      * @return bool

  295.      */

  296.     function StartAnalysis($optimize=TRUE)

  297.     {

  298.         if( !$this->isLoadDic )

  299.         {

  300.             $this->LoadDict();

  301.         }

  302.         $this->simpleResult = $this->finallyResult = array();

  303.         $this->sourceString .= chr(0).chr(32);

  304.         $slen = strlen($this->sourceString);

  305.         $sbcArr = array();

  306.         $j = 0;

  307.         //全角与半角字符对照表

  308.         for($i=0xFF00; $i < 0xFF5F; $i++)

  309.         {

  310.             $scb = 0x20 + $j;

  311.             $j++;

  312.             $sbcArr[$i] = $scb;

  313.         }

  314.         //对字符串进行粗分

  315.         $onstr = '';

  316.         $lastc = 1; //1 中/韩/日文, 2 英文/数字/符号('.', '@', '#', '+'), 3 ANSI符号 4 纯数字 5 非ANSI符号或不支持字符

  317.         $s = 0;

  318.         $ansiWordMatch = "[0-9a-z@#%\+\.-]";

  319.         $notNumberMatch = "[a-z@#%\+]";

  320.         for($i=0; $i < $slen; $i++)

  321.         {

  322.             $c = $this->sourceString[$i].$this->sourceString[++$i];

  323.             $cn = hexdec(bin2hex($c));

  324.             $cn = isset($sbcArr[$cn]) ? $sbcArr[$cn] : $cn;

  325.             //ANSI字符

  326.             if($cn < 0x80)

  327.             {

  328.                 if( preg_match('/'.$ansiWordMatch.'/i', chr($cn)) )

  329.                 {

  330.                     if( $lastc != 2 && $onstr != '') {

  331.                         $this->simpleResult[$s]['w'] = $onstr;

  332.                         $this->simpleResult[$s]['t'] = $lastc;

  333.                         $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  334.                         $s++;

  335.                         $onstr = '';

  336.                     }

  337.                     $lastc = 2;

  338.                     $onstr .= chr(0).chr($cn);

  339.                 } else {

  340.                     if( $onstr != '' )

  341.                     {

  342.                         $this->simpleResult[$s]['w'] = $onstr;

  343.                         if( $lastc==2 )

  344.                         {

  345.                             if( !preg_match('/'.$notNumberMatch.'/i', iconv(UCS2, 'utf-8', $onstr)) ) $lastc = 4;

  346.                         }

  347.                         $this->simpleResult[$s]['t'] = $lastc;

  348.                         if( $lastc != 4 ) $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  349.                         $s++;

  350.                     }

  351.                     $onstr = '';

  352.                     $lastc = 3;

  353.                     if($cn < 31)

  354.                     {

  355.                         continue;

  356.                     } else {

  357.                         $this->simpleResult[$s]['w'] = chr(0).chr($cn);

  358.                         $this->simpleResult[$s]['t'] = 3;

  359.                         $s++;

  360.                     }

  361.                 }

  362.             }

  363.             //普通字符

  364.             else

  365.             {

  366.                 //正常文字

  367.                 if( ($cn>0x3FFF && $cn < 0x9FA6) || ($cn>0xF8FF && $cn < 0xFA2D)

  368.                     || ($cn>0xABFF && $cn < 0xD7A4) || ($cn>0x3040 && $cn < 0x312B) )

  369.                 {

  370.                     if( $lastc != 1 && $onstr != '')

  371.                     {

  372.                         $this->simpleResult[$s]['w'] = $onstr;

  373.                         if( $lastc==2 )

  374.                         {

  375.                             if( !preg_match('/'.$notNumberMatch.'/i', iconv(UCS2, 'utf-8', $onstr)) ) $lastc = 4;

  376.                         }

  377.                         $this->simpleResult[$s]['t'] = $lastc;

  378.                         if( $lastc != 4 ) $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  379.                         $s++;

  380.                         $onstr = '';

  381.                     }

  382.                     $lastc = 1;

  383.                     $onstr .= $c;

  384.                 }

  385.                 //特殊符号

  386.                 else

  387.                 {

  388.                     if( $onstr != '' )

  389.                     {

  390.                         $this->simpleResult[$s]['w'] = $onstr;

  391.                         if( $lastc==2 )

  392.                         {

  393.                             if( !preg_match('/'.$notNumberMatch.'/i', iconv(UCS2, 'utf-8', $onstr)) ) $lastc = 4;

  394.                         }

  395.                         $this->simpleResult[$s]['t'] = $lastc;

  396.                         if( $lastc != 4 ) $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  397.                         $s++;

  398.                     }

  399.                     //检测书名

  400.                     if( $cn == 0x300A )

  401.                     {

  402.                         $tmpw = '';

  403.                         $n = 1;

  404.                         $isok = FALSE;

  405.                         $ew = chr(0x30).chr(0x0B);

  406.                         while(TRUE)

  407.                         {

  408. 							if(!isset($this->sourceString[$i+$n]) && !isset($this->sourceString[$i+$n+1]))

  409. 							break;

  410.                             $w = $this->sourceString[$i+$n].$this->sourceString[$i+$n+1];

  411.                             if( $w == $ew )

  412.                             {

  413.                                 $this->simpleResult[$s]['w'] = $c;

  414.                                 $this->simpleResult[$s]['t'] = 5;

  415.                                 $s++;

  416.                         

  417.                                 $this->simpleResult[$s]['w'] = $tmpw;

  418.                                 $this->newWords[$tmpw] = 1;

  419.                                 if( !isset($this->newWords[$tmpw]) )

  420.                                 {

  421.                                     $this->foundWordStr .= $this->_out_string_encoding($tmpw).'/nb, ';

  422.                                     $this->SetWordInfos($tmpw, array('c'=>1, 'm'=>'nb'));

  423.                                 }

  424.                                 $this->simpleResult[$s]['t'] = 13;

  425.                                 $s++;

  426.                                 //最大切分模式对书名继续分词

  427.                                 if( $this->differMax )

  428.                                 {

  429.                                     $this->simpleResult[$s]['w'] = $tmpw;

  430.                                     $this->simpleResult[$s]['t'] = 21;

  431.                                     $this->_deep_analysis($tmpw, $lastc, $s, $optimize);

  432.                                     $s++;

  433.                                 }

  434.                                 $this->simpleResult[$s]['w'] = $ew;

  435.                                 $this->simpleResult[$s]['t'] =  5;

  436.                                 $s++;

  437.                                 $i = $i + $n + 1;

  438.                                 $isok = TRUE;

  439.                                 $onstr = '';

  440.                                 $lastc = 5;

  441.                                 break;

  442.                             } else {

  443.                                 $n = $n+2;

  444.                                 $tmpw .= $w;

  445.                                 if( strlen($tmpw) > 60 )

  446.                                 {

  447.                                     break;

  448.                                 }

  449.                             }

  450.                         }//while

  451.                         if( !$isok )

  452.                         {

  453.                             $this->simpleResult[$s]['w'] = $c;

  454.                               $this->simpleResult[$s]['t'] = 5;

  455.                               $s++;

  456.                               $onstr = '';

  457.                             $lastc = 5;

  458.                         }

  459.                         continue;

  460.                     }

  461.                     $onstr = '';

  462.                     $lastc = 5;

  463.                     if( $cn==0x3000 )

  464.                     {

  465.                         continue;

  466.                     } else {

  467.                         $this->simpleResult[$s]['w'] = $c;

  468.                         $this->simpleResult[$s]['t'] = 5;

  469.                         $s++;

  470.                     }

  471.                 }//2byte symbol

  472.             }//end 2byte char

  473.         }//end for

  474.         //处理分词后的结果

  475.         $this->_sort_finally_result();

  476.     }

  477.     /**

  478.      * 深入分词

  479.      * @parem $str

  480.      * @parem $ctype (2 英文类， 3 中/韩/日文类)

  481.      * @parem $spos   当前粗分结果游标

  482.      * @return bool

  483.      */

  484.     function _deep_analysis( &$str, $ctype, $spos, $optimize=TRUE )

  485.     {

  486.         //中文句子

  487.         if( $ctype==1 )

  488.         {

  489.             $slen = strlen($str);

  490.             //小于系统配置分词要求长度的句子

  491.             if( $slen < $this->notSplitLen )

  492.             {

  493.                 $tmpstr = '';

  494.                 $lastType = 0;

  495.                 if( $spos > 0 ) $lastType = $this->simpleResult[$spos-1]['t'];

  496.                 if($slen < 5)

  497.                 {

  498.                       //echo iconv(UCS2, 'utf-8', $str).'<br>';

  499.                       if( $lastType==4 && ( isset($this->addonDic['u'][$str]) || isset($this->addonDic['u'][substr($str, 0, 2)]) ) )

  500.                       {

  501.                             $str2 = '';

  502.                             if( !isset($this->addonDic['u'][$str]) && isset($this->addonDic['s'][substr($str, 2, 2)]) )

  503.                             {

  504.                                 $str2 = substr($str, 2, 2);

  505.                                 $str  = substr($str, 0, 2);

  506.                             }

  507.                             $ww = $this->simpleResult[$spos - 1]['w'].$str;

  508.                             $this->simpleResult[$spos - 1]['w'] = $ww;

  509.                             $this->simpleResult[$spos - 1]['t'] = 4;

  510.                             if( !isset($this->newWords[$this->simpleResult[$spos - 1]['w']]) )

  511.                             {

  512.                                 $this->foundWordStr .= $this->_out_string_encoding( $ww ).'/mu, ';

  513.                                 $this->SetWordInfos($ww, array('c'=>1, 'm'=>'mu'));

  514.                             }

  515.                             $this->simpleResult[$spos]['w'] = '';

  516.                             if( $str2 != '' )

  517.                             {

  518.                                 $this->finallyResult[$spos-1][] = $ww;

  519.                                 $this->finallyResult[$spos-1][] = $str2;

  520.                             }

  521.                        } else {

  522.                               $this->finallyResult[$spos][] = $str;

  523.                        }

  524.                 } else {

  525.                       $this->_deep_analysis_cn( $str, $ctype, $spos, $slen, $optimize );

  526.                 }

  527.             }

  528.             //正常长度的句子，循环进行分词处理

  529.             else {

  530.                 $this->_deep_analysis_cn( $str, $ctype, $spos, $slen, $optimize );

  531.             }

  532.         }

  533.         //英文句子，转为小写

  534.         else {

  535.             if( $this->toLower ) {

  536.                 $this->finallyResult[$spos][] = strtolower($str);

  537.             } else {

  538.                 $this->finallyResult[$spos][] = $str;

  539.             }

  540.         }

  541.     }

  542.     /**

  543.      * 中文的深入分词

  544.      * @parem $str

  545.      * @return void

  546.      */

  547.     function _deep_analysis_cn( &$str, $lastec, $spos, $slen, $optimize=TRUE )

  548.     {

  549.         $quote1 = chr(0x20).chr(0x1C);

  550.         $tmparr = array();

  551.         $hasw = 0;

  552.         //如果前一个词为 “ ， 并且字符串小于3个字符当成一个词处理

  553.         if( $spos > 0 && $slen < 11 && $this->simpleResult[$spos-1]['w']==$quote1 )

  554.         {

  555.             $tmparr[] = $str;

  556.             if( !isset($this->newWords[$str]) )

  557.             {

  558.                 $this->foundWordStr .= $this->_out_string_encoding($str).'/nq, ';

  559.                 $this->SetWordInfos($str, array('c'=>1, 'm'=>'nq'));

  560.             }

  561.             if( !$this->differMax )

  562.             {

  563.                 $this->finallyResult[$spos][] = $str;

  564.                 return ;

  565.             }

  566.         }

  567.         //进行切分

  568.         for($i=$slen-1; $i > 0; $i -= 2)

  569.         {

  570.             //单个词

  571.             $nc = $str[$i-1].$str[$i];

  572.             //是否已经到最后两个字

  573.             if( $i <= 2 )

  574.             {

  575.                 $tmparr[] = $nc;

  576.                 $i = 0;

  577.                 break;

  578.             }

  579.             $isok = FALSE;

  580.             $i = $i + 1;

  581.             for($k=$this->dicWordMax; $k>1; $k=$k-2)

  582.             {

  583.                 if($i < $k) continue;

  584.                 $w = substr($str, $i-$k, $k);

  585.                 if( strlen($w) <= 2 )

  586.                 {

  587.                     $i = $i - 1;

  588.                     break;

  589.                 }

  590.                 if( $this->IsWord( $w ) )

  591.                 {

  592.                     $tmparr[] = $w;

  593.                     $i = $i - $k + 1;

  594.                     $isok = TRUE;

  595.                     break;

  596.                 }

  597.             }

  598.             //echo '<hr />';

  599.             //没适合词

  600.             if(!$isok) $tmparr[] = $nc;

  601.         }

  602.         $wcount = count($tmparr);

  603.         if( $wcount==0 ) return ;

  604.         $this->finallyResult[$spos] = array_reverse($tmparr);

  605.         //优化结果(岐义处理、新词、数词、人名识别等)

  606.         if( $optimize )

  607.         {

  608.             $this->_optimize_result( $this->finallyResult[$spos], $spos );

  609.         }

  610.     }

  611.     /**

  612.     * 对最终分词结果进行优化（把simpleresult结果合并，并尝试新词识别、数词合并等）

  613.     * @parem $optimize 是否优化合并的结果

  614.     * @return bool

  615.     */

  616.     //t = 1 中/韩/日文, 2 英文/数字/符号('.', '@', '#', '+'), 3 ANSI符号 4 纯数字 5 非ANSI符号或不支持字符

  617.     function _optimize_result( &$smarr, $spos )

  618.     {

  619.         $newarr = array();

  620.         $prePos = $spos - 1;

  621.         $arlen = count($smarr);

  622.         $i = $j = 0;

  623.         //检测数量词

  624.         if( $prePos > -1 && !isset($this->finallyResult[$prePos]) )

  625.         {

  626.             $lastw = $this->simpleResult[$prePos]['w'];

  627.             $lastt = $this->simpleResult[$prePos]['t'];

  628.               if( ($lastt==4 || isset( $this->addonDic['c'][$lastw] )) && isset( $this->addonDic['u'][$smarr[0]] ) )

  629.               {

  630.                  $this->simpleResult[$prePos]['w'] = $lastw.$smarr[0];

  631.                  $this->simpleResult[$prePos]['t'] = 4;

  632.                  if( !isset($this->newWords[ $this->simpleResult[$prePos]['w'] ]) )

  633.                  {

  634.                      $this->foundWordStr .= $this->_out_string_encoding( $this->simpleResult[$prePos]['w'] ).'/mu, ';

  635.                      $this->SetWordInfos($this->simpleResult[$prePos]['w'], array('c'=>1, 'm'=>'mu'));

  636.                  }

  637.                  $smarr[0] = '';

  638.                  $i++;

  639.               }

  640.        }

  641.        for(; $i < $arlen; $i++)

  642.        {

  643.             

  644.             if( !isset( $smarr[$i+1] ) )

  645.             {

  646.                 $newarr[$j] = $smarr[$i];

  647.                 break;

  648.             }

  649.             $cw = $smarr[$i];

  650.             $nw = $smarr[$i+1];

  651.             $ischeck = FALSE;

  652.             //检测数量词

  653.             if( isset( $this->addonDic['c'][$cw] ) && isset( $this->addonDic['u'][$nw] ) )

  654.             {

  655.                 //最大切分时保留合并前的词

  656.                 if($this->differMax)

  657.                 {

  658.                         $newarr[$j] = chr(0).chr(0x28);

  659.                         $j++;

  660.                         $newarr[$j] = $cw;

  661.                         $j++;

  662.                         $newarr[$j] = $nw;

  663.                         $j++;

  664.                         $newarr[$j] = chr(0).chr(0x29);

  665.                         $j++;

  666.                 }

  667.                 $newarr[$j] = $cw.$nw;

  668.                 if( !isset($this->newWords[$newarr[$j]]) )

  669.                 {

  670.                     $this->foundWordStr .= $this->_out_string_encoding( $newarr[$j] ).'/mu, ';

  671.                     $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'mu'));

  672.                 }

  673.                 $j++; $i++; $ischeck = TRUE;

  674.             }

  675.             //检测前导词(通常是姓)

  676.             else if( isset( $this->addonDic['n'][ $smarr[$i] ] ) )

  677.             {

  678.                 $is_rs = FALSE;

  679.                 //词语是副词或介词或频率很高的词不作为人名

  680.                 if( strlen($nw)==4 )

  681.                 {

  682.                     $winfos = $this->GetWordInfos($nw);

  683.                     if(isset($winfos['m']) && ($winfos['m']=='r' || $winfos['m']=='c' || $winfos['c']>500) )

  684.                     {

  685.                          $is_rs = TRUE;

  686.                     }

  687.                 }

  688.                 if( !isset($this->addonDic['s'][$nw]) && strlen($nw)<5 && !$is_rs )

  689.                 {

  690.                     $newarr[$j] = $cw.$nw;

  691.                     //echo iconv(UCS2, 'utf-8', $newarr[$j])."<br>";

  692.                     //尝试检测第三个词

  693.                     if( strlen($nw)==2 && isset($smarr[$i+2]) && strlen($smarr[$i+2])==2 && !isset( $this->addonDic['s'][$smarr[$i+2]] ) )

  694.                     {

  695.                         $newarr[$j] .= $smarr[$i+2];

  696.                         $i++;

  697.                     }

  698.                     if( !isset($this->newWords[$newarr[$j]]) )

  699.                     {

  700.                         $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'nr'));

  701.                         $this->foundWordStr .= $this->_out_string_encoding($newarr[$j]).'/nr, ';

  702.                     }

  703.                     //为了防止错误，保留合并前的姓名

  704.                     if(strlen($nw)==4)

  705.                     {

  706.                         $j++;

  707.                         $newarr[$j] = chr(0).chr(0x28);

  708.                         $j++;

  709.                         $newarr[$j] = $cw;

  710.                         $j++;

  711.                         $newarr[$j] = $nw;

  712.                         $j++;

  713.                         $newarr[$j] = chr(0).chr(0x29);

  714.                     }

  715.                     $j++; $i++; $ischeck = TRUE;

  716.                 }

  717.             }

  718.             //检测后缀词(地名等)

  719.             else if( isset($this->addonDic['a'][$nw]) )

  720.             {

  721.                 $is_rs = FALSE;

  722.                 //词语是副词或介词不作为前缀

  723.                 if( strlen($cw)>2 )

  724.                 {

  725.                     $winfos = $this->GetWordInfos($cw);

  726.                     if(isset($winfos['m']) && ($winfos['m']=='a' || $winfos['m']=='r' || $winfos['m']=='c' || $winfos['c']>500) )

  727.                     {

  728.                          $is_rs = TRUE;

  729.                     }

  730.                 }

  731.                 if( !isset($this->addonDic['s'][$cw]) && !$is_rs )

  732.                 {

  733.                     $newarr[$j] = $cw.$nw;

  734.                     if( !isset($this->newWords[$newarr[$j]]) )

  735.                     {

  736.                         $this->foundWordStr .= $this->_out_string_encoding($newarr[$j]).'/na, ';

  737.                         $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'na'));

  738.                     }

  739.                     $i++; $j++; $ischeck = TRUE;

  740.                 }

  741.             }

  742.             //新词识别（暂无规则）

  743.             else if($this->unitWord)

  744.             {

  745.                 if(strlen($cw)==2 && strlen($nw)==2 

  746.                 && !isset($this->addonDic['s'][$cw]) && !isset($this->addonDic['t'][$cw]) && !isset($this->addonDic['a'][$cw]) 

  747.                 && !isset($this->addonDic['s'][$nw]) && !isset($this->addonDic['c'][$nw]))

  748.                 {

  749.                     $newarr[$j] = $cw.$nw;

  750.                     //尝试检测第三个词

  751.                     if( isset($smarr[$i+2]) && strlen($smarr[$i+2])==2 && (isset( $this->addonDic['a'][$smarr[$i+2]] ) || isset( $this->addonDic['u'][$smarr[$i+2]] )) )

  752.                     {

  753.                         $newarr[$j] .= $smarr[$i+2];

  754.                         $i++;

  755.                     }

  756.                     if( !isset($this->newWords[$newarr[$j]]) )

  757.                     {

  758.                         $this->foundWordStr .= $this->_out_string_encoding($newarr[$j]).'/ms, ';

  759.                         $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'ms'));

  760.                     }

  761.                     $i++; $j++; $ischeck = TRUE;

  762.                 }

  763.             }

  764.             //不符合规则

  765.             if( !$ischeck )

  766.             {

  767.                 $newarr[$j] = $cw;

  768.                   //二元消岐处理——最大切分模式

  769.                 if( $this->differMax && !isset($this->addonDic['s'][$cw]) && strlen($cw) < 5 && strlen($nw) < 7)

  770.                 {

  771.                     $slen = strlen($nw);

  772.                     $hasDiff = FALSE;

  773.                     for($y=2; $y <= $slen-2; $y=$y+2)

  774.                     {

  775.                         $nhead = substr($nw, $y-2, 2);

  776.                         $nfont = $cw.substr($nw, 0, $y-2);

  777.                         if( $this->IsWord( $nfont.$nhead ) )

  778.                         {

  779.                             if( strlen($cw) > 2 ) $j++;

  780.                             $hasDiff = TRUE;

  781.                             $newarr[$j] = $nfont.$nhead;

  782.                         }

  783.                     }

  784.                 }

  785.                 $j++;

  786.             }

  787.        }//end for

  788.        $smarr =  $newarr;

  789.     }

  790.     /**

  791.     * 转换最终分词结果到 finallyResult 数组

  792.     * @return void

  793.     */

  794.     function _sort_finally_result()

  795.     {

  796.           $newarr = array();

  797.         $i = 0;

  798.         foreach($this->simpleResult as $k=>$v)

  799.         {

  800.             if( empty($v['w']) ) continue;

  801.             if( isset($this->finallyResult[$k]) && count($this->finallyResult[$k]) > 0 )

  802.             {

  803.                 foreach($this->finallyResult[$k] as $w)

  804.                 {

  805.                     if(!empty($w))

  806.                     {

  807.                         $newarr[$i]['w'] = $w;

  808.                         $newarr[$i]['t'] = 20;

  809.                         $i++;

  810.                     }

  811.                 }

  812.             }

  813.             else if($v['t'] != 21)

  814.             {

  815.                 $newarr[$i]['w'] = $v['w'];

  816.                 $newarr[$i]['t'] = $v['t'];

  817.                 $i++;

  818.             }

  819.         }

  820.         $this->finallyResult = $newarr;

  821.         $newarr = '';

  822.     }

  823.     /**

  824.      * 把uncode字符串转换为输出字符串

  825.      * @parem str

  826.      * return string

  827.      */

  828.     function _out_string_encoding( &$str )

  829.     {

  830.         $rsc = $this->_source_result_charset();

  831.         if( $rsc==1 ) {

  832.             $rsstr = iconv(UCS2, 'utf-8', $str);

  833.         }

  834.         else if( $rsc==2 ) {

  835.             $rsstr = iconv('utf-8', 'gb18030', iconv(UCS2, 'utf-8', $str) );

  836.         } else {

  837.             $rsstr = iconv('utf-8', 'big5', iconv(UCS2, 'utf-8', $str) );

  838.         }

  839.         return $rsstr;

  840.     }

  841.     /**

  842.      * 获取最终结果字符串（用空格分开后的分词结果）

  843.      * @return string

  844.      */

  845.     function GetFinallyResult($spword=' ', $word_meanings=FALSE)

  846.     {

  847.         $rsstr = '';

  848.         foreach($this->finallyResult as $v)

  849.         {

  850.             if( $this->resultType==2 && ($v['t']==3 || $v['t']==5) )

  851.             {

  852.                 continue;

  853.             }

  854.             $m = '';

  855.             if( $word_meanings )

  856.             {

  857.                 $m = $this->GetWordProperty($v['w']);

  858.             }

  859.             $w = $this->_out_string_encoding($v['w']);

  860.             if( $w != ' ' )

  861.             {

  862.                 if($word_meanings) {

  863.                     $rsstr .= $spword.$w.$m;

  864.                 } else {

  865.                     $rsstr .= $spword.$w;

  866.                 }

  867.             }

  868.         }

  869.         return $rsstr;

  870.     }

  871.     /**

  872.      * 获取粗分结果，不包含粗分属性

  873.      * @return array()

  874.      */

  875.     function GetSimpleResult()

  876.     {

  877.         $rearr = array();

  878.         foreach($this->simpleResult as $k=>$v)

  879.         {

  880.             if( empty($v['w']) ) continue;

  881.             $w = $this->_out_string_encoding($v['w']);

  882.             if( $w != ' ' ) $rearr[] = $w;

  883.         }

  884.         return $rearr;

  885.     }

  886.     /**

  887.      * 获取粗分结果，包含粗分属性（1中文词句、2 ANSI词汇（包括全角），3 ANSI标点符号（包括全角），4数字（包括全角），5 中文标点或无法识别字符）

  888.      * @return array()

  889.      */

  890.     function GetSimpleResultAll()

  891.     {

  892.         $rearr = array();

  893.         foreach($this->simpleResult as $k=>$v)

  894.         {

  895.             $w = $this->_out_string_encoding($v['w']);

  896.             if( $w != ' ' )

  897.             {

  898.                 $rearr[$k]['w'] = $w;

  899.                 $rearr[$k]['t'] = $v['t'];

  900.             }

  901.         }

  902.         return $rearr;

  903.     }

  904.     /**

  905.      * 获取索引hash数组

  906.      * @return array('word'=>count,)

  907.      */

  908.     function GetFinallyIndex()

  909.     {

  910.         $rearr = array();

  911.         foreach($this->finallyResult as $v)

  912.         {

  913.             if( $this->resultType==2 && ($v['t']==3 || $v['t']==5) )

  914.             {

  915.                 continue;

  916.             }

  917.             $w = $this->_out_string_encoding($v['w']);

  918.             if( $w == ' ' )

  919.             {

  920.                 continue;

  921.             }

  922.             if( isset($rearr[$w]) )

  923.             {

  924.                  $rearr[$w]++;

  925.             } else {

  926.                  $rearr[$w] = 1;

  927.             }

  928.         }

  929.         return $rearr;

  930.     }

  931.     /**

  932.      * 获得保存目标编码

  933.      * @return int

  934.      */

  935.     function _source_result_charset()

  936.     {

  937.         if( preg_match("/^utf/", $this->targetCharSet) ) {

  938.            $rs = 1;

  939.         }

  940.         else if( preg_match("/^gb/", $this->targetCharSet) ) {

  941.            $rs = 2;

  942.         }

  943.         else if( preg_match("/^big/", $this->targetCharSet) ) {

  944.            $rs = 3;

  945.         }

  946.         else {

  947.             $rs = 4;

  948.         }

  949.         return $rs;

  950.     }

  951.      /**

  952.      * 编译词典

  953.      * @parem $sourcefile utf-8编码的文本词典数据文件<参见范例dict/not-build/base_dic_full.txt>

  954.      * 注意, 需要PHP开放足够的内存才能完成操作

  955.      * @return void

  956.      */

  957.     function MakeDict( $source_file, $target_file='' )

  958.     {

  959.         $target_file = ($target_file=='' ? $this->mainDicFile : $target_file);

  960.         $allk = array();

  961.         $fp = fopen($source_file, 'r');

  962.         while( $line = fgets($fp, 512) )

  963.         {

  964.             if( $line[0]=='@' ) continue;

  965.             list($w, $r, $a) = explode(',', $line);

  966.             $a = trim( $a );

  967.             $w = iconv('utf-8', UCS2, $w);

  968.             $k = $this->_get_index( $w );

  969.             if( isset($allk[ $k ]) )

  970.                 $allk[ $k ][ $w ] = array($r, $a);

  971.             else

  972.                 $allk[ $k ][ $w ] = array($r, $a);

  973.         }

  974.         fclose( $fp );

  975.         $fp = fopen($target_file, 'w');

  976.         $heade_rarr = array();

  977.         $alldat = '';

  978.         $start_pos = $this->mask_value * 8;

  979.         foreach( $allk as $k => $v )

  980.         {

  981.             $dat  = serialize( $v );

  982.             $dlen = strlen($dat);

  983.             $alldat .= $dat;

  984.             $heade_rarr[ $k ][0] = $start_pos;

  985.             $heade_rarr[ $k ][1] = $dlen;

  986.             $heade_rarr[ $k ][2] = count( $v );

  987.             $start_pos += $dlen;

  988.         }

  989.         unset( $allk );

  990.         for($i=0; $i < $this->mask_value; $i++)

  991.         {

  992.             if( !isset($heade_rarr[$i]) )

  993.             {

  994.                 $heade_rarr[$i] = array(0, 0, 0);

  995.             }

  996.             fwrite($fp, pack("Inn", $heade_rarr[$i][0], $heade_rarr[$i][1], $heade_rarr[$i][2]));

  997.         }

  998.         fwrite( $fp, $alldat);

  999.         fclose( $fp );

  1000.     } 

  1001.      /**

  1002.      * 导出词典的词条

  1003.      * @parem $targetfile 保存位置

  1004.      * @return void

  1005.      */

  1006.     function ExportDict( $targetfile )

  1007.     {

  1008.         if( !$this->mainDicHand )

  1009.         {

  1010.             $this->mainDicHand = fopen($this->mainDicFile, 'rw');

  1011.         }

  1012.         $fp = fopen($targetfile, 'w');

  1013.         for($i=0; $i <= $this->mask_value; $i++)

  1014.         {

  1015.             $move_pos = $i * 8;

  1016.             fseek($this->mainDicHand, $move_pos, SEEK_SET);

  1017.             $dat = fread($this->mainDicHand, 8);

  1018.             $arr = unpack('I1s/n1l/n1c', $dat);

  1019.             if( $arr['l'] == 0 )

  1020.             {

  1021.                 continue;

  1022.             }

  1023.             fseek($this->mainDicHand, $arr['s'], SEEK_SET);

  1024.             $data = @unserialize(fread($this->mainDicHand, $arr['l']));

  1025.             if( !is_array($data) ) continue;

  1026.             foreach($data as $k => $v)

  1027.             {

  1028.                 $w = iconv(UCS2, 'utf-8', $k);

  1029.                 fwrite($fp, "{$w},{$v[0]},{$v[1]}\n");

  1030.             }

  1031.         }

  1032.         fclose( $fp );

  1033.         return TRUE;

  1034.      }

  1035. 	function InportDict( $targetfile )

  1036.     {

  1037.      	if(!ini_set('memory_limit', '128M'))

  1038. 			exit('设置内存错误，请到官网下载解压版的base_dic_full.dic!');

  1039.      	require_once(DEDEINC.'/libraries/zip.class.php');

  1040.      	$zip = new zip();

  1041.      	//echo $targetfile;

  1042.      	$unpackagefile = array_keys($zip->Extract($targetfile,DEDEINC.'/data/'));

  1043.      	//exit();

  1044.      	$this->MakeDict(DEDEINC.'/data/'.$unpackagefile[0]);

  1045.      	unlink(DEDEINC.'/data/'.$unpackagefile[0]);

  1046.      	return true;

  1047.     }

  1048. }