DedeBIZ
/
DedeV6
Watch 2
Star 0
Fork 0
Code Pull Requests 0 Releases 25 Activity
国内流行的内容管理系统（CMS）多端全媒体解决方案 https://www.dedebiz.com
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
440 Commits
3 Branches
110MB
Tree: afc2911867
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'afc2911867'
${ noResults }
DedeV6/src/include/splitword.class.php

1135 lines
38KB
Raw Blame History 

  1. <?php

  2. if (!defined('DEDEINC')) exit('dedebiz');

  3. /**

  4.  * Unicode编码词典的php分词器

  5.  *

  6.  *  1、只适用于php5，必要函数 iconv

  7.  *  2、本程序是使用RMM逆向匹配算法进行分词的，词库需要特别编译，本类里提供了 MakeDict() 方法

  8.  *  3、简单操作流程：SetSource -> StartAnalysis -> Get***Result

  9.  *  4、对主词典使用特殊格式进行编码, 不需要载入词典到内存操作

  10.  *

  11.  * @version        $Id: splitword.class.php 2 11:45 2011-2-14 itplato $

  12.  * @package        DedeBIZ.Libraries

  13.  * @copyright      Copyright (c) 2022, DedeBIZ.COM

  14.  * @license        https://www.dedebiz.com/license

  15.  * @link           https://www.dedebiz.com

  16.  */

  17. 

  18. //常量定义

  19. define('_SP_', chr(0xFF).chr(0xFE)); 

  20. define('UCS2', 'ucs-2be');

  21. class SplitWord

  22. {

  23.     

  24.     //hash算法选项

  25.     var $mask_value = 0xFFFF;

  26.     

  27.     //输入和输出的字符编码（只允许 utf-8、gbk/gb2312/gb18030、big5 三种类型）  

  28.     var $sourceCharSet = 'utf-8';

  29.     var $targetCharSet = 'utf-8';

  30.     

  31.     //生成的分词结果数据类型 1 为全部， 2为 词典词汇及单个中日韩简繁字符及英文， 3 为词典词汇及英文

  32.     var $resultType = 1;

  33.     

  34.     //句子长度小于这个数值时不拆分，notSplitLen = n(个汉字) * 2 + 1

  35.     var $notSplitLen = 5;

  36.     

  37.     //把英文单词全部转小写

  38.     var $toLower = FALSE;

  39.     

  40.     //使用最大切分模式对二元词进行消岐

  41.     var $differMax = FALSE;

  42.     

  43.     //尝试合并单字

  44.     var $unitWord = TRUE;

  45.     

  46.     //初始化类时直接加载词典

  47.     var $loadInit = TRUE;

  48.     

  49.     //使用热门词优先模式进行消岐

  50.     var $differFreq = FALSE;

  51.     

  52.     //被转换为unicode的源字符串

  53.     var $sourceString = '';

  54.     

  55.     //附加词典

  56.     var $addonDic = array();

  57.     var $addonDicFile = 'data/words_addons.dic';

  58.     

  59.     //主词典 

  60.     var $dicStr = '';

  61.     var $mainDic = array();

  62.     var $mainDicHand = FALSE;

  63.     var $mainDicInfos = array();

  64.     var $mainDicFile = 'data/base_dic_full.dic';

  65.     //是否直接载入词典（选是载入速度较慢，但解析较快；选否载入较快，但解析较慢，需要时才会载入特定的词条）

  66.     var $mainDicFileZip = 'data/base_dic_full.zip';

  67.     var $isLoadAll = FALSE;

  68.     var $isUnpacked = FALSE;

  69.     

  70.     //主词典词语最大长度 x / 2

  71.     var $dicWordMax = 14;

  72.     //粗分后的数组（通常是截取句子等用途）

  73.     var $simpleResult = array();

  74.     //最终结果(用空格分开的词汇列表)

  75.     var $finallyResult = '';

  76.     

  77.     //是否已经载入词典

  78.     var $isLoadDic = FALSE;

  79.     //系统识别或合并的新词

  80.     var $newWords = array();

  81.     var $foundWordStr = '';

  82.     //词库载入时间

  83.     var $loadTime = 0;

  84.     

  85.     /**

  86.      * 构造函数

  87.      * @param $source_charset

  88.      * @param $target_charset

  89.      * @param $load_alldic 

  90.      * @param $source

  91.      *

  92.      * @return void

  93.      */

  94.     function __construct($source_charset='utf-8', $target_charset='utf-8', $load_all=TRUE, $source='')

  95.     {

  96.         $this->SetSource( $source, $source_charset, $target_charset );

  97.         $this->isLoadAll = $load_all;

  98.         if(file_exists(DEDEINC.'/'.$this->mainDicFile)) $this->isUnpacked = TRUE;

  99.         if($this->loadInit) $this->LoadDict();

  100.     }

  101.     

  102.     function SplitWord($source_charset='utf-8', $target_charset='utf-8', $load_all=TRUE, $source='')

  103.     {

  104.         $this->__construct($source_charset, $target_charset, $load_all, $source);

  105.     }

  106.     

  107.    /**

  108.     * 析构函数

  109.     */

  110.     function __destruct()

  111.     {

  112.         if( $this->mainDicHand !== FALSE )

  113.         {

  114.             @fclose( $this->mainDicHand );

  115.         }

  116.     }

  117.     

  118.     /**

  119.      * 根据字符串计算key索引

  120.      * @param $key

  121.      * @return short int

  122.      */

  123.     function _get_index( $key )

  124.     {

  125.         $l = strlen($key);

  126.         $h = 0x238f13af;

  127.         while ($l--)

  128.         {

  129.             $h += ($h << 5);

  130.             $h ^= ord($key[$l]);

  131.             $h &= 0x7fffffff;

  132.         }

  133.         return ($h % $this->mask_value);

  134.     }

  135.     

  136.     /**

  137.      * 从文件获得词

  138.      * @param $key

  139.      * @param $type (类型 word 或 key_groups)

  140.      * @return short int

  141.      */

  142.     function GetWordInfos( $key, $type='word' )

  143.     {

  144.         if( !$this->mainDicHand )

  145.         {

  146.             $this->mainDicHand = fopen($this->mainDicFile, 'r');

  147.         }

  148.         $p = 0;

  149.         $keynum = $this->_get_index( $key );

  150.         if( isset($this->mainDicInfos[ $keynum ]) )

  151.         {

  152.             $data = $this->mainDicInfos[ $keynum ];

  153.         }

  154.         else

  155.         {

  156.             //rewind( $this->mainDicHand );

  157.             $move_pos = $keynum * 8;

  158.             fseek($this->mainDicHand, $move_pos, SEEK_SET);

  159.             $dat = fread($this->mainDicHand, 8);

  160.             $arr = unpack('I1s/n1l/n1c', $dat);

  161.             if( $arr['l'] == 0 )

  162.             {

  163.                 return FALSE;

  164.             }

  165.             fseek($this->mainDicHand, $arr['s'], SEEK_SET);

  166.             $data = @unserialize(fread($this->mainDicHand, $arr['l']));

  167.             $this->mainDicInfos[ $keynum ] = $data;

  168.        }

  169.        if( !is_array($data) || !isset($data[$key]) ) 

  170.        {

  171.            return FALSE;

  172.        }

  173.        return ($type=='word' ? $data[$key] : $data);

  174.     }

  175.     

  176.     /**

  177.      * 设置源字符串

  178.      * @param $source

  179.      * @param $source_charset

  180.      * @param $target_charset

  181.      *

  182.      * @return bool

  183.      */

  184.     function SetSource( $source, $source_charset='utf-8', $target_charset='utf-8' )

  185.     {

  186.         $this->sourceCharSet = strtolower($source_charset);

  187.         $this->targetCharSet = strtolower($target_charset);

  188.         $this->simpleResult = array();

  189.         $this->finallyResult = array();

  190.         $this->finallyIndex = array();

  191.         if( $source != '' )

  192.         {

  193.             $rs = TRUE;

  194.             if( preg_match("/^utf/", $source_charset) ) {

  195.                 $this->sourceString = @iconv('utf-8', UCS2, $source);

  196.             }

  197.             else if( preg_match("/^gb/", $source_charset) ) {

  198.                 $this->sourceString = @iconv('utf-8', UCS2, iconv('gb18030', 'utf-8', $source));

  199.             }

  200.             else if( preg_match("/^big/", $source_charset) ) {

  201.                 $this->sourceString = @iconv('utf-8', UCS2, iconv('big5', 'utf-8', $source));

  202.             }

  203.             else {

  204.                 $rs = FALSE;

  205.             }

  206.         }

  207.         else

  208.         {

  209.            $rs = FALSE;

  210.         }

  211.         return $rs;

  212.     }

  213.     

  214.     /**

  215.      * 设置结果类型(只在获取finallyResult才有效)

  216.      * @param $rstype 1 为全部， 2去除特殊符号

  217.      *

  218.      * @return void

  219.      */

  220.     function SetResultType( $rstype )

  221.     {

  222.         $this->resultType = $rstype;

  223.     }

  224.     

  225.     /**

  226.      * 载入词典

  227.      *

  228.      * @return void

  229.      */

  230.     function LoadDict( $maindic='' )

  231.     {

  232. 		$this->addonDicFile = DEDEINC.'/'.$this->addonDicFile;

  233. 		$this->mainDicFile = DEDEINC.'/'.$this->mainDicFile;

  234. 		$this->mainDicFileZip = DEDEINC.'/'.$this->mainDicFileZip;

  235.         $startt = microtime(TRUE);

  236.         //正常读取文件

  237.         $dicAddon = $this->addonDicFile;

  238.         if($maindic=='' || !file_exists($maindic) )

  239.         {

  240.             $dicWords = $this->mainDicFile ;

  241.         }

  242.         else

  243.         {

  244.             $dicWords = $maindic;

  245.             $this->mainDicFile = $maindic;

  246.         }

  247.         

  248.         //加载主词典（只打开）

  249.         if($this->isUnpacked){

  250.         	$this->mainDicHand = fopen($dicWords, 'r');

  251.         }else{

  252.         	$this->InportDict($this->mainDicFileZip);

  253.         }

  254.         

  255.         //载入副词典

  256.         $hw = '';

  257.         $ds = file($dicAddon);

  258.         foreach($ds as $d)

  259.         {

  260.             $d = trim($d);

  261.             if($d=='') continue;

  262.             $estr = substr($d, 1, 1);

  263.             if( $estr==':' ) {

  264.                 $hw = substr($d, 0, 1);

  265.             }

  266.             else

  267.             {

  268.                 $spstr = _SP_;

  269.                 $spstr = iconv(UCS2, 'utf-8', $spstr);

  270.                 $ws = explode(',', $d);

  271.                 $wall = iconv('utf-8', UCS2, join($spstr, $ws));

  272.                 $ws = explode(_SP_, $wall);

  273.                 foreach($ws as $estr)

  274.                 {

  275.                     $this->addonDic[$hw][$estr] = strlen($estr);

  276.                 }

  277.             }

  278.         }

  279.         $this->loadTime = microtime(TRUE) - $startt;

  280.         $this->isLoadDic = TRUE;

  281.     }

  282.     

  283.    /**

  284.     * 检测某个词是否存在

  285.     */

  286.     function IsWord( $word )

  287.     {

  288.          $winfos = $this->GetWordInfos( $word );

  289.          return ($winfos !== FALSE);

  290.     }

  291.     

  292.     /**

  293.      * 获得某个词的词性及词频信息

  294.      * @parem $word unicode编码的词

  295.      * @return void

  296.      */

  297.      function GetWordProperty($word)

  298.      {

  299.         if( strlen($word)<4 )

  300.         {

  301.             return '/s';

  302.         }

  303.         $infos = $this->GetWordInfos($word);

  304.         return isset($infos[1]) ? "/{$infos[1]}{$infos[0]}" : "/s";

  305.      }

  306.     

  307.     /**

  308.      * 指定某词的词性信息（通常是新词）

  309.      * @parem $word unicode编码的词

  310.      * @parem $infos array('c' => 词频, 'm' => 词性);

  311.      * @return void;

  312.      */

  313.     function SetWordInfos($word, $infos)

  314.     {

  315.         if( strlen($word)<4 )

  316.         {

  317.             return ;

  318.         }

  319.         if( isset($this->mainDicInfos[$word]) )

  320.         {

  321.             $this->newWords[$word]++;

  322.             $this->mainDicInfos[$word]['c']++;

  323.         }

  324.         else

  325.         {

  326.             $this->newWords[$word] = 1;

  327.             $this->mainDicInfos[$word] = $infos;

  328.         }

  329.     }

  330.     

  331.     /**

  332.      * 开始执行分析

  333.      * @parem bool optimize 是否对结果进行优化

  334.      * @return bool

  335.      */

  336.     function StartAnalysis($optimize=TRUE)

  337.     {

  338.         if( !$this->isLoadDic )

  339.         {

  340.             $this->LoadDict();

  341.         }

  342.         $this->simpleResult = $this->finallyResult = array();

  343.         $this->sourceString .= chr(0).chr(32);

  344.         $slen = strlen($this->sourceString);

  345.         $sbcArr = array();

  346.         $j = 0;

  347.         //全角与半角字符对照表

  348.         for($i=0xFF00; $i < 0xFF5F; $i++)

  349.         {

  350.             $scb = 0x20 + $j;

  351.             $j++;

  352.             $sbcArr[$i] = $scb;

  353.         }

  354.         //对字符串进行粗分

  355.         $onstr = '';

  356.         $lastc = 1; //1 中/韩/日文, 2 英文/数字/符号('.', '@', '#', '+'), 3 ANSI符号 4 纯数字 5 非ANSI符号或不支持字符

  357.         $s = 0;

  358.         $ansiWordMatch = "[0-9a-z@#%\+\.-]";

  359.         $notNumberMatch = "[a-z@#%\+]";

  360.         for($i=0; $i < $slen; $i++)

  361.         {

  362.             $c = $this->sourceString[$i].$this->sourceString[++$i];

  363.             $cn = hexdec(bin2hex($c));

  364.             $cn = isset($sbcArr[$cn]) ? $sbcArr[$cn] : $cn;

  365.             //ANSI字符

  366.             if($cn < 0x80)

  367.             {

  368.                 if( preg_match('/'.$ansiWordMatch.'/i', chr($cn)) )

  369.                 {

  370.                     if( $lastc != 2 && $onstr != '') {

  371.                         $this->simpleResult[$s]['w'] = $onstr;

  372.                         $this->simpleResult[$s]['t'] = $lastc;

  373.                         $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  374.                         $s++;

  375.                         $onstr = '';

  376.                     }

  377.                     $lastc = 2;

  378.                     $onstr .= chr(0).chr($cn);

  379.                 }

  380.                 else

  381.                 {

  382.                     if( $onstr != '' )

  383.                     {

  384.                         $this->simpleResult[$s]['w'] = $onstr;

  385.                         if( $lastc==2 )

  386.                         {

  387.                             if( !preg_match('/'.$notNumberMatch.'/i', iconv(UCS2, 'utf-8', $onstr)) ) $lastc = 4;

  388.                         }

  389.                         $this->simpleResult[$s]['t'] = $lastc;

  390.                         if( $lastc != 4 ) $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  391.                         $s++;

  392.                     }

  393.                     $onstr = '';

  394.                     $lastc = 3;

  395.                     if($cn < 31)

  396.                     {

  397.                         continue;

  398.                     }

  399.                     else

  400.                     {

  401.                         $this->simpleResult[$s]['w'] = chr(0).chr($cn);

  402.                         $this->simpleResult[$s]['t'] = 3;

  403.                         $s++;

  404.                     }

  405.                 }

  406.             }

  407.             //普通字符

  408.             else

  409.             {

  410.                 //正常文字

  411.                 if( ($cn>0x3FFF && $cn < 0x9FA6) || ($cn>0xF8FF && $cn < 0xFA2D)

  412.                     || ($cn>0xABFF && $cn < 0xD7A4) || ($cn>0x3040 && $cn < 0x312B) )

  413.                 {

  414.                     if( $lastc != 1 && $onstr != '')

  415.                     {

  416.                         $this->simpleResult[$s]['w'] = $onstr;

  417.                         if( $lastc==2 )

  418.                         {

  419.                             if( !preg_match('/'.$notNumberMatch.'/i', iconv(UCS2, 'utf-8', $onstr)) ) $lastc = 4;

  420.                         }

  421.                         $this->simpleResult[$s]['t'] = $lastc;

  422.                         if( $lastc != 4 ) $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  423.                         $s++;

  424.                         $onstr = '';

  425.                     }

  426.                     $lastc = 1;

  427.                     $onstr .= $c;

  428.                 }

  429.                 //特殊符号

  430.                 else

  431.                 {

  432.                     if( $onstr != '' )

  433.                     {

  434.                         $this->simpleResult[$s]['w'] = $onstr;

  435.                         if( $lastc==2 )

  436.                         {

  437.                             if( !preg_match('/'.$notNumberMatch.'/i', iconv(UCS2, 'utf-8', $onstr)) ) $lastc = 4;

  438.                         }

  439.                         $this->simpleResult[$s]['t'] = $lastc;

  440.                         if( $lastc != 4 ) $this->_deep_analysis($onstr, $lastc, $s, $optimize);

  441.                         $s++;

  442.                     }

  443.                     

  444.                     //检测书名

  445.                     if( $cn == 0x300A )

  446.                     {

  447.                         $tmpw = '';

  448.                         $n = 1;

  449.                         $isok = FALSE;

  450.                         $ew = chr(0x30).chr(0x0B);

  451.                         while(TRUE)

  452.                         {

  453. 							if(!isset($this->sourceString[$i+$n]) && !isset($this->sourceString[$i+$n+1]))

  454. 							break;

  455.                             $w = $this->sourceString[$i+$n].$this->sourceString[$i+$n+1];

  456.                             if( $w == $ew )

  457.                             {

  458.                                 $this->simpleResult[$s]['w'] = $c;

  459.                                 $this->simpleResult[$s]['t'] = 5;

  460.                                 $s++;

  461.                         

  462.                                 $this->simpleResult[$s]['w'] = $tmpw;

  463.                                 $this->newWords[$tmpw] = 1;

  464.                                 if( !isset($this->newWords[$tmpw]) )

  465.                                 {

  466.                                     $this->foundWordStr .= $this->_out_string_encoding($tmpw).'/nb, ';

  467.                                     $this->SetWordInfos($tmpw, array('c'=>1, 'm'=>'nb'));

  468.                                 }

  469.                                 $this->simpleResult[$s]['t'] = 13;

  470.                                 

  471.                                 $s++;

  472. 

  473.                                 //最大切分模式对书名继续分词

  474.                                 if( $this->differMax )

  475.                                 {

  476.                                     $this->simpleResult[$s]['w'] = $tmpw;

  477.                                     $this->simpleResult[$s]['t'] = 21;

  478.                                     $this->_deep_analysis($tmpw, $lastc, $s, $optimize);

  479.                                     $s++;

  480.                                 }

  481.                                 

  482.                                 $this->simpleResult[$s]['w'] = $ew;

  483.                                 $this->simpleResult[$s]['t'] =  5;

  484.                                 $s++;

  485.                         

  486.                                 $i = $i + $n + 1;

  487.                                 $isok = TRUE;

  488.                                 $onstr = '';

  489.                                 $lastc = 5;

  490.                                 break;

  491.                             }

  492.                             else

  493.                             {

  494.                                 $n = $n+2;

  495.                                 $tmpw .= $w;

  496.                                 if( strlen($tmpw) > 60 )

  497.                                 {

  498.                                     break;

  499.                                 }

  500.                             }

  501.                         }//while

  502.                         if( !$isok )

  503.                         {

  504.                             $this->simpleResult[$s]['w'] = $c;

  505.                               $this->simpleResult[$s]['t'] = 5;

  506.                               $s++;

  507.                               $onstr = '';

  508.                             $lastc = 5;

  509.                         }

  510.                         continue;

  511.                     }

  512.                     

  513.                     $onstr = '';

  514.                     $lastc = 5;

  515.                     if( $cn==0x3000 )

  516.                     {

  517.                         continue;

  518.                     }

  519.                     else

  520.                     {

  521.                         $this->simpleResult[$s]['w'] = $c;

  522.                         $this->simpleResult[$s]['t'] = 5;

  523.                         $s++;

  524.                     }

  525.                 }//2byte symbol

  526.                 

  527.             }//end 2byte char

  528.         

  529.         }//end for

  530.         

  531.         //处理分词后的结果

  532.         $this->_sort_finally_result();

  533.     }

  534.     

  535.     /**

  536.      * 深入分词

  537.      * @parem $str

  538.      * @parem $ctype (2 英文类， 3 中/韩/日文类)

  539.      * @parem $spos   当前粗分结果游标

  540.      * @return bool

  541.      */

  542.     function _deep_analysis( &$str, $ctype, $spos, $optimize=TRUE )

  543.     {

  544. 

  545.         //中文句子

  546.         if( $ctype==1 )

  547.         {

  548.             $slen = strlen($str);

  549.             //小于系统配置分词要求长度的句子

  550.             if( $slen < $this->notSplitLen )

  551.             {

  552.                 $tmpstr = '';

  553.                 $lastType = 0;

  554.                 if( $spos > 0 ) $lastType = $this->simpleResult[$spos-1]['t'];

  555.                 if($slen < 5)

  556.                 {

  557.                       //echo iconv(UCS2, 'utf-8', $str).'<br/>';

  558.                       if( $lastType==4 && ( isset($this->addonDic['u'][$str]) || isset($this->addonDic['u'][substr($str, 0, 2)]) ) )

  559.                       {

  560.                               $str2 = '';

  561.                               if( !isset($this->addonDic['u'][$str]) && isset($this->addonDic['s'][substr($str, 2, 2)]) )

  562.                               {

  563.                                      $str2 = substr($str, 2, 2);

  564.                                      $str  = substr($str, 0, 2);

  565.                               }

  566.                               $ww = $this->simpleResult[$spos - 1]['w'].$str;

  567.                               $this->simpleResult[$spos - 1]['w'] = $ww;

  568.                               $this->simpleResult[$spos - 1]['t'] = 4;

  569.                               if( !isset($this->newWords[$this->simpleResult[$spos - 1]['w']]) )

  570.                               {

  571.                                      $this->foundWordStr .= $this->_out_string_encoding( $ww ).'/mu, ';

  572.                                      $this->SetWordInfos($ww, array('c'=>1, 'm'=>'mu'));

  573.                               }

  574.                               $this->simpleResult[$spos]['w'] = '';

  575.                               if( $str2 != '' )

  576.                               {

  577.                                      $this->finallyResult[$spos-1][] = $ww;

  578.                                      $this->finallyResult[$spos-1][] = $str2;

  579.                               }

  580.                        }

  581.                        else {

  582.                               $this->finallyResult[$spos][] = $str;

  583.                        }

  584.                 }

  585.                 else

  586.                 {

  587.                       $this->_deep_analysis_cn( $str, $ctype, $spos, $slen, $optimize );

  588.                 }

  589.             }

  590.             //正常长度的句子，循环进行分词处理

  591.             else

  592.             {

  593.                 $this->_deep_analysis_cn( $str, $ctype, $spos, $slen, $optimize );

  594.             }

  595.         }

  596.         //英文句子，转为小写

  597.         else

  598.         {

  599.             if( $this->toLower ) {

  600.                 $this->finallyResult[$spos][] = strtolower($str);

  601.             }

  602.             else {

  603.                 $this->finallyResult[$spos][] = $str;

  604.             }

  605.         }

  606.     }

  607.     

  608.     /**

  609.      * 中文的深入分词

  610.      * @parem $str

  611.      * @return void

  612.      */

  613.     function _deep_analysis_cn( &$str, $lastec, $spos, $slen, $optimize=TRUE )

  614.     {

  615.         $quote1 = chr(0x20).chr(0x1C);

  616.         $tmparr = array();

  617.         $hasw = 0;

  618.         //如果前一个词为 “ ， 并且字符串小于3个字符当成一个词处理。

  619.         if( $spos > 0 && $slen < 11 && $this->simpleResult[$spos-1]['w']==$quote1 )

  620.         {

  621.             $tmparr[] = $str;

  622.             if( !isset($this->newWords[$str]) )

  623.             {

  624.                 $this->foundWordStr .= $this->_out_string_encoding($str).'/nq, ';

  625.                 $this->SetWordInfos($str, array('c'=>1, 'm'=>'nq'));

  626.             }

  627.             if( !$this->differMax )

  628.             {

  629.                 $this->finallyResult[$spos][] = $str;

  630.                 return ;

  631.             }

  632.         }

  633.         //进行切分

  634.         for($i=$slen-1; $i > 0; $i -= 2)

  635.         {

  636.             //单个词

  637.             $nc = $str[$i-1].$str[$i];

  638.             //是否已经到最后两个字

  639.             if( $i <= 2 )

  640.             {

  641.                 $tmparr[] = $nc;

  642.                 $i = 0;

  643.                 break;

  644.             }

  645.             $isok = FALSE;

  646.             $i = $i + 1;

  647.             for($k=$this->dicWordMax; $k>1; $k=$k-2)

  648.             {

  649.                 if($i < $k) continue;

  650.                 $w = substr($str, $i-$k, $k);

  651.                 if( strlen($w) <= 2 )

  652.                 {

  653.                     $i = $i - 1;

  654.                     break;

  655.                 }

  656.                 if( $this->IsWord( $w ) )

  657.                 {

  658.                     $tmparr[] = $w;

  659.                     $i = $i - $k + 1;

  660.                     $isok = TRUE;

  661.                     break;

  662.                 }

  663.             }

  664.             //echo '<hr />';

  665.             //没适合词

  666.             if(!$isok) $tmparr[] = $nc;

  667.         }

  668.         $wcount = count($tmparr);

  669.         if( $wcount==0 ) return ;

  670.         $this->finallyResult[$spos] = array_reverse($tmparr);

  671.         //优化结果(岐义处理、新词、数词、人名识别等)

  672.         if( $optimize )

  673.         {

  674.             $this->_optimize_result( $this->finallyResult[$spos], $spos );

  675.         }

  676.     }

  677.     

  678.     /**

  679.     * 对最终分词结果进行优化（把simpleresult结果合并，并尝试新词识别、数词合并等）

  680.     * @parem $optimize 是否优化合并的结果

  681.     * @return bool

  682.     */

  683.     //t = 1 中/韩/日文, 2 英文/数字/符号('.', '@', '#', '+'), 3 ANSI符号 4 纯数字 5 非ANSI符号或不支持字符

  684.     function _optimize_result( &$smarr, $spos )

  685.     {

  686.         $newarr = array();

  687.         $prePos = $spos - 1;

  688.         $arlen = count($smarr);

  689.         $i = $j = 0;

  690.         //检测数量词

  691.         if( $prePos > -1 && !isset($this->finallyResult[$prePos]) )

  692.         {

  693.             $lastw = $this->simpleResult[$prePos]['w'];

  694.             $lastt = $this->simpleResult[$prePos]['t'];

  695.               if( ($lastt==4 || isset( $this->addonDic['c'][$lastw] )) && isset( $this->addonDic['u'][$smarr[0]] ) )

  696.               {

  697.                  $this->simpleResult[$prePos]['w'] = $lastw.$smarr[0];

  698.                  $this->simpleResult[$prePos]['t'] = 4;

  699.                  if( !isset($this->newWords[ $this->simpleResult[$prePos]['w'] ]) )

  700.                  {

  701.                      $this->foundWordStr .= $this->_out_string_encoding( $this->simpleResult[$prePos]['w'] ).'/mu, ';

  702.                      $this->SetWordInfos($this->simpleResult[$prePos]['w'], array('c'=>1, 'm'=>'mu'));

  703.                  }

  704.                  $smarr[0] = '';

  705.                  $i++;

  706.               }

  707.        }

  708.        for(; $i < $arlen; $i++)

  709.        {

  710.             

  711.             if( !isset( $smarr[$i+1] ) )

  712.             {

  713.                 $newarr[$j] = $smarr[$i];

  714.                 break;

  715.             }

  716.             $cw = $smarr[$i];

  717.             $nw = $smarr[$i+1];

  718.             $ischeck = FALSE;

  719.             //检测数量词

  720.             if( isset( $this->addonDic['c'][$cw] ) && isset( $this->addonDic['u'][$nw] ) )

  721.             {

  722.                 //最大切分时保留合并前的词

  723.                 if($this->differMax)

  724.                 {

  725.                         $newarr[$j] = chr(0).chr(0x28);

  726.                         $j++;

  727.                         $newarr[$j] = $cw;

  728.                         $j++;

  729.                         $newarr[$j] = $nw;

  730.                         $j++;

  731.                         $newarr[$j] = chr(0).chr(0x29);

  732.                         $j++;

  733.                 }

  734.                 $newarr[$j] = $cw.$nw;

  735.                 if( !isset($this->newWords[$newarr[$j]]) )

  736.                 {

  737.                     $this->foundWordStr .= $this->_out_string_encoding( $newarr[$j] ).'/mu, ';

  738.                     $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'mu'));

  739.                 }

  740.                 $j++; $i++; $ischeck = TRUE;

  741.             }

  742.             //检测前导词(通常是姓)

  743.             else if( isset( $this->addonDic['n'][ $smarr[$i] ] ) )

  744.             {

  745.                 $is_rs = FALSE;

  746.                 //词语是副词或介词或频率很高的词不作为人名

  747.                 if( strlen($nw)==4 )

  748.                 {

  749.                     $winfos = $this->GetWordInfos($nw);

  750.                     if(isset($winfos['m']) && ($winfos['m']=='r' || $winfos['m']=='c' || $winfos['c']>500) )

  751.                     {

  752.                          $is_rs = TRUE;

  753.                     }

  754.                 }

  755.                 if( !isset($this->addonDic['s'][$nw]) && strlen($nw)<5 && !$is_rs )

  756.                 {

  757.                     $newarr[$j] = $cw.$nw;

  758.                     //echo iconv(UCS2, 'utf-8', $newarr[$j])."<br />";

  759.                     //尝试检测第三个词

  760.                     if( strlen($nw)==2 && isset($smarr[$i+2]) && strlen($smarr[$i+2])==2 && !isset( $this->addonDic['s'][$smarr[$i+2]] ) )

  761.                     {

  762.                         $newarr[$j] .= $smarr[$i+2];

  763.                         $i++;

  764.                     }

  765.                     if( !isset($this->newWords[$newarr[$j]]) )

  766.                     {

  767.                         $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'nr'));

  768.                         $this->foundWordStr .= $this->_out_string_encoding($newarr[$j]).'/nr, ';

  769.                     }

  770.                     //为了防止错误，保留合并前的姓名

  771.                     if(strlen($nw)==4)

  772.                     {

  773.                         $j++;

  774.                         $newarr[$j] = chr(0).chr(0x28);

  775.                         $j++;

  776.                         $newarr[$j] = $cw;

  777.                         $j++;

  778.                         $newarr[$j] = $nw;

  779.                         $j++;

  780.                         $newarr[$j] = chr(0).chr(0x29);

  781.                     }

  782.                     

  783.                     $j++; $i++; $ischeck = TRUE;

  784.                 }

  785.             }

  786.             //检测后缀词(地名等)

  787.             else if( isset($this->addonDic['a'][$nw]) )

  788.             {

  789.                 $is_rs = FALSE;

  790.                 //词语是副词或介词不作为前缀

  791.                 if( strlen($cw)>2 )

  792.                 {

  793.                     $winfos = $this->GetWordInfos($cw);

  794.                     if(isset($winfos['m']) && ($winfos['m']=='a' || $winfos['m']=='r' || $winfos['m']=='c' || $winfos['c']>500) )

  795.                     {

  796.                          $is_rs = TRUE;

  797.                     }

  798.                 }

  799.                 if( !isset($this->addonDic['s'][$cw]) && !$is_rs )

  800.                 {

  801.                     $newarr[$j] = $cw.$nw;

  802.                     if( !isset($this->newWords[$newarr[$j]]) )

  803.                     {

  804.                         $this->foundWordStr .= $this->_out_string_encoding($newarr[$j]).'/na, ';

  805.                         $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'na'));

  806.                     }

  807.                     $i++; $j++; $ischeck = TRUE;

  808.                 }

  809.             }

  810.             //新词识别（暂无规则）

  811.             else if($this->unitWord)

  812.             {

  813.                 if(strlen($cw)==2 && strlen($nw)==2 

  814.                 && !isset($this->addonDic['s'][$cw]) && !isset($this->addonDic['t'][$cw]) && !isset($this->addonDic['a'][$cw]) 

  815.                 && !isset($this->addonDic['s'][$nw]) && !isset($this->addonDic['c'][$nw]))

  816.                 {

  817.                     $newarr[$j] = $cw.$nw;

  818.                     //尝试检测第三个词

  819.                     if( isset($smarr[$i+2]) && strlen($smarr[$i+2])==2 && (isset( $this->addonDic['a'][$smarr[$i+2]] ) || isset( $this->addonDic['u'][$smarr[$i+2]] )) )

  820.                     {

  821.                         $newarr[$j] .= $smarr[$i+2];

  822.                         $i++;

  823.                     }

  824.                     if( !isset($this->newWords[$newarr[$j]]) )

  825.                     {

  826.                         $this->foundWordStr .= $this->_out_string_encoding($newarr[$j]).'/ms, ';

  827.                         $this->SetWordInfos($newarr[$j], array('c'=>1, 'm'=>'ms'));

  828.                     }

  829.                     $i++; $j++; $ischeck = TRUE;

  830.                 }

  831.             }

  832.             

  833.             //不符合规则

  834.             if( !$ischeck )

  835.             {

  836.                 $newarr[$j] = $cw;

  837.                   //二元消岐处理——最大切分模式

  838.                 if( $this->differMax && !isset($this->addonDic['s'][$cw]) && strlen($cw) < 5 && strlen($nw) < 7)

  839.                 {

  840.                     $slen = strlen($nw);

  841.                     $hasDiff = FALSE;

  842.                     for($y=2; $y <= $slen-2; $y=$y+2)

  843.                     {

  844.                         $nhead = substr($nw, $y-2, 2);

  845.                         $nfont = $cw.substr($nw, 0, $y-2);

  846.                         if( $this->IsWord( $nfont.$nhead ) )

  847.                         {

  848.                             if( strlen($cw) > 2 ) $j++;

  849.                             $hasDiff = TRUE;

  850.                             $newarr[$j] = $nfont.$nhead;

  851.                         }

  852.                     }

  853.                 }

  854.                 $j++;

  855.             }

  856.             

  857.        }//end for

  858.        $smarr =  $newarr;

  859.     }

  860.     

  861.     /**

  862.     * 转换最终分词结果到 finallyResult 数组

  863.     * @return void

  864.     */

  865.     function _sort_finally_result()

  866.     {

  867.           $newarr = array();

  868.         $i = 0;

  869.         foreach($this->simpleResult as $k=>$v)

  870.         {

  871.             if( empty($v['w']) ) continue;

  872.             if( isset($this->finallyResult[$k]) && count($this->finallyResult[$k]) > 0 )

  873.             {

  874.                 foreach($this->finallyResult[$k] as $w)

  875.                 {

  876.                     if(!empty($w))

  877.                     {

  878.                         $newarr[$i]['w'] = $w;

  879.                         $newarr[$i]['t'] = 20;

  880.                         $i++;

  881.                     }

  882.                 }

  883.             }

  884.             else if($v['t'] != 21)

  885.             {

  886.                 $newarr[$i]['w'] = $v['w'];

  887.                 $newarr[$i]['t'] = $v['t'];

  888.                 $i++;

  889.             }

  890.         }

  891.         $this->finallyResult = $newarr;

  892.         $newarr = '';

  893.       }

  894.     

  895.     /**

  896.      * 把uncode字符串转换为输出字符串

  897.      * @parem str

  898.      * return string

  899.      */

  900.      function _out_string_encoding( &$str )

  901.      {

  902.         $rsc = $this->_source_result_charset();

  903.         if( $rsc==1 ) {

  904.             $rsstr = iconv(UCS2, 'utf-8', $str);

  905.         }

  906.         else if( $rsc==2 ) {

  907.             $rsstr = iconv('utf-8', 'gb18030', iconv(UCS2, 'utf-8', $str) );

  908.         }

  909.         else{

  910.             $rsstr = iconv('utf-8', 'big5', iconv(UCS2, 'utf-8', $str) );

  911.         }

  912.         return $rsstr;

  913.      }

  914.     

  915.     /**

  916.      * 获取最终结果字符串（用空格分开后的分词结果）

  917.      * @return string

  918.      */

  919.      function GetFinallyResult($spword=' ', $word_meanings=FALSE)

  920.      {

  921.         $rsstr = '';

  922.         foreach($this->finallyResult as $v)

  923.         {

  924.             if( $this->resultType==2 && ($v['t']==3 || $v['t']==5) )

  925.             {

  926.                 continue;

  927.             }

  928.             $m = '';

  929.             if( $word_meanings )

  930.             {

  931.                 $m = $this->GetWordProperty($v['w']);

  932.             }

  933.             $w = $this->_out_string_encoding($v['w']);

  934.             if( $w != ' ' )

  935.             {

  936.                 if($word_meanings) {

  937.                     $rsstr .= $spword.$w.$m;

  938.                 }

  939.                 else {

  940.                     $rsstr .= $spword.$w;

  941.                 }

  942.             }

  943.         }

  944.         return $rsstr;

  945.      }

  946.      

  947.     /**

  948.      * 获取粗分结果，不包含粗分属性

  949.      * @return array()

  950.      */

  951.      function GetSimpleResult()

  952.      {

  953.         $rearr = array();

  954.         foreach($this->simpleResult as $k=>$v)

  955.         {

  956.             if( empty($v['w']) ) continue;

  957.             $w = $this->_out_string_encoding($v['w']);

  958.             if( $w != ' ' ) $rearr[] = $w;

  959.         }

  960.         return $rearr;

  961.      }

  962.      

  963.     /**

  964.      * 获取粗分结果，包含粗分属性（1中文词句、2 ANSI词汇（包括全角），3 ANSI标点符号（包括全角），4数字（包括全角），5 中文标点或无法识别字符）

  965.      * @return array()

  966.      */

  967.      function GetSimpleResultAll()

  968.      {

  969.         $rearr = array();

  970.         foreach($this->simpleResult as $k=>$v)

  971.         {

  972.             $w = $this->_out_string_encoding($v['w']);

  973.             if( $w != ' ' )

  974.             {

  975.                 $rearr[$k]['w'] = $w;

  976.                 $rearr[$k]['t'] = $v['t'];

  977.             }

  978.         }

  979.         return $rearr;

  980.      }

  981.      

  982.     /**

  983.      * 获取索引hash数组

  984.      * @return array('word'=>count,...)

  985.      */

  986.      function GetFinallyIndex()

  987.      {

  988.         $rearr = array();

  989.         foreach($this->finallyResult as $v)

  990.         {

  991.             if( $this->resultType==2 && ($v['t']==3 || $v['t']==5) )

  992.             {

  993.                 continue;

  994.             }

  995.             $w = $this->_out_string_encoding($v['w']);

  996.             if( $w == ' ' )

  997.             {

  998.                 continue;

  999.             }

  1000.             if( isset($rearr[$w]) )

  1001.             {

  1002.                  $rearr[$w]++;

  1003.             }

  1004.             else

  1005.             {

  1006.                  $rearr[$w] = 1;

  1007.             }

  1008.         }

  1009.         return $rearr;

  1010.      }

  1011.      

  1012.     /**

  1013.      * 获得保存目标编码

  1014.      * @return int

  1015.      */

  1016.      function _source_result_charset()

  1017.      {

  1018.         if( preg_match("/^utf/", $this->targetCharSet) ) {

  1019.            $rs = 1;

  1020.         }

  1021.         else if( preg_match("/^gb/", $this->targetCharSet) ) {

  1022.            $rs = 2;

  1023.         }

  1024.         else if( preg_match("/^big/", $this->targetCharSet) ) {

  1025.            $rs = 3;

  1026.         }

  1027.         else {

  1028.             $rs = 4;

  1029.         }

  1030.         return $rs;

  1031.      }

  1032.      

  1033.      /**

  1034.      * 编译词典

  1035.      * @parem $sourcefile utf-8编码的文本词典数据文件<参见范例dict/not-build/base_dic_full.txt>

  1036.      * 注意, 需要PHP开放足够的内存才能完成操作

  1037.      * @return void

  1038.      */

  1039.      function MakeDict( $source_file, $target_file='' )

  1040.      {

  1041.         $target_file = ($target_file=='' ? $this->mainDicFile : $target_file);

  1042.         $allk = array();

  1043.         $fp = fopen($source_file, 'r');

  1044.         while( $line = fgets($fp, 512) )

  1045.         {

  1046.             if( $line[0]=='@' ) continue;

  1047.             list($w, $r, $a) = explode(',', $line);

  1048.             $a = trim( $a );

  1049.             $w = iconv('utf-8', UCS2, $w);

  1050.             $k = $this->_get_index( $w );

  1051.             if( isset($allk[ $k ]) )

  1052.                 $allk[ $k ][ $w ] = array($r, $a);

  1053.             else

  1054.                 $allk[ $k ][ $w ] = array($r, $a);

  1055.         }

  1056.         fclose( $fp );

  1057.         $fp = fopen($target_file, 'w');

  1058.         $heade_rarr = array();

  1059.         $alldat = '';

  1060.         $start_pos = $this->mask_value * 8;

  1061.         foreach( $allk as $k => $v )

  1062.         {

  1063.             $dat  = serialize( $v );

  1064.             $dlen = strlen($dat);

  1065.             $alldat .= $dat;

  1066.         

  1067.             $heade_rarr[ $k ][0] = $start_pos;

  1068.             $heade_rarr[ $k ][1] = $dlen;

  1069.             $heade_rarr[ $k ][2] = count( $v );

  1070.         

  1071.             $start_pos += $dlen;

  1072.         }

  1073.         unset( $allk );

  1074.         for($i=0; $i < $this->mask_value; $i++)

  1075.         {

  1076.             if( !isset($heade_rarr[$i]) )

  1077.             {

  1078.                 $heade_rarr[$i] = array(0, 0, 0);

  1079.             }

  1080.             fwrite($fp, pack("Inn", $heade_rarr[$i][0], $heade_rarr[$i][1], $heade_rarr[$i][2]));

  1081.         }

  1082.         fwrite( $fp, $alldat);

  1083.         fclose( $fp );

  1084.      }

  1085.      

  1086.      /**

  1087.      * 导出词典的词条

  1088.      * @parem $targetfile 保存位置

  1089.      * @return void

  1090.      */

  1091.      function ExportDict( $targetfile )

  1092.      {

  1093.         if( !$this->mainDicHand )

  1094.         {

  1095.             $this->mainDicHand = fopen($this->mainDicFile, 'rw');

  1096.         }

  1097.         $fp = fopen($targetfile, 'w');

  1098.         for($i=0; $i <= $this->mask_value; $i++)

  1099.         {

  1100.             $move_pos = $i * 8;

  1101.             fseek($this->mainDicHand, $move_pos, SEEK_SET);

  1102.             $dat = fread($this->mainDicHand, 8);

  1103.             $arr = unpack('I1s/n1l/n1c', $dat);

  1104.             if( $arr['l'] == 0 )

  1105.             {

  1106.                 continue;

  1107.             }

  1108.             fseek($this->mainDicHand, $arr['s'], SEEK_SET);

  1109.             $data = @unserialize(fread($this->mainDicHand, $arr['l']));

  1110.             if( !is_array($data) ) continue;

  1111.             foreach($data as $k => $v)

  1112.             {

  1113.                 $w = iconv(UCS2, 'utf-8', $k);

  1114.                 fwrite($fp, "{$w},{$v[0]},{$v[1]}\n");

  1115.             }

  1116.         }

  1117.         fclose( $fp );

  1118.         return TRUE;

  1119.      }

  1120.      

  1121. 	function InportDict( $targetfile )

  1122.      {

  1123.      	if(!ini_set('memory_limit', '128M'))

  1124. 			exit('设置内存错误，请到dede官网下载解压版的base_dic_full.dic!');

  1125.      	require_once(DEDEINC.'/zip.class.php');

  1126.      	$zip = new zip();

  1127.      	//echo $targetfile;

  1128.      	$unpackagefile = array_keys($zip->Extract($targetfile,DEDEINC.'/data/'));

  1129.      	//exit();

  1130.      	$this->MakeDict(DEDEINC.'/data/'.$unpackagefile[0]);

  1131.      	unlink(DEDEINC.'/data/'.$unpackagefile[0]);

  1132.      	return true;

  1133.      }

  1134. }