DedeBIZ
/
DedeV6
Watch 2
Star 0
Fork 0
Code Pull Requests 0 Releases 25 Activity
国内流行的内容管理系统（CMS）多端全媒体解决方案 https://www.dedebiz.com
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
177 Commits
3 Branches
110MB
Tree: 814d406b07
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '814d406b07'
${ noResults }
DedeV6/src/include/dedecollection.class.php

1212 lines
41KB
Raw Blame History 

  1. <?php   if(!defined('DEDEINC')) exit('dedecms');

  2. /**

  3.  * Dede采集类

  4.  *

  5.  * @version        $Id: dedecollection.class.php 1 20:20 2010年7月7日Z tianya $

  6.  * @package        DedeCMS.Libraries

  7.  * @copyright      Copyright (c) 2020, DedeBIZ.COM

  8.  * @license        https://www.dedebiz.com/license/v6

  9.  * @link           https://www.dedebiz.com

  10.  */

  11. 

  12. require_once(DEDEINC."/dedecollection.func.php"); //采集扩展函数

  13. require_once(DEDEINC."/image.func.php");

  14. require_once(DEDEINC."/dedehtml2.class.php");

  15. @set_time_limit(0);

  16. 

  17. /**

  18.  * Dede采集类

  19.  *

  20.  * @package          DedeCollection

  21.  * @subpackage       DedeCMS.Libraries

  22.  * @link             https://www.dedebiz.com

  23.  */

  24. class DedeCollection

  25. {

  26.     var $artNotes = array();  //文章采集的字段信息

  27.     var $spNotes = array();  //文章采集的字段信息

  28.     var $lists = array();     //采集节点的来源列表处理信息

  29.     var $noteInfos = array(); //采集节点的基本配置信息

  30.     var $dsql = '';

  31.     var $noteId = '';

  32.     var $cDedeHtml = '';

  33.     var $cHttpDown = '';

  34.     var $mediaCount = 0;

  35.     var $tmpUnitValue = '';

  36.     var $tmpLinks = array();

  37.     var $tmpHtml = '';

  38.     var $breImage = '';

  39.     var $errString = '';

  40. 

  41.     //兼容php5构造函数

  42.     function __construct()

  43.     {

  44.         $this->dsql = $GLOBALS['dsql'];

  45.         $this->cHttpDown = new DedeHttpDown();

  46.         $this->cDedeHtml = new DedeHtml2();

  47.     }

  48. 

  49.     function DedeCollection()

  50.     {

  51.         $this->__construct();

  52.     }

  53. 

  54.     //析放资源

  55.     function Close()

  56.     {

  57.     }

  58. 

  59.     /**

  60.      *  从数据库里载入某个节点

  61.      *

  62.      * @access    public

  63.      * @param     int   $nid  采集节点ID

  64.      * @return    void

  65.      */

  66.     function LoadNote($nid)

  67.     {

  68.         $this->noteId = $nid;

  69.         $row = $this->dsql->GetOne("SELECT * FROM `#@__co_note` WHERE nid='$nid'");

  70.         $this->LoadListConfig($row['listconfig']);

  71.         $this->LoadItemConfig($row['itemconfig']);

  72.     }

  73. 

  74.     /**

  75.      *  分析基本节点的及索引配置信息

  76.      *

  77.      * @access    public

  78.      * @param     string  $configString  配置字符串

  79.      * @return    void

  80.      */

  81.     function LoadListConfig($configString)

  82.     {

  83.         $dtp = new DedeTagParse();

  84.         $dtp2 = new DedeTagParse();

  85.         $dtp->LoadString($configString);

  86.         for($i=0; $i<=$dtp->Count; $i++)

  87.         {

  88.             $ctag = $dtp->CTags[$i];

  89. 

  90.             //item 配置

  91.             //节点基本信息

  92.             if($ctag->GetName()=="noteinfo")

  93.             {

  94.                 $this->noteInfos['notename'] = $ctag->GetAtt('notename');

  95.                 $this->noteInfos['matchtype'] = $ctag->GetAtt('matchtype');

  96.                 $this->noteInfos['channelid'] = $ctag->GetAtt('channelid');

  97.                 $this->noteInfos['refurl'] = $ctag->GetAtt('refurl');

  98.                 $this->noteInfos['sourcelang'] = $ctag->GetAtt('sourcelang');

  99.                 $this->noteInfos['cosort'] = $ctag->GetAtt('cosort');

  100.                 $this->noteInfos['isref'] = $ctag->GetAtt('isref');

  101.                 $this->noteInfos['exptime'] = $ctag->GetAtt('exptime');

  102.             }

  103. 

  104.             //list 配置

  105.             //要采集的列表页的信息

  106.             else if($ctag->GetName()=="listrule")

  107.             {

  108.                 $this->lists['sourcetype'] = $ctag->GetAtt('sourcetype');

  109.                 $this->lists['rssurl'] = $ctag->GetAtt('rssurl');

  110.                 $this->lists['regxurl'] = $ctag->GetAtt('regxurl');

  111.                 $this->lists['startid'] = $ctag->GetAtt('startid');

  112.                 $this->lists['endid'] = $ctag->GetAtt('endid');

  113.                 $this->lists['addv'] = $ctag->GetAtt('addv');

  114.                 $this->lists['urlrule'] = $ctag->GetAtt('urlrule');

  115.                 $this->lists['musthas'] = $ctag->GetAtt('musthas');

  116.                 $this->lists['nothas'] = $ctag->GetAtt('nothas');

  117.                 $this->lists['listpic'] = $ctag->GetAtt('listpic');

  118.                 $this->lists['usemore'] =  $ctag->GetAtt('usemore');

  119.                 $dtp2->LoadString($ctag->GetInnerText());

  120.                 for($j=0; $j<=$dtp2->Count; $j++)

  121.                 {

  122.                     $ctag2 = $dtp2->CTags[$j];

  123.                     $tname = $ctag2->GetName();

  124.                     if($tname=='addurls')

  125.                     {

  126.                         $this->lists['addurls'] = trim($ctag2->GetInnerText());

  127.                     }

  128.                     else if($tname=='regxrule')

  129.                     {

  130.                         $this->lists['regxrule'] = trim($ctag2->GetInnerText());

  131.                     }

  132.                     else if($tname=='areastart')

  133.                     {

  134.                         $this->lists['areastart'] = trim($ctag2->GetInnerText());

  135.                     }

  136.                     else if($tname=='areaend')

  137.                     {

  138.                         $this->lists['areaend'] = trim($ctag2->GetInnerText());

  139.                     }

  140.                     else if($tname=='batchrule')

  141.                     {

  142.                         $this->lists['batchrule'] = trim($ctag2->GetInnerText());

  143.                     }

  144.                 }

  145. 

  146.                 //分析列表网址

  147.                 if($this->lists['sourcetype'] != 'rss')

  148.                 {

  149.                     $this->lists['url'] = GetUrlFromListRule($this->lists['regxurl'],$this->lists['addurls'],

  150.                     $this->lists['startid'],$this->lists['endid'],$this->lists['addv'],$this->lists['usemore'],$this->lists['batchrule']);

  151.                 }

  152.                 else

  153.                 {

  154.                     $this->lists['url'] = $this->lists['rssurl'];

  155.                 }

  156.             }

  157.         }//End Loop

  158. 

  159.         $dtp->Clear();

  160.         $dtp2->Clear();

  161.     }

  162. 

  163.     /**

  164.      *  分析采集文章页的字段的设置

  165.      *

  166.      * @access    public

  167.      * @param     string  $configString  配置字符串

  168.      * @return    void

  169.      */

  170.     function LoadItemConfig($configString)

  171.     {

  172.         $dtp = new DedeTagParse();

  173.         $dtp2 = new DedeTagParse();

  174.         $dtp->LoadString($configString);

  175.         for($i=0; $i<=$dtp->Count; $i++)

  176.         {

  177.             $ctag = $dtp->CTags[$i];

  178.             if($ctag->GetName()=='sppage')

  179.             {

  180.                 $this->artNotes['sppage'] = $ctag->GetInnerText();

  181.                 $this->artNotes['sptype'] = $ctag->GetAtt('sptype');

  182.                 $this->spNotes['srul'] = $ctag->GetAtt('srul');

  183.                 $this->spNotes['erul'] = $ctag->GetAtt('erul');

  184.             }

  185.             else if($ctag->GetName()=='previewurl')

  186.             {

  187.                 $this->artNotes['previewurl'] = $ctag->GetInnerText();

  188.             }

  189.             else if($ctag->GetName()=='keywordtrim')

  190.             {

  191.                 $this->artNotes['keywordtrim'] = $ctag->GetInnerText();

  192.             }

  193.             else if($ctag->GetName()=='descriptiontrim')

  194.             {

  195.                 $this->artNotes['descriptiontrim'] = $ctag->GetInnerText();

  196.             }

  197.             else if($ctag->GetName()=='item')

  198.             {

  199.                 $field = $ctag->GetAtt('field');

  200.                 if($field == '')

  201.                 {

  202.                     continue;

  203.                 }

  204.                 $this->artNotes[$field]['value'] = $ctag->GetAtt('value');

  205.                 $this->artNotes[$field]['isunit'] = $ctag->GetAtt('isunit');

  206.                 $this->artNotes[$field]['isdown'] = $ctag->GetAtt('isdown');

  207.                 $this->artNotes[$field]['trim'] = array();

  208.                 $this->artNotes[$field]['match'] = '';

  209.                 $this->artNotes[$field]['function'] = '';

  210.                 $t = 0;

  211.                 $dtp2->LoadString($ctag->GetInnerText());

  212.                 for($k=0; $k<=$dtp2->Count; $k++)

  213.                 {

  214.                     $ctag2 = $dtp2->CTags[$k];

  215.                     if($ctag2->GetName()=='trim')

  216.                     {

  217.                         $this->artNotes[$field]['trim'][$t][0] = str_replace('#n#','&nbsp;',$ctag2->GetInnerText());

  218.                         $this->artNotes[$field]['trim'][$t][1] = $ctag2->GetAtt('replace');

  219.                         $t++;

  220.                     }

  221.                     else if($ctag2->GetName()=='match')

  222.                     {

  223.                         $this->artNotes[$field]['match'] = str_replace('#n#','&nbsp;',$ctag2->GetInnerText());

  224.                     }

  225.                     else if($ctag2->GetName()=='function')

  226.                     {

  227.                         $this->artNotes[$field]['function'] = $ctag2->GetInnerText();

  228.                     }

  229.                 }

  230.             }

  231.         }//End Loop

  232. 

  233.         $dtp->Clear();

  234.         $dtp2->Clear();

  235.     }

  236. 

  237.     /**

  238.      *  下载其中一个网址，并保存

  239.      *

  240.      * @access    public

  241.      * @param     int  $aid  文档ID

  242.      * @param     string  $dourl  操作地址

  243.      * @param     string  $litpic  缩略图

  244.      * @param     bool  $issave  是否保存

  245.      * @return    string

  246.      */

  247.     function DownUrl($aid, $dourl, $litpic='', $issave=TRUE)

  248.     {

  249.         $this->tmpLinks = array();

  250.         $this->tmpUnitValue = '';

  251.         $this->breImage = '';

  252.         $this->tmpHtml = $this->DownOnePage($dourl);

  253. 

  254.         //检测是否有分页字段，并预先处理

  255.         if(!empty($this->artNotes['sppage']))

  256.         {

  257.             $noteid = '';

  258.             foreach($this->artNotes as $k=>$sarr)

  259.             {

  260.                 if(isset($sarr['isunit']) && $sarr['isunit']==1)

  261.                 {

  262.                     $noteid = $k;

  263.                     break;

  264.                 }

  265.             }

  266.             

  267.             $this->GetSpPage($dourl, $noteid, $this->tmpHtml);

  268. 

  269.             if(preg_match("/#p#/i", $this->tmpUnitValue))

  270.             {

  271.                 if ($this->artNotes["sptype"] != 'diyrule')

  272.                 {

  273.                     $this->tmpUnitValue = '副标题#e#'.$this->tmpUnitValue;

  274.                 }

  275.             }

  276.         }

  277. 

  278.         //处理字段

  279.         $body = $this->GetPageFields($dourl, $issave, $litpic);

  280. 

  281.         //保存资料到数据库

  282.         if($issave)

  283.         {

  284.             $query = " UPDATE `#@__co_htmls` SET dtime='".time()."',result='".addslashes($body)."',isdown='1' WHERE aid='$aid' ";

  285.             if(!$this->dsql->ExecuteNoneQuery($query))

  286.             {

  287.                 echo $this->dsql->GetError();

  288.             }

  289.             return $body;

  290.         }

  291.         return $body;

  292.     }

  293.     

  294.     // 解析地址

  295.     function GetUrl($uri)

  296.     {

  297.         $arr = $tmp = array();

  298. 

  299.         // query

  300.         $x = array_pad( explode( '?', $uri ), 2, false );

  301.         $arr['query'] = ( $x[1] )? $x[1] : '' ;

  302. 

  303.         // resource

  304.         $x         = array_pad( explode( '/', $x[0] ), 2, false );

  305.         $x_last = array_pop( $x );

  306.         if( strpos( $x_last, '.' ) === false )

  307.         {

  308.             $arr['resource'] = '';

  309.             $x[] = $x_last;

  310.         }

  311.         else

  312.         {

  313.             $arr['resource'] = $x_last;

  314.             $tmp = @explode('.', $arr['resource']);

  315.             $arr['file'] = @$tmp[0];

  316.             $arr['ext'] = '.'.@$tmp[1];

  317.         }

  318. 

  319.         // path    

  320.         $arr['path'] = implode( '/', $x );

  321.         if( substr( $arr['path'], -1 ) !== '/' ) $arr['path'] .= '/';

  322. 

  323.         // url

  324.         $arr['url'] = $uri;

  325. 

  326.         return $arr;

  327.     }

  328. 

  329.     /**

  330.      *  获取分页区域的内容

  331.      *

  332.      * @access    public

  333.      * @param     string  $dourl  操作地址

  334.      * @param     string  $noteid  节点ID

  335.      * @param     string  $html  html内容

  336.      * @param     int  $step  步骤

  337.      * @return    string

  338.      */

  339.     function GetSpPage($dourl, $noteid, $html, $step=0)

  340.     {

  341.         $sarr = $this->artNotes[$noteid];

  342.         

  343.         $linkareaHtml = $this->GetHtmlArea('[内容]', $this->artNotes['sppage'], $html);

  344.         if($linkareaHtml=='')

  345.         {

  346.             if($this->tmpUnitValue=='')

  347.             {

  348.                 $this->tmpUnitValue .= $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  349.             }

  350.             else

  351.             {

  352.                 $this->tmpUnitValue .= "#p#副标题#e#".$this->GetHtmlArea('[内容]', $sarr['match'], $html);

  353.             }

  354.             if ($this->artNotes["sptype"] != 'diyrule') return;

  355.             

  356.         }

  357.         

  358. 

  359.         //完整的分页列表

  360.         if($this->artNotes["sptype"]=='full' || $this->artNotes["sptype"]=='')

  361.         {

  362.             $this->tmpUnitValue .= $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  363.             $this->cDedeHtml->GetLinkType = "link";

  364.             $this->cDedeHtml->SetSource($linkareaHtml, $dourl, 'link');

  365.             foreach($this->cDedeHtml->Links as $k=>$t)

  366.             {

  367.                 $k = $this->cDedeHtml->FillUrl($k);

  368.                 if($k==$dourl)

  369.                 {

  370.                     continue;

  371.                 }

  372.                 $nhtml = $this->DownOnePage($k);

  373.                 if($nhtml!='')

  374.                 {

  375.                     $ct = trim($this->GetHtmlArea('[内容]', $sarr['match'], $nhtml));

  376.                     if($ct!='')

  377.                     {

  378.                         $this->tmpUnitValue .= "#p#副标题#e#".$ct;

  379.                     }

  380.                 }

  381.             }

  382.         } 

  383.         else if ($this->artNotes["sptype"] == 'diyrule')

  384.         {

  385.             $maxpage = 10;

  386.             $urlinfo = $this->GetUrl($dourl);

  387.             $testurl = str_replace(array_keys($urlinfo), array_values($urlinfo), $this->artNotes['sppage']);

  388.             $testurl = str_ireplace('{p}', '~p~', $testurl);

  389.             $testurl = str_replace(array('{', '}'), '', $testurl);

  390.             $lastchash = md5($html);

  391.             for($i=$this->spNotes['srul']; $i <= $this->spNotes['erul']; $i++)

  392.             {

  393.                 $tempurl = str_replace('~p~', $i, $testurl);

  394.                 $tempurl = $this->cDedeHtml->FillUrl($tempurl);

  395. 

  396.                 $nhtml = $this->DownOnePage($tempurl);

  397.                 $newchash = md5($nhtml);

  398.                 if ($newchash == $lastchash) continue;

  399.                 $lastchash = $newchash;

  400.                 

  401.                 if($nhtml!='')

  402.                 {

  403.                     $ct = trim($this->GetHtmlArea('[内容]', $sarr['match'], $nhtml));

  404.                     if($ct!='')

  405.                     {

  406.                         $this->tmpUnitValue .= "#p#副标题#e#".$ct;

  407.                         // echo $this->tmpUnitValue;exit;

  408.                     }

  409.                 }

  410.             }

  411.         }

  412.         //上下页形式或不完整的分页列表

  413.         else

  414.         {

  415.             if($step>50)

  416.             {

  417.                 return;

  418.             }

  419.             if($step==0)

  420.             {

  421.                 $this->tmpUnitValue .= $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  422.             }

  423.             $this->cDedeHtml->GetLinkType = "link";

  424.             $this->cDedeHtml->SetSource($linkareaHtml, $dourl, 'link');

  425.             $hasLink = FALSE;

  426.             foreach($this->cDedeHtml->Links as $k=>$t)

  427.             {

  428.                 $k = $this->cDedeHtml->FillUrl($k);

  429.                 if(in_array($k, $this->tmpLinks))

  430.                 {

  431.                     CONTINUE;

  432.                 }

  433.                 else{

  434.                     $nhtml = $this->DownOnePage($k);

  435.                     if($nhtml!='')

  436.                     {

  437.                         $ct = trim($this->GetHtmlArea('[内容]',$sarr['match'],$nhtml));

  438.                         if($ct!='')

  439.                         {

  440.                             $this->tmpUnitValue .= "#p#副标题#e#".$ct;

  441.                         }

  442.                     }

  443.                     $hasLink = TRUE;

  444.                     $this->tmpLinks[] = $k;

  445.                     $dourl = $k;

  446.                     $step++;

  447.                 }

  448.             }

  449.             if($hasLink)

  450.             {

  451.                 $this->GetSpPage($dourl, $noteid, $nhtml, $step);

  452.             }

  453.         }

  454.     }

  455. 

  456.     /**

  457.      *  获取特定区域的HTML

  458.      *

  459.      * @access    public

  460.      * @param     string  $sptag  区域标记

  461.      * @param     string  $areaRule  地址规则

  462.      * @param     string  $html  html代码

  463.      * @return    string

  464.      */

  465.     function GetHtmlArea($sptag, &$areaRule, &$html)

  466.     {

  467.         //用正则表达式的模式匹配

  468.         if($this->noteInfos['matchtype']=='regex')

  469.         {

  470.             $areaRule = str_replace("/", "\\/", $areaRule);

  471.             $areaRules = explode($sptag, $areaRule);

  472.             $arr = array();

  473.             if($html==''||$areaRules[0]=='')

  474.             {

  475.                 return '';

  476.             }

  477.             preg_match('#'.$areaRules[0]."(.*)".$areaRules[1]."#isU", $html, $arr);

  478.             return empty($arr[1]) ? '' : trim($arr[1]);

  479.         }

  480. 

  481.         //用字符串模式匹配

  482.         else

  483.         {

  484.             $areaRules = explode($sptag,$areaRule);

  485.             if($html=='' || $areaRules[0]=='')

  486.             {

  487.                 return '';

  488.             }

  489.             $posstart = @strpos($html,$areaRules[0]);

  490.             if($posstart===FALSE)

  491.             {

  492.                 return '';

  493.             }

  494.             $posstart = $posstart + strlen($areaRules[0]);

  495.             $posend = @strpos($html,$areaRules[1],$posstart);

  496.             if($posend > $posstart && $posend!==FALSE)

  497.             {

  498.                 //return substr($html,$posstart+strlen($areaRules[0]),$posend-$posstart-strlen($areaRules[0]));

  499.                 return substr($html,$posstart,$posend-$posstart);

  500.             }

  501.             else

  502.             {

  503.                 return '';

  504.             }

  505.         }

  506.     }

  507. 

  508.     /**

  509.      *  下载指定网址

  510.      *

  511.      * @access    public

  512.      * @param     string  $dourl  下载地址

  513.      */

  514.     function DownOnePage($dourl)

  515.     {

  516.         $this->cHttpDown->OpenUrl($dourl);

  517.         $html = $this->cHttpDown->GetHtml();

  518.         $this->cHttpDown->Close();

  519.         $this->ChangeCode($html);

  520.         return $html;

  521.     }

  522. 

  523.     /**

  524.      *  下载特定资源，并保存为指定文件

  525.      *

  526.      * @access    public

  527.      * @param     string  $dourl  操作地址

  528.      * @param     string  $mtype  附件类型

  529.      * @param     string  $islitpic  是否缩略图

  530.      * @return    string

  531.      */

  532.     function DownMedia($dourl, $mtype='img', $islitpic=FALSE)

  533.     {

  534.         global $notckpic;

  535.         if(empty($notckpic))

  536.         {

  537.             $notckpic = 0;

  538.         }

  539. 

  540.         //检测是否已经下载此文件

  541.         $wi = FALSE;

  542.         $tofile = $filename = '';

  543.         if($notckpic==0)

  544.         {

  545.             $row = $this->dsql->GetOne("SELECT hash,tofile FROM `#@__co_mediaurls` WHERE nid='{$this->noteId}' AND hash='".md5($dourl)."' ");

  546.             if(isset($row['tofile']))

  547.             {

  548.                 $tofile = $filename = $row['tofile'];

  549.             }

  550.         }

  551. 

  552.         //如果不存在，下载文件

  553.         if($tofile=='' || !file_exists($GLOBALS['cfg_basedir'].$filename))

  554.         {

  555.             $filename = $this->GetRndName($dourl,$mtype);

  556.             if(!preg_match("#^\/#", $filename))

  557.             {

  558.                 $filename = "/".$filename;

  559.             }

  560. 

  561.             //防盗链模式

  562.             if($this->noteInfos['isref']=='yes' && $this->noteInfos['refurl']!='')

  563.             {

  564.                 if($this->noteInfos['exptime']=='')

  565.                 {

  566.                     $this->noteInfos['exptime'] = 10;

  567.                 }

  568.                 DownImageKeep($dourl,$this->noteInfos['refurl'],$GLOBALS['cfg_basedir'].$filename,'',0,$this->Item['exptime']);

  569.             }

  570. 

  571.             //普通模式

  572.             else

  573.             {

  574.                 $this->cHttpDown->OpenUrl($dourl);

  575.                 $this->cHttpDown->SaveToBin($GLOBALS['cfg_basedir'].$filename);

  576.                 $this->cHttpDown->Close();

  577.             }

  578. 

  579.             //下载文件成功，保存记录

  580.             if(file_exists($GLOBALS['cfg_basedir'].$filename))

  581.             {

  582.                 if($tofile=='')

  583.                 {

  584.                     $query = "INSERT INTO `#@__co_mediaurls`(nid,hash,tofile) VALUES ('".$this->noteId."', '".md5($dourl)."', '".addslashes($filename)."');";

  585.                 }

  586.                 else

  587.                 {

  588.                     $query = "UPDATE `#@__co_mediaurls` SET tofile='".addslashes($filename)."' WHERE hash='".md5($dourl)."' ";

  589.                 }

  590.                 $this->dsql->ExecuteNoneQuery($query);

  591.             }

  592.         }

  593. 

  594.         //如果下载图片失败或图片不存在，返回网址

  595.         if(!file_exists($GLOBALS['cfg_basedir'].$filename))

  596.         {

  597.             return $dourl;

  598.         }

  599. 

  600.         //生成缩略图

  601.         if($mtype=='img' && !$islitpic && $this->breImage=='')

  602.         {

  603.             $this->breImage = $filename;

  604.             if(!preg_match("#^http:\/\/#", $this->breImage) && file_exists($GLOBALS['cfg_basedir'].$filename))

  605.             {

  606.                 $filenames = explode('/',$filename);

  607.                 $filenamed = $filenames[count($filenames)-1];

  608.                 $nfilename = str_replace('.','_lit.',$filenamed);

  609.                 $nfilename = str_replace($filenamed,$nfilename,$filename);

  610.                 if(@copy($GLOBALS['cfg_basedir'].$filename, $GLOBALS['cfg_basedir'].$nfilename))

  611.                 {

  612.                     ImageResize($GLOBALS['cfg_basedir'].$nfilename,$GLOBALS['cfg_ddimg_width'],$GLOBALS['cfg_ddimg_height']);

  613.                     $this->breImage = $nfilename;

  614.                 }

  615.             }

  616.         }

  617.         if($mtype=='img' && !$islitpic)

  618.         {

  619.             @WaterImg($GLOBALS['cfg_basedir'].$filename,'collect');

  620.         }

  621.         return $filename;

  622.     }

  623. 

  624.     /**

  625.      *  获得下载媒体的随机名称

  626.      *

  627.      * @access    public

  628.      * @param     string  $url  地址

  629.      * @param     string  $v  值

  630.      * @return    string

  631.      */

  632.     function GetRndName($url, $v)

  633.     {

  634.         global $cfg_image_dir,$cfg_dir_purview;

  635.         $this->mediaCount++;

  636.         $mnum = $this->mediaCount;

  637.         $timedir = "c".MyDate("ymd",time());

  638.         //存放路径

  639.         $fullurl = preg_replace("#\/{1,}#", "/", $cfg_image_dir."/");

  640.         if(!is_dir($GLOBALS['cfg_basedir']."/$fullurl"))

  641.         {

  642.             MkdirAll($GLOBALS['cfg_basedir']."/$fullurl", $cfg_dir_purview);

  643.         }

  644. 

  645.         $fullurl = $fullurl.$timedir."/";

  646.         if(!is_dir($GLOBALS['cfg_basedir']."/$fullurl"))

  647.         {

  648.             MkdirAll($GLOBALS['cfg_basedir']."/$fullurl", $cfg_dir_purview);

  649.         }

  650. 

  651.         //文件名称

  652.         $timename = str_replace('.','', ExecTime());

  653.         $threadnum = 0;

  654.         if(isset($_GET['threadnum']))

  655.         {

  656.             $threadnum = intval($_GET['threadnum']);

  657.         }

  658.         $filename = dd2char($timename.$threadnum.'-'.$mnum.mt_rand(1000,9999));

  659. 

  660.         //分配扩展名

  661.         $urls = explode('.',$url);

  662.         if($v=='img')

  663.         {

  664.             $shortname = '.jpg';

  665.             if(preg_match("#\.gif$#i", $url))

  666.             {

  667.                 $shortname = '.gif';

  668.             }

  669.             else if(preg_match("#\.png$#i", $url))

  670.             {

  671.                 $shortname = '.png';

  672.             }

  673.         }

  674.         else if($v=='embed')

  675.         {

  676.             $shortname = '.swf';

  677.         }

  678.         else

  679.         {

  680.             $shortname = '';

  681.         }

  682.         $fullname = $fullurl.$filename.$shortname;

  683.         return preg_replace("#\/{1,}#", "/", $fullname);

  684.     }

  685. 

  686.     /**

  687.      *  按载入的网页内容获取规则，从一个HTML文件中获取内容

  688.      *

  689.      * @access    public

  690.      * @param     string  $dourl  操作地址

  691.      * @param     string  $needDown  需要下载

  692.      * @param     string  $litpic  缩略图

  693.      * @return    string

  694.      */

  695.     function GetPageFields($dourl, $needDown, $litpic='')

  696.     {

  697.         global $cfg_auot_description;

  698.         if($this->tmpHtml == '')

  699.         {

  700.             return '';

  701.         }

  702.         $artitem = '';

  703.         $isPutUnit = FALSE;

  704.         $tmpLtKeys = array();

  705.         $inarr = array();

  706. 

  707.         //自动分析关键字和摘要

  708.         preg_match("#<meta[\s]+name=['\"]keywords['\"] content=['\"](.*)['\"]#isU", $this->tmpHtml, $inarr);

  709.         preg_match("#<meta[\s]+content=['\"](.*)['\"] name=['\"]keywords['\"]#isU", $this->tmpHtml, $inarr2);

  710.         if(!isset($inarr[1]) && isset($inarr2[1]))

  711.         {

  712.             $inarr[1] = $inarr2[1];

  713.         }

  714.         if(isset($inarr[1]))

  715.         {

  716.             $keywords = trim(cn_substr(html2text($inarr[1]),30));

  717.             $keywords = preg_replace("#".$this->artNotes['keywordtrim']."#isU",'',$keywords);

  718.             if(!preg_match("#,#", $keywords))

  719.             {

  720.                 $keywords = str_replace(' ', ',', $keywords);

  721.             }

  722.             $artitem .= "{dede:field name='keywords'}".$keywords."{/dede:field}\r\n";

  723.         }

  724.         else

  725.         {

  726.             $artitem .= "{dede:field name='keywords'}{/dede:field}\r\n";

  727.         }

  728.         // preg_match("#<meta[\s]+name=['\"]description['\"] content=['\"](.*)['\"]#isU", $this->tmpHtml, $inarr);

  729.         // preg_match("#<meta[\s]+content=['\"](.*)['\"] name=['\"]description['\"]#isU", $this->tmpHtml, $inarr2);

  730.         preg_match("#<meta[\s]+name=['\"]description['\"][\s]+content=['\"]([^>]*?)['\"]#iU", $this->tmpHtml, $inarr);

  731.         preg_match("#<meta[\s]+content=['\"]([^>]*?)['\"][\s]+name=['\"]description['\"]#iU", $this->tmpHtml, $inarr2);

  732.         if(!isset($inarr[1]) && isset($inarr2[1]))

  733.         {

  734.             $inarr[1] = $inarr2[1];

  735.         }

  736.         if(isset($inarr[1]))

  737.         {

  738.             $description = trim(cn_substr(html2text($inarr[1]),$cfg_auot_description));

  739.             $description = preg_replace("/".$this->artNotes['descriptiontrim']."/isU",'',$description);

  740.             $artitem .= "{dede:field name='description'}".$description."{/dede:field}\r\n";

  741.         }

  742.         else

  743.         {

  744.             $artitem .= "{dede:field name='description'}{/dede:field}\r\n";

  745.         }

  746. 

  747.         foreach($this->artNotes as $k=>$sarr)

  748.         {

  749.             //可能出现意外的情况

  750.             if($k=='sppage' || $k=='sptype')

  751.             {

  752.                 continue;

  753.             }

  754.             if(!is_array($sarr))

  755.             {

  756.                 continue;

  757.             }

  758. 

  759.             //特殊的规则或没匹配选项

  760.             if($sarr['match']=='' || trim($sarr['match'])=='[内容]')

  761.             {

  762.                 if($sarr['value']!='[内容]')

  763.                 {

  764.                     $v = trim($sarr['value']);

  765.                 }

  766.                 else

  767.                 {

  768.                     $v = '';

  769.                 }

  770.             }

  771.             else

  772.             {

  773.                 //分多页的内容

  774.                 if($this->tmpUnitValue!='' && !$isPutUnit && $sarr['isunit']==1)

  775.                 {

  776.                     $v = $this->tmpUnitValue;

  777.                     $isPutUnit = TRUE;

  778.                 }

  779.                 else

  780.                 {

  781.                     $v = $this->GetHtmlArea('[内容]',$sarr['match'],$this->tmpHtml);

  782.                 }

  783. 

  784.                 //过滤内容规则

  785.                 if(isset($sarr['trim']) && $v!='')

  786.                 {

  787.                     foreach($sarr['trim'] as $nv)

  788.                     {

  789.                         if($nv[0]=='')

  790.                         {

  791.                             continue;

  792.                         }

  793.                         $nvs = str_replace("/", "\\/", $nv[0]);

  794.                         $v = preg_replace("#".$nvs."#isU", $nv[1], $v);

  795.                     }

  796.                 }

  797. 

  798.                 //是否下载远程资源

  799.                 if($needDown)

  800.                 {

  801.                     if($sarr['isdown'] == '1')

  802.                     {

  803.                         $v = $this->DownMedias($v, $dourl);

  804.                     }

  805.                 }

  806.                 else

  807.                 {

  808.                     if($sarr['isdown'] == '1')

  809.                     {

  810.                         $v = $this->MediasReplace($v, $dourl);

  811.                     }

  812.                 }

  813.             }

  814.             $v = trim($v);

  815. 

  816.             //用户自行对内容进行处理的接口

  817.             if($sarr['function'] != '')

  818.             {

  819.                 $tmpLtKeys[$k]['v'] = $v;

  820.                 $tmpLtKeys[$k]['f'] = $sarr['function'];

  821.             }

  822.             else

  823.             {

  824.                 $v = preg_replace("#(　)$#", '', $v);

  825.                 $v = preg_replace("#[\r\n\t ]{1,}$#", '', $v);

  826.                 $artitem .= "{dede:field name='$k'}$v{/dede:field}\r\n";

  827.             }

  828.         }//End Foreach

  829. 

  830.         //处理带函数的项目

  831.         foreach($tmpLtKeys as $k=>$sarr)

  832.         {

  833.             $v = $this->RunPHP($sarr['v'],$sarr['f']);

  834.             $v = preg_replace("#(　)$#", '', $v);

  835.             $v = preg_replace("#[\r\n\t ]{1,}$#", '', $v);

  836.             $artitem .= "{dede:field name='$k'}$v{/dede:field}\r\n";

  837.         }

  838.         if($litpic!='' && $this->lists['listpic']==1)

  839.         {

  840.             $artitem .= "{dede:field name='litpic'}".$this->DownMedia($litpic,'img',TRUE)."{/dede:field}\r\n";

  841.         }

  842.         else

  843.         {

  844.             $artitem .= "{dede:field name='litpic'}".$this->breImage."{/dede:field}\r\n";

  845.         }

  846.         return $artitem;

  847.     }

  848. 

  849.     /**

  850.      *  下载内容里的资源

  851.      *

  852.      * @access    public

  853.      * @param     string  $html  html内容

  854.      * @param     string  $url  地址

  855.      * @return    string

  856.      */

  857.     function DownMedias(&$html, $url)

  858.     {

  859.         $this->cDedeHtml->SetSource($html,$url,'media');

  860. 

  861.         //下载标记里的图片和flash

  862.         foreach($this->cDedeHtml->Medias as $k=>$v)

  863.         {

  864.             $furl = $this->cDedeHtml->FillUrl($k);

  865.             if($v=='embed' && !preg_match("#\.(swf)\?(.*)$#i", $k)&& !preg_match("#\.(swf)$#i", $k))

  866.             {

  867.                 continue;

  868.             }

  869.             $okurl = $this->DownMedia($furl, $v);

  870.             $html = str_replace($k, $okurl, $html);

  871.         }

  872. 

  873.         //下载超链接里的图片

  874.         foreach($this->cDedeHtml->Links as $v=>$k)

  875.         {

  876.             if(preg_match("#\.(jpg|gif|png)\?(.*)$#i",$v) || preg_match("#\.(jpg|gif|png)$#i", $v))

  877.             {

  878.                 $m = "img";

  879.             }

  880.             else if(preg_match("#\.(swf)\?(.*)$#i", $v) || preg_match("#\.(swf)$#i", $v))

  881.             {

  882.                 $m = "embed";

  883.             }

  884.             else

  885.             {

  886.                 continue;

  887.             }

  888.             $furl = $this->cDedeHtml->FillUrl($v);

  889.             $okurl = $this->DownMedia($furl, $m);

  890.             $html = str_replace($v, $okurl, $html);

  891.         }

  892.         return $html;

  893.     }

  894. 

  895.     /**

  896.      *  仅替换内容里的资源为绝对网址

  897.      *

  898.      * @access    public

  899.      * @param     string  $html  html内容

  900.      * @param     string  $dourl  操作地址

  901.      * @return    string

  902.      */

  903.     function MediasReplace(&$html, $dourl)

  904.     {

  905.         $this->cDedeHtml->SetSource($html, $dourl, 'media');

  906.         foreach($this->cDedeHtml->Medias as $k=>$v)

  907.         {

  908.             $k = trim($k);

  909.             $okurl = $this->cDedeHtml->FillUrl($k);

  910.             $html = str_replace($k, $okurl, $html);

  911.         }

  912.         return $html;

  913.     }

  914. 

  915.     //测试列表

  916.     function Testlists(&$dourl)

  917.     {

  918.         $links = array();

  919. 

  920.         //从RSS中获取网址

  921.         if($this->lists['sourcetype']=='rss')

  922.         {

  923.             $dourl = $this->lists['rssurl'];

  924.             $links = GetRssLinks($dourl);

  925.             return $links;

  926.         }

  927. 

  928.         //正常情况

  929.         if(isset($this->lists['url'][0][0]))

  930.         {

  931.             $dourl = $this->lists['url'][0][0];

  932.         }

  933.         else

  934.         {

  935.             $dourl = '';

  936.             $this->errString = "配置中指定列表的网址错误!\r\n";

  937.             return $links;

  938.         }

  939.         $dhtml = new DedeHtml2();

  940.         $html = $this->DownOnePage($dourl);

  941.         if($html=='')

  942.         {

  943.             $this->errString = "读取网址： $dourl 时失败！\r\n";

  944.             return $links;

  945.         }

  946.         if( trim($this->lists['areastart']) !='' && trim($this->lists['areaend']) != '' )

  947.         {

  948.             $areabody = $this->lists['areastart'].'[var:区域]'.$this->lists['areaend'];

  949.             $html = $this->GetHtmlArea('[var:区域]',$areabody,$html);

  950.         }

  951.         $t1 = ExecTime();

  952.         $dhtml->SetSource($html,$dourl,'link');

  953. 		$this->lists['musthas'] = str_replace('/', '\/', $this->lists['musthas']);

  954.         foreach($dhtml->Links as $s)

  955.         {

  956.             if($this->lists['nothas']!='')

  957.             {

  958.                 if( preg_match("#".$this->lists['nothas']."#i", $s['link']) )

  959.                 {

  960.                     continue;

  961.                 }

  962.             }

  963.             if($this->lists['musthas']!='')

  964.             {

  965.                 if( !preg_match("#".$this->lists['musthas']."#i", $s['link']) )

  966.                 {

  967.                     continue;

  968.                 }

  969.             }

  970.             $links[] = $s;

  971.         }

  972.         return $links;

  973.     }

  974. 

  975.     /**

  976.      *  测试文章规则

  977.      *

  978.      * @access    public

  979.      * @param     $dourl  操作地址

  980.      * @return    string

  981.      */

  982.     function TestArt($dourl)

  983.     {

  984.         return $this->DownUrl(0, $dourl, '', FALSE);

  985.     }

  986. 

  987.     /**

  988.      *  采集种子网址

  989.      *

  990.      * @access    public

  991.      * @param     int  $islisten  是否监听

  992.      * @param     int  $glstart  采集开始

  993.      * @param     int  $pagesize  分页尺寸

  994.      * @return    string

  995.      */

  996.     function GetSourceUrl($islisten=0, $glstart=0, $pagesize=10)

  997.     {

  998.         //在第一页中进行预处理

  999.         //“下载种子网址的未下载内容”的模式不需要经过采集种子网址的步骤

  1000.         if($glstart==0)

  1001.         {

  1002.             //重新采集所有内容模式

  1003.             if($islisten == -1)

  1004.             {

  1005.                 $this->dsql->ExecuteNoneQuery("DELETE FROM `#@__co_urls` WHERE nid='".$this->noteId."'");

  1006.                 $this->dsql->ExecuteNoneQuery("DELETE FROM `#@__co_htmls` WHERE nid='".$this->noteId."' ");

  1007.             }

  1008.             //监听模式(保留未导出的内容、保留节点的历史网址记录)

  1009.             else

  1010.             {

  1011.                 $this->dsql->ExecuteNoneQuery("DELETE FROM `#@__co_htmls` WHERE nid='".$this->noteId."' AND isexport=1 ");

  1012.             }

  1013.         }

  1014. 

  1015.         //从RSS中获取种子

  1016.         if($this->lists['sourcetype']=='rss')

  1017.         {

  1018.             $links = GetRssLinks($this->lists['rssurl']);

  1019.             //if($this->noteInfos['cosort']!='asc')

  1020.             $tmplink = krsort($links);

  1021.             $lk = 0;

  1022.             foreach($links as $v)

  1023.             {

  1024.                 if($islisten==1)

  1025.                 {

  1026.                     $lrow = $this->dsql->GetOne("SELECT * FROM `#@__co_urls` WHERE nid='{$this->noteId}' AND hash='".md5($v['link'])."' ");

  1027.                     if(is_array($lrow))

  1028.                     {

  1029.                         continue;

  1030.                     }

  1031.                 }

  1032.                 $lk++;

  1033.                 if($mytotal > 0 && $lk >= $mytotal) break;

  1034.                 $inquery = "INSERT INTO `#@__co_htmls` (`nid` ,`typeid`, `title` , `litpic` , `url` , `dtime` , `isdown` , `isexport` , `result`)

  1035.                     VALUES ('{$this->noteId}' , '0', '".addslashes($v['title'])."' , '".addslashes($v['image'])."' , '".addslashes($v['link'])."' , 'dtime' , '0' , '0' , ''); ";

  1036.                 $this->dsql->ExecuteNoneQuery($inquery);

  1037. 

  1038.                 $inquery = "INSERT INTO `#@__co_urls`(hash,nid) VALUES ('".md5($v['link'])."','{$this->noteId}');";

  1039.                 $this->dsql->ExecuteNoneQuery($inquery);

  1040.             }

  1041.             return 0;

  1042.         }

  1043.         else

  1044.         {

  1045.             $tmplink = array();

  1046.             $arrStart = 0;

  1047.             $moviePostion = 0;

  1048.             $endpos = $glstart + $pagesize;

  1049.             $totallen = count($this->lists['url']);

  1050. 			//dump($this->lists['url']);exit;

  1051.             foreach($this->lists['url'] as $k=>$cururls)

  1052.             {

  1053.                 //$status = FALSE;

  1054.                 $urlnum = 0;

  1055.                 $cururl = $cururls[0];

  1056.                 $typeid = (empty($cururls[1]) ? 0 : $cururls[1]);

  1057.                 $moviePostion++;

  1058. 				

  1059.                 if($moviePostion > $endpos)

  1060.                 {

  1061.                     break;

  1062.                 }

  1063.                 if($moviePostion > $glstart)

  1064.                 {

  1065.                     $html = $this->DownOnePage($cururl);

  1066.                     if( trim($this->lists['areastart']) !='' && trim($this->lists['areaend']) != '' )

  1067.                     {

  1068.                         $areabody = $this->lists['areastart'].'[var:区域]'.$this->lists['areaend'];

  1069.                         $html = $this->GetHtmlArea('[var:区域]',$areabody,$html);

  1070.                     }

  1071.                     $this->cDedeHtml->SetSource($html, $cururl, 'link');

  1072.                     $lk = 0;

  1073.                     foreach($this->cDedeHtml->Links as $k=>$v)

  1074.                     {

  1075.                         if($this->lists['nothas']!='')

  1076.                         {

  1077.                             if( preg_match("#".$this->lists['nothas']."#", $v['link']) )

  1078.                             {

  1079.                                 continue;

  1080.                             }

  1081.                         }

  1082.                         if($this->lists['musthas']!='')

  1083.                         {

  1084.                             if( !preg_match("#".$this->lists['musthas']."#i", $v['link']) )

  1085.                             {

  1086.                                 continue;

  1087.                             }

  1088.                         }

  1089.                         $tmplink[$arrStart][0] = $v;

  1090.                         $tmplink[$arrStart][1] = $typeid;

  1091.                         $arrStart++;

  1092.                         $lk++;

  1093.                     }

  1094.                     $this->cDedeHtml->Clear();

  1095.                 }

  1096.             }//foreach

  1097.             //if($this->noteInfos['cosort']!='asc')

  1098. 

  1099.             krsort($tmplink);

  1100.             $unum = count($tmplink);

  1101.             if($unum>0)

  1102.             {

  1103.                 //echo "完成本次种子网址抓取，共找到：{$unum} 个记录!<br/>\r\n";

  1104.                 foreach($tmplink as $vs)

  1105.                 {

  1106.                     $v = $vs[0];

  1107.                     $typeid = $vs[1];

  1108.                     if($islisten==1)

  1109.                     {

  1110.                         $lrow = $this->dsql->GetOne("SELECT * FROM `#@__co_urls` WHERE nid='{$this->noteId}' AND hash='".md5($v['link'])."' ");

  1111.                         if(is_array($lrow))

  1112.                         {

  1113.                             continue;

  1114.                         }

  1115.                     }

  1116.                     $inquery = "INSERT INTO `#@__co_htmls` (`nid` ,`typeid`, `title` , `litpic` , `url` , `dtime` , `isdown` , `isexport` , `result`)

  1117.                     VALUES ('{$this->noteId}' ,'$typeid', '".addslashes($v['title'])."' , '".addslashes($v['image'])."' , '".addslashes($v['link'])."' , '".time()."' , '0' , '0' , ''); ";

  1118.                     $this->dsql->ExecuteNoneQuery($inquery);

  1119. 

  1120.                     $inquery = "INSERT INTO `#@__co_urls`(hash,nid) VALUES ('".md5($v['link'])."','{$this->noteId}');";

  1121.                     $this->dsql->ExecuteNoneQuery($inquery);

  1122.                 }

  1123.                 if($endpos >= $totallen)

  1124.                 {

  1125.                     return 0;

  1126.                 }

  1127.                 else

  1128.                 {

  1129.                     return ($totallen-$endpos);

  1130.                 }

  1131.             }

  1132.             else

  1133.             {

  1134.                 //仅在第一批采集时出错才返回

  1135.                 if($glstart==0)

  1136.                 {

  1137.                     return -1;

  1138.                 }

  1139. 

  1140.                 //在其它页出错照常采集后面内容

  1141.                 if($endpos >= $totallen)

  1142.                 {

  1143.                     return 0;

  1144.                 }

  1145.                 else

  1146.                 {

  1147.                     return ($totallen-$endpos);

  1148.                 }

  1149.             }

  1150.         }

  1151.     }

  1152. 

  1153.     /**

  1154.      *  用扩展函数处理采集到的原始数据

  1155.      *

  1156.      * @access    public

  1157.      * @param     string  $fvalue  值

  1158.      * @param     string  $phpcode  PHP代码

  1159.      * @return    string

  1160.      */

  1161.     function RunPHP($fvalue, $phpcode)

  1162.     {

  1163.         $DedeMeValue = $fvalue;

  1164.         $phpcode = preg_replace("#'@me'|\"@me\"|@me#isU", '$DedeMeValue', $phpcode);

  1165.         if(preg_match("#@body#i", $phpcode))

  1166.         {

  1167.             $DedeBodyValue = $this->tmpHtml;

  1168.             $phpcode = preg_replace("#'@body'|\"@body\"|@body#isU", '$DedeBodyValue', $phpcode);

  1169.         }

  1170.         if(preg_match("#@litpic#i", $phpcode))

  1171.         {

  1172.             $DedeLitPicValue = $this->breImage;

  1173.             $phpcode = preg_replace("#'@litpic'|\"@litpic\"|@litpic#isU", '$DedeLitPicValue', $phpcode);

  1174.         }

  1175.         eval($phpcode.";");

  1176.         return $DedeMeValue;

  1177.     }

  1178. 

  1179.     /**

  1180.      *  编码转换

  1181.      *

  1182.      * @access    public

  1183.      * @param     string  $str  字符串

  1184.      * @return    string

  1185.      */

  1186.     function ChangeCode(&$str)

  1187.     {

  1188.         global $cfg_soft_lang;

  1189.         if($cfg_soft_lang=='utf-8')

  1190.         {

  1191.             if($this->noteInfos["sourcelang"]=="gb2312")

  1192.             {

  1193.                 $str = gb2utf8($str);

  1194.             }

  1195.             if($this->noteInfos["sourcelang"]=="big5")

  1196.             {

  1197.                 $str = gb2utf8(big52gb($str));

  1198.             }

  1199.         }

  1200.         else

  1201.         {

  1202.             if($this->noteInfos["sourcelang"]=="utf-8")

  1203.             {

  1204.                 $str = utf82gb($str);

  1205.             }

  1206.             if($this->noteInfos["sourcelang"]=="big5")

  1207.             {

  1208.                 $str = big52gb($str);

  1209.             }

  1210.         }

  1211.     }

  1212. }//End Class