DedeBIZ
/
DedeV6
Watch 2
Star 0
Fork 0
Code Pull Requests 0 Releases 25 Activity
国内流行的内容管理系统（CMS）多端全媒体解决方案 https://www.dedebiz.com
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
281 Commits
3 Branches
85MB
Tree: cb81d0de69
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cb81d0de69'
${ noResults }
DedeV6/src/include/dedecollection.class.php

1019 lines
39KB
Raw Blame History 

  1. <?php if (!defined('DEDEINC')) exit('dedebiz');

  2. /**

  3.  * Dede采集类

  4.  *

  5.  * @version        $Id: dedecollection.class.php 1 20:20 2010年7月7日Z tianya $

  6.  * @package        DedeBIZ.Libraries

  7.  * @copyright      Copyright (c) 2021, DedeBIZ.COM

  8.  * @license        https://www.dedebiz.com/license

  9.  * @link           https://www.dedebiz.com

  10.  */

  11. 

  12. require_once(DEDEINC . "/dedecollection.func.php"); //采集扩展函数

  13. require_once(DEDEINC . "/image.func.php");

  14. require_once(DEDEINC . "/dedehtml2.class.php");

  15. @set_time_limit(0);

  16. 

  17. /**

  18.  * Dede采集类

  19.  *

  20.  * @package          DedeCollection

  21.  * @subpackage       DedeBIZ.Libraries

  22.  * @link             https://www.dedebiz.com

  23.  */

  24. class DedeCollection

  25. {

  26.     var $artNotes = array();  //文章采集的字段信息

  27.     var $spNotes = array();  //文章采集的字段信息

  28.     var $lists = array();     //采集节点的来源列表处理信息

  29.     var $noteInfos = array(); //采集节点的基本配置信息

  30.     var $dsql = '';

  31.     var $noteId = '';

  32.     var $cDedeHtml = '';

  33.     var $cHttpDown = '';

  34.     var $mediaCount = 0;

  35.     var $tmpUnitValue = '';

  36.     var $tmpLinks = array();

  37.     var $tmpHtml = '';

  38.     var $breImage = '';

  39.     var $errString = '';

  40. 

  41.     //兼容php5构造函数

  42.     function __construct()

  43.     {

  44.         $this->dsql = $GLOBALS['dsql'];

  45.         $this->cHttpDown = new DedeHttpDown();

  46.         $this->cDedeHtml = new DedeHtml2();

  47.     }

  48. 

  49.     function DedeCollection()

  50.     {

  51.         $this->__construct();

  52.     }

  53. 

  54.     //析放资源

  55.     function Close()

  56.     {

  57.     }

  58. 

  59.     /**

  60.      *  从数据库里载入某个节点

  61.      *

  62.      * @access    public

  63.      * @param     int   $nid  采集节点ID

  64.      * @return    void

  65.      */

  66.     function LoadNote($nid)

  67.     {

  68.         $this->noteId = $nid;

  69.         $row = $this->dsql->GetOne("SELECT * FROM `#@__co_note` WHERE nid='$nid'");

  70.         $this->LoadListConfig($row['listconfig']);

  71.         $this->LoadItemConfig($row['itemconfig']);

  72.     }

  73. 

  74.     /**

  75.      *  分析基本节点的及索引配置信息

  76.      *

  77.      * @access    public

  78.      * @param     string  $configString  配置字符串

  79.      * @return    void

  80.      */

  81.     function LoadListConfig($configString)

  82.     {

  83.         $dtp = new DedeTagParse();

  84.         $dtp2 = new DedeTagParse();

  85.         $dtp->LoadString($configString);

  86.         for ($i = 0; $i <= $dtp->Count; $i++) {

  87.             $ctag = $dtp->CTags[$i];

  88. 

  89.             //item 配置

  90.             //节点基本信息

  91.             if ($ctag->GetName() == "noteinfo") {

  92.                 $this->noteInfos['notename'] = $ctag->GetAtt('notename');

  93.                 $this->noteInfos['matchtype'] = $ctag->GetAtt('matchtype');

  94.                 $this->noteInfos['channelid'] = $ctag->GetAtt('channelid');

  95.                 $this->noteInfos['refurl'] = $ctag->GetAtt('refurl');

  96.                 $this->noteInfos['sourcelang'] = $ctag->GetAtt('sourcelang');

  97.                 $this->noteInfos['cosort'] = $ctag->GetAtt('cosort');

  98.                 $this->noteInfos['isref'] = $ctag->GetAtt('isref');

  99.                 $this->noteInfos['exptime'] = $ctag->GetAtt('exptime');

  100.             }

  101. 

  102.             //list 配置

  103.             //要采集的列表页的信息

  104.             else if ($ctag->GetName() == "listrule") {

  105.                 $this->lists['sourcetype'] = $ctag->GetAtt('sourcetype');

  106.                 $this->lists['rssurl'] = $ctag->GetAtt('rssurl');

  107.                 $this->lists['regxurl'] = $ctag->GetAtt('regxurl');

  108.                 $this->lists['startid'] = $ctag->GetAtt('startid');

  109.                 $this->lists['endid'] = $ctag->GetAtt('endid');

  110.                 $this->lists['addv'] = $ctag->GetAtt('addv');

  111.                 $this->lists['urlrule'] = $ctag->GetAtt('urlrule');

  112.                 $this->lists['musthas'] = $ctag->GetAtt('musthas');

  113.                 $this->lists['nothas'] = $ctag->GetAtt('nothas');

  114.                 $this->lists['listpic'] = $ctag->GetAtt('listpic');

  115.                 $this->lists['usemore'] =  $ctag->GetAtt('usemore');

  116.                 $dtp2->LoadString($ctag->GetInnerText());

  117.                 for ($j = 0; $j <= $dtp2->Count; $j++) {

  118.                     $ctag2 = $dtp2->CTags[$j];

  119.                     $tname = $ctag2->GetName();

  120.                     if ($tname == 'addurls') {

  121.                         $this->lists['addurls'] = trim($ctag2->GetInnerText());

  122.                     } else if ($tname == 'regxrule') {

  123.                         $this->lists['regxrule'] = trim($ctag2->GetInnerText());

  124.                     } else if ($tname == 'areastart') {

  125.                         $this->lists['areastart'] = trim($ctag2->GetInnerText());

  126.                     } else if ($tname == 'areaend') {

  127.                         $this->lists['areaend'] = trim($ctag2->GetInnerText());

  128.                     } else if ($tname == 'batchrule') {

  129.                         $this->lists['batchrule'] = trim($ctag2->GetInnerText());

  130.                     }

  131.                 }

  132. 

  133.                 //分析列表网址

  134.                 if ($this->lists['sourcetype'] != 'rss') {

  135.                     $this->lists['url'] = GetUrlFromListRule(

  136.                         $this->lists['regxurl'],

  137.                         $this->lists['addurls'],

  138.                         $this->lists['startid'],

  139.                         $this->lists['endid'],

  140.                         $this->lists['addv'],

  141.                         $this->lists['usemore'],

  142.                         $this->lists['batchrule']

  143.                     );

  144.                 } else {

  145.                     $this->lists['url'] = $this->lists['rssurl'];

  146.                 }

  147.             }

  148.         } //End Loop

  149. 

  150.         $dtp->Clear();

  151.         $dtp2->Clear();

  152.     }

  153. 

  154.     /**

  155.      *  分析采集文章页的字段的设置

  156.      *

  157.      * @access    public

  158.      * @param     string  $configString  配置字符串

  159.      * @return    void

  160.      */

  161.     function LoadItemConfig($configString)

  162.     {

  163.         $dtp = new DedeTagParse();

  164.         $dtp2 = new DedeTagParse();

  165.         $dtp->LoadString($configString);

  166.         for ($i = 0; $i <= $dtp->Count; $i++) {

  167.             $ctag = $dtp->CTags[$i];

  168.             if ($ctag->GetName() == 'sppage') {

  169.                 $this->artNotes['sppage'] = $ctag->GetInnerText();

  170.                 $this->artNotes['sptype'] = $ctag->GetAtt('sptype');

  171.                 $this->spNotes['srul'] = $ctag->GetAtt('srul');

  172.                 $this->spNotes['erul'] = $ctag->GetAtt('erul');

  173.             } else if ($ctag->GetName() == 'previewurl') {

  174.                 $this->artNotes['previewurl'] = $ctag->GetInnerText();

  175.             } else if ($ctag->GetName() == 'keywordtrim') {

  176.                 $this->artNotes['keywordtrim'] = $ctag->GetInnerText();

  177.             } else if ($ctag->GetName() == 'descriptiontrim') {

  178.                 $this->artNotes['descriptiontrim'] = $ctag->GetInnerText();

  179.             } else if ($ctag->GetName() == 'item') {

  180.                 $field = $ctag->GetAtt('field');

  181.                 if ($field == '') {

  182.                     continue;

  183.                 }

  184.                 $this->artNotes[$field]['value'] = $ctag->GetAtt('value');

  185.                 $this->artNotes[$field]['isunit'] = $ctag->GetAtt('isunit');

  186.                 $this->artNotes[$field]['isdown'] = $ctag->GetAtt('isdown');

  187.                 $this->artNotes[$field]['trim'] = array();

  188.                 $this->artNotes[$field]['match'] = '';

  189.                 $this->artNotes[$field]['function'] = '';

  190.                 $t = 0;

  191.                 $dtp2->LoadString($ctag->GetInnerText());

  192.                 for ($k = 0; $k <= $dtp2->Count; $k++) {

  193.                     $ctag2 = $dtp2->CTags[$k];

  194.                     if ($ctag2->GetName() == 'trim') {

  195.                         $this->artNotes[$field]['trim'][$t][0] = str_replace('#n#', '&nbsp;', $ctag2->GetInnerText());

  196.                         $this->artNotes[$field]['trim'][$t][1] = $ctag2->GetAtt('replace');

  197.                         $t++;

  198.                     } else if ($ctag2->GetName() == 'match') {

  199.                         $this->artNotes[$field]['match'] = str_replace('#n#', '&nbsp;', $ctag2->GetInnerText());

  200.                     } else if ($ctag2->GetName() == 'function') {

  201.                         $this->artNotes[$field]['function'] = $ctag2->GetInnerText();

  202.                     }

  203.                 }

  204.             }

  205.         } //End Loop

  206. 

  207.         $dtp->Clear();

  208.         $dtp2->Clear();

  209.     }

  210. 

  211.     /**

  212.      *  下载其中一个网址，并保存

  213.      *

  214.      * @access    public

  215.      * @param     int  $aid  文档ID

  216.      * @param     string  $dourl  操作地址

  217.      * @param     string  $litpic  缩略图

  218.      * @param     bool  $issave  是否保存

  219.      * @return    string

  220.      */

  221.     function DownUrl($aid, $dourl, $litpic = '', $issave = TRUE)

  222.     {

  223.         $this->tmpLinks = array();

  224.         $this->tmpUnitValue = '';

  225.         $this->breImage = '';

  226.         $this->tmpHtml = $this->DownOnePage($dourl);

  227. 

  228.         //检测是否有分页字段，并预先处理

  229.         if (!empty($this->artNotes['sppage'])) {

  230.             $noteid = '';

  231.             foreach ($this->artNotes as $k => $sarr) {

  232.                 if (isset($sarr['isunit']) && $sarr['isunit'] == 1) {

  233.                     $noteid = $k;

  234.                     break;

  235.                 }

  236.             }

  237. 

  238.             $this->GetSpPage($dourl, $noteid, $this->tmpHtml);

  239. 

  240.             if (preg_match("/#p#/i", $this->tmpUnitValue)) {

  241.                 if ($this->artNotes["sptype"] != 'diyrule') {

  242.                     $this->tmpUnitValue = '副标题#e#' . $this->tmpUnitValue;

  243.                 }

  244.             }

  245.         }

  246. 

  247.         //处理字段

  248.         $body = $this->GetPageFields($dourl, $issave, $litpic);

  249. 

  250.         //保存资料到数据库

  251.         if ($issave) {

  252.             $query = " UPDATE `#@__co_htmls` SET dtime='" . time() . "',result='" . addslashes($body) . "',isdown='1' WHERE aid='$aid' ";

  253.             if (!$this->dsql->ExecuteNoneQuery($query)) {

  254.                 echo $this->dsql->GetError();

  255.             }

  256.             return $body;

  257.         }

  258.         return $body;

  259.     }

  260. 

  261.     // 解析地址

  262.     function GetUrl($uri)

  263.     {

  264.         $arr = $tmp = array();

  265. 

  266.         // query

  267.         $x = array_pad(explode('?', $uri), 2, false);

  268.         $arr['query'] = ($x[1]) ? $x[1] : '';

  269. 

  270.         // resource

  271.         $x         = array_pad(explode('/', $x[0]), 2, false);

  272.         $x_last = array_pop($x);

  273.         if (strpos($x_last, '.') === false) {

  274.             $arr['resource'] = '';

  275.             $x[] = $x_last;

  276.         } else {

  277.             $arr['resource'] = $x_last;

  278.             $tmp = @explode('.', $arr['resource']);

  279.             $arr['file'] = @$tmp[0];

  280.             $arr['ext'] = '.' . @$tmp[1];

  281.         }

  282. 

  283.         // path    

  284.         $arr['path'] = implode('/', $x);

  285.         if (substr($arr['path'], -1) !== '/') $arr['path'] .= '/';

  286. 

  287.         // url

  288.         $arr['url'] = $uri;

  289. 

  290.         return $arr;

  291.     }

  292. 

  293.     /**

  294.      *  获取分页区域的内容

  295.      *

  296.      * @access    public

  297.      * @param     string  $dourl  操作地址

  298.      * @param     string  $noteid  节点ID

  299.      * @param     string  $html  html内容

  300.      * @param     int  $step  步骤

  301.      * @return    string

  302.      */

  303.     function GetSpPage($dourl, $noteid, $html, $step = 0)

  304.     {

  305.         $sarr = $this->artNotes[$noteid];

  306. 

  307.         $linkareaHtml = $this->GetHtmlArea('[内容]', $this->artNotes['sppage'], $html);

  308.         if ($linkareaHtml == '') {

  309.             if ($this->tmpUnitValue == '') {

  310.                 $this->tmpUnitValue .= $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  311.             } else {

  312.                 $this->tmpUnitValue .= "#p#副标题#e#" . $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  313.             }

  314.             if ($this->artNotes["sptype"] != 'diyrule') return;

  315.         }

  316. 

  317. 

  318.         //完整的分页列表

  319.         if ($this->artNotes["sptype"] == 'full' || $this->artNotes["sptype"] == '') {

  320.             $this->tmpUnitValue .= $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  321.             $this->cDedeHtml->GetLinkType = "link";

  322.             $this->cDedeHtml->SetSource($linkareaHtml, $dourl, 'link');

  323.             foreach ($this->cDedeHtml->Links as $k => $t) {

  324.                 $k = $this->cDedeHtml->FillUrl($k);

  325.                 if ($k == $dourl) {

  326.                     continue;

  327.                 }

  328.                 $nhtml = $this->DownOnePage($k);

  329.                 if ($nhtml != '') {

  330.                     $ct = trim($this->GetHtmlArea('[内容]', $sarr['match'], $nhtml));

  331.                     if ($ct != '') {

  332.                         $this->tmpUnitValue .= "#p#副标题#e#" . $ct;

  333.                     }

  334.                 }

  335.             }

  336.         } else if ($this->artNotes["sptype"] == 'diyrule') {

  337.             $maxpage = 10;

  338.             $urlinfo = $this->GetUrl($dourl);

  339.             $testurl = str_replace(array_keys($urlinfo), array_values($urlinfo), $this->artNotes['sppage']);

  340.             $testurl = str_ireplace('{p}', '~p~', $testurl);

  341.             $testurl = str_replace(array('{', '}'), '', $testurl);

  342.             $lastchash = md5($html);

  343.             for ($i = $this->spNotes['srul']; $i <= $this->spNotes['erul']; $i++) {

  344.                 $tempurl = str_replace('~p~', $i, $testurl);

  345.                 $tempurl = $this->cDedeHtml->FillUrl($tempurl);

  346. 

  347.                 $nhtml = $this->DownOnePage($tempurl);

  348.                 $newchash = md5($nhtml);

  349.                 if ($newchash == $lastchash) continue;

  350.                 $lastchash = $newchash;

  351. 

  352.                 if ($nhtml != '') {

  353.                     $ct = trim($this->GetHtmlArea('[内容]', $sarr['match'], $nhtml));

  354.                     if ($ct != '') {

  355.                         $this->tmpUnitValue .= "#p#副标题#e#" . $ct;

  356.                         // echo $this->tmpUnitValue;exit;

  357.                     }

  358.                 }

  359.             }

  360.         }

  361.         //上下页形式或不完整的分页列表

  362.         else {

  363.             if ($step > 50) {

  364.                 return;

  365.             }

  366.             if ($step == 0) {

  367.                 $this->tmpUnitValue .= $this->GetHtmlArea('[内容]', $sarr['match'], $html);

  368.             }

  369.             $this->cDedeHtml->GetLinkType = "link";

  370.             $this->cDedeHtml->SetSource($linkareaHtml, $dourl, 'link');

  371.             $hasLink = FALSE;

  372.             foreach ($this->cDedeHtml->Links as $k => $t) {

  373.                 $k = $this->cDedeHtml->FillUrl($k);

  374.                 if (in_array($k, $this->tmpLinks)) {

  375.                     continue;

  376.                 } else {

  377.                     $nhtml = $this->DownOnePage($k);

  378.                     if ($nhtml != '') {

  379.                         $ct = trim($this->GetHtmlArea('[内容]', $sarr['match'], $nhtml));

  380.                         if ($ct != '') {

  381.                             $this->tmpUnitValue .= "#p#副标题#e#" . $ct;

  382.                         }

  383.                     }

  384.                     $hasLink = TRUE;

  385.                     $this->tmpLinks[] = $k;

  386.                     $dourl = $k;

  387.                     $step++;

  388.                 }

  389.             }

  390.             if ($hasLink) {

  391.                 $this->GetSpPage($dourl, $noteid, $nhtml, $step);

  392.             }

  393.         }

  394.     }

  395. 

  396.     /**

  397.      *  获取特定区域的HTML

  398.      *

  399.      * @access    public

  400.      * @param     string  $sptag  区域标记

  401.      * @param     string  $areaRule  地址规则

  402.      * @param     string  $html  html代码

  403.      * @return    string

  404.      */

  405.     function GetHtmlArea($sptag, &$areaRule, &$html)

  406.     {

  407.         //用正则表达式的模式匹配

  408.         if ($this->noteInfos['matchtype'] == 'regex') {

  409.             $areaRule = str_replace("/", "\\/", $areaRule);

  410.             $areaRules = explode($sptag, $areaRule);

  411.             $arr = array();

  412.             if ($html == '' || $areaRules[0] == '') {

  413.                 return '';

  414.             }

  415.             preg_match('#' . $areaRules[0] . "(.*)" . $areaRules[1] . "#isU", $html, $arr);

  416.             return empty($arr[1]) ? '' : trim($arr[1]);

  417.         }

  418. 

  419.         //用字符串模式匹配

  420.         else {

  421.             $areaRules = explode($sptag, $areaRule);

  422.             if ($html == '' || $areaRules[0] == '') {

  423.                 return '';

  424.             }

  425.             $posstart = @strpos($html, $areaRules[0]);

  426.             if ($posstart === FALSE) {

  427.                 return '';

  428.             }

  429.             $posstart = $posstart + strlen($areaRules[0]);

  430.             $posend = @strpos($html, $areaRules[1], $posstart);

  431.             if ($posend > $posstart && $posend !== FALSE) {

  432.                 //return substr($html,$posstart+strlen($areaRules[0]),$posend-$posstart-strlen($areaRules[0]));

  433.                 return substr($html, $posstart, $posend - $posstart);

  434.             } else {

  435.                 return '';

  436.             }

  437.         }

  438.     }

  439. 

  440.     /**

  441.      *  下载指定网址

  442.      *

  443.      * @access    public

  444.      * @param     string  $dourl  下载地址

  445.      */

  446.     function DownOnePage($dourl)

  447.     {

  448.         $this->cHttpDown->OpenUrl($dourl);

  449.         $html = $this->cHttpDown->GetHtml();

  450.         $this->cHttpDown->Close();

  451.         $this->ChangeCode($html);

  452.         return $html;

  453.     }

  454. 

  455.     /**

  456.      *  下载特定资源，并保存为指定文件

  457.      *

  458.      * @access    public

  459.      * @param     string  $dourl  操作地址

  460.      * @param     string  $mtype  附件类型

  461.      * @param     string  $islitpic  是否缩略图

  462.      * @return    string

  463.      */

  464.     function DownMedia($dourl, $mtype = 'img', $islitpic = FALSE)

  465.     {

  466.         global $notckpic;

  467.         if (empty($notckpic)) {

  468.             $notckpic = 0;

  469.         }

  470. 

  471.         //检测是否已经下载此文件

  472.         $wi = FALSE;

  473.         $tofile = $filename = '';

  474.         if ($notckpic == 0) {

  475.             $row = $this->dsql->GetOne("SELECT hash,tofile FROM `#@__co_mediaurls` WHERE nid='{$this->noteId}' AND hash='" . md5($dourl) . "' ");

  476.             if (isset($row['tofile'])) {

  477.                 $tofile = $filename = $row['tofile'];

  478.             }

  479.         }

  480. 

  481.         //如果不存在，下载文件

  482.         if ($tofile == '' || !file_exists($GLOBALS['cfg_basedir'] . $filename)) {

  483.             $filename = $this->GetRndName($dourl, $mtype);

  484.             if (!preg_match("#^\/#", $filename)) {

  485.                 $filename = "/" . $filename;

  486.             }

  487. 

  488.             //防盗链模式

  489.             if ($this->noteInfos['isref'] == 'yes' && $this->noteInfos['refurl'] != '') {

  490.                 if ($this->noteInfos['exptime'] == '') {

  491.                     $this->noteInfos['exptime'] = 10;

  492.                 }

  493.                 DownImageKeep($dourl, $this->noteInfos['refurl'], $GLOBALS['cfg_basedir'] . $filename, '', 0, $this->Item['exptime']);

  494.             }

  495. 

  496.             //普通模式

  497.             else {

  498.                 $this->cHttpDown->OpenUrl($dourl);

  499.                 $this->cHttpDown->SaveToBin($GLOBALS['cfg_basedir'] . $filename);

  500.                 $this->cHttpDown->Close();

  501.             }

  502. 

  503.             //下载文件成功，保存记录

  504.             if (file_exists($GLOBALS['cfg_basedir'] . $filename)) {

  505.                 if ($tofile == '') {

  506.                     $query = "INSERT INTO `#@__co_mediaurls`(nid,hash,tofile) VALUES ('" . $this->noteId . "', '" . md5($dourl) . "', '" . addslashes($filename) . "');";

  507.                 } else {

  508.                     $query = "UPDATE `#@__co_mediaurls` SET tofile='" . addslashes($filename) . "' WHERE hash='" . md5($dourl) . "' ";

  509.                 }

  510.                 $this->dsql->ExecuteNoneQuery($query);

  511.             }

  512.         }

  513. 

  514.         //如果下载图片失败或图片不存在，返回网址

  515.         if (!file_exists($GLOBALS['cfg_basedir'] . $filename)) {

  516.             return $dourl;

  517.         }

  518. 

  519.         //生成缩略图

  520.         if ($mtype == 'img' && !$islitpic && $this->breImage == '') {

  521.             $this->breImage = $filename;

  522.             if (!preg_match("#^http:\/\/#", $this->breImage) && file_exists($GLOBALS['cfg_basedir'] . $filename)) {

  523.                 $filenames = explode('/', $filename);

  524.                 $filenamed = $filenames[count($filenames) - 1];

  525.                 $nfilename = str_replace('.', '_lit.', $filenamed);

  526.                 $nfilename = str_replace($filenamed, $nfilename, $filename);

  527.                 if (@copy($GLOBALS['cfg_basedir'] . $filename, $GLOBALS['cfg_basedir'] . $nfilename)) {

  528.                     ImageResize($GLOBALS['cfg_basedir'] . $nfilename, $GLOBALS['cfg_ddimg_width'], $GLOBALS['cfg_ddimg_height']);

  529.                     $this->breImage = $nfilename;

  530.                 }

  531.             }

  532.         }

  533.         if ($mtype == 'img' && !$islitpic) {

  534.             @WaterImg($GLOBALS['cfg_basedir'] . $filename, 'collect');

  535.         }

  536.         return $filename;

  537.     }

  538. 

  539.     /**

  540.      *  获得下载媒体的随机名称

  541.      *

  542.      * @access    public

  543.      * @param     string  $url  地址

  544.      * @param     string  $v  值

  545.      * @return    string

  546.      */

  547.     function GetRndName($url, $v)

  548.     {

  549.         global $cfg_image_dir, $cfg_dir_purview;

  550.         $this->mediaCount++;

  551.         $mnum = $this->mediaCount;

  552.         $timedir = "c" . MyDate("ymd", time());

  553.         //存放路径

  554.         $fullurl = preg_replace("#\/{1,}#", "/", $cfg_image_dir . "/");

  555.         if (!is_dir($GLOBALS['cfg_basedir'] . "/$fullurl")) {

  556.             MkdirAll($GLOBALS['cfg_basedir'] . "/$fullurl", $cfg_dir_purview);

  557.         }

  558. 

  559.         $fullurl = $fullurl . $timedir . "/";

  560.         if (!is_dir($GLOBALS['cfg_basedir'] . "/$fullurl")) {

  561.             MkdirAll($GLOBALS['cfg_basedir'] . "/$fullurl", $cfg_dir_purview);

  562.         }

  563. 

  564.         //文件名称

  565.         $timename = str_replace('.', '', ExecTime());

  566.         $threadnum = 0;

  567.         if (isset($_GET['threadnum'])) {

  568.             $threadnum = intval($_GET['threadnum']);

  569.         }

  570.         $filename = dd2char($timename . $threadnum . '-' . $mnum . mt_rand(1000, 9999));

  571. 

  572.         //分配扩展名

  573.         $urls = explode('.', $url);

  574.         if ($v == 'img') {

  575.             $shortname = '.jpg';

  576.             if (preg_match("#\.gif$#i", $url)) {

  577.                 $shortname = '.gif';

  578.             } else if (preg_match("#\.png$#i", $url)) {

  579.                 $shortname = '.png';

  580.             }

  581.         } else if ($v == 'embed') {

  582.             $shortname = '.swf';

  583.         } else {

  584.             $shortname = '';

  585.         }

  586.         $fullname = $fullurl . $filename . $shortname;

  587.         return preg_replace("#\/{1,}#", "/", $fullname);

  588.     }

  589. 

  590.     /**

  591.      *  按载入的网页内容获取规则，从一个HTML文件中获取内容

  592.      *

  593.      * @access    public

  594.      * @param     string  $dourl  操作地址

  595.      * @param     string  $needDown  需要下载

  596.      * @param     string  $litpic  缩略图

  597.      * @return    string

  598.      */

  599.     function GetPageFields($dourl, $needDown, $litpic = '')

  600.     {

  601.         global $cfg_auot_description;

  602.         if ($this->tmpHtml == '') {

  603.             return '';

  604.         }

  605.         $artitem = '';

  606.         $isPutUnit = FALSE;

  607.         $tmpLtKeys = array();

  608.         $inarr = array();

  609. 

  610.         //自动分析关键字和摘要

  611.         preg_match("#<meta[\s]+name=['\"]keywords['\"] content=['\"](.*)['\"]#isU", $this->tmpHtml, $inarr);

  612.         preg_match("#<meta[\s]+content=['\"](.*)['\"] name=['\"]keywords['\"]#isU", $this->tmpHtml, $inarr2);

  613.         if (!isset($inarr[1]) && isset($inarr2[1])) {

  614.             $inarr[1] = $inarr2[1];

  615.         }

  616.         if (isset($inarr[1])) {

  617.             $keywords = trim(cn_substr(html2text($inarr[1]), 30));

  618.             $keywords = preg_replace("#" . $this->artNotes['keywordtrim'] . "#isU", '', $keywords);

  619.             if (!preg_match("#,#", $keywords)) {

  620.                 $keywords = str_replace(' ', ',', $keywords);

  621.             }

  622.             $artitem .= "{dede:field name='keywords'}" . $keywords . "{/dede:field}\r\n";

  623.         } else {

  624.             $artitem .= "{dede:field name='keywords'}{/dede:field}\r\n";

  625.         }

  626.         // preg_match("#<meta[\s]+name=['\"]description['\"] content=['\"](.*)['\"]#isU", $this->tmpHtml, $inarr);

  627.         // preg_match("#<meta[\s]+content=['\"](.*)['\"] name=['\"]description['\"]#isU", $this->tmpHtml, $inarr2);

  628.         preg_match("#<meta[\s]+name=['\"]description['\"][\s]+content=['\"]([^>]*?)['\"]#iU", $this->tmpHtml, $inarr);

  629.         preg_match("#<meta[\s]+content=['\"]([^>]*?)['\"][\s]+name=['\"]description['\"]#iU", $this->tmpHtml, $inarr2);

  630.         if (!isset($inarr[1]) && isset($inarr2[1])) {

  631.             $inarr[1] = $inarr2[1];

  632.         }

  633.         if (isset($inarr[1])) {

  634.             $description = trim(cn_substr(html2text($inarr[1]), $cfg_auot_description));

  635.             $description = preg_replace("/" . $this->artNotes['descriptiontrim'] . "/isU", '', $description);

  636.             $artitem .= "{dede:field name='description'}" . $description . "{/dede:field}\r\n";

  637.         } else {

  638.             $artitem .= "{dede:field name='description'}{/dede:field}\r\n";

  639.         }

  640. 

  641.         foreach ($this->artNotes as $k => $sarr) {

  642.             //可能出现意外的情况

  643.             if ($k == 'sppage' || $k == 'sptype') {

  644.                 continue;

  645.             }

  646.             if (!is_array($sarr)) {

  647.                 continue;

  648.             }

  649. 

  650.             //特殊的规则或没匹配选项

  651.             if ($sarr['match'] == '' || trim($sarr['match']) == '[内容]') {

  652.                 if ($sarr['value'] != '[内容]') {

  653.                     $v = trim($sarr['value']);

  654.                 } else {

  655.                     $v = '';

  656.                 }

  657.             } else {

  658.                 //分多页的内容

  659.                 if ($this->tmpUnitValue != '' && !$isPutUnit && $sarr['isunit'] == 1) {

  660.                     $v = $this->tmpUnitValue;

  661.                     $isPutUnit = TRUE;

  662.                 } else {

  663.                     $v = $this->GetHtmlArea('[内容]', $sarr['match'], $this->tmpHtml);

  664.                 }

  665. 

  666.                 //过滤内容规则

  667.                 if (isset($sarr['trim']) && $v != '') {

  668.                     foreach ($sarr['trim'] as $nv) {

  669.                         if ($nv[0] == '') {

  670.                             continue;

  671.                         }

  672.                         $nvs = str_replace("/", "\\/", $nv[0]);

  673.                         $v = preg_replace("#" . $nvs . "#isU", $nv[1], $v);

  674.                     }

  675.                 }

  676. 

  677.                 //是否下载远程资源

  678.                 if ($needDown) {

  679.                     if ($sarr['isdown'] == '1') {

  680.                         $v = $this->DownMedias($v, $dourl);

  681.                     }

  682.                 } else {

  683.                     if ($sarr['isdown'] == '1') {

  684.                         $v = $this->MediasReplace($v, $dourl);

  685.                     }

  686.                 }

  687.             }

  688.             $v = trim($v);

  689. 

  690.             //用户自行对内容进行处理的接口

  691.             if ($sarr['function'] != '') {

  692.                 $tmpLtKeys[$k]['v'] = $v;

  693.                 $tmpLtKeys[$k]['f'] = $sarr['function'];

  694.             } else {

  695.                 $v = preg_replace("#(　)$#", '', $v);

  696.                 $v = preg_replace("#[\r\n\t ]{1,}$#", '', $v);

  697.                 $artitem .= "{dede:field name='$k'}$v{/dede:field}\r\n";

  698.             }

  699.         } //End Foreach

  700. 

  701.         //处理带函数的项目

  702.         foreach ($tmpLtKeys as $k => $sarr) {

  703.             $v = $this->RunPHP($sarr['v'], $sarr['f']);

  704.             $v = preg_replace("#(　)$#", '', $v);

  705.             $v = preg_replace("#[\r\n\t ]{1,}$#", '', $v);

  706.             $artitem .= "{dede:field name='$k'}$v{/dede:field}\r\n";

  707.         }

  708.         if ($litpic != '' && $this->lists['listpic'] == 1) {

  709.             $artitem .= "{dede:field name='litpic'}" . $this->DownMedia($litpic, 'img', TRUE) . "{/dede:field}\r\n";

  710.         } else {

  711.             $artitem .= "{dede:field name='litpic'}" . $this->breImage . "{/dede:field}\r\n";

  712.         }

  713.         return $artitem;

  714.     }

  715. 

  716.     /**

  717.      *  下载内容里的资源

  718.      *

  719.      * @access    public

  720.      * @param     string  $html  html内容

  721.      * @param     string  $url  地址

  722.      * @return    string

  723.      */

  724.     function DownMedias(&$html, $url)

  725.     {

  726.         $this->cDedeHtml->SetSource($html, $url, 'media');

  727. 

  728.         //下载标记里的图片和flash

  729.         foreach ($this->cDedeHtml->Medias as $k => $v) {

  730.             $furl = $this->cDedeHtml->FillUrl($k);

  731.             if ($v == 'embed' && !preg_match("#\.(swf)\?(.*)$#i", $k) && !preg_match("#\.(swf)$#i", $k)) {

  732.                 continue;

  733.             }

  734.             $okurl = $this->DownMedia($furl, $v);

  735.             $html = str_replace($k, $okurl, $html);

  736.         }

  737. 

  738.         //下载超链接里的图片

  739.         foreach ($this->cDedeHtml->Links as $v => $k) {

  740.             if (preg_match("#\.(jpg|gif|png)\?(.*)$#i", $v) || preg_match("#\.(jpg|gif|png)$#i", $v)) {

  741.                 $m = "img";

  742.             } else if (preg_match("#\.(swf)\?(.*)$#i", $v) || preg_match("#\.(swf)$#i", $v)) {

  743.                 $m = "embed";

  744.             } else {

  745.                 continue;

  746.             }

  747.             $furl = $this->cDedeHtml->FillUrl($v);

  748.             $okurl = $this->DownMedia($furl, $m);

  749.             $html = str_replace($v, $okurl, $html);

  750.         }

  751.         return $html;

  752.     }

  753. 

  754.     /**

  755.      *  仅替换内容里的资源为绝对网址

  756.      *

  757.      * @access    public

  758.      * @param     string  $html  html内容

  759.      * @param     string  $dourl  操作地址

  760.      * @return    string

  761.      */

  762.     function MediasReplace(&$html, $dourl)

  763.     {

  764.         $this->cDedeHtml->SetSource($html, $dourl, 'media');

  765.         foreach ($this->cDedeHtml->Medias as $k => $v) {

  766.             $k = trim($k);

  767.             $okurl = $this->cDedeHtml->FillUrl($k);

  768.             $html = str_replace($k, $okurl, $html);

  769.         }

  770.         return $html;

  771.     }

  772. 

  773.     //测试列表

  774.     function Testlists(&$dourl)

  775.     {

  776.         $links = array();

  777. 

  778.         //从RSS中获取网址

  779.         if ($this->lists['sourcetype'] == 'rss') {

  780.             $dourl = $this->lists['rssurl'];

  781.             $links = GetRssLinks($dourl);

  782.             return $links;

  783.         }

  784. 

  785.         //正常情况

  786.         if (isset($this->lists['url'][0][0])) {

  787.             $dourl = $this->lists['url'][0][0];

  788.         } else {

  789.             $dourl = '';

  790.             $this->errString = "配置中指定列表的网址错误!\r\n";

  791.             return $links;

  792.         }

  793.         $dhtml = new DedeHtml2();

  794.         $html = $this->DownOnePage($dourl);

  795.         if ($html == '') {

  796.             $this->errString = "读取网址： $dourl 时失败！\r\n";

  797.             return $links;

  798.         }

  799.         if (trim($this->lists['areastart']) != '' && trim($this->lists['areaend']) != '') {

  800.             $areabody = $this->lists['areastart'] . '[var:区域]' . $this->lists['areaend'];

  801.             $html = $this->GetHtmlArea('[var:区域]', $areabody, $html);

  802.         }

  803.         $t1 = ExecTime();

  804.         $dhtml->SetSource($html, $dourl, 'link');

  805.         $this->lists['musthas'] = str_replace('/', '\/', $this->lists['musthas']);

  806.         foreach ($dhtml->Links as $s) {

  807.             if ($this->lists['nothas'] != '') {

  808.                 if (preg_match("#" . $this->lists['nothas'] . "#i", $s['link'])) {

  809.                     continue;

  810.                 }

  811.             }

  812.             if ($this->lists['musthas'] != '') {

  813.                 if (!preg_match("#" . $this->lists['musthas'] . "#i", $s['link'])) {

  814.                     continue;

  815.                 }

  816.             }

  817.             $links[] = $s;

  818.         }

  819.         return $links;

  820.     }

  821. 

  822.     /**

  823.      *  测试文章规则

  824.      *

  825.      * @access    public

  826.      * @param     $dourl  操作地址

  827.      * @return    string

  828.      */

  829.     function TestArt($dourl)

  830.     {

  831.         return $this->DownUrl(0, $dourl, '', FALSE);

  832.     }

  833. 

  834.     /**

  835.      *  采集种子网址

  836.      *

  837.      * @access    public

  838.      * @param     int  $islisten  是否监听

  839.      * @param     int  $glstart  采集开始

  840.      * @param     int  $pagesize  分页尺寸

  841.      * @return    string

  842.      */

  843.     function GetSourceUrl($islisten = 0, $glstart = 0, $pagesize = 10)

  844.     {

  845.         //在第一页中进行预处理

  846.         //“下载种子网址的未下载内容”的模式不需要经过采集种子网址的步骤

  847.         if ($glstart == 0) {

  848.             //重新采集所有内容模式

  849.             if ($islisten == -1) {

  850.                 $this->dsql->ExecuteNoneQuery("DELETE FROM `#@__co_urls` WHERE nid='" . $this->noteId . "'");

  851.                 $this->dsql->ExecuteNoneQuery("DELETE FROM `#@__co_htmls` WHERE nid='" . $this->noteId . "' ");

  852.             }

  853.             //监听模式(保留未导出的内容、保留节点的历史网址记录)

  854.             else {

  855.                 $this->dsql->ExecuteNoneQuery("DELETE FROM `#@__co_htmls` WHERE nid='" . $this->noteId . "' AND isexport=1 ");

  856.             }

  857.         }

  858. 

  859.         //从RSS中获取种子

  860.         if ($this->lists['sourcetype'] == 'rss') {

  861.             $links = GetRssLinks($this->lists['rssurl']);

  862.             //if($this->noteInfos['cosort']!='asc')

  863.             $tmplink = krsort($links);

  864.             $lk = 0;

  865.             foreach ($links as $v) {

  866.                 if ($islisten == 1) {

  867.                     $lrow = $this->dsql->GetOne("SELECT * FROM `#@__co_urls` WHERE nid='{$this->noteId}' AND hash='" . md5($v['link']) . "' ");

  868.                     if (is_array($lrow)) {

  869.                         continue;

  870.                     }

  871.                 }

  872.                 $lk++;

  873.                 if ($mytotal > 0 && $lk >= $mytotal) break;

  874.                 $inquery = "INSERT INTO `#@__co_htmls` (`nid` ,`typeid`, `title` , `litpic` , `url` , `dtime` , `isdown` , `isexport` , `result`)

  875.                     VALUES ('{$this->noteId}' , '0', '" . addslashes($v['title']) . "' , '" . addslashes($v['image']) . "' , '" . addslashes($v['link']) . "' , 'dtime' , '0' , '0' , ''); ";

  876.                 $this->dsql->ExecuteNoneQuery($inquery);

  877. 

  878.                 $inquery = "INSERT INTO `#@__co_urls`(hash,nid) VALUES ('" . md5($v['link']) . "','{$this->noteId}');";

  879.                 $this->dsql->ExecuteNoneQuery($inquery);

  880.             }

  881.             return 0;

  882.         } else {

  883.             $tmplink = array();

  884.             $arrStart = 0;

  885.             $moviePostion = 0;

  886.             $endpos = $glstart + $pagesize;

  887.             $totallen = count($this->lists['url']);

  888.             //dump($this->lists['url']);exit;

  889.             foreach ($this->lists['url'] as $k => $cururls) {

  890.                 //$status = FALSE;

  891.                 $urlnum = 0;

  892.                 $cururl = $cururls[0];

  893.                 $typeid = (empty($cururls[1]) ? 0 : $cururls[1]);

  894.                 $moviePostion++;

  895. 

  896.                 if ($moviePostion > $endpos) {

  897.                     break;

  898.                 }

  899.                 if ($moviePostion > $glstart) {

  900.                     $html = $this->DownOnePage($cururl);

  901.                     if (trim($this->lists['areastart']) != '' && trim($this->lists['areaend']) != '') {

  902.                         $areabody = $this->lists['areastart'] . '[var:区域]' . $this->lists['areaend'];

  903.                         $html = $this->GetHtmlArea('[var:区域]', $areabody, $html);

  904.                     }

  905.                     $this->cDedeHtml->SetSource($html, $cururl, 'link');

  906.                     $lk = 0;

  907.                     foreach ($this->cDedeHtml->Links as $k => $v) {

  908.                         if ($this->lists['nothas'] != '') {

  909.                             if (preg_match("#" . $this->lists['nothas'] . "#", $v['link'])) {

  910.                                 continue;

  911.                             }

  912.                         }

  913.                         if ($this->lists['musthas'] != '') {

  914.                             if (!preg_match("#" . $this->lists['musthas'] . "#i", $v['link'])) {

  915.                                 continue;

  916.                             }

  917.                         }

  918.                         $tmplink[$arrStart][0] = $v;

  919.                         $tmplink[$arrStart][1] = $typeid;

  920.                         $arrStart++;

  921.                         $lk++;

  922.                     }

  923.                     $this->cDedeHtml->Clear();

  924.                 }

  925.             } //foreach

  926.             //if($this->noteInfos['cosort']!='asc')

  927. 

  928.             krsort($tmplink);

  929.             $unum = count($tmplink);

  930.             if ($unum > 0) {

  931.                 //echo "完成本次种子网址抓取，共找到：{$unum} 个记录!<br/>\r\n";

  932.                 foreach ($tmplink as $vs) {

  933.                     $v = $vs[0];

  934.                     $typeid = $vs[1];

  935.                     if ($islisten == 1) {

  936.                         $lrow = $this->dsql->GetOne("SELECT * FROM `#@__co_urls` WHERE nid='{$this->noteId}' AND hash='" . md5($v['link']) . "' ");

  937.                         if (is_array($lrow)) {

  938.                             continue;

  939.                         }

  940.                     }

  941.                     $inquery = "INSERT INTO `#@__co_htmls` (`nid` ,`typeid`, `title` , `litpic` , `url` , `dtime` , `isdown` , `isexport` , `result`)

  942.                     VALUES ('{$this->noteId}' ,'$typeid', '" . addslashes($v['title']) . "' , '" . addslashes($v['image']) . "' , '" . addslashes($v['link']) . "' , '" . time() . "' , '0' , '0' , ''); ";

  943.                     $this->dsql->ExecuteNoneQuery($inquery);

  944. 

  945.                     $inquery = "INSERT INTO `#@__co_urls`(hash,nid) VALUES ('" . md5($v['link']) . "','{$this->noteId}');";

  946.                     $this->dsql->ExecuteNoneQuery($inquery);

  947.                 }

  948.                 if ($endpos >= $totallen) {

  949.                     return 0;

  950.                 } else {

  951.                     return ($totallen - $endpos);

  952.                 }

  953.             } else {

  954.                 //仅在第一批采集时出错才返回

  955.                 if ($glstart == 0) {

  956.                     return -1;

  957.                 }

  958. 

  959.                 //在其它页出错照常采集后面内容

  960.                 if ($endpos >= $totallen) {

  961.                     return 0;

  962.                 } else {

  963.                     return ($totallen - $endpos);

  964.                 }

  965.             }

  966.         }

  967.     }

  968. 

  969.     /**

  970.      *  用扩展函数处理采集到的原始数据

  971.      *

  972.      * @access    public

  973.      * @param     string  $fvalue  值

  974.      * @param     string  $phpcode  PHP代码

  975.      * @return    string

  976.      */

  977.     function RunPHP($fvalue, $phpcode)

  978.     {

  979.         $DedeMeValue = $fvalue;

  980.         $phpcode = preg_replace("#'@me'|\"@me\"|@me#isU", '$DedeMeValue', $phpcode);

  981.         if (preg_match("#@body#i", $phpcode)) {

  982.             $DedeBodyValue = $this->tmpHtml;

  983.             $phpcode = preg_replace("#'@body'|\"@body\"|@body#isU", '$DedeBodyValue', $phpcode);

  984.         }

  985.         if (preg_match("#@litpic#i", $phpcode)) {

  986.             $DedeLitPicValue = $this->breImage;

  987.             $phpcode = preg_replace("#'@litpic'|\"@litpic\"|@litpic#isU", '$DedeLitPicValue', $phpcode);

  988.         }

  989.         eval($phpcode . ";");

  990.         return $DedeMeValue;

  991.     }

  992. 

  993.     /**

  994.      *  编码转换

  995.      *

  996.      * @access    public

  997.      * @param     string  $str  字符串

  998.      * @return    string

  999.      */

  1000.     function ChangeCode(&$str)

  1001.     {

  1002.         global $cfg_soft_lang;

  1003.         if ($cfg_soft_lang == 'utf-8') {

  1004.             if ($this->noteInfos["sourcelang"] == "gb2312") {

  1005.                 $str = gb2utf8($str);

  1006.             }

  1007.             if ($this->noteInfos["sourcelang"] == "big5") {

  1008.                 $str = gb2utf8(big52gb($str));

  1009.             }

  1010.         } else {

  1011.             if ($this->noteInfos["sourcelang"] == "utf-8") {

  1012.                 $str = utf82gb($str);

  1013.             }

  1014.             if ($this->noteInfos["sourcelang"] == "big5") {

  1015.                 $str = big52gb($str);

  1016.             }

  1017.         }

  1018.     }

  1019. }//End Class