phpspider\core\phpspider::collect_page()

Complexity

Conditions	21
Paths	1444

Size

Total Lines	140
Code Lines	64

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

<?php
    public function collect_page() 
    {
        $get_collect_url_num = $this->get_collect_url_num();

Are you sure the assignment to $get_collect_url_num is correct as $this->get_collect_url_num() targeting phpspider\core\phpspider::get_collect_url_num() seems to always return null.

        log::info("Find pages: {$get_collect_url_num} ");

        $queue_lsize = $this->queue_lsize();

Are you sure the assignment to $queue_lsize is correct as $this->queue_lsize() targeting phpspider\core\phpspider::queue_lsize() seems to always return null.

        log::info("Waiting for collect pages: {$queue_lsize} ");

        $get_collected_url_num = $this->get_collected_url_num();

Are you sure the assignment to $get_collected_url_num is correct as $this->get_collected_url_num() targeting phpspider\core\phpspider::get_collected_url_num() seems to always return null.

        log::info("Collected pages: {$get_collected_url_num} ");

        // 多任务的时候输出爬虫序号
        if (self::$tasknum > 1) 
        {
            log::info("Current task id: ".self::$taskid);
        }

        // 先进先出
        $link = $this->queue_rpop();

Are you sure the assignment to $link is correct as $this->queue_rpop() targeting phpspider\core\phpspider::queue_rpop() seems to always return null.

        $link = $this->link_uncompress($link);

Are you sure the assignment to $link is correct as $this->link_uncompress($link) targeting phpspider\core\phpspider::link_uncompress() seems to always return null.

        $url = $link['url'];

        // 标记为已爬取网页
        $this->incr_collected_url_num($url);

        // 爬取页面开始时间
        $page_time_start = microtime(true);

        requests::$input_encoding = null;
        $html = $this->request_url($url, $link);

Are you sure the assignment to $html is correct as $this->request_url($url, $link) targeting phpspider\core\phpspider::request_url() seems to always return null.

        if (!$html) 

$html is of type void, thus it always evaluated to false.

        {
            return false;

The expression return false returns the type false which is incompatible with the documented return type void.

        }
        // 当前正在爬取的网页页面的对象
        $page = array(
            'url'     => $url,
            'raw'     => $html,
            'request' => array(
                'url'          => $url,
                'method'       => $link['method'],
                'headers'      => $link['headers'],
                'params'       => $link['params'],
                'context_data' => $link['context_data'],
                'try_num'      => $link['try_num'],
                'max_try'      => $link['max_try'],
                'depth'        => $link['depth'],
                'taskid'       => self::$taskid,
            ),
        );
        unset($html);

        //--------------------------------------------------------------------------------
        // 处理回调函数
        //--------------------------------------------------------------------------------

        // 判断当前网页是否被反爬虫了, 需要开发者实现 
        if ($this->is_anti_spider) 
        {
            $is_anti_spider = call_user_func($this->is_anti_spider, $url, $page['raw'], $this);
            // 如果在回调函数里面判断被反爬虫并且返回true
            if ($is_anti_spider) 
            {
                return false;
            }
        }

        // 在一个网页下载完成之后调用. 主要用来对下载的网页进行处理.
        // 比如下载了某个网页, 希望向网页的body中添加html标签
        if ($this->on_download_page) 
        {
            $return = call_user_func($this->on_download_page, $page, $this);
            // 针对那些老是忘记return的人
            if (isset($return)) $page = $return;
        }

        // 是否从当前页面分析提取URL
        // 回调函数如果返回false表示不需要再从此网页中发现待爬url
        $is_find_url = true;
        if ($link['url_type'] == 'scan_page') 
        {
            if ($this->on_scan_page) 
            {
                $return = call_user_func($this->on_scan_page, $page, $page['raw'], $this);
                if (isset($return)) $is_find_url = $return;
            }
        }
        elseif ($link['url_type'] == 'list_page') 
        {
            if ($this->on_list_page) 
            {
                $return = call_user_func($this->on_list_page, $page, $page['raw'], $this);
                if (isset($return)) $is_find_url = $return;
            }
        }
        elseif ($link['url_type'] == 'content_page') 
        {
            if ($this->on_content_page) 
            {
                $return = call_user_func($this->on_content_page, $page, $page['raw'], $this);
                if (isset($return)) $is_find_url = $return;
            }
        }

        // on_scan_page、on_list_page、on_content_page 返回false表示不需要再从此网页中发现待爬url
        if ($is_find_url) 
        {
            // 如果深度没有超过最大深度, 获取下一级URL
            if (self::$configs['max_depth'] == 0 || $link['depth'] < self::$configs['max_depth']) 
            {
                // 分析提取HTML页面中的URL
                $this->get_urls($page['raw'], $url, $link['depth'] + 1);
            }
        }

        // 如果是内容页, 分析提取HTML页面中的字段
        // 列表页也可以提取数据的, source_type: urlcontext, 未实现
        if ($link['url_type'] == 'content_page') 
        {
            $this->get_html_fields($page['raw'], $url, $page);
        }

        // 如果当前深度大于缓存的, 更新缓存
        $this->incr_depth_num($link['depth']);

        // 处理页面耗时时间
        $time_run = round(microtime(true) - $page_time_start, 3);
        log::debug("Success process page {$url} in {$time_run} s");

        $spider_time_run = util::time2second(intval(microtime(true) - self::$time_start));
        log::info("Spider running in {$spider_time_run}");

        // 爬虫爬取每个网页的时间间隔, 单位: 毫秒
        if (!isset(self::$configs['interval'])) 
        {
            // 默认睡眠100毫秒, 太快了会被认为是ddos
            self::$configs['interval'] = 100;
        }
        usleep(self::$configs['interval'] * 1000);
    }