ricardosierra / phpspider

core/phpspider.bak20170807.php

Spacing +104 added lines, -104 removed lines

@@ -15,7 +15,7 @@ 
 
 namespace phpspider\core;
 
-require_once __DIR__ . '/constants.php';
+require_once __DIR__.'/constants.php';
 
 use phpspider\core\requests;
 use phpspider\core\selector;
@@ -352,7 +352,7 @@ 
     function __construct($configs = array())
     {
         // 产生时钟云，解决php7下面ctrl+c无法停止bug
-        declare(ticks = 1);
+        declare(ticks=1);
 
         // 先打开以显示验证报错内容
         log::$log_show = true;
@@ -369,25 +369,25 @@ 
             exit;
         }
 
-        $configs['name']        = isset($configs['name'])        ? $configs['name']        : 'phpspider';
-        $configs['proxies']     = isset($configs['proxies'])     ? $configs['proxies']     : '';
-        $configs['user_agent']  = isset($configs['user_agent'])  ? $configs['user_agent']  : self::AGENT_PC;
+        $configs['name']        = isset($configs['name']) ? $configs['name'] : 'phpspider';
+        $configs['proxies']     = isset($configs['proxies']) ? $configs['proxies'] : '';
+        $configs['user_agent']  = isset($configs['user_agent']) ? $configs['user_agent'] : self::AGENT_PC;
         $configs['user_agents'] = isset($configs['user_agents']) ? $configs['user_agents'] : null;
-        $configs['client_ip']   = isset($configs['client_ip'])   ? $configs['client_ip']   : null;
-        $configs['client_ips']  = isset($configs['client_ips'])  ? $configs['client_ips']  : null;
-        $configs['interval']    = isset($configs['interval'])    ? $configs['interval']    : self::INTERVAL;
-        $configs['timeout']     = isset($configs['timeout'])     ? $configs['timeout']     : self::TIMEOUT;
-        $configs['max_try']     = isset($configs['max_try'])     ? $configs['max_try']     : self::MAX_TRY;
-        $configs['max_depth']   = isset($configs['max_depth'])   ? $configs['max_depth']   : 0;
-        $configs['max_fields']  = isset($configs['max_fields'])  ? $configs['max_fields']  : 0;
-        $configs['export']      = isset($configs['export'])      ? $configs['export']      : array();
+        $configs['client_ip']   = isset($configs['client_ip']) ? $configs['client_ip'] : null;
+        $configs['client_ips']  = isset($configs['client_ips']) ? $configs['client_ips'] : null;
+        $configs['interval']    = isset($configs['interval']) ? $configs['interval'] : self::INTERVAL;
+        $configs['timeout']     = isset($configs['timeout']) ? $configs['timeout'] : self::TIMEOUT;
+        $configs['max_try']     = isset($configs['max_try']) ? $configs['max_try'] : self::MAX_TRY;
+        $configs['max_depth']   = isset($configs['max_depth']) ? $configs['max_depth'] : 0;
+        $configs['max_fields']  = isset($configs['max_fields']) ? $configs['max_fields'] : 0;
+        $configs['export']      = isset($configs['export']) ? $configs['export'] : array();
 
         // csv、sql、db
-        self::$export_type  = isset($configs['export']['type'])  ? $configs['export']['type']  : '';
-        self::$export_file  = isset($configs['export']['file'])  ? $configs['export']['file']  : '';
+        self::$export_type  = isset($configs['export']['type']) ? $configs['export']['type'] : '';
+        self::$export_file  = isset($configs['export']['file']) ? $configs['export']['file'] : '';
         self::$export_table = isset($configs['export']['table']) ? $configs['export']['table'] : '';
-        self::$db_config    = isset($configs['db_config'])       ? $configs['db_config']       : array();
-        self::$queue_config = isset($configs['queue_config'])    ? $configs['queue_config']    : array();
+        self::$db_config    = isset($configs['db_config']) ? $configs['db_config'] : array();
+        self::$queue_config = isset($configs['queue_config']) ? $configs['queue_config'] : array();
 
         // 是否设置了并发任务数, 并且大于1, 而且不是windows环境
         if (isset($configs['tasknum']) && $configs['tasknum'] > 1 && !util::is_win()) 
@@ -604,7 +604,7 @@ 
         $command2 = isset($argv[2]) ? $argv[2] : '';
 
         // 根据命令做相应处理
-        switch($command)
+        switch ($command)
         {
             // 启动 phpspider
         case 'start':
@@ -766,7 +766,7 @@ 
             // 显示最后结果
             log::$log_show = true;
 
-            $spider_time_run = util::time2second(intval(microtime(true) - self::$time_start));
+            $spider_time_run = util::time2second(intval(microtime(true)-self::$time_start));
             log::note("Spider finished in {$spider_time_run}");
 
             $get_collected_url_num = $this->get_collected_url_num();
@@ -795,12 +795,12 @@ 
         // 检查PHP版本
         if (version_compare(PHP_VERSION, '5.3.0', 'lt')) 
         {
-            log::error('PHP 5.3+ is required, currently installed version is: ' . phpversion());
+            log::error('PHP 5.3+ is required, currently installed version is: '.phpversion());
             exit;
         }
 
         // 检查CURL扩展
-        if(!function_exists('curl_init'))
+        if (!function_exists('curl_init'))
         {
             log::error("The curl extension was not found");
             exit;
@@ -861,7 +861,7 @@ 
             exit;
         }
 
-        foreach ( self::$configs['scan_urls'] as $url ) 
+        foreach (self::$configs['scan_urls'] as $url) 
         {
             // 只检查配置中的入口URL, 通过 add_scan_url 添加的不检查了.
             if (!$this->is_scan_page($url))
@@ -917,7 +917,7 @@ 
         //--------------------------------------------------------------------------------
 
         // 添加入口URL到队列
-        foreach ( self::$configs['scan_urls'] as $url ) 
+        foreach (self::$configs['scan_urls'] as $url) 
         {
             // false 表示不允许重复
             $this->add_scan_url($url, null, false);
@@ -965,13 +965,13 @@ 
         $pid = pcntl_fork();
 
         // 主进程记录子进程pid
-        if($pid > 0)
+        if ($pid > 0)
         {
             // 暂时没用
             //self::$taskpids[$taskid] = $pid;
         }
         // 子进程运行
-        elseif(0 === $pid)
+        elseif (0 === $pid)
         {
             log::warn("Fork children task({$taskid}) successful...");
 
@@ -999,7 +999,7 @@ 
     {
         queue::set_connect('default', self::$queue_config);
         queue::init(); 
-        while( $queue_lsize = $this->queue_lsize() )
+        while ($queue_lsize = $this->queue_lsize())
         { 
             // 如果是主任务
             if (self::$taskmaster) 
@@ -1008,7 +1008,7 @@ 
                 if (self::$tasknum > 1 && !self::$fork_task_complete) 
                 {
                     // 主进程采集到两倍于任务数时, 生成子任务一起采集
-                    if ( $queue_lsize > self::$tasknum*2 ) 
+                    if ($queue_lsize > self::$tasknum * 2) 
                     {
                         self::$fork_task_complete = true;
 
@@ -1037,7 +1037,7 @@ 
             else 
             {
                 // 如果队列中的网页比任务数2倍多, 子任务可以采集, 否则等待...
-                if ( $queue_lsize > self::$tasknum*2 ) 
+                if ($queue_lsize > self::$tasknum * 2) 
                 {
                     // 抓取页面
                     $this->collect_page();
@@ -1176,7 +1176,7 @@ 
             if (self::$configs['max_depth'] == 0 || $link['depth'] < self::$configs['max_depth']) 
             {
                 // 分析提取HTML页面中的URL
-                $this->get_urls($page['raw'], $url, $link['depth'] + 1);
+                $this->get_urls($page['raw'], $url, $link['depth']+1);
             }
         }
 
@@ -1191,10 +1191,10 @@ 
         $this->incr_depth_num($link['depth']);
 
         // 处理页面耗时时间
-        $time_run = round(microtime(true) - $page_time_start, 3);
+        $time_run = round(microtime(true)-$page_time_start, 3);
         log::debug("Success process page {$url} in {$time_run} s");
 
-        $spider_time_run = util::time2second(intval(microtime(true) - self::$time_start));
+        $spider_time_run = util::time2second(intval(microtime(true)-self::$time_start));
         log::info("Spider running in {$spider_time_run}");
 
         // 爬虫爬取每个网页的时间间隔, 单位: 毫秒
@@ -1314,12 +1314,12 @@ 
                     log::error("Failed to download page {$url}");
                     self::$collect_fail++;
                 }
-                elseif (in_array($http_code, array('0','502','503','429'))) 
+                elseif (in_array($http_code, array('0', '502', '503', '429'))) 
                 {
                     // 采集次数加一
                     $link['try_num']++;
                     // 抓取次数 小于 允许抓取失败次数
-                    if ( $link['try_num'] <= $link['max_try'] ) 
+                    if ($link['try_num'] <= $link['max_try']) 
                     {
                         // 扔到队列头部去, 继续采集
                         $this->queue_rpush($link);
@@ -1337,7 +1337,7 @@ 
         }
 
         // 爬取页面耗时时间
-        $time_run = round(microtime(true) - $time_start, 3);
+        $time_run = round(microtime(true)-$time_start, 3);
         log::debug("Success download page {$url} in {$time_run} s");
         self::$collect_succ++;
 
@@ -1382,7 +1382,7 @@ 
 
         foreach ($urls as $key=>$url) 
         {
-            $urls[$key] = str_replace(array("\"", "'",'&amp;'), array("",'','&'), $url);
+            $urls[$key] = str_replace(array("\"", "'", '&amp;'), array("", '', '&'), $url);
         }
 
         //--------------------------------------------------------------------------------
@@ -1458,12 +1458,12 @@ 
 
         // 排除JavaScript的连接
         //if (strpos($url, "javascript:") !== false) 
-        if( preg_match("@^(javascript:|#|'|\")@i", $url) || $url == '')
+        if (preg_match("@^(javascript:|#|'|\")@i", $url) || $url == '')
         {
             return false;
         }
         // 排除没有被解析成功的语言标签
-        if(substr($url, 0, 3) == '<%=')
+        if (substr($url, 0, 3) == '<%=')
         {
             return false;
         }
@@ -1482,46 +1482,46 @@ 
         $domain = $parse_url['host'];
         $path = empty($parse_url['path']) ? '' : $parse_url['path'];
         $base_url_path = $domain.$path;
-        $base_url_path = preg_replace("/\/([^\/]*)\.(.*)$/","/",$base_url_path);
-        $base_url_path = preg_replace("/\/$/",'',$base_url_path);
+        $base_url_path = preg_replace("/\/([^\/]*)\.(.*)$/", "/", $base_url_path);
+        $base_url_path = preg_replace("/\/$/", '', $base_url_path);
 
         $i = $path_step = 0;
         $dstr = $pstr = '';
-        $pos = strpos($url,'#');
-        if($pos > 0)
+        $pos = strpos($url, '#');
+        if ($pos > 0)
         {
             // 去掉#和后面的字符串
             $url = substr($url, 0, $pos);
         }
 
         // 京东变态的都是 //www.jd.com/111.html
-        if(substr($url, 0, 2) == '//')
+        if (substr($url, 0, 2) == '//')
         {
             $url = str_replace("//", "", $url);
         }
         // /1234.html
-        elseif($url[0] == '/')
+        elseif ($url[0] == '/')
         {
             $url = $domain.$url;
         }
         // ./1234.html、../1234.html 这种类型的
-        elseif($url[0] == '.')
+        elseif ($url[0] == '.')
         {
-            if(!isset($url[2]))
+            if (!isset($url[2]))
             {
                 return false;
             }
             else
             {
-                $urls = explode('/',$url);
-                foreach($urls as $u)
+                $urls = explode('/', $url);
+                foreach ($urls as $u)
                 {
-                    if( $u == '..' )
+                    if ($u == '..')
                     {
                         $path_step++;
                     }
                     // 遇到 ., 不知道为什么不直接写$u == '.', 貌似一样的
-                    else if( $i < count($urls)-1 )
+                    else if ($i < count($urls)-1)
                     {
                         $dstr .= $urls[$i].'/';
                     }
@@ -1531,29 +1531,29 @@ 
                     }
                     $i++;
                 }
-                $urls = explode('/',$base_url_path);
-                if(count($urls) <= $path_step)
+                $urls = explode('/', $base_url_path);
+                if (count($urls) <= $path_step)
                 {
                     return false;
                 }
                 else
                 {
                     $pstr = '';
-                    for($i=0;$i<count($urls)-$path_step;$i++){ $pstr .= $urls[$i].'/'; }
+                    for ($i = 0; $i < count($urls)-$path_step; $i++) { $pstr .= $urls[$i].'/'; }
                     $url = $pstr.$dstr;
                 }
             }
         }
         else 
         {
-            if( strtolower(substr($url, 0, 7))=='http://' )
+            if (strtolower(substr($url, 0, 7)) == 'http://')
             {
-                $url = preg_replace('#^http://#i','',$url);
+                $url = preg_replace('#^http://#i', '', $url);
                 $scheme = "http";
             }
-            else if( strtolower(substr($url, 0, 8))=='https://' )
+            else if (strtolower(substr($url, 0, 8)) == 'https://')
             {
-                $url = preg_replace('#^https://#i','',$url);
+                $url = preg_replace('#^https://#i', '', $url);
                 $scheme = "https";
             }
             else
@@ -1650,16 +1650,16 @@ 
     public function link_uncompress($link)
     {
         $link = array(
-            'url'          => isset($link['url'])          ? $link['url']          : '',             
-            'url_type'     => isset($link['url_type'])     ? $link['url_type']     : '',             
-            'method'       => isset($link['method'])       ? $link['method']       : 'get',             
-            'headers'      => isset($link['headers'])      ? $link['headers']      : array(),    
-            'params'       => isset($link['params'])       ? $link['params']       : array(),           
+            'url'          => isset($link['url']) ? $link['url'] : '',             
+            'url_type'     => isset($link['url_type']) ? $link['url_type'] : '',             
+            'method'       => isset($link['method']) ? $link['method'] : 'get',             
+            'headers'      => isset($link['headers']) ? $link['headers'] : array(),    
+            'params'       => isset($link['params']) ? $link['params'] : array(),           
             'context_data' => isset($link['context_data']) ? $link['context_data'] : '',                
-            'proxies'      => isset($link['proxies'])      ? $link['proxies']      : self::$configs['proxies'],             
-            'try_num'      => isset($link['try_num'])      ? $link['try_num']      : 0,                 
-            'max_try'      => isset($link['max_try'])      ? $link['max_try']      : self::$configs['max_try'],
-            'depth'        => isset($link['depth'])        ? $link['depth']        : 0,             
+            'proxies'      => isset($link['proxies']) ? $link['proxies'] : self::$configs['proxies'],             
+            'try_num'      => isset($link['try_num']) ? $link['try_num'] : 0,                 
+            'max_try'      => isset($link['max_try']) ? $link['max_try'] : self::$configs['max_try'],
+            'depth'        => isset($link['depth']) ? $link['depth'] : 0,             
         );
 
         return $link;
@@ -1704,12 +1704,12 @@ 
                     exit(0);
                 }
 
-                if (version_compare(PHP_VERSION,'5.4.0','<'))
+                if (version_compare(PHP_VERSION, '5.4.0', '<'))
                 {
                     $fields_str = json_encode($fields);
-                    $fields_str = preg_replace_callback( "#\\\u([0-9a-f]{4})#i", function($matchs) {
+                    $fields_str = preg_replace_callback("#\\\u([0-9a-f]{4})#i", function($matchs) {
                         return iconv('UCS-2BE', 'UTF-8', pack('H4', $matchs[1]));
-                    }, $fields_str ); 
+                    }, $fields_str); 
                 } 
                 else
                 {
@@ -1775,7 +1775,7 @@ 
             if (!empty($conf['selector'])) 
             {
                 // 如果这个field是上一个field的附带连接
-                if (isset($conf['source_type']) && $conf['source_type']=='attached_url') 
+                if (isset($conf['source_type']) && $conf['source_type'] == 'attached_url') 
                 {
                     // 取出上个field的内容作为连接, 内容分页是不进队列直接下载网页的
                     if (!empty($fields[$conf['attached_url']])) 
@@ -1802,15 +1802,15 @@ 
                 }
 
                 // 没有设置抽取规则的类型 或者 设置为 xpath
-                if (!isset($conf['selector_type']) || $conf['selector_type']=='xpath') 
+                if (!isset($conf['selector_type']) || $conf['selector_type'] == 'xpath') 
                 {
                     $values = $this->get_fields_xpath($html, $conf['selector'], $conf['name']);
                 }
-                elseif ($conf['selector_type']=='css') 
+                elseif ($conf['selector_type'] == 'css') 
                 {
                     $values = $this->get_fields_css($html, $conf['selector'], $conf['name']);
                 }
-                elseif ($conf['selector_type']=='regex') 
+                elseif ($conf['selector_type'] == 'regex') 
                 {
                     $values = $this->get_fields_regex($html, $conf['selector'], $conf['name']);
                 }
@@ -1957,7 +1957,7 @@ 
 
                 $config = self::$db_config;
                 @mysqli_connect($config['host'], $config['user'], $config['pass'], $config['name'], $config['port']);
-                if(mysqli_connect_errno())
+                if (mysqli_connect_errno())
                 {
                     log::error("Export data to a database need Mysql support, Error: ".mysqli_connect_error());
                     exit;
@@ -1994,7 +1994,7 @@ 
             $msg .= "Do you want to continue? [Y/n]";
             fwrite(STDOUT, $msg);
             $arg = strtolower(trim(fgets(STDIN)));
-            $arg = empty($arg) || !in_array($arg, array('y','n')) ? 'y' : $arg;
+            $arg = empty($arg) || !in_array($arg, array('y', 'n')) ? 'y' : $arg;
             if ($arg == 'n') 
             {
                 foreach ($keys as $key) 
@@ -2034,9 +2034,9 @@ 
     public function set_task_status()
     {
         // 每采集成功一个页面, 生成当前进程状态到文件, 供主进程使用
-        $mem = round(memory_get_usage(true)/(1024*1024),2);
-        $use_time = microtime(true) - self::$time_start; 
-        $speed = round((self::$collect_succ + self::$collect_fail) / $use_time, 2);
+        $mem = round(memory_get_usage(true) / (1024 * 1024), 2);
+        $use_time = microtime(true)-self::$time_start; 
+        $speed = round((self::$collect_succ+self::$collect_fail) / $use_time, 2);
         $status = array(
             'id' => self::$taskid,
             'pid' => self::$taskpid,
@@ -2609,7 +2609,7 @@ 
     {
         static $last_lines = 0;
 
-        if(!is_null($force_clear_lines)) 
+        if (!is_null($force_clear_lines)) 
         {
             $last_lines = $force_clear_lines;
         }
@@ -2617,19 +2617,19 @@ 
         // 获取终端宽度
         $toss = $status = null;
         $term_width = exec('tput cols', $toss, $status);
-        if($status || empty($term_width)) 
+        if ($status || empty($term_width)) 
         {
             $term_width = 64; // Arbitrary fall-back term width.
         }
 
         $line_count = 0;
-        foreach(explode("\n", $message) as $line) 
+        foreach (explode("\n", $message) as $line) 
         {
             $line_count += count(str_split($line, $term_width));
         }
 
         // Erasure MAGIC: Clear as many lines as the last output had.
-        for($i = 0; $i < $last_lines; $i++) 
+        for ($i = 0; $i < $last_lines; $i++) 
         {
             // Return to the beginning of the line
             echo "\r";
@@ -2666,16 +2666,16 @@ 
         $display_str = "\033[1A\n\033[K-----------------------------\033[47;30m PHPSPIDER \033[0m-----------------------------\n\033[0m";
         //$display_str = "-----------------------------\033[47;30m PHPSPIDER \033[0m-----------------------------\n\033[0m";
         $run_time_str = util::time2second(time()-self::$time_start, false);
-        $display_str .= 'PHPSpider version:' . self::VERSION . "          PHP version:" . PHP_VERSION . "\n";
-        $display_str .= 'start time:'. date('Y-m-d H:i:s', self::$time_start).'   run ' . $run_time_str . " \n";
+        $display_str .= 'PHPSpider version:'.self::VERSION."          PHP version:".PHP_VERSION."\n";
+        $display_str .= 'start time:'.date('Y-m-d H:i:s', self::$time_start).'   run '.$run_time_str." \n";
 
-        $display_str .= 'spider name: ' . self::$configs['name'] . "\n";
+        $display_str .= 'spider name: '.self::$configs['name']."\n";
         if (self::$multiserver) 
         {
-            $display_str .= 'server id: ' . self::$serverid."\n";
+            $display_str .= 'server id: '.self::$serverid."\n";
         }
-        $display_str .= 'task number: ' . self::$tasknum . "\n";
-        $display_str .= 'load average: ' . implode(", ", $loadavg) . "\n";
+        $display_str .= 'task number: '.self::$tasknum."\n";
+        $display_str .= 'load average: '.implode(", ", $loadavg)."\n";
         $display_str .= "document: https://doc.phpspider.org\n";
 
         $display_str .= $this->display_task_ui();
@@ -2705,12 +2705,12 @@ 
     {
         $display_str = "-------------------------------\033[47;30m TASKS \033[0m-------------------------------\n";
 
-        $display_str .= "\033[47;30mtaskid\033[0m". str_pad('', self::$taskid_length+2-strlen('taskid')). 
-            "\033[47;30mtaskpid\033[0m". str_pad('', self::$pid_length+2-strlen('taskpid')). 
-            "\033[47;30mmem\033[0m". str_pad('', self::$mem_length+2-strlen('mem')). 
-            "\033[47;30mcollect succ\033[0m". str_pad('', self::$urls_length-strlen('collect succ')). 
-            "\033[47;30mcollect fail\033[0m". str_pad('', self::$urls_length-strlen('collect fail')). 
-            "\033[47;30mspeed\033[0m". str_pad('', self::$speed_length+2-strlen('speed')). 
+        $display_str .= "\033[47;30mtaskid\033[0m".str_pad('', self::$taskid_length+2-strlen('taskid')). 
+            "\033[47;30mtaskpid\033[0m".str_pad('', self::$pid_length+2-strlen('taskpid')). 
+            "\033[47;30mmem\033[0m".str_pad('', self::$mem_length+2-strlen('mem')). 
+            "\033[47;30mcollect succ\033[0m".str_pad('', self::$urls_length-strlen('collect succ')). 
+            "\033[47;30mcollect fail\033[0m".str_pad('', self::$urls_length-strlen('collect fail')). 
+            "\033[47;30mspeed\033[0m".str_pad('', self::$speed_length+2-strlen('speed')). 
             "\n";
 
         // "\033[32;40m [OK] \033[0m"
@@ -2738,12 +2738,12 @@ 
     {
         $display_str = "-------------------------------\033[47;30m SERVER \033[0m------------------------------\n";
 
-        $display_str .= "\033[47;30mserver\033[0m". str_pad('', self::$server_length+2-strlen('serverid')). 
-            "\033[47;30mtasknum\033[0m". str_pad('', self::$tasknum_length+2-strlen('tasknum')). 
-            "\033[47;30mmem\033[0m". str_pad('', self::$mem_length+2-strlen('mem')). 
-            "\033[47;30mcollect succ\033[0m". str_pad('', self::$urls_length-strlen('collect succ')). 
-            "\033[47;30mcollect fail\033[0m". str_pad('', self::$urls_length-strlen('collect fail')). 
-            "\033[47;30mspeed\033[0m". str_pad('', self::$speed_length+2-strlen('speed')). 
+        $display_str .= "\033[47;30mserver\033[0m".str_pad('', self::$server_length+2-strlen('serverid')). 
+            "\033[47;30mtasknum\033[0m".str_pad('', self::$tasknum_length+2-strlen('tasknum')). 
+            "\033[47;30mmem\033[0m".str_pad('', self::$mem_length+2-strlen('mem')). 
+            "\033[47;30mcollect succ\033[0m".str_pad('', self::$urls_length-strlen('collect succ')). 
+            "\033[47;30mcollect fail\033[0m".str_pad('', self::$urls_length-strlen('collect fail')). 
+            "\033[47;30mspeed\033[0m".str_pad('', self::$speed_length+2-strlen('speed')). 
             "\n";
 
         $server_list_json = queue::get("server_list");
@@ -2784,11 +2784,11 @@ 
     {
         $display_str = "---------------------------\033[47;30m COLLECT STATUS \033[0m--------------------------\n";
 
-        $display_str .= "\033[47;30mfind pages\033[0m". str_pad('', 16-strlen('find pages')). 
-            "\033[47;30mqueue\033[0m". str_pad('', 14-strlen('queue')). 
-            "\033[47;30mcollected\033[0m". str_pad('', 15-strlen('collected')). 
-            "\033[47;30mfields\033[0m". str_pad('', 15-strlen('fields')). 
-            "\033[47;30mdepth\033[0m". str_pad('', 12-strlen('depth')). 
+        $display_str .= "\033[47;30mfind pages\033[0m".str_pad('', 16-strlen('find pages')). 
+            "\033[47;30mqueue\033[0m".str_pad('', 14-strlen('queue')). 
+            "\033[47;30mcollected\033[0m".str_pad('', 15-strlen('collected')). 
+            "\033[47;30mfields\033[0m".str_pad('', 15-strlen('fields')). 
+            "\033[47;30mdepth\033[0m".str_pad('', 12-strlen('depth')). 
             "\n";
 
         $collect   = $this->get_collect_url_num();

core/requests.php

Spacing +60 added lines, -60 removed lines

@@ -72,21 +72,21 @@ 
     public static $output_encoding = null;
     public static $cookies         = array(); // array of cookies to pass
     // $cookies['username'] = "seatle";
-    public static $rawheaders = array();                        // array of raw headers to send
-    public static $domain_cookies = array();                    // array of cookies for domain to pass
-    public static $hosts = array();                             // random host binding for make request faster
-    public static $headers = array();                           // headers returned from server sent here
-    public static $useragents = array("requests/2.0.0");        // random agent we masquerade as
-    public static $client_ips = array();                        // random ip we masquerade as
-    public static $proxies = array();                           // random proxy ip
-    public static $raw = "";                                    // head + body content returned from server sent here
-    public static $head = "";                                   // head content
-    public static $content = "";                                // The body before encoding
-    public static $text = "";                                   // The body after encoding
-    public static $info = array();                              // curl info
-    public static $history = 302;                               // http request status before redirect. ex:30x
-    public static $status_code = 0;                             // http request status
-    public static $error = "";                                  // error messages sent here
+    public static $rawheaders = array(); // array of raw headers to send
+    public static $domain_cookies = array(); // array of cookies for domain to pass
+    public static $hosts = array(); // random host binding for make request faster
+    public static $headers = array(); // headers returned from server sent here
+    public static $useragents = array("requests/2.0.0"); // random agent we masquerade as
+    public static $client_ips = array(); // random ip we masquerade as
+    public static $proxies = array(); // random proxy ip
+    public static $raw = ""; // head + body content returned from server sent here
+    public static $head = ""; // head content
+    public static $content = ""; // The body before encoding
+    public static $text = ""; // The body after encoding
+    public static $info = array(); // curl info
+    public static $history = 302; // http request status before redirect. ex:30x
+    public static $status_code = 0; // http request status
+    public static $error = ""; // error messages sent here
 
     /**
      * set timeout
@@ -289,7 +289,7 @@ 
         {
             return false;
         }
-        if ( empty($domain) ) 
+        if (empty($domain)) 
         {
             self::$cookies = array();
         }
@@ -541,17 +541,17 @@ 
      */
     public static function init()
     {
-        if (!is_resource ( self::$ch ))
+        if (!is_resource(self::$ch))
         {
-            self::$ch = curl_init ();
-            curl_setopt( self::$ch, CURLOPT_RETURNTRANSFER, true );
-            curl_setopt( self::$ch, CURLOPT_HEADER, false );
-            curl_setopt( self::$ch, CURLOPT_USERAGENT, "phpspider-requests/".self::VERSION );
+            self::$ch = curl_init();
+            curl_setopt(self::$ch, CURLOPT_RETURNTRANSFER, true);
+            curl_setopt(self::$ch, CURLOPT_HEADER, false);
+            curl_setopt(self::$ch, CURLOPT_USERAGENT, "phpspider-requests/".self::VERSION);
             // 如果设置了两个时间，就分开设置
             if (is_array(self::$timeout)) 
             {
-                curl_setopt( self::$ch, CURLOPT_CONNECTTIMEOUT, self::$timeout[0] );
-                curl_setopt( self::$ch, CURLOPT_TIMEOUT, self::$timeout[1]);
+                curl_setopt(self::$ch, CURLOPT_CONNECTTIMEOUT, self::$timeout[0]);
+                curl_setopt(self::$ch, CURLOPT_TIMEOUT, self::$timeout[1]);
             }
             else 
             {
@@ -560,7 +560,7 @@ 
             }
             curl_setopt(self::$ch, CURLOPT_MAXREDIRS, 5); //maximum number of redirects allowed
             // 在多线程处理场景下使用超时选项时，会忽略signals对应的处理函数，但是无耐的是还有小概率的crash情况发生
-            curl_setopt( self::$ch, CURLOPT_NOSIGNAL, true);
+            curl_setopt(self::$ch, CURLOPT_NOSIGNAL, true);
         }
         return self::$ch;
     }
@@ -570,7 +570,7 @@ 
      */
     public static function get($url, $fields = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         return self::request($url, 'get', $fields, NULL, $allow_redirects, $cert);
     }
 
@@ -593,19 +593,19 @@ 
      */
     public static function post($url, $fields = array(), $files = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         return self::request($url, 'POST', $fields, $files, $allow_redirects, $cert);
     }
 
     public static function put($url, $fields = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         return self::request($url, 'PUT', $fields, $allow_redirects, $cert);
     }
 
     public static function delete($url, $fields = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         return self::request($url, 'DELETE', $fields, $allow_redirects, $cert);
     }
 
@@ -614,19 +614,19 @@ 
     // 此方法经常被用来测试超文本链接的有效性，可访问性，和最近的改变。.
     public static function head($url, $fields = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         self::request($url, 'HEAD', $fields, $allow_redirects, $cert);
     }
 
     public static function options($url, $fields = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         return self::request($url, 'OPTIONS', $fields, $allow_redirects, $cert);
     }
 
     public static function patch($url, $fields = array(), $allow_redirects = true, $cert = NULL)
     {
-        self::init ();
+        self::init();
         return self::request($url, 'PATCH', $fields, $allow_redirects, $cert);
     }
 
@@ -645,7 +645,7 @@ 
     public static function request($url, $method = 'GET', $fields = array(), $files = array(), $allow_redirects = true, $cert = NULL)
     {
         $method = strtoupper($method);
-        if(!self::_is_url($url))
+        if (!self::_is_url($url))
         {
             self::$error = "You have requested URL ({$url}) is not a valid HTTP address";
             return false;
@@ -679,7 +679,7 @@ 
             }
         }
 
-        curl_setopt( self::$ch, CURLOPT_URL, $url );
+        curl_setopt(self::$ch, CURLOPT_URL, $url);
 
         if ($method != 'GET')
         {
@@ -692,13 +692,13 @@ 
                 // CURLOPT_POST会把上傳的文件类型设为 multipart/form-data
                 // 把CURLOPT_POSTFIELDS的内容按multipart/form-data 的形式编码
                 // CURLOPT_CUSTOMREQUEST可以按指定内容上传
-                if ( isset($tmpheaders['content-type']) && $tmpheaders['content-type'] == 'application/json' ) 
+                if (isset($tmpheaders['content-type']) && $tmpheaders['content-type'] == 'application/json') 
                 {
-                    curl_setopt( self::$ch, CURLOPT_CUSTOMREQUEST, $method ); 
+                    curl_setopt(self::$ch, CURLOPT_CUSTOMREQUEST, $method); 
                 }
                 else 
                 {
-                    curl_setopt( self::$ch, CURLOPT_POST, true );
+                    curl_setopt(self::$ch, CURLOPT_POST, true);
                 }
 
                 $file_fields = array();
@@ -724,16 +724,16 @@ 
             else
             {
                 self::$rawheaders['X-HTTP-Method-Override'] = $method;
-                curl_setopt( self::$ch, CURLOPT_CUSTOMREQUEST, $method ); 
+                curl_setopt(self::$ch, CURLOPT_CUSTOMREQUEST, $method); 
             }
 
-            if ( $method == 'POST' ) 
+            if ($method == 'POST') 
             {
                 // 不是上传文件的，用http_build_query, 能实现更好的兼容性，更小的请求数据包
-                if ( empty($file_fields) ) 
+                if (empty($file_fields)) 
                 {
                     // post方式
-                    if ( is_array($fields) ) 
+                    if (is_array($fields)) 
                     {
                         $fields = http_build_query($fields);
                     }
@@ -741,7 +741,7 @@ 
                 else 
                 {
                     // 有post数据
-                    if ( is_array($fields) && !empty($fields) ) 
+                    if (is_array($fields) && !empty($fields)) 
                     {
                         // 某些server可能会有问题
                         $fields = array_merge($fields, $file_fields);
@@ -753,13 +753,13 @@ 
                 }
 
                 // 不能直接传数组，不知道是什么Bug，会非常慢
-                curl_setopt( self::$ch, CURLOPT_POSTFIELDS, $fields );
+                curl_setopt(self::$ch, CURLOPT_POSTFIELDS, $fields);
             }
         }
 
         $cookies = self::get_cookies();
         $domain_cookies = self::get_cookies($domain);
-        $cookies =  array_merge($cookies, $domain_cookies);
+        $cookies = array_merge($cookies, $domain_cookies);
         // 是否设置了cookie
         if (!empty($cookies)) 
         {
@@ -773,13 +773,13 @@ 
 
         if (!empty(self::$useragents)) 
         {
-            $key = rand(0, count(self::$useragents) - 1);
+            $key = rand(0, count(self::$useragents)-1);
             self::$rawheaders['User-Agent'] = self::$useragents[$key];
         }
 
         if (!empty(self::$client_ips)) 
         {
-            $key                                 = rand(0, count(self::$client_ips) - 1);
+            $key                                 = rand(0, count(self::$client_ips)-1);
             self::$rawheaders['CLIENT-IP']       = self::$client_ips[$key];
             self::$rawheaders['X-FORWARDED-FOR'] = self::$client_ips[$key];
         }
@@ -791,10 +791,10 @@ 
             {
                 $http_headers[] = $k.': '.$v;
             }
-            curl_setopt( self::$ch, CURLOPT_HTTPHEADER, $http_headers );
+            curl_setopt(self::$ch, CURLOPT_HTTPHEADER, $http_headers);
         }
 
-        curl_setopt( self::$ch, CURLOPT_ENCODING, 'gzip' );
+        curl_setopt(self::$ch, CURLOPT_ENCODING, 'gzip');
 
         // 关闭验证
         if ($scheme == 'https') 
@@ -805,33 +805,33 @@ 
 
         if (self::$proxies)
         {
-            $key = rand(0, count(self::$proxies) - 1);
+            $key = rand(0, count(self::$proxies)-1);
             $proxy = self::$proxies[$key];
-            curl_setopt( self::$ch, CURLOPT_PROXY, $proxy );
+            curl_setopt(self::$ch, CURLOPT_PROXY, $proxy);
         }
 
         // header + body，header 里面有 cookie
-        curl_setopt( self::$ch, CURLOPT_HEADER, true );
+        curl_setopt(self::$ch, CURLOPT_HEADER, true);
         // 请求跳转后的内容
         if ($allow_redirects)
         {
-            curl_setopt( self::$ch, CURLOPT_FOLLOWLOCATION, true);
+            curl_setopt(self::$ch, CURLOPT_FOLLOWLOCATION, true);
         }
 
-        self::$raw = curl_exec ( self::$ch );
+        self::$raw = curl_exec(self::$ch);
         // 真实url
         //$location = curl_getinfo( self::$ch, CURLINFO_EFFECTIVE_URL);
-        self::$info = curl_getinfo( self::$ch );
+        self::$info = curl_getinfo(self::$ch);
         //print_r(self::$info);
         self::$status_code = self::$info['http_code'];
         if (self::$raw === false)
         {
-            self::$error = 'Curl error: ' . curl_error( self::$ch );
+            self::$error = 'Curl error: '.curl_error(self::$ch);
             //trigger_error(self::$error, E_USER_WARNING);
         }
 
         // 关闭句柄
-        curl_close( self::$ch );
+        curl_close(self::$ch);
 
         // 请求成功之后才把URL存起来
         list($header, $text) = self::split_header_body();
@@ -861,7 +861,7 @@ 
     // 获取 mimetype
     public static function get_mimetype($filepath)
     {
-        $fp  = finfo_open(FILEINFO_MIME);
+        $fp = finfo_open(FILEINFO_MIME);
         $mime = finfo_file($fp, $filepath);
         finfo_close($fp);
         $arr  = explode(';', $mime);
@@ -883,7 +883,7 @@ 
     {
         // 构造post数据
         $data = '';
-        $delimiter = '-------------' . uniqid();
+        $delimiter = '-------------'.uniqid();
         // 表单数据
         foreach ($post_fields as $name => $content) 
         {
@@ -941,11 +941,11 @@ 
         {
             $out = self::$output_encoding;
         }
-        if ( ! isset($out))
+        if (!isset($out))
         {
             $out = 'UTF-8';
         }
-        if ( ! in_array($mode, $valid))
+        if (!in_array($mode, $valid))
         {
             throw new Exception('invalid mode, mode='.$mode);
         }
@@ -965,7 +965,7 @@ 
         }
 
         $pattern = '/(<meta[^>]*?charset=([\"\']?))([a-z\d_\-]*)(\2[^>]*?>)/is';
-        if ( ! isset($in))
+        if (!isset($in))
         {
             $n = preg_match($pattern, $html, $in);
             if ($n > 0)