PiedWeb / SeoPocketCrawler

Crawler.php

<?php

include 'vendor/autoload.php';

// ---------------
// Configure
// ---------------
$config = [
    'start' => [
        'alias' => 's',
        'help' => 'Define where the crawl start.',
    ],
    'limit' => [
        'alias' => 'l',
        'default' => 5,
        'help' => 'Define where a depth limit for the crawler (default 5).',
        'filter' => 'int',
    ],
    'ignore' => [
        'alias' => 'i',
        'default' => null,
        'help' => 'Virtual Robots.txt wich will be interpreted for this crawl (could be a string or an URL).',
    ],
    'user-agent' => [
        'alias' => 'u',
        'default' => 'PockectCr. From PiedWeb',
        'help' => 'Define the user-agent used during the crawl',
    ],
    'verbose' => [
        'alias' => 'v',
        'default' => 1,
        'help' => 'Display debugging information (0/1, default 1).',
        'filter' => 'int',
    ],

];

$CliArgs = new CliArgs\CliArgs($config);
if ($CliArgs->isFlagExists('help', 'h')) echo $CliArgs->getHelp('help');

$startUrl = $CliArgs->getArg('start');
$limit    = intval($CliArgs->getArg('limit'));

$ignore   = $CliArgs->getArg('ignore') ?? '';
if (filter_var($ignore, FILTER_VALIDATE_URL)) $ignore = \PiedWeb\Curl\Request::get($ignore);

$userAgent = (string) $CliArgs->getArg('user-agent');
$debug     = $CliArgs->getArg('verbose');

// ---------------
// Crawler working
// ---------------
if ($startUrl === null) {
    throw new \Exception('--start is required');
}

if ($debug) echo PHP_EOL.PHP_EOL.PHP_EOL.'    Crawl starting'.PHP_EOL.'    '.$startUrl.PHP_EOL.PHP_EOL.PHP_EOL.PHP_EOL;

$crawl = new \PiedWeb\SeoPocketCrawler\Crawler($startUrl, $ignore, $limit, $userAgent);

The type PiedWeb\SeoPocketCrawler\Crawler was not found. Maybe you did not declare it correctly or list all dependencies?

$crawl->crawl($debug);

if ($debug) {
    echo PHP_EOL.PHP_EOL.'---------------'.PHP_EOL.PHP_EOL.'    '.'Crawl succeed'.PHP_EOL;
    echo '    '.'You can find your data in '.PHP_EOL.$crawl->getDataFolder().'/index.csv'.PHP_EOL;
    echo '    You can find source code from html crawled pages in '.$crawl->getCacheFolder();
    echo PHP_EOL.PHP_EOL;
}