emmanuelroecker / php-linkchecker

        $robotstxt = explode("\n", $robotstxt);

        $result    = [];

        foreach ($robotstxt as $line) {

            if (preg_match('/^\s*Sitemap:(.*)/i', $line, $match)) {

                $urlsitemap = trim($match[1]);

                $response   = $this->client->get($urlsitemap, ['exceptions' => false]);

                if ($response->getStatusCode() != 200) {

                    $result['sitemap']['error'][] = $urlsitemap;

                } else {

                    $result['sitemap']['ok'][$urlsitemap] = $this->checkSitemap($response->getBody()->getContents());

                }

            }

            if (preg_match('/^\s*Disallow:(.*)/i', $line, $match)) {

                $urldisallow = trim($match[1]);

                $response    = $this->client->get($urldisallow, ['exceptions' => false]);

                }

            }

            if (preg_match('/^\s*Disallow:(.*)/i', $line, $match)) {

                $urldisallow = trim($match[1]);

                $response    = $this->client->get($urldisallow, ['exceptions' => false]);

                if (($response->getStatusCode() != 200) && ($response->getStatusCode() != 403)) {

                    $result['disallow']['error'][] = $urldisallow;

                } else {

                    $result['disallow']['ok'][] = $urldisallow;

                }

            }

        }

        return $result;