spatie /
crawler
| @@ 564-579 (lines=16) @@ | ||
| 561 | return $this->crawledUrlCount >= $this->maximumCrawlCount; |
|
| 562 | } |
|
| 563 | ||
| 564 | protected function mayIndex(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool |
|
| 565 | { |
|
| 566 | if (! $this->respectRobots) { |
|
| 567 | return true; |
|
| 568 | } |
|
| 569 | ||
| 570 | if (! $robotsHeaders->mayIndex()) { |
|
| 571 | return false; |
|
| 572 | } |
|
| 573 | ||
| 574 | if (! $robotsMeta->mayIndex()) { |
|
| 575 | return false; |
|
| 576 | } |
|
| 577 | ||
| 578 | return true; |
|
| 579 | } |
|
| 580 | ||
| 581 | protected function mayFollow(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool |
|
| 582 | { |
|
| @@ 581-596 (lines=16) @@ | ||
| 578 | return true; |
|
| 579 | } |
|
| 580 | ||
| 581 | protected function mayFollow(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool |
|
| 582 | { |
|
| 583 | if (! $this->respectRobots) { |
|
| 584 | return true; |
|
| 585 | } |
|
| 586 | ||
| 587 | if (! $robotsHeaders->mayFollow()) { |
|
| 588 | return false; |
|
| 589 | } |
|
| 590 | ||
| 591 | if (! $robotsMeta->mayFollow()) { |
|
| 592 | return false; |
|
| 593 | } |
|
| 594 | ||
| 595 | return true; |
|
| 596 | } |
|
| 597 | } |
|
| 598 | ||
