|
1
|
|
|
<?php |
|
2
|
|
|
/** |
|
3
|
|
|
* @author Matthijs van den Bos <[email protected]> |
|
4
|
|
|
* @copyright 2013 Matthijs van den Bos |
|
5
|
|
|
*/ |
|
6
|
|
|
|
|
7
|
|
|
namespace VDB\Spider\PersistenceHandler; |
|
8
|
|
|
|
|
9
|
|
|
use Symfony\Component\Finder\Finder; |
|
10
|
|
|
use VDB\Spider\Resource; |
|
11
|
|
|
|
|
12
|
|
|
class FileRawResponsePersistenceHandler implements PersistenceHandlerInterface |
|
13
|
|
|
{ |
|
14
|
|
|
/** |
|
15
|
|
|
* @var string the path where all spider results should be persisted. |
|
16
|
|
|
* The results will be grouped in a directory by spider ID. |
|
17
|
|
|
*/ |
|
18
|
|
|
private $path = ''; |
|
19
|
|
|
|
|
20
|
|
|
private $spiderId = ''; |
|
21
|
|
|
|
|
22
|
|
|
private $totalSizePersisted = 0; |
|
23
|
|
|
|
|
24
|
|
|
/** @var \Iterator */ |
|
25
|
|
|
private $iterator; |
|
26
|
|
|
|
|
27
|
|
|
/** @var Finder */ |
|
28
|
|
|
private $finder; |
|
29
|
|
|
|
|
30
|
|
|
/** |
|
31
|
|
|
* @param string $path the path where all spider results should be persisted. |
|
32
|
|
|
* The results will be grouped in a directory by spider ID. |
|
33
|
|
|
*/ |
|
34
|
|
|
public function __construct($path) |
|
35
|
|
|
{ |
|
36
|
|
|
$this->path = $path; |
|
37
|
|
|
} |
|
38
|
|
|
|
|
39
|
|
View Code Duplication |
public function setSpiderId($spiderId) |
|
|
|
|
|
|
40
|
|
|
{ |
|
41
|
|
|
$this->spiderId = $spiderId; |
|
42
|
|
|
|
|
43
|
|
|
// create the path |
|
44
|
|
|
if (!file_exists($this->getResultPath())) { |
|
45
|
|
|
mkdir($this->getResultPath(), 0700, true); |
|
46
|
|
|
} |
|
47
|
|
|
} |
|
48
|
|
|
|
|
49
|
|
|
public function count() |
|
50
|
|
|
{ |
|
51
|
|
|
return $this->getFinder()->count(); |
|
52
|
|
|
} |
|
53
|
|
|
|
|
54
|
|
|
private function getResultPath() |
|
55
|
|
|
{ |
|
56
|
|
|
return $this->path . DIRECTORY_SEPARATOR . $this->spiderId . DIRECTORY_SEPARATOR; |
|
57
|
|
|
} |
|
58
|
|
|
|
|
59
|
|
|
public function persist(Resource $resource) |
|
60
|
|
|
{ |
|
61
|
|
|
$fileName = urlencode($resource->getUri()->toString()); |
|
62
|
|
|
$file = new \SplFileObject($this->getResultPath() . $fileName, 'w'); |
|
63
|
|
|
$rawResponse = $resource->getResponse()->__toString(); |
|
64
|
|
|
$this->totalSizePersisted += $file->fwrite($rawResponse); |
|
65
|
|
|
} |
|
66
|
|
|
|
|
67
|
|
View Code Duplication |
private function getFinder() |
|
|
|
|
|
|
68
|
|
|
{ |
|
69
|
|
|
if (!$this->finder instanceof Finder) { |
|
70
|
|
|
$this->finder = Finder::create()->files()->in($this->getResultPath()); |
|
71
|
|
|
} |
|
72
|
|
|
return $this->finder; |
|
73
|
|
|
} |
|
74
|
|
|
|
|
75
|
|
|
private function getIterator() |
|
|
|
|
|
|
76
|
|
|
{ |
|
77
|
|
|
if (!$this->iterator instanceof \Iterator) { |
|
78
|
|
|
$this->iterator = $this->getFinder()->getIterator(); |
|
79
|
|
|
} |
|
80
|
|
|
return $this->iterator; |
|
81
|
|
|
} |
|
82
|
|
|
|
|
83
|
|
|
/** |
|
84
|
|
|
* @return Resource |
|
85
|
|
|
*/ |
|
86
|
|
|
public function current() |
|
87
|
|
|
{ |
|
88
|
|
|
return $this->getIterator()->current()->getContents(); |
|
89
|
|
|
} |
|
90
|
|
|
|
|
91
|
|
|
/** |
|
92
|
|
|
* @return void |
|
93
|
|
|
*/ |
|
94
|
|
|
public function next() |
|
95
|
|
|
{ |
|
96
|
|
|
$this->getIterator()->next(); |
|
97
|
|
|
} |
|
98
|
|
|
|
|
99
|
|
|
/** |
|
100
|
|
|
* @return int |
|
101
|
|
|
*/ |
|
102
|
|
|
public function key() |
|
103
|
|
|
{ |
|
104
|
|
|
return $this->getIterator()->key(); |
|
105
|
|
|
} |
|
106
|
|
|
|
|
107
|
|
|
/** |
|
108
|
|
|
* @return boolean |
|
109
|
|
|
*/ |
|
110
|
|
|
public function valid() |
|
111
|
|
|
{ |
|
112
|
|
|
return $this->getIterator()->valid(); |
|
113
|
|
|
} |
|
114
|
|
|
|
|
115
|
|
|
/** |
|
116
|
|
|
* @return void |
|
117
|
|
|
*/ |
|
118
|
|
|
public function rewind() |
|
119
|
|
|
{ |
|
120
|
|
|
$this->getIterator()->rewind(); |
|
121
|
|
|
} |
|
122
|
|
|
} |
|
123
|
|
|
|
mvdbos /
php-spider
Loading history...
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.