|
1
|
|
|
<?php
|
|
2
|
|
|
|
|
3
|
|
|
namespace Zrashwani\NewsScrapper\Adapters;
|
|
4
|
|
|
|
|
5
|
|
|
use \Symfony\Component\DomCrawler\Crawler;
|
|
6
|
|
|
|
|
7
|
|
|
/**
|
|
8
|
|
|
* Adapter to extract page data according to default html tags
|
|
9
|
|
|
* @author Zeid Rashwani <zrashwani.com>
|
|
10
|
|
|
*/
|
|
11
|
|
|
class DefaultAdapter extends AbstractAdapter
|
|
12
|
|
|
{
|
|
13
|
|
|
|
|
14
|
|
|
/**
|
|
15
|
|
|
* extract title information from crawler object
|
|
16
|
|
|
* @param Crawler $crawler
|
|
17
|
|
|
* @return string
|
|
18
|
|
|
*/
|
|
19
|
|
View Code Duplication |
public function extractTitle(Crawler $crawler)
|
|
|
|
|
|
|
20
|
|
|
{
|
|
21
|
|
|
$ret = null;
|
|
22
|
|
|
|
|
23
|
|
|
$crawler->filterXPath('//head/title')
|
|
24
|
|
|
->each(
|
|
25
|
|
|
function(Crawler $node) use (&$ret) {
|
|
26
|
|
|
$ret = $node->text();
|
|
27
|
|
|
}
|
|
28
|
|
|
);
|
|
29
|
|
|
|
|
30
|
|
|
return $ret;
|
|
31
|
|
|
}
|
|
32
|
|
|
|
|
33
|
|
|
/**
|
|
34
|
|
|
* extract image url from crawler open graph
|
|
35
|
|
|
* @param Crawler $crawler
|
|
36
|
|
|
* @return string
|
|
37
|
|
|
*/
|
|
38
|
|
|
public function extractImage(Crawler $crawler)
|
|
39
|
|
|
{
|
|
40
|
|
|
$ret = null;
|
|
41
|
|
|
$theAdapter = $this;
|
|
42
|
|
|
|
|
43
|
|
|
$crawler->filterXPath('//img')
|
|
44
|
|
|
->each(
|
|
45
|
|
|
function(Crawler $node) use (&$ret, $theAdapter) {
|
|
46
|
|
|
$img_src = $theAdapter->normalizeLink($node->attr('src'));
|
|
47
|
|
|
$width_org = $height_org = 0;
|
|
48
|
|
|
|
|
49
|
|
|
$url = pathinfo($img_src);
|
|
50
|
|
|
list($width, $height) = getimagesize($url['dirname'].'/'.($url['basename']));
|
|
51
|
|
|
|
|
52
|
|
View Code Duplication |
if (empty($ret) === false) {
|
|
|
|
|
|
|
53
|
|
|
$url_ret = pathinfo($ret);
|
|
54
|
|
|
list($width_org, $height_org) = getimagesize(
|
|
55
|
|
|
$url_ret['dirname'].
|
|
56
|
|
|
'/'.($url_ret['basename'])
|
|
57
|
|
|
);
|
|
58
|
|
|
}
|
|
59
|
|
|
|
|
60
|
|
|
if ($width > $width_org && $height > $height_org) {
|
|
61
|
|
|
$ret = $img_src;
|
|
62
|
|
|
}
|
|
63
|
|
|
}
|
|
64
|
|
|
);
|
|
65
|
|
|
|
|
66
|
|
|
return $ret;
|
|
67
|
|
|
}
|
|
68
|
|
|
|
|
69
|
|
|
/**
|
|
70
|
|
|
* extract page description standard meta tags
|
|
71
|
|
|
* @param Crawler $crawler
|
|
72
|
|
|
* @return string
|
|
73
|
|
|
*/
|
|
74
|
|
View Code Duplication |
public function extractDescription(Crawler $crawler)
|
|
|
|
|
|
|
75
|
|
|
{
|
|
76
|
|
|
$ret = null;
|
|
77
|
|
|
|
|
78
|
|
|
$crawler->filterXPath("//head/meta[@name='description']")
|
|
79
|
|
|
->each(
|
|
80
|
|
|
function(Crawler $node) use (&$ret) {
|
|
81
|
|
|
$ret = $node->attr('content');
|
|
82
|
|
|
}
|
|
83
|
|
|
);
|
|
84
|
|
|
|
|
85
|
|
|
return $ret;
|
|
86
|
|
|
}
|
|
87
|
|
|
|
|
88
|
|
|
/**
|
|
89
|
|
|
* extract keywords out of crawler object
|
|
90
|
|
|
* @param Crawler $crawler
|
|
91
|
|
|
* @return array
|
|
92
|
|
|
*/
|
|
93
|
|
View Code Duplication |
public function extractKeywords(Crawler $crawler)
|
|
|
|
|
|
|
94
|
|
|
{
|
|
95
|
|
|
$ret = array();
|
|
96
|
|
|
|
|
97
|
|
|
$crawler->filterXPath("//head/meta[@name='keywords']")
|
|
98
|
|
|
->each(
|
|
99
|
|
|
function(Crawler $node) use (&$ret) {
|
|
100
|
|
|
$node_txt = trim($node->attr('content'));
|
|
101
|
|
|
if (!empty($node_txt)) {
|
|
102
|
|
|
$ret = explode(',', $node_txt);
|
|
103
|
|
|
}
|
|
104
|
|
|
}
|
|
105
|
|
|
);
|
|
106
|
|
|
$ret = $this->normalizeKeywords($ret);
|
|
107
|
|
|
|
|
108
|
|
|
return $ret;
|
|
109
|
|
|
}
|
|
110
|
|
|
|
|
111
|
|
|
/**
|
|
112
|
|
|
* extrcting body of page article by selecting <article> tag with longest content
|
|
113
|
|
|
* @param Crawler $crawler
|
|
114
|
|
|
* @return string
|
|
115
|
|
|
*/
|
|
116
|
|
|
public function extractBody(Crawler $crawler)
|
|
117
|
|
|
{
|
|
118
|
|
|
$ret = null;
|
|
119
|
|
|
|
|
120
|
|
|
$crawler->filterXPath("//article")
|
|
121
|
|
|
->each(
|
|
122
|
|
|
function(Crawler $node) use (&$ret) {
|
|
123
|
|
|
|
|
124
|
|
|
$node_txt = $node->text();
|
|
125
|
|
|
if (strlen($node_txt) > strlen($ret)) {
|
|
126
|
|
|
$ret = $this->normalizeHtml($node->html());
|
|
127
|
|
|
}
|
|
128
|
|
|
}
|
|
129
|
|
|
);
|
|
130
|
|
|
|
|
131
|
|
|
return $ret;
|
|
132
|
|
|
}
|
|
133
|
|
|
|
|
134
|
|
|
/**
|
|
135
|
|
|
* extract publish date of page, by examining the first <time> tag in document
|
|
136
|
|
|
* @param Crawler $crawler
|
|
137
|
|
|
* @return \DateTime
|
|
138
|
|
|
*/
|
|
139
|
|
|
public function extractPublishDate(Crawler $crawler)
|
|
140
|
|
|
{
|
|
141
|
|
|
$date_str = null;
|
|
142
|
|
|
|
|
143
|
|
|
$crawler->filterXPath("//meta[@name='pubdate']")
|
|
144
|
|
|
->each(
|
|
145
|
|
|
function(Crawler $node) use (&$date_str) {
|
|
146
|
|
|
if (empty($date_str) === true) {
|
|
147
|
|
|
$date_str = $node->attr('content');
|
|
148
|
|
|
}
|
|
149
|
|
|
}
|
|
150
|
|
|
);
|
|
151
|
|
|
|
|
152
|
|
|
try {
|
|
153
|
|
|
if (!is_null($date_str)) {
|
|
154
|
|
|
$ret = \DateTime::createFromFormat('Ymd', $date_str);
|
|
155
|
|
|
$ret->setTime(0, 0, 0);
|
|
156
|
|
|
return $ret->format(\DateTime::ISO8601);
|
|
157
|
|
|
}
|
|
158
|
|
|
} catch (\Exception $ex) {
|
|
159
|
|
|
error_log('invalid date'); //invalid date format
|
|
160
|
|
|
}
|
|
161
|
|
|
|
|
162
|
|
|
return null;
|
|
163
|
|
|
}
|
|
164
|
|
|
|
|
165
|
|
|
/**
|
|
166
|
|
|
* extracting author information from html metadata
|
|
167
|
|
|
* @param Crawler $crawler
|
|
168
|
|
|
* @return string
|
|
169
|
|
|
*/
|
|
170
|
|
View Code Duplication |
public function extractAuthor(Crawler $crawler)
|
|
|
|
|
|
|
171
|
|
|
{
|
|
172
|
|
|
$ret = null;
|
|
173
|
|
|
$crawler->filterXPath("//head/meta[@name='author']")
|
|
174
|
|
|
->each(
|
|
175
|
|
|
function(Crawler $node) use (&$ret) {
|
|
176
|
|
|
$ret = $node->attr('content');
|
|
177
|
|
|
}
|
|
178
|
|
|
);
|
|
179
|
|
|
|
|
180
|
|
|
|
|
181
|
|
|
return $ret;
|
|
182
|
|
|
}
|
|
183
|
|
|
}
|
|
184
|
|
|
|
zrashwani /
news-scrapper
Loading history...
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.