|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace Zrashwani\NewsScrapper\Adapters; |
|
4
|
|
|
|
|
5
|
|
|
use \Symfony\Component\DomCrawler\Crawler; |
|
6
|
|
|
|
|
7
|
|
|
/** |
|
8
|
|
|
* Adapter to extract news base on json-ld format based on schema.org vocabulary specifications |
|
9
|
|
|
* @link http://www.w3.org/TR/json-ld/#embedding-json-ld-in-html-documents json-ld W3C recommendation |
|
10
|
|
|
* @author Zeid Rashwani <zrashwani.com> |
|
11
|
|
|
*/ |
|
12
|
|
|
class JsonLDAdapter extends AbstractAdapter |
|
13
|
|
|
{ |
|
14
|
|
|
private $json_cached = array(); |
|
15
|
|
|
private $crawler_cached = null; |
|
16
|
|
|
|
|
17
|
|
View Code Duplication |
public function extractTitle(Crawler $crawler) |
|
|
|
|
|
|
18
|
|
|
{ |
|
19
|
|
|
$article_data = $this->getJsonData($crawler); |
|
20
|
|
|
$ret = isset($article_data['headline']) ? $article_data['headline'] : null; |
|
21
|
|
|
|
|
22
|
|
|
return $ret; |
|
23
|
|
|
} |
|
24
|
|
|
|
|
25
|
|
|
public function extractImage(Crawler $crawler) |
|
26
|
|
|
{ |
|
27
|
|
|
$ret = null; |
|
28
|
|
|
$article_data = $this->getJsonData($crawler); |
|
29
|
|
|
|
|
30
|
|
|
if (isset($article_data['image']) === true) { |
|
31
|
|
|
$ret = $article_data['image']; |
|
32
|
|
|
} elseif (isset($article_data['thumbnailUrl']) === true) { |
|
33
|
|
|
$ret = $article_data['thumbnailUrl']; |
|
34
|
|
|
} |
|
35
|
|
|
|
|
36
|
|
|
return $ret; |
|
37
|
|
|
} |
|
38
|
|
|
|
|
39
|
|
View Code Duplication |
public function extractDescription(Crawler $crawler) |
|
|
|
|
|
|
40
|
|
|
{ |
|
41
|
|
|
$article_data = $this->getJsonData($crawler); |
|
42
|
|
|
$ret = isset($article_data['description']) ? $article_data['description'] : null; |
|
43
|
|
|
|
|
44
|
|
|
return $ret; |
|
45
|
|
|
} |
|
46
|
|
|
|
|
47
|
|
|
public function extractKeywords(Crawler $crawler) |
|
48
|
|
|
{ |
|
49
|
|
|
$article_data = $this->getJsonData($crawler); |
|
50
|
|
|
$ret = isset($article_data['keywords']) ? $article_data['keywords'] : array(); |
|
51
|
|
|
|
|
52
|
|
|
if (!is_array($ret)) { |
|
53
|
|
|
$ret = explode(',', $ret); |
|
54
|
|
|
} |
|
55
|
|
|
|
|
56
|
|
|
return $ret; |
|
57
|
|
|
} |
|
58
|
|
|
|
|
59
|
|
|
/** |
|
60
|
|
|
* extracting body is not implemented for this adapter, |
|
61
|
|
|
* json-ld don't have this data |
|
62
|
|
|
* @param Crawler $crawler |
|
63
|
|
|
* @return null |
|
64
|
|
|
*/ |
|
65
|
|
|
public function extractBody(Crawler $crawler) |
|
66
|
|
|
{ |
|
67
|
|
|
return; |
|
68
|
|
|
} |
|
69
|
|
|
|
|
70
|
|
|
/** |
|
71
|
|
|
* extracting publish date based on "datePublished" or "dateCreated" information |
|
72
|
|
|
* @param Crawler $crawler |
|
73
|
|
|
* @return string |
|
74
|
|
|
*/ |
|
75
|
|
|
public function extractPublishDate(Crawler $crawler) |
|
76
|
|
|
{ |
|
77
|
|
|
$date_str = null; |
|
78
|
|
|
$article_data = $this->getJsonData($crawler); |
|
79
|
|
|
|
|
80
|
|
|
if (isset($article_data['datePublished']) === true) { |
|
81
|
|
|
$date_str = $article_data['datePublished']; |
|
82
|
|
|
} elseif (isset($article_data['dateCreated']) === true) { |
|
83
|
|
|
$date_str = $article_data['dateCreated']; |
|
84
|
|
|
} |
|
85
|
|
|
|
|
86
|
|
|
if (!is_null($date_str)) { |
|
87
|
|
|
$ret = new \DateTime($date_str); |
|
88
|
|
|
return $ret->format(\DateTime::ISO8601); |
|
89
|
|
|
} else { |
|
90
|
|
|
return null; |
|
91
|
|
|
} |
|
92
|
|
|
} |
|
93
|
|
|
|
|
94
|
|
|
/** |
|
95
|
|
|
* extracting author name through json "author" or "creator" information |
|
96
|
|
|
* @param Crawler $crawler |
|
97
|
|
|
* @return string |
|
98
|
|
|
*/ |
|
99
|
|
|
public function extractAuthor(Crawler $crawler) |
|
100
|
|
|
{ |
|
101
|
|
|
$ret = null; |
|
102
|
|
|
$article_data = $this->getJsonData($crawler); |
|
103
|
|
|
$author_data = null; |
|
104
|
|
|
|
|
105
|
|
|
if (isset($article_data['author'])) { |
|
106
|
|
|
$author_data = $article_data['author']; |
|
107
|
|
|
} elseif (isset($article_data['creator'])) { |
|
108
|
|
|
$author_data = $article_data['creator']; |
|
109
|
|
|
} |
|
110
|
|
|
|
|
111
|
|
|
if ($author_data !== null) { |
|
112
|
|
|
if (is_array($author_data) === true) { |
|
113
|
|
|
if (isset($author_data['@type']) === true && $author_data['@type'] === 'Person') { |
|
114
|
|
|
$ret = $author_data['name']; |
|
115
|
|
|
} else { |
|
116
|
|
|
$ret = implode(', ', $author_data); |
|
117
|
|
|
} |
|
118
|
|
|
} else { |
|
119
|
|
|
$ret = $author_data; |
|
120
|
|
|
} |
|
121
|
|
|
} |
|
122
|
|
|
|
|
123
|
|
|
|
|
124
|
|
|
return $ret; |
|
125
|
|
|
} |
|
126
|
|
|
|
|
127
|
|
|
/** |
|
128
|
|
|
* getting json data decoded as array from crawler object |
|
129
|
|
|
* @param Crawler $crawler |
|
130
|
|
|
* @return array |
|
131
|
|
|
*/ |
|
132
|
|
|
protected function getJsonData(Crawler $crawler) |
|
133
|
|
|
{ |
|
134
|
|
|
if (count($this->json_cached) && $crawler === $this->crawler_cached) { //avoid executing xpath several times |
|
135
|
|
|
return $this->json_cached; |
|
136
|
|
|
} |
|
137
|
|
|
|
|
138
|
|
|
$ret = array(); |
|
139
|
|
|
$crawler->filterXPath('//script[@type="application/ld+json"]') |
|
140
|
|
|
->each(function(Crawler $node) use (&$ret) { |
|
141
|
|
|
$json_content = trim($node->text()); |
|
142
|
|
|
if (empty($json_content) === true && $node->attr('src')) { |
|
|
|
|
|
|
143
|
|
|
$script_path = $this->normalizeLink($node->attr('src')); |
|
144
|
|
|
$json_content = file_get_contents($script_path); |
|
145
|
|
|
} |
|
146
|
|
|
|
|
147
|
|
|
$ret = json_decode($json_content, true); |
|
148
|
|
|
}); |
|
149
|
|
|
|
|
150
|
|
|
$valid_article = $this->checkIfArticle($ret); |
|
151
|
|
|
if ($valid_article) { |
|
152
|
|
|
$this->json_cached = $ret; |
|
153
|
|
|
$this->crawler_cached = $crawler; |
|
154
|
|
|
} |
|
155
|
|
|
|
|
156
|
|
|
return $this->json_cached; |
|
157
|
|
|
} |
|
158
|
|
|
|
|
159
|
|
|
/** |
|
160
|
|
|
* check if json-ld array passed represents a valid article type |
|
161
|
|
|
* based on schema.org vocabulary specification |
|
162
|
|
|
* @param array $article_data |
|
163
|
|
|
* @return boolean |
|
164
|
|
|
*/ |
|
165
|
|
|
protected function checkIfArticle(array $article_data) |
|
166
|
|
|
{ |
|
167
|
|
|
$article_types = ['Article', 'NewsArticle', 'Report', 'ScholarlyArticle', |
|
168
|
|
|
'MedicalScholarlyArticle', 'SocialMediaPosting', |
|
169
|
|
|
'BlogPosting', 'LiveBlogPosting', |
|
170
|
|
|
'DiscussionForumPosting', 'TechArticle', |
|
171
|
|
|
'APIReference']; |
|
172
|
|
|
|
|
173
|
|
|
if (isset($article_data['@context']) && |
|
174
|
|
|
$article_data['@context'] == 'http://schema.org' && |
|
175
|
|
|
isset($article_data['@type']) && |
|
176
|
|
|
in_array($article_data['@type'], $article_types)) { |
|
177
|
|
|
return true; |
|
178
|
|
|
} else { |
|
179
|
|
|
return false; |
|
180
|
|
|
} |
|
181
|
|
|
} |
|
182
|
|
|
} |
|
183
|
|
|
|
zrashwani /
news-scrapper
Loading history...
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.