|
1
|
|
|
'use strict'; |
|
2
|
|
|
|
|
3
|
|
|
const driver = options => { |
|
4
|
|
|
const Wappalyzer = require('./wappalyzer'); |
|
5
|
|
|
const request = require('request'); |
|
6
|
|
|
const url = require('url'); |
|
7
|
|
|
const fs = require('fs'); |
|
8
|
|
|
const Browser = require('zombie'); |
|
9
|
|
|
|
|
10
|
|
|
const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json')); |
|
11
|
|
|
|
|
12
|
|
|
return { |
|
13
|
|
|
analyze: pageUrl => { |
|
14
|
|
|
const origPageUrl = url.parse(pageUrl); |
|
15
|
|
|
const analyzedPageUrls = []; |
|
16
|
|
|
const apps = []; |
|
17
|
|
|
|
|
18
|
|
|
const wappalyzer = new Wappalyzer(); |
|
19
|
|
|
|
|
20
|
|
|
wappalyzer.apps = json.apps; |
|
21
|
|
|
wappalyzer.categories = json.categories; |
|
22
|
|
|
|
|
23
|
|
|
wappalyzer.driver.log = (message, source, type) => { |
|
24
|
|
|
if ( Boolean(options.debug) ) { |
|
25
|
|
|
console.log('[wappalyzer ' + type + ']', '[' + source + ']', message); |
|
26
|
|
|
} |
|
27
|
|
|
}; |
|
28
|
|
|
|
|
29
|
|
|
wappalyzer.driver.displayApps = detected => { |
|
30
|
|
|
Object.keys(detected).forEach(appName => { |
|
31
|
|
|
const app = detected[appName]; |
|
32
|
|
|
|
|
33
|
|
|
var categories = []; |
|
34
|
|
|
|
|
35
|
|
|
app.props.cats.forEach(id => { |
|
36
|
|
|
var category = {}; |
|
37
|
|
|
|
|
38
|
|
|
category[id] = wappalyzer.categories[id].name; |
|
39
|
|
|
|
|
40
|
|
|
categories.push(category) |
|
41
|
|
|
}); |
|
42
|
|
|
|
|
43
|
|
|
if ( !apps.some(detectedApp => detectedApp.name === app.name) ) { |
|
44
|
|
|
apps.push({ |
|
45
|
|
|
name: app.name, |
|
46
|
|
|
confidence: app.confidenceTotal.toString(), |
|
47
|
|
|
version: app.version, |
|
48
|
|
|
icon: app.props.icon || 'default.svg', |
|
49
|
|
|
website: app.props.website, |
|
50
|
|
|
categories |
|
51
|
|
|
}); |
|
52
|
|
|
} |
|
53
|
|
|
}); |
|
54
|
|
|
}; |
|
55
|
|
|
|
|
56
|
|
|
const browser = new Browser({ |
|
57
|
|
|
userAgent: options.userAgent, |
|
58
|
|
|
waitDuration: options.maxWait + 'ms', |
|
59
|
|
|
}); |
|
60
|
|
|
|
|
61
|
|
|
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); |
|
62
|
|
|
|
|
63
|
|
|
const fetch = (pageUrl, index, depth) => { |
|
64
|
|
|
return new Promise(async (resolve, reject) => { |
|
65
|
|
|
// Return when the URL is a duplicate or maxUrls has been reached |
|
66
|
|
|
if ( analyzedPageUrls.indexOf(pageUrl.href) !== -1 || analyzedPageUrls.length >= options.maxUrls ) { |
|
67
|
|
|
return resolve(); |
|
68
|
|
|
} |
|
69
|
|
|
|
|
70
|
|
|
analyzedPageUrls.push(pageUrl.href); |
|
71
|
|
|
|
|
72
|
|
|
wappalyzer.log('depth: ' + depth + '; delay: ' + ( options.delay * index ) + 'ms; url: ' + pageUrl.href, 'driver'); |
|
73
|
|
|
|
|
74
|
|
|
// Be nice |
|
75
|
|
|
if ( options.delay ) { |
|
76
|
|
|
await sleep(options.delay * index); |
|
77
|
|
|
} |
|
78
|
|
|
|
|
79
|
|
|
browser.visit(pageUrl.href, error => { |
|
80
|
|
|
if ( !browser.resources['0'] || !browser.resources['0'].response ) { |
|
81
|
|
|
wappalyzer.log('No response from server', 'browser', 'error'); |
|
82
|
|
|
|
|
83
|
|
|
return resolve(); |
|
84
|
|
|
} |
|
85
|
|
|
|
|
86
|
|
|
browser.wait() |
|
87
|
|
|
.catch(error => wappalyzer.log(error.message, 'browser')) |
|
88
|
|
|
.finally(() => { |
|
89
|
|
|
wappalyzer.driver.document = browser.document; |
|
90
|
|
|
|
|
91
|
|
|
const headers = {}; |
|
92
|
|
|
|
|
93
|
|
|
browser.resources['0'].response.headers._headers.forEach(header => { |
|
94
|
|
|
if ( !headers[header[0]] ){ |
|
95
|
|
|
headers[header[0]] = []; |
|
96
|
|
|
} |
|
97
|
|
|
|
|
98
|
|
|
headers[header[0]].push(header[1]); |
|
99
|
|
|
}); |
|
100
|
|
|
|
|
101
|
|
|
const vars = Object.getOwnPropertyNames(browser.window); |
|
102
|
|
|
const html = browser.html(); |
|
103
|
|
|
const scripts = Array.prototype.slice |
|
104
|
|
|
.apply(browser.document.scripts) |
|
105
|
|
|
.filter(s => s.src) |
|
106
|
|
|
.map(s => s.src); |
|
107
|
|
|
|
|
108
|
|
|
wappalyzer.analyze(pageUrl.hostname, pageUrl.href, { |
|
109
|
|
|
headers, |
|
110
|
|
|
html, |
|
111
|
|
|
env: vars, |
|
112
|
|
|
scripts |
|
113
|
|
|
}); |
|
114
|
|
|
|
|
115
|
|
|
resolve(browser); |
|
116
|
|
|
}); |
|
117
|
|
|
}); |
|
118
|
|
|
}); |
|
119
|
|
|
}; |
|
120
|
|
|
|
|
121
|
|
|
const crawl = async (pageUrl, index, depth) => { |
|
122
|
|
|
try { |
|
123
|
|
|
const browser = await fetch(pageUrl, index, depth); |
|
124
|
|
|
|
|
125
|
|
|
if ( options.recursive && depth < options.maxDepth && browser ) { |
|
126
|
|
|
const links = Array.from(browser.body.getElementsByTagName('a')).filter(link => link.hostname === origPageUrl.hostname); |
|
127
|
|
|
|
|
128
|
|
|
await Promise.all(links.map(async (link, index) => { |
|
129
|
|
|
link.hash = ''; |
|
130
|
|
|
|
|
131
|
|
|
return crawl(link, index, depth + 1); |
|
132
|
|
|
})); |
|
133
|
|
|
} |
|
134
|
|
|
|
|
135
|
|
|
return Promise.resolve(apps); |
|
136
|
|
|
} catch (error) { |
|
137
|
|
|
return Promise.reject(error); |
|
138
|
|
|
} |
|
139
|
|
|
}; |
|
140
|
|
|
|
|
141
|
|
|
return crawl(origPageUrl, 1, 1); |
|
142
|
|
|
} |
|
143
|
|
|
}; |
|
144
|
|
|
}; |
|
145
|
|
|
|
|
146
|
|
|
module.exports = driver; |
|
147
|
|
|
|
AliasIO /
wappalyzer
