fetch.js ➔ _findHTMLCharset - Code Metrics - Inspection of "Update and rename main.yml to ci.yml" - antograssiot/core - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 80b04f...98ef0e )

by GRASSIOT

created 2019-10-29 20:39 UTC

fetch.js ➔ _findHTMLCharset A

↳ Parent: src/Bridge/Symfony/Bundle/Resources/public/fetch/fetch.js

Complexity

Conditions

Size

Total Lines	22
Code Lines	12

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	12
c	0
b	0
f	0
dl	0
loc	22
rs	9.3333
cc	5

'use strict';

var http = require('http');
var https = require('https');
var urllib = require('url');
var utillib = require('util');
var zlib = require('zlib');
var dns = require('dns');
var Stream = require('stream').Readable;
var CookieJar = require('./cookiejar').CookieJar;
var encodinglib = require('encoding');
var net = require('net');

var USE_ALLOC = typeof Buffer.alloc === 'function';


exports.FetchStream = FetchStream;
exports.CookieJar = CookieJar;
exports.fetchUrl = fetchUrl;

function FetchStream(url, options) {
    Stream.call(this);

    options = options || {};

    this.url = url;
    if (!this.url) {
        return this.emit('error', new Error('url not defined'));
    }

    this.userAgent = options.userAgent || 'FetchStream';

    this._redirect_count = 0;

    this.options = options || {};
    this.normalizeOptions();

    // prevent errors before 'error' handler is set by defferring actions
    if (typeof setImmediate !== 'undefined') {

        setImmediate(this.runStream.bind(this, url));
    } else {
        process.nextTick(this.runStream.bind(this, url));
    }
    this.responseBuffer = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');

    this.ended = false;
    this.readyToRead = 0;

}
utillib.inherits(FetchStream, Stream);

FetchStream.prototype._read = function (size) {
    if (this.ended && this.responseBuffer.length === 0) {
        this.push(null);
        return;
    }
    this.readyToRead += size;
    this.drainBuffer();
};

FetchStream.prototype.drainBuffer = function () {
    if (this.readyToRead === 0) {
        return;
    }
    if (this.responseBuffer.length === 0) {
        return;
    }
    var push;
    var rest;
    var restSize;

    if (this.responseBuffer.length > this.readyToRead) {
        push = USE_ALLOC ? Buffer.alloc(this.readyToRead, '', 'binary') : new Buffer(this.readyToRead, 'binary');

        this.responseBuffer.copy(push, 0, 0, this.readyToRead);
        restSize = this.responseBuffer.length - this.readyToRead;
        rest = USE_ALLOC ? Buffer.alloc(restSize, '', 'binary') : new Buffer(restSize, 'binary');
        this.responseBuffer.copy(rest, 0, this.readyToRead);
    } else {
        push = this.responseBuffer;
        rest = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
    }
    this.responseBuffer = rest;
    this.readyToRead = 0;
    if (this.options.encoding) {
        this.push(push, this.options.encoding);
    } else {
        this.push(push);
    }
};

FetchStream.prototype.destroy = function (ex) {
    this.emit('destroy', ex);
};

FetchStream.prototype.normalizeOptions = function () {

    // cookiejar
    this.cookieJar = this.options.cookieJar || new CookieJar();

    // default redirects - 10
    // if disableRedirect is set, then 0
    if (!this.options.disableRedirect && typeof this.options.maxRedirects !== 'number' &&
        !(this.options.maxRedirects instanceof Number)) {
        this.options.maxRedirects = 10;
    } else if (this.options.disableRedirects) {
        this.options.maxRedirects = 0;
    }

    // normalize header keys
    // HTTP and HTTPS takes in key names in case insensitive but to find
    // an exact value from an object key name needs to be case sensitive
    // so we're just lowercasing all input keys
    this.options.headers = this.options.headers || {};

    var keys = Object.keys(this.options.headers);
    var newheaders = {};
    var i;

    for (i = keys.length - 1; i >= 0; i--) {
        newheaders[keys[i].toLowerCase().trim()] = this.options.headers[keys[i]];
    }

    this.options.headers = newheaders;

    if (!this.options.headers['user-agent']) {
        this.options.headers['user-agent'] = this.userAgent;
    }

    if (!this.options.headers.pragma) {
        this.options.headers.pragma = 'no-cache';
    }

    if (!this.options.headers['cache-control']) {
        this.options.headers['cache-control'] = 'no-cache';
    }

    if (!this.options.disableGzip) {
        this.options.headers['accept-encoding'] = 'gzip, deflate';
    } else {
        delete this.options.headers['accept-encoding'];
    }

    // max length for the response,
    // if not set, default is Infinity
    if (!this.options.maxResponseLength) {
        this.options.maxResponseLength = Infinity;
    }

    // method:
    // defaults to GET, or when payload present to POST
    if (!this.options.method) {
        this.options.method = this.options.payload || this.options.payloadSize ? 'POST' : 'GET';
    }

    // set cookies
    // takes full cookie definition strings as params
    if (this.options.cookies) {
        for (i = 0; i < this.options.cookies.length; i++) {
            this.cookieJar.setCookie(this.options.cookies[i], this.url);
        }
    }

    // rejectUnauthorized
    if (typeof this.options.rejectUnauthorized === 'undefined') {
        this.options.rejectUnauthorized = true;
    }
};

FetchStream.prototype.parseUrl = function (url) {
    var urlparts = urllib.parse(url, false, true),
        transport,
        urloptions = {
            host: urlparts.hostname || urlparts.host,
            port: urlparts.port,
            path: urlparts.pathname + (urlparts.search || '') || '/',
            method: this.options.method,
            rejectUnauthorized: this.options.rejectUnauthorized
        };

    switch (urlparts.protocol) {
        case 'https:':
            transport = https;
            break;
        case 'http:':
        default:
            transport = http;
            break;
    }

    if (transport === https) {
        if('agentHttps' in this.options){
            urloptions.agent = this.options.agentHttps;
        }
        if('agent' in this.options){
            urloptions.agent = this.options.agent;
        }
    } else {
        if('agentHttp' in this.options){
            urloptions.agent = this.options.agentHttp;
        }
        if('agent' in this.options){
            urloptions.agent = this.options.agent;
        }
    }

    if (!urloptions.port) {
        switch (urlparts.protocol) {
            case 'https:':
                urloptions.port = 443;
                break;
            case 'http:':
            default:
                urloptions.port = 80;
                break;
        }
    }

    urloptions.headers = this.options.headers || {};

    if (urlparts.auth) {
        var buf = USE_ALLOC ? Buffer.alloc(Buffer.byteLength(urlparts.auth), urlparts.auth) : new Buffer(urlparts.auth);

        urloptions.headers.Authorization = 'Basic ' + buf.toString('base64');
    }

    return {
        urloptions: urloptions,
        transport: transport
    };
};

FetchStream.prototype.setEncoding = function (encoding) {
    this.options.encoding = encoding;
};

FetchStream.prototype.runStream = function (url) {
    var url_data = this.parseUrl(url),
        cookies = this.cookieJar.getCookies(url);

    if (cookies) {
        url_data.urloptions.headers.cookie = cookies;
    } else {
        delete url_data.urloptions.headers.cookie;
    }

    if (this.options.payload) {
        url_data.urloptions.headers['content-length'] = Buffer.byteLength(this.options.payload || '', 'utf-8');

    }

    if (this.options.payloadSize) {
        url_data.urloptions.headers['content-length'] = this.options.payloadSize;
    }

    if (this.options.asyncDnsLoookup) {
        var dnsCallback = (function (err, addresses) {
            if (err) {
                this.emit('error', err);
                return;
            }

            url_data.urloptions.headers.host = url_data.urloptions.hostname || url_data.urloptions.host;
            url_data.urloptions.hostname = addresses[0];
            url_data.urloptions.host = url_data.urloptions.headers.host + (url_data.urloptions.port ? ':' + url_data.urloptions.port : '');

            this._runStream(url_data, url);
        }).bind(this);

        if (net.isIP(url_data.urloptions.host)) {
            dnsCallback(null, [url_data.urloptions.host]);
        } else {
            dns.resolve4(url_data.urloptions.host, dnsCallback);
        }
    } else {
        this._runStream(url_data, url);
    }
};

FetchStream.prototype._runStream = function (url_data, url) {

    var req = url_data.transport.request(url_data.urloptions, (function (res) {

        // catch new cookies before potential redirect
        if (Array.isArray(res.headers['set-cookie'])) {
            for (var i = 0; i < res.headers['set-cookie'].length; i++) {
                this.cookieJar.setCookie(res.headers['set-cookie'][i], url);
            }
        }

        if ([301, 302, 303, 307, 308].indexOf(res.statusCode) >= 0) {
            if (!this.options.disableRedirects && this.options.maxRedirects > this._redirect_count && res.headers.location) {
                this._redirect_count++;
                req.destroy();
                this.runStream(urllib.resolve(url, res.headers.location));
                return;

            }
        }

        this.meta = {
            status: res.statusCode,
            responseHeaders: res.headers,
            finalUrl: url,
            redirectCount: this._redirect_count,
            cookieJar: this.cookieJar
        };

        var curlen = 0,
            maxlen,

            receive = (function (chunk) {
                if (curlen + chunk.length > this.options.maxResponseLength) {
                    maxlen = this.options.maxResponseLength - curlen;
                } else {
                    maxlen = chunk.length;
                }

                if (maxlen <= 0) {
                    return;
                }

                curlen += Math.min(maxlen, chunk.length);
                if (maxlen >= chunk.length) {
                    if (this.responseBuffer.length === 0) {
                        this.responseBuffer = chunk;
                    } else {
                        this.responseBuffer = Buffer.concat([this.responseBuffer, chunk]);

                    }
                } else {
                    this.responseBuffer = Buffer.concat([this.responseBuffer, chunk], this.responseBuffer.length + maxlen);
                }
                this.drainBuffer();
            }).bind(this),

            error = (function (e) {
                this.ended = true;
                this.emit('error', e);
                this.drainBuffer();
            }).bind(this),

            end = (function () {
                this.ended = true;
                if (this.responseBuffer.length === 0) {
                    this.push(null);
                }
            }).bind(this),

            unpack = (function (type, res) {
                var z = zlib['create' + type]();
                z.on('data', receive);
                z.on('error', error);
                z.on('end', end);
                res.pipe(z);
            }).bind(this);


        this.emit('meta', this.meta);

        if (res.headers['content-encoding']) {
            switch (res.headers['content-encoding'].toLowerCase().trim()) {
                case 'gzip':
                    return unpack('Gunzip', res);
                case 'deflate':
                    return unpack('InflateRaw', res);
            }
        }

        res.on('data', receive);
        res.on('end', end);


    }).bind(this));

    req.on('error', (function (e) {
        this.emit('error', e);
    }).bind(this));

    if (this.options.timeout) {
        req.setTimeout(this.options.timeout, req.abort.bind(req));
    }
    this.on('destroy', req.abort.bind(req));

    if (this.options.payload) {
        req.end(this.options.payload);
    } else if (this.options.payloadStream) {
        this.options.payloadStream.pipe(req);
        this.options.payloadStream.resume();
    } else {
        req.end();
    }
};

function fetchUrl(url, options, callback) {
    if (!callback && typeof options === 'function') {
        callback = options;
        options = undefined;
    }
    options = options || {};

    var fetchstream = new FetchStream(url, options),
        response_data, chunks = [],
        length = 0,
        curpos = 0,
        buffer,
        content_type,
        callbackFired = false;

    fetchstream.on('meta', function (meta) {
        response_data = meta;
        content_type = _parseContentType(meta.responseHeaders['content-type']);
    });

    fetchstream.on('data', function (chunk) {
        if (chunk) {
            chunks.push(chunk);
            length += chunk.length;
        }
    });

    fetchstream.on('error', function (error) {
        if (error && error.code === 'HPE_INVALID_CONSTANT') {
            // skip invalid formatting errors
            return;
        }
        if (callbackFired) {
            return;
        }
        callbackFired = true;
        callback(error);
    });

    fetchstream.on('end', function () {
        if (callbackFired) {
            return;

        }
        callbackFired = true;

        buffer = USE_ALLOC ? Buffer.alloc(length) : new Buffer(length);

        for (var i = 0, len = chunks.length; i < len; i++) {
            chunks[i].copy(buffer, curpos);

            curpos += chunks[i].length;
        }

        if (content_type.mimeType === 'text/html') {
            content_type.charset = _findHTMLCharset(buffer) || content_type.charset;
        }

        content_type.charset = (options.overrideCharset || content_type.charset || 'utf-8').trim().toLowerCase();


        if (!options.disableDecoding && !content_type.charset.match(/^utf-?8$/i)) {
            buffer = encodinglib.convert(buffer, 'UTF-8', content_type.charset);
        }

        if (options.outputEncoding) {
            return callback(null, response_data, buffer.toString(options.outputEncoding));
        } else {

            return callback(null, response_data, buffer);
        }

    });
}

function _parseContentType(str) {
    if (!str) {
        return {};
    }
    var parts = str.split(';'),
        mimeType = parts.shift(),
        charset, chparts;

    for (var i = 0, len = parts.length; i < len; i++) {
        chparts = parts[i].split('=');
        if (chparts.length > 1) {
            if (chparts[0].trim().toLowerCase() === 'charset') {
                charset = chparts[1];
            }
        }
    }

    return {
        mimeType: (mimeType || '').trim().toLowerCase(),
        charset: (charset || 'UTF-8').trim().toLowerCase() // defaults to UTF-8
    };
}

function _findHTMLCharset(htmlbuffer) {

    var body = htmlbuffer.toString('ascii'),
        input, meta, charset;

    if ((meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i))) {
        input = meta[0];
    }

    if (input) {
        charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
        if (charset) {
            charset = (charset[1] || '').trim().toLowerCase();
        }
    }

    if (!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))) {
        charset = (meta[1] || '').trim().toLowerCase();
    }

    return charset;

}


1			'use strict';
2
3			var http = require('http');
4			var https = require('https');
5			var urllib = require('url');
6			var utillib = require('util');
7			var zlib = require('zlib');
8			var dns = require('dns');
9			var Stream = require('stream').Readable;
10			var CookieJar = require('./cookiejar').CookieJar;
11			var encodinglib = require('encoding');
12			var net = require('net');
13
14			var USE_ALLOC = typeof Buffer.alloc === 'function';
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
15
16			exports.FetchStream = FetchStream;
17			exports.CookieJar = CookieJar;
18			exports.fetchUrl = fetchUrl;
19
20			function FetchStream(url, options) {
21			Stream.call(this);
22
23			options = options \|\| {};
24
25			this.url = url;
26			if (!this.url) {
27			return this.emit('error', new Error('url not defined'));
28			}
29
30			this.userAgent = options.userAgent \|\| 'FetchStream';
31
32			this._redirect_count = 0;
33
34			this.options = options \|\| {};
35			this.normalizeOptions();
36
37			// prevent errors before 'error' handler is set by defferring actions
38			if (typeof setImmediate !== 'undefined') {
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `setImmediate` seems to be never declared. If this is a global, consider adding a `/** global: setImmediate */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
39			setImmediate(this.runStream.bind(this, url));
40			} else {
41			process.nextTick(this.runStream.bind(this, url));
42			}
43			this.responseBuffer = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
44			this.ended = false;
45			this.readyToRead = 0;
			0 ignored issues – show Best Practice introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report There is no `return` statement in this branch, but you do return something in other branches. Did you maybe miss it? If you do not want to return anything, consider adding `return undefined;` explicitly. Loading history...
46			}
47			utillib.inherits(FetchStream, Stream);
48
49			FetchStream.prototype._read = function (size) {
50			if (this.ended && this.responseBuffer.length === 0) {
51			this.push(null);
52			return;
53			}
54			this.readyToRead += size;
55			this.drainBuffer();
56			};
57
58			FetchStream.prototype.drainBuffer = function () {
59			if (this.readyToRead === 0) {
60			return;
61			}
62			if (this.responseBuffer.length === 0) {
63			return;
64			}
65			var push;
66			var rest;
67			var restSize;
68
69			if (this.responseBuffer.length > this.readyToRead) {
70			push = USE_ALLOC ? Buffer.alloc(this.readyToRead, '', 'binary') : new Buffer(this.readyToRead, 'binary');
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
71			this.responseBuffer.copy(push, 0, 0, this.readyToRead);
72			restSize = this.responseBuffer.length - this.readyToRead;
73			rest = USE_ALLOC ? Buffer.alloc(restSize, '', 'binary') : new Buffer(restSize, 'binary');
74			this.responseBuffer.copy(rest, 0, this.readyToRead);
75			} else {
76			push = this.responseBuffer;
77			rest = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
78			}
79			this.responseBuffer = rest;
80			this.readyToRead = 0;
81			if (this.options.encoding) {
82			this.push(push, this.options.encoding);
83			} else {
84			this.push(push);
85			}
86			};
87
88			FetchStream.prototype.destroy = function (ex) {
89			this.emit('destroy', ex);
90			};
91
92			FetchStream.prototype.normalizeOptions = function () {
93
94			// cookiejar
95			this.cookieJar = this.options.cookieJar \|\| new CookieJar();
96
97			// default redirects - 10
98			// if disableRedirect is set, then 0
99			if (!this.options.disableRedirect && typeof this.options.maxRedirects !== 'number' &&
100			!(this.options.maxRedirects instanceof Number)) {
101			this.options.maxRedirects = 10;
102			} else if (this.options.disableRedirects) {
103			this.options.maxRedirects = 0;
104			}
105
106			// normalize header keys
107			// HTTP and HTTPS takes in key names in case insensitive but to find
108			// an exact value from an object key name needs to be case sensitive
109			// so we're just lowercasing all input keys
110			this.options.headers = this.options.headers \|\| {};
111
112			var keys = Object.keys(this.options.headers);
113			var newheaders = {};
114			var i;
115
116			for (i = keys.length - 1; i >= 0; i--) {
117			newheaders[keys[i].toLowerCase().trim()] = this.options.headers[keys[i]];
118			}
119
120			this.options.headers = newheaders;
121
122			if (!this.options.headers['user-agent']) {
123			this.options.headers['user-agent'] = this.userAgent;
124			}
125
126			if (!this.options.headers.pragma) {
127			this.options.headers.pragma = 'no-cache';
128			}
129
130			if (!this.options.headers['cache-control']) {
131			this.options.headers['cache-control'] = 'no-cache';
132			}
133
134			if (!this.options.disableGzip) {
135			this.options.headers['accept-encoding'] = 'gzip, deflate';
136			} else {
137			delete this.options.headers['accept-encoding'];
138			}
139
140			// max length for the response,
141			// if not set, default is Infinity
142			if (!this.options.maxResponseLength) {
143			this.options.maxResponseLength = Infinity;
144			}
145
146			// method:
147			// defaults to GET, or when payload present to POST
148			if (!this.options.method) {
149			this.options.method = this.options.payload \|\| this.options.payloadSize ? 'POST' : 'GET';
150			}
151
152			// set cookies
153			// takes full cookie definition strings as params
154			if (this.options.cookies) {
155			for (i = 0; i < this.options.cookies.length; i++) {
156			this.cookieJar.setCookie(this.options.cookies[i], this.url);
157			}
158			}
159
160			// rejectUnauthorized
161			if (typeof this.options.rejectUnauthorized === 'undefined') {
162			this.options.rejectUnauthorized = true;
163			}
164			};
165
166			FetchStream.prototype.parseUrl = function (url) {
167			var urlparts = urllib.parse(url, false, true),
168			transport,
169			urloptions = {
170			host: urlparts.hostname \|\| urlparts.host,
171			port: urlparts.port,
172			path: urlparts.pathname + (urlparts.search \|\| '') \|\| '/',
173			method: this.options.method,
174			rejectUnauthorized: this.options.rejectUnauthorized
175			};
176
177			switch (urlparts.protocol) {
178			case 'https:':
179			transport = https;
180			break;
181			case 'http:':
182			default:
183			transport = http;
184			break;
185			}
186
187			if (transport === https) {
188			if('agentHttps' in this.options){
189			urloptions.agent = this.options.agentHttps;
190			}
191			if('agent' in this.options){
192			urloptions.agent = this.options.agent;
193			}
194			} else {
195			if('agentHttp' in this.options){
196			urloptions.agent = this.options.agentHttp;
197			}
198			if('agent' in this.options){
199			urloptions.agent = this.options.agent;
200			}
201			}
202
203			if (!urloptions.port) {
204			switch (urlparts.protocol) {
205			case 'https:':
206			urloptions.port = 443;
207			break;
208			case 'http:':
209			default:
210			urloptions.port = 80;
211			break;
212			}
213			}
214
215			urloptions.headers = this.options.headers \|\| {};
216
217			if (urlparts.auth) {
218			var buf = USE_ALLOC ? Buffer.alloc(Buffer.byteLength(urlparts.auth), urlparts.auth) : new Buffer(urlparts.auth);
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
219			urloptions.headers.Authorization = 'Basic ' + buf.toString('base64');
220			}
221
222			return {
223			urloptions: urloptions,
224			transport: transport
225			};
226			};
227
228			FetchStream.prototype.setEncoding = function (encoding) {
229			this.options.encoding = encoding;
230			};
231
232			FetchStream.prototype.runStream = function (url) {
233			var url_data = this.parseUrl(url),
234			cookies = this.cookieJar.getCookies(url);
235
236			if (cookies) {
237			url_data.urloptions.headers.cookie = cookies;
238			} else {
239			delete url_data.urloptions.headers.cookie;
240			}
241
242			if (this.options.payload) {
243			url_data.urloptions.headers['content-length'] = Buffer.byteLength(this.options.payload \|\| '', 'utf-8');
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
244			}
245
246			if (this.options.payloadSize) {
247			url_data.urloptions.headers['content-length'] = this.options.payloadSize;
248			}
249
250			if (this.options.asyncDnsLoookup) {
251			var dnsCallback = (function (err, addresses) {
252			if (err) {
253			this.emit('error', err);
254			return;
255			}
256
257			url_data.urloptions.headers.host = url_data.urloptions.hostname \|\| url_data.urloptions.host;
258			url_data.urloptions.hostname = addresses[0];
259			url_data.urloptions.host = url_data.urloptions.headers.host + (url_data.urloptions.port ? ':' + url_data.urloptions.port : '');
260
261			this._runStream(url_data, url);
262			}).bind(this);
263
264			if (net.isIP(url_data.urloptions.host)) {
265			dnsCallback(null, [url_data.urloptions.host]);
266			} else {
267			dns.resolve4(url_data.urloptions.host, dnsCallback);
268			}
269			} else {
270			this._runStream(url_data, url);
271			}
272			};
273
274			FetchStream.prototype._runStream = function (url_data, url) {
275
276			var req = url_data.transport.request(url_data.urloptions, (function (res) {
277
278			// catch new cookies before potential redirect
279			if (Array.isArray(res.headers['set-cookie'])) {
280			for (var i = 0; i < res.headers['set-cookie'].length; i++) {
281			this.cookieJar.setCookie(res.headers['set-cookie'][i], url);
282			}
283			}
284
285			if ([301, 302, 303, 307, 308].indexOf(res.statusCode) >= 0) {
286			if (!this.options.disableRedirects && this.options.maxRedirects > this._redirect_count && res.headers.location) {
287			this._redirect_count++;
288			req.destroy();
289			this.runStream(urllib.resolve(url, res.headers.location));
290			return;
			0 ignored issues – show Comprehensibility Best Practice introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report Are you sure this return statement is not missing an argument? If this is intended, consider adding an explicit `undefined` like `return undefined;`. Loading history...
291			}
292			}
293
294			this.meta = {
295			status: res.statusCode,
296			responseHeaders: res.headers,
297			finalUrl: url,
298			redirectCount: this._redirect_count,
299			cookieJar: this.cookieJar
300			};
301
302			var curlen = 0,
303			maxlen,
304
305			receive = (function (chunk) {
306			if (curlen + chunk.length > this.options.maxResponseLength) {
307			maxlen = this.options.maxResponseLength - curlen;
308			} else {
309			maxlen = chunk.length;
310			}
311
312			if (maxlen <= 0) {
313			return;
314			}
315
316			curlen += Math.min(maxlen, chunk.length);
317			if (maxlen >= chunk.length) {
318			if (this.responseBuffer.length === 0) {
319			this.responseBuffer = chunk;
320			} else {
321			this.responseBuffer = Buffer.concat([this.responseBuffer, chunk]);
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
322			}
323			} else {
324			this.responseBuffer = Buffer.concat([this.responseBuffer, chunk], this.responseBuffer.length + maxlen);
325			}
326			this.drainBuffer();
327			}).bind(this),
328
329			error = (function (e) {
330			this.ended = true;
331			this.emit('error', e);
332			this.drainBuffer();
333			}).bind(this),
334
335			end = (function () {
336			this.ended = true;
337			if (this.responseBuffer.length === 0) {
338			this.push(null);
339			}
340			}).bind(this),
341
342			unpack = (function (type, res) {
343			var z = zlib['create' + type]();
344			z.on('data', receive);
345			z.on('error', error);
346			z.on('end', end);
347			res.pipe(z);
348			}).bind(this);
			0 ignored issues – show unused-code introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The call to `bind` does not seem necessary since the function does not use `this`. Consider calling it directly. Loading history...
349
350			this.emit('meta', this.meta);
351
352			if (res.headers['content-encoding']) {
353			switch (res.headers['content-encoding'].toLowerCase().trim()) {
354			case 'gzip':
355			return unpack('Gunzip', res);
356			case 'deflate':
357			return unpack('InflateRaw', res);
358			}
359			}
360
361			res.on('data', receive);
362			res.on('end', end);
			0 ignored issues – show Best Practice introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report There is no `return` statement in this branch, but you do return something in other branches. Did you maybe miss it? If you do not want to return anything, consider adding `return undefined;` explicitly. Loading history...
363
364			}).bind(this));
365
366			req.on('error', (function (e) {
367			this.emit('error', e);
368			}).bind(this));
369
370			if (this.options.timeout) {
371			req.setTimeout(this.options.timeout, req.abort.bind(req));
372			}
373			this.on('destroy', req.abort.bind(req));
374
375			if (this.options.payload) {
376			req.end(this.options.payload);
377			} else if (this.options.payloadStream) {
378			this.options.payloadStream.pipe(req);
379			this.options.payloadStream.resume();
380			} else {
381			req.end();
382			}
383			};
384
385			function fetchUrl(url, options, callback) {
386			if (!callback && typeof options === 'function') {
387			callback = options;
388			options = undefined;
389			}
390			options = options \|\| {};
391
392			var fetchstream = new FetchStream(url, options),
393			response_data, chunks = [],
394			length = 0,
395			curpos = 0,
396			buffer,
397			content_type,
398			callbackFired = false;
399
400			fetchstream.on('meta', function (meta) {
401			response_data = meta;
402			content_type = _parseContentType(meta.responseHeaders['content-type']);
403			});
404
405			fetchstream.on('data', function (chunk) {
406			if (chunk) {
407			chunks.push(chunk);
408			length += chunk.length;
409			}
410			});
411
412			fetchstream.on('error', function (error) {
413			if (error && error.code === 'HPE_INVALID_CONSTANT') {
414			// skip invalid formatting errors
415			return;
416			}
417			if (callbackFired) {
418			return;
419			}
420			callbackFired = true;
421			callback(error);
422			});
423
424			fetchstream.on('end', function () {
425			if (callbackFired) {
426			return;
			0 ignored issues – show Comprehensibility Best Practice introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report Are you sure this return statement is not missing an argument? If this is intended, consider adding an explicit `undefined` like `return undefined;`. Loading history...
427			}
428			callbackFired = true;
429
430			buffer = USE_ALLOC ? Buffer.alloc(length) : new Buffer(length);
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `Buffer` seems to be never declared. If this is a global, consider adding a `/** global: Buffer */` comment. This checks looks for references to variables that have not been declared. This is most likey a typographical error or a variable has been renamed. To learn more about declaring variables in Javascript, see the MDN. Loading history...
431			for (var i = 0, len = chunks.length; i < len; i++) {
432			chunks[i].copy(buffer, curpos);
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `curpos` is changed as part of the for loop for example by `chunks.i.length` on line `433`. Only the value of the last iteration will be visible in this function if it is called after the loop. Loading history...
433			curpos += chunks[i].length;
434			}
435
436			if (content_type.mimeType === 'text/html') {
437			content_type.charset = _findHTMLCharset(buffer) \|\| content_type.charset;
438			}
439
440			content_type.charset = (options.overrideCharset \|\| content_type.charset \|\| 'utf-8').trim().toLowerCase();
441
442
443			if (!options.disableDecoding && !content_type.charset.match(/^utf-?8$/i)) {
444			buffer = encodinglib.convert(buffer, 'UTF-8', content_type.charset);
445			}
446
447			if (options.outputEncoding) {
448			return callback(null, response_data, buffer.toString(options.outputEncoding));
449			} else {
			0 ignored issues – show Comprehensibility introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report `else` is not necessary here since all `if` branches return, consider removing it to reduce nesting and make code more readable. Loading history...
450			return callback(null, response_data, buffer);
451			}
452
453			});
454			}
455
456			function _parseContentType(str) {
457			if (!str) {
458			return {};
459			}
460			var parts = str.split(';'),
461			mimeType = parts.shift(),
462			charset, chparts;
463
464			for (var i = 0, len = parts.length; i < len; i++) {
465			chparts = parts[i].split('=');
466			if (chparts.length > 1) {
467			if (chparts[0].trim().toLowerCase() === 'charset') {
468			charset = chparts[1];
469			}
470			}
471			}
472
473			return {
474			mimeType: (mimeType \|\| '').trim().toLowerCase(),
475			charset: (charset \|\| 'UTF-8').trim().toLowerCase() // defaults to UTF-8
476			};
477			}
478
479			function _findHTMLCharset(htmlbuffer) {
480
481			var body = htmlbuffer.toString('ascii'),
482			input, meta, charset;
483
484			if ((meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i))) {
485			input = meta[0];
486			}
487
488			if (input) {
489			charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
490			if (charset) {
491			charset = (charset[1] \|\| '').trim().toLowerCase();
492			}
493			}
494
495			if (!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))) {
496			charset = (meta[1] \|\| '').trim().toLowerCase();
497			}
498
499			return charset;
			0 ignored issues – show Bug introduced 2018-11-16 13:29 UTC by Report Bug Copy Issue Report The variable `charset` does not seem to be initialized in case `input` on line `488` is `false`. Are you sure this can never be the case? Loading history...
500			}
501

antograssiot / core

Push — master ( 80b04f...98ef0e )

fetch.js ➔ _findHTMLCharset A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like