nightfury7 / universalbasics.js

src/ub.files.csv.js

/** global: UB */
//removeIf(nodejs)




var arrayFuncs = {


	/** Fast and simple CSV encoder.
		Supports multi-line values and values with double quotes.
		Automatic/smart escaping like Excel-generated CSVs. */
	encodeCSV: function(headers, trimValues, columnar, seperator = ",") {
		var linesData = this;
		
		if (columnar) {
			linesData = linesData.transpose();
		}
		
		var sb = [];
		
		// per line
		for (var l = (headers != null ? -1 : 0), ll = linesData.length - 1; l <= ll; l++) {
			
			// fetch header / line data words
			var words = l === -1 ? headers : linesData[l];
			
			// per word
			for (var v = 0, vl = words.length - 1; v <= vl; v++) {
				
				// if value given
				if (words[v] != null) {
					
					// convert value to text
					var word = words[v].toString();
					if (trimValues) {
						word = word.trim();
					}
					
					// add word
					if (word.indexOf('"') > -1) {
						
						// escape quotes, and enclose in quotes if word has a quote
						sb.push('"');
						sb.push(word.split("\"").join("\"\""));
						sb.push('"');
						
					} else if (word.indexOf(',') > -1) {
						
						// enclose in quotes if word has comma
						sb.push('"');
						sb.push(word);
						sb.push('"');
						
					} else {
						
						// add straightaway otherwise
						sb.push(word);
					}
				}
				
				// add seperator
				if (v < vl) {
					sb.push(seperator);
				}
			}
			
			// add newline
			if (l < ll) {
				sb.push("\r\n");
			}
		}
		
		return sb.join("");
	},
	

	/** Fast and simple TSV encoder.
		Supports multi-line values and values with double quotes.
		Automatic/smart escaping like Excel-generated TSV. */
	encodeTSV: function(headers, trimValues, columnar) {
		var linesData = this;
		return linesData.encodeCSV(headers, trimValues, columnar, "\t");
	},

    none:null,
}

// register funcs
UB.registerFuncs(Array.prototype, arrayFuncs);



var stringFuncs = {

	/** Decodes the given CSV file string and returns the cell data as an array.
	 *  Extremely robust and fast CSV parser. Only parser that works with all bizarre but valid test files.
	 * 
	 * 3 modes are available:
	 * if `headers` is null - All cells are returned as 2D array. (default)
	 * if `headers` is given and 0 length - The first row is stored in `headers`, remaining cells and returned as 2D array.
	 * if `headers` is given and >0 length - All rows are returned as objects, with the given headers treated as the prop names for the objects.
	 */
	decodeCSV: function (headers, trimValues, columnar = false, seperator = ",") {
		var csvString = this.toString();
	
		// cut String into lines
		var lines = csvString.trim().splitLines();
		var sep = seperator == "auto" ? UB.CSV_detectSeperator(csvString, lines.length) : seperator;
		
		// config
		var hasHeaders = headers != null;
		var returnAsObjs = hasHeaders && headers.exists();

		// status
		var inQuoted = false;
		
		// result
		var linesData = [];
		var word = [];
		var tempHeaders = [];
		var lineWords = [];

		// per line
		for (var l = 0, ll = lines.length; l < ll; l++) {
			var line = lines[l];
			var isHeader = (l === 0 && hasHeaders);
			
			// if we are in quoted text
			if (inQuoted) {
				
				// keep taking chars
				
			}else{
				
				// save words into headers array / new array
				lineWords = [];
				if (isHeader){
					if (returnAsObjs){
						lineWords = tempHeaders;
					}else{
						lineWords = headers;
					}
				}
				if (!isHeader) {
					linesData.push(lineWords);
				}
				
			}
			
			// per char
			for (var c = 0, clast = line.length - 1; c <= clast; c++) {
				var ch = line.charAt(c);
				
				// if we are in quoted text
				if (inQuoted) {
					
					// quotes..
					if (ch == "\"") {
						
						// quote may be escaped
						if (line.charAt(c + 1) == "\"") {
							c++;
							word.push("\"");
						}else {
							
							// quote means ending quoted text
							inQuoted = false;
						}
						
						continue;
					}
					
					// normal char
					word.push(ch);
					
					
				}else {
					
					// quote means beginning quoted text
					if (ch == "\""){
						inQuoted = true;
						continue;
					}
					
					// comma means end of word
					if (ch == sep) {
						lineWords.push(trimValues ? word.join("").trim() : word.join(""));
						word = [];
						continue;
					}
					
					// normal char
					word.push(ch);
					
					// newline means end of word
					if (c == clast) {
						lineWords.push(trimValues ? word.join("").trim() : word.join(""));
						word = [];
					}
					
				}
				
			}
			
			// at end of line take word
			if (!inQuoted && word.Length > 0) {
				lineWords.push(trimValues ? word.join("").trim() : word.join(""));
				word = [];
			}
		}

		// convert array to objs
		if (returnAsObjs){

			// go thru all rows
			for (var l = 0, ll = linesData.length; l < ll; l++) {

The variable ll already seems to be declared on line 125. Consider using another variable name or omitting the var keyword.

The variable l already seems to be declared on line 125. Consider using another variable name or omitting the var keyword.

				var row = linesData[l];
				var obj = {};

				// convert all cells to obj props
				for (var h = 0, hl = headers.length; h < hl; h++) {
					var header = headers[h];
					obj[header] = row[h];
				}
				linesData[l] = obj;
			}
		}
		
		// convert 2D array to columnar
		if (columnar && !returnAsObjs) {
			linesData = linesData.transpose();
		}

		return linesData;
	},

	/** Decodes the given TSV file string and returns the cell data as an array.
	 *  Extremely robust and fast TSV parser. Only parser that works with all bizarre but valid test files.
	 * 
	 * 3 modes are available:
	 * if `headers` is null - All cells are returned as 2D array. (default)
	 * if `headers` is given and 0 length - The first row is stored in `headers`, remaining cells and returned as 2D array.
	 * if `headers` is given and >0 length - All rows are returned as objects, with the given headers treated as the prop names for the objects.
	 */
	decodeTSV: function (headers, trimValues, columnar = false) {
		var tsvString = this.toString();
		return tsvString.decodeCSV(headers, trimValues, columnar, "\t");
	},

    none:null,
}

// register funcs
UB.registerFuncs(String.prototype, stringFuncs);





/* File Utils - NodeJS only */

var fs = require('fs');
var pathUtil = require('path');

arrayFuncs = {

	saveToCSV: function(filePath, headers = null, trimValues = true, columnar = false, fixedSep = ",") {
		var data = this;

		// write CSV to string
		var str = data.encodeCSV(headers, trimValues, columnar, fixedSep);
		
		// save data as string via filestream
		str.saveToText(filePath);
	},

    none:null,
}

// register funcs
UB.registerFuncs(Array.prototype, arrayFuncs);


stringFuncs = {

	/**
	 * Loads the given CSV file, decoding the cell data and returning it as an array.
	 * Extremely robust and fast CSV parser. Only parser that works with all bizarre but valid test files.
	 * 
	 * 3 modes are available:
	 * if `headers` is null - All cells are returned as 2D array. (default)
	 * if `headers` is given and 0 length - The first row is stored in `headers`, remaining cells and returned as 2D array.
	 * if `headers` is given and >0 length - All rows are returned as objects, with the given headers treated as the prop names for the objects.
	 */
	loadCSV: function(encoding = "utf8", headers = null, trimValues = true, columnar = false, seperator = "auto") {

		// load text file
		var file = this.toString();
		var text = file.loadText(encoding);
		if (text == null) {
			return null;
		}
		
		// parse CSV string into Array
		return text.decodeCSV(headers, trimValues, columnar, seperator);
	},

    none:null,
}

// register funcs
UB.registerFuncs(String.prototype, stringFuncs);

//endRemoveIf(nodejs)





// UTILS
UB.CSV_seperators = [',', ';', ':', '\t'];

UB.CSV_detectSeperator = function(csvString, rowCount) {
	var sepCount = UB.newArray(0, UB.CSV_seperators.length);

	var character;

	var quoted = false;
	var firstChar = true;
	var foundAny = false;

	var c = 0;
	var cl = csvString.length - 50;  // skip last few chars
	while (c < cl) {
		character = csvString.charAt(c);
		c++;

		switch (character) {
			case '"':
				if (quoted) {
					if (csvString.charAt(c + 1) != '"') {  // Value is quoted and current character is " and next character is not ".
						quoted = false;
					} else {
						c++;  // Value is quoted and current and next characters are "" - read (skip) peeked qoute.
					}
				} else {
					if (firstChar) {  // Set value as quoted only if this quote is the first char in the value.
						quoted = true;
					}
				}
				break;
			case '\r':
			case '\n':
				if (!quoted) {
					firstChar = true;
					continue;
				}
				break;
			default:
				if (!quoted) {
					var index = UB.CSV_seperators.indexOf(character);
					if (index != -1) {
						sepCount[index]++;
						firstChar = true;
						foundAny = true;
						continue;
					}
				}
				break;
		}

		if (firstChar) {
			firstChar = false;
		}
	}

	return !foundAny ? ',' : UB.CSV_seperators[sepCount.indexOfMax()];
}