569 lines
16 KiB
TypeScript
569 lines
16 KiB
TypeScript
import { Converter } from "../src/Converter";
|
|
import csv from "../src";
|
|
var assert = require("assert");
|
|
var fs = require("fs");
|
|
var sandbox = require('sinon').sandbox.create();
|
|
describe("testCSVConverter2", function () {
|
|
afterEach(function () {
|
|
sandbox.restore();
|
|
});
|
|
|
|
it("should convert from large csv string", function (done) {
|
|
var csvStr = fs.readFileSync(__dirname + "/data/large-csv-sample.csv", "utf8");
|
|
var conv = new Converter({
|
|
});
|
|
conv.fromString(csvStr).then(function (res) {
|
|
assert(res.length === 5290);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should set eol", function (done) {
|
|
var rs = fs.createReadStream(__dirname + "/data/large-csv-sample.csv");
|
|
var conv = new Converter({
|
|
eol: "\n"
|
|
});
|
|
var count = 0;
|
|
conv.subscribe(function (resultJson, index) {
|
|
count++;
|
|
assert(resultJson);
|
|
// assert(row.length === 2);
|
|
assert(index >= 0);
|
|
});
|
|
conv.on("error", function () {
|
|
console.log(arguments);
|
|
});
|
|
conv.then(function (result) {
|
|
assert(result);
|
|
assert(count === 5290);
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should convert tsv String", function (done) {
|
|
var tsv = __dirname + "/data/dataTsv";
|
|
var csvStr = fs.readFileSync(tsv, "utf8");
|
|
var conv = new Converter({
|
|
delimiter: "\t",
|
|
"checkType": false
|
|
});
|
|
conv.fromString(csvStr).then(function (res) {
|
|
assert(res);
|
|
assert.equal(res.length, 200);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should allow customised header with nohead csv string.", function (done) {
|
|
var testData = __dirname + "/data/noheadercsv";
|
|
var rs = fs.readFileSync(testData, "utf8");
|
|
var conv = new Converter({
|
|
noheader: true,
|
|
headers: ["a", "b", "c", "e", "f", "g"]
|
|
});
|
|
conv.fromString(rs).then(function (json) {
|
|
assert.equal(json[0].field7, 40);
|
|
assert.equal(json[0].a, "CC102-PDMI-001");
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should parse fromFile", function (done) {
|
|
var csvFile = __dirname + "/data/large-csv-sample.csv";
|
|
var conv = new Converter({
|
|
});
|
|
conv.fromFile(csvFile).then(function (res) {
|
|
assert.equal(res.length, 5290);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should fromFile should emit error", function (done) {
|
|
var csvFile = __dirname + "/data/dataWithUnclosedQuotes";
|
|
var conv = new Converter({
|
|
});
|
|
conv.fromFile(csvFile).then(function (res) {
|
|
|
|
done();
|
|
}, function (err) {
|
|
assert(err);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should parse no header with dynamic column number", function (done) {
|
|
var testData = __dirname + "/data/noheaderWithVaryColumnNum";
|
|
var rs = fs.readFileSync(testData, "utf8");
|
|
var conv = new Converter({
|
|
noheader: true
|
|
});
|
|
conv.fromString(rs).then(function (json) {
|
|
assert.equal(json.length, 2);
|
|
assert.equal(json[1].field4, 7);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should parse tabsv data with dynamic columns", function (done) {
|
|
var testData = __dirname + "/data/tabsv";
|
|
var rs = fs.readFileSync(testData, "utf8");
|
|
var conv = new Converter({
|
|
delimiter: "\t"
|
|
});
|
|
conv.fromString(rs).then(function (json) {
|
|
assert.equal(json[0].Idevise, "");
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should use first line break as eol", function (done) {
|
|
var testData = __dirname + "/data/testEol";
|
|
var conv = new Converter({
|
|
noheader: true
|
|
});
|
|
conv.fromFile(testData).then(function (json) {
|
|
assert(json);
|
|
done();
|
|
});
|
|
});
|
|
|
|
|
|
it("should detect delimiter", function (done) {
|
|
var testData = __dirname + "/data/dataWithAutoDelimiter";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter({ delimiter: "auto" });
|
|
conv.then(function (res) {
|
|
assert.equal(res[0].col1, "Mini. Sectt:hisar S.O");
|
|
assert.equal(res[1].col1, "#Mini. Sectt");
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should emit delimiter event", function (done) {
|
|
var testData = __dirname + "/data/dataWithAutoDelimiter";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter({ delimiter: "auto" });
|
|
var delimiterCallback = sandbox.spy(function (delimiter) {
|
|
assert.equal(delimiter, ":");
|
|
});
|
|
conv.on("delimiter", delimiterCallback);
|
|
conv.then(function () {
|
|
assert.equal(delimiterCallback.callCount, 1);
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should emit delimiter event when no header", function (done) {
|
|
var testData = __dirname + "/data/dataWithAutoDelimiter";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter({ delimiter: "auto", noheader: true });
|
|
var delimiterCallback = sandbox.spy(function (delimiter) {
|
|
assert.equal(delimiter, ":");
|
|
});
|
|
conv.on("delimiter", delimiterCallback);
|
|
conv.then(function () {
|
|
assert.equal(delimiterCallback.callCount, 1);
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
// it("should not emit delimiter event when delimiter is specified", function (done) {
|
|
// var testData = __dirname + "/data/columnArray";
|
|
// var rs = fs.createReadStream(testData);
|
|
// var conv = new Converter();
|
|
// conv.on("delimiter", function (delimiter) {
|
|
// assert.fail("delimiter event should not have been emitted");
|
|
// });
|
|
// conv.then(function () {
|
|
// done();
|
|
// });
|
|
|
|
// rs.pipe(conv);
|
|
// });
|
|
|
|
it("should stripe out whitespaces if trim is true", function (done) {
|
|
var testData = __dirname + "/data/dataWithWhiteSpace";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter({ trim: true });
|
|
conv.then(function (res) {
|
|
assert.equal(res[0]["Column 1"], "Column1Row1");
|
|
assert.equal(res[0]["Column 2"], "Column2Row1");
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should convert triple quotes correctly", function (done) {
|
|
var testData = __dirname + "/data/dataWithTripleQoutes";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter({ trim: true });
|
|
conv.then(function (res) {
|
|
assert.equal(res[0].Description, "ac, abs, moon");
|
|
assert.equal(res[1].Model, "Venture \"Extended Edition\"");
|
|
assert.equal(res[2].Model, "Venture \"Extended Edition, Very Large\"");
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
|
|
|
|
it("should pre process raw data in the line", function (done) {
|
|
var testData = __dirname + "/data/quoteTolerant";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter();
|
|
conv.preRawData(function (d) {
|
|
return d.replace('THICK', 'THIN');
|
|
});
|
|
conv.then(function (res) {
|
|
assert(res[0].Description.indexOf('THIN') > -1);
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should pre process by line in the line", function (done) {
|
|
var testData = __dirname + "/data/quoteTolerant";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter();
|
|
conv.preFileLine(function (line, lineNumber) {
|
|
if (lineNumber === 1) {
|
|
line = line.replace('THICK', 'THIN');
|
|
}
|
|
return line;
|
|
});
|
|
|
|
conv.then(function (res) {
|
|
assert(res[0].Description.indexOf('THIN') > -1);
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should support object mode", function (done) {
|
|
var testData = __dirname + "/data/complexJSONCSV";
|
|
var rs = fs.createReadStream(testData);
|
|
var conv = new Converter({}, {
|
|
objectMode: true
|
|
});
|
|
conv.on("data", function (d) {
|
|
assert(typeof d === "object");
|
|
});
|
|
conv.then(function (res) {
|
|
assert(res);
|
|
assert(res.length > 0);
|
|
done();
|
|
});
|
|
rs.pipe(conv);
|
|
});
|
|
|
|
it("should get delimiter automatically if there is no header", function (done) {
|
|
var test_converter = new Converter({
|
|
delimiter: 'auto',
|
|
headers: ['col1', 'col2'],
|
|
noheader: true,
|
|
checkColumn: true
|
|
});
|
|
|
|
var my_data = 'first_val\tsecond_val';
|
|
test_converter.fromString(my_data).then(function (result) {
|
|
assert.equal(result.length, 1);
|
|
assert.equal(result[0].col1, "first_val");
|
|
assert.equal(result[0].col2, "second_val");
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should process escape chars", function (done) {
|
|
var test_converter = new Converter({
|
|
escape: "\\",
|
|
checkType: true
|
|
});
|
|
|
|
var testData = __dirname + "/data/dataWithSlashEscape";
|
|
var rs = fs.createReadStream(testData);
|
|
test_converter.then(function (res) {
|
|
assert.equal(res[0].raw.hello, "world");
|
|
assert.equal(res[0].raw.test, true);
|
|
done();
|
|
});
|
|
rs.pipe(test_converter);
|
|
});
|
|
|
|
it("should output ndjson format", function (done) {
|
|
var conv = new Converter();
|
|
conv.fromString("a,b,c\n1,2,3\n4,5,6")
|
|
.on("data", function (d) {
|
|
d = d.toString();
|
|
assert.equal(d[d.length - 1], "\n");
|
|
})
|
|
.on("done", done);
|
|
});
|
|
|
|
it("should parse from stream", function (done) {
|
|
var testData = __dirname + "/data/complexJSONCSV";
|
|
var rs = fs.createReadStream(testData);
|
|
csv()
|
|
.fromStream(rs)
|
|
.then(function (res) {
|
|
assert(res);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should set output as csv", function (done) {
|
|
var testData = __dirname + "/data/complexJSONCSV";
|
|
var rs = fs.createReadStream(testData);
|
|
var numOfRow = 0;
|
|
csv({ output: "csv" })
|
|
.fromStream(rs)
|
|
.subscribe(function (row, idx) {
|
|
numOfRow++;
|
|
assert(row);
|
|
assert(idx >= 0);
|
|
})
|
|
|
|
.on("done", function (error) {
|
|
assert(!error);
|
|
assert.equal(2, numOfRow);
|
|
assert(numOfRow !== 0);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should transform with subscribe function", function (done) {
|
|
var testData = __dirname + "/data/complexJSONCSV";
|
|
var rs = fs.createReadStream(testData);
|
|
var numOfRow = 0;
|
|
var numOfJson = 0;
|
|
csv()
|
|
.fromStream(rs)
|
|
.subscribe(function (json, idx) {
|
|
json.a = "test";
|
|
assert(idx >= 0);
|
|
})
|
|
.on("data", function (d) {
|
|
const j = JSON.parse(d.toString());
|
|
assert.equal(j.a, "test");
|
|
})
|
|
.on("end", function () {
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should parse a complex JSON", function (done) {
|
|
var converter = new Converter({ checkType: true });
|
|
var r = fs.createReadStream(__dirname + "/data/complexJSONCSV");
|
|
converter.then(function (res) {
|
|
assert(res);
|
|
assert(res.length === 2);
|
|
assert(res[0].fieldA.title === "Food Factory");
|
|
assert(res[0].fieldA.children.length === 2);
|
|
assert(res[0].fieldA.children[0].name === "Oscar");
|
|
assert(res[0].fieldA.children[0].id === 23);
|
|
assert(res[0].fieldA.children[1].name === "Tikka");
|
|
assert.equal(res[0].fieldA.children[1].employee.length, 2);
|
|
assert(res[0].fieldA.children[1].employee[0].name === "Tim", JSON.stringify(res[0].fieldA.children[1].employee[0]));
|
|
assert(res[0].fieldA.address.length === 2);
|
|
assert(res[0].fieldA.address[0] === "3 Lame Road");
|
|
assert(res[0].fieldA.address[1] === "Grantstown");
|
|
assert(res[0].description === "A fresh new food factory", res[0].description);
|
|
done();
|
|
});
|
|
r.pipe(converter);
|
|
});
|
|
|
|
it("should allow flatKey to change parse behaviour", function (done) {
|
|
var conv = new Converter({
|
|
flatKeys: true
|
|
});
|
|
conv.fromString("a.b,b.d,c.a\n1,2,3\n4,5,6").subscribe(function (d) {
|
|
assert(d["a.b"]);
|
|
assert(d["b.d"]);
|
|
assert(d["c.a"]);
|
|
})
|
|
.on("done", done);
|
|
});
|
|
it("should allow flat mods to change parse behaviour", function (done) {
|
|
var conv = new Converter({
|
|
colParser: {
|
|
"a.b": {
|
|
flat: true
|
|
}
|
|
}
|
|
});
|
|
conv.fromString("a.b,b.d,c.a\n1,2,3\n4,5,6").subscribe(function (d) {
|
|
assert(d["a.b"]);
|
|
})
|
|
.on("done", done);
|
|
});
|
|
|
|
it("should process long header", function (done) {
|
|
var testData = __dirname + "/data/longHeader";
|
|
var rs = fs.createReadStream(testData, { highWaterMark: 100 });
|
|
var numOfRow = 0;
|
|
var numOfJson = 0;
|
|
csv({}, { highWaterMark: 100 })
|
|
.fromStream(rs)
|
|
.subscribe(function (res, idx) {
|
|
numOfJson++;
|
|
assert.equal(res.Date, '8/26/16');
|
|
assert(idx >= 0);
|
|
})
|
|
.on("done", function () {
|
|
assert(numOfJson === 1);
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should parse #139", function (done) {
|
|
var rs = fs.createReadStream(__dirname + "/data/data#139");
|
|
csv()
|
|
.fromStream(rs)
|
|
.then(function (res) {
|
|
assert.equal(res[1].field3, "9001009395 9001009990");
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should ignore column", function (done) {
|
|
var rs = fs.createReadStream(__dirname + "/data/dataWithQoutes");
|
|
var headerEmitted = false;
|
|
csv({
|
|
ignoreColumns: /TIMESTAMP/
|
|
})
|
|
.fromStream(rs)
|
|
.on("header", function (header) {
|
|
assert.equal(header.indexOf("TIMESTAMP"), -1);
|
|
assert.equal(header.indexOf("UPDATE"), 0);
|
|
if (headerEmitted) {
|
|
throw ("header event should only happen once")
|
|
}
|
|
headerEmitted = true;
|
|
})
|
|
// .on("csv", function (row, idx) {
|
|
// if (!headerEmitted) {
|
|
// throw ("header should be emitted before any data events");
|
|
// }
|
|
// assert(idx >= 0);
|
|
// if (idx === 1) {
|
|
// assert.equal(row[0], "n");
|
|
// }
|
|
// })
|
|
.subscribe(function (j, idx) {
|
|
assert(!j.TIMESTAMP);
|
|
assert(idx >= 0);
|
|
})
|
|
.on("done", function () {
|
|
assert(headerEmitted);
|
|
done();
|
|
});
|
|
});
|
|
it("should keep space around comma in csv", function () {
|
|
const str = `"Name","Number"
|
|
"John , space", 1234
|
|
"Mr. , space", 4321
|
|
`;
|
|
return csv().fromString(str)
|
|
.then((data) => {
|
|
assert.equal(data[0].Name, "John , space");
|
|
assert.equal(data[1].Name, "Mr. , space");
|
|
})
|
|
})
|
|
|
|
it("should include column", function (done) {
|
|
var rs = fs.createReadStream(__dirname + "/data/dataWithQoutes");
|
|
csv({
|
|
includeColumns: /TIMESTAMP/
|
|
})
|
|
.fromStream(rs)
|
|
.on("header", function (header) {
|
|
assert.equal(header.indexOf("TIMESTAMP"), 0);
|
|
assert.equal(header.indexOf("UPDATE"), -1);
|
|
assert.equal(header.length, 1);
|
|
})
|
|
.subscribe(function (j, idx) {
|
|
assert(idx >= 0);
|
|
if (idx === 1) {
|
|
assert.equal(j.TIMESTAMP, "abc, def, ccc");
|
|
}
|
|
assert(!j.UID)
|
|
assert(!j['BYTES SENT'])
|
|
})
|
|
.on("done", function () {
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should allow headers and include columns to be given as reference to the same var", function (done) {
|
|
var rs = fs.createReadStream(__dirname + "/data/complexJSONCSV");
|
|
var headers = [
|
|
'first',
|
|
'second',
|
|
'third',
|
|
];
|
|
|
|
var expected = headers;
|
|
|
|
csv({
|
|
headers: headers,
|
|
includeColumns: /(first|second|third)/,
|
|
})
|
|
.fromStream(rs)
|
|
.on("header", function (header) {
|
|
expected.forEach(function (value, index) {
|
|
assert.equal(header.indexOf(value), index);
|
|
});
|
|
})
|
|
.subscribe(function (j, idx) {
|
|
assert(idx >= 0);
|
|
assert.equal(expected.length, Object.keys(j).length);
|
|
expected.forEach(function (attribute) {
|
|
assert(j.hasOwnProperty(attribute));
|
|
});
|
|
})
|
|
.on("done", function () {
|
|
done();
|
|
});
|
|
});
|
|
|
|
it("should leave provided params objects unmutated", function() {
|
|
var rs = fs.createReadStream(__dirname + "/data/complexJSONCSV");
|
|
var includeColumns = [
|
|
'fieldA.title',
|
|
'description',
|
|
];
|
|
|
|
|
|
return csv({
|
|
includeColumns: /(fieldA\.title|description)/,
|
|
})
|
|
.fromStream(rs)
|
|
.on("json", function(j, idx) {
|
|
assert(idx >= 0);
|
|
})
|
|
.on("header", function(header) {
|
|
includeColumns.forEach(function (value, index) {
|
|
assert.equal(index, header.indexOf(value));
|
|
});
|
|
})
|
|
});
|
|
|
|
it("should only call done once", function (done) {
|
|
var counter=0;
|
|
csv()
|
|
.fromString('"a","b", "c""')
|
|
.on('done',function(){
|
|
counter++;
|
|
});
|
|
setTimeout(function(){
|
|
assert.equal(counter,1);
|
|
done();
|
|
},100);
|
|
})
|
|
});
|