223 lines
27 KiB
JavaScript
223 lines
27 KiB
JavaScript
|
"use strict";
|
||
|
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||
|
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||
|
|
};
|
||
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
||
|
|
var src_1 = __importDefault(require("../src"));
|
||
|
|
var assert = require("assert");
|
||
|
|
var fs = require("fs");
|
||
|
|
var sinon_1 = require("sinon");
|
||
|
|
var sb = sinon_1.sandbox.create();
|
||
|
|
describe("testCSVConverter3", function () {
|
||
|
|
afterEach(function () {
|
||
|
|
sb.restore();
|
||
|
|
});
|
||
|
|
it("should parse large csv file with UTF-8 without spliting characters", function (done) {
|
||
|
|
var testData = __dirname + "/data/large-utf8.csv";
|
||
|
|
var rs = fs.createReadStream(testData);
|
||
|
|
var csvConverter = src_1.default({});
|
||
|
|
var count = 0;
|
||
|
|
csvConverter.preRawData(function (csvRawData) {
|
||
|
|
assert(csvRawData.charCodeAt(0) < 2000);
|
||
|
|
return csvRawData;
|
||
|
|
});
|
||
|
|
csvConverter.on("data", function () {
|
||
|
|
count++;
|
||
|
|
});
|
||
|
|
csvConverter.then(function () {
|
||
|
|
assert(count === 5290);
|
||
|
|
done();
|
||
|
|
});
|
||
|
|
rs.pipe(csvConverter);
|
||
|
|
});
|
||
|
|
it("should setup customise type convert function", function (done) {
|
||
|
|
src_1.default({
|
||
|
|
checkType: true,
|
||
|
|
colParser: {
|
||
|
|
"column1": "string",
|
||
|
|
"column5": function (item, head, resultRow, row, i) {
|
||
|
|
assert.equal(item, '{"hello":"world"}');
|
||
|
|
assert.equal(head, "column5"),
|
||
|
|
assert(resultRow);
|
||
|
|
assert(row);
|
||
|
|
assert.equal(i, 5);
|
||
|
|
return "hello world";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
})
|
||
|
|
.fromFile(__dirname + "/data/dataWithType")
|
||
|
|
.subscribe(function (json) {
|
||
|
|
assert.equal(typeof json.column1, "string");
|
||
|
|
assert.equal(json.column5, "hello world");
|
||
|
|
assert.strictEqual(json["name#!"], false);
|
||
|
|
assert.strictEqual(json["column9"], true);
|
||
|
|
})
|
||
|
|
.on('done', function () {
|
||
|
|
done();
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should accept pipe as quote", function (done) {
|
||
|
|
src_1.default({
|
||
|
|
quote: "|",
|
||
|
|
output: "csv"
|
||
|
|
})
|
||
|
|
.fromFile(__dirname + "/data/pipeAsQuote")
|
||
|
|
.subscribe(function (csv) {
|
||
|
|
assert.equal(csv[2], "blahhh, blah");
|
||
|
|
})
|
||
|
|
.on('done', function () {
|
||
|
|
done();
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("emit file not exists error when try to open a non-exists file", function () {
|
||
|
|
var called = false;
|
||
|
|
var cb = sb.spy(function (err) {
|
||
|
|
assert(err.toString().indexOf("File does not exist") > -1);
|
||
|
|
});
|
||
|
|
return src_1.default()
|
||
|
|
.fromFile("somefile")
|
||
|
|
.subscribe(function (csv) {
|
||
|
|
})
|
||
|
|
.on("error", cb)
|
||
|
|
.then(function () {
|
||
|
|
assert(false);
|
||
|
|
}, function (err) {
|
||
|
|
assert.equal(cb.callCount, 1);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should include column that is both included and excluded", function () {
|
||
|
|
return src_1.default({
|
||
|
|
includeColumns: /b/,
|
||
|
|
ignoreColumns: /a|b/
|
||
|
|
})
|
||
|
|
.fromString("a,b,c\n1,2,3\n4,5,6")
|
||
|
|
.subscribe(function (d) {
|
||
|
|
assert(d.b);
|
||
|
|
assert(!d.a);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should allow async preLine hook", function () {
|
||
|
|
return src_1.default()
|
||
|
|
.preFileLine(function (line) {
|
||
|
|
return new Promise(function (resolve, reject) {
|
||
|
|
setTimeout(function () {
|
||
|
|
resolve(line + "changed");
|
||
|
|
}, 20);
|
||
|
|
});
|
||
|
|
})
|
||
|
|
.fromString("a,b\n1,2")
|
||
|
|
.subscribe(function (d) {
|
||
|
|
assert(d.bchanged);
|
||
|
|
assert.equal(d.bchanged, "2changed");
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should allow async subscribe function", function () {
|
||
|
|
return src_1.default({ trim: true })
|
||
|
|
.fromString("a,b,c\n 1,2,3\n 4,5,6")
|
||
|
|
.subscribe(function (d) {
|
||
|
|
return new Promise(function (resolve, reject) {
|
||
|
|
setTimeout(function () {
|
||
|
|
d.a = 10;
|
||
|
|
resolve();
|
||
|
|
}, 20);
|
||
|
|
});
|
||
|
|
})
|
||
|
|
.then(function (d) {
|
||
|
|
assert.equal(d[0].a, 10);
|
||
|
|
assert.equal(d[1].a, 10);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should propagate value to next then", function () {
|
||
|
|
return src_1.default({ trim: true })
|
||
|
|
.fromString("a,b,c\n 1,2,3\n 4,5,6")
|
||
|
|
.then(undefined, undefined)
|
||
|
|
.then(function (d) {
|
||
|
|
assert.equal(d.length, 2);
|
||
|
|
assert.equal(d[0].a, "1");
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should propagate error to next then", function () {
|
||
|
|
return src_1.default({ trim: true })
|
||
|
|
.fromFile(__dirname + "/data/dataWithUnclosedQuotes")
|
||
|
|
.then(undefined, undefined)
|
||
|
|
.then(function () {
|
||
|
|
assert(false);
|
||
|
|
}, function (err) {
|
||
|
|
assert(err);
|
||
|
|
assert.equal(err.err, "unclosed_quote");
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should fallback to text is number can not be parsed", function () {
|
||
|
|
return src_1.default({
|
||
|
|
colParser: {
|
||
|
|
"a": "number"
|
||
|
|
}
|
||
|
|
})
|
||
|
|
.fromString("a,b,c\n 1,2,3\n fefe,5,6")
|
||
|
|
.then(function (d) {
|
||
|
|
assert.strictEqual(d[0].a, 1);
|
||
|
|
assert.equal(d[1].a, "fefe");
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should omit a column", function () {
|
||
|
|
return src_1.default({
|
||
|
|
colParser: {
|
||
|
|
"a": "omit"
|
||
|
|
}
|
||
|
|
})
|
||
|
|
.fromString("a,b,c\n 1,2,3\n fefe,5,6")
|
||
|
|
.then(function (d) {
|
||
|
|
assert.strictEqual(d[0].a, undefined);
|
||
|
|
assert.equal(d[1].a, undefined);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("could turn off quote and should trim even quote is turned off", function () {
|
||
|
|
return src_1.default({
|
||
|
|
quote: "off",
|
||
|
|
trim: true
|
||
|
|
})
|
||
|
|
.fromString("a,b,c\n \"1\",\"2\",\"3\"\n \"fefe,5\",6")
|
||
|
|
.then(function (d) {
|
||
|
|
assert.equal(d[0].a, '"1"');
|
||
|
|
assert.equal(d[0].b, '"2"');
|
||
|
|
assert.equal(d[1].a, '"fefe');
|
||
|
|
assert.equal(d[1].b, '5"');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should allow ignoreEmpty with checkColumn", function () {
|
||
|
|
return src_1.default({
|
||
|
|
checkColumn: true,
|
||
|
|
ignoreEmpty: true
|
||
|
|
})
|
||
|
|
.fromString("date,altitude,airtime\n 2016-07-08,2000,23\n \n 2016-07-09,3000,43")
|
||
|
|
.then(function (data) {
|
||
|
|
}, function (err) {
|
||
|
|
console.log(err);
|
||
|
|
assert(!err);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should allow quotes without content", function () {
|
||
|
|
var data = "a|^^|^b^";
|
||
|
|
return src_1.default({
|
||
|
|
delimiter: '|',
|
||
|
|
quote: '^',
|
||
|
|
noheader: true,
|
||
|
|
})
|
||
|
|
.fromString(data)
|
||
|
|
.then(function (jsonObj) {
|
||
|
|
assert.equal(jsonObj[0].field2, "");
|
||
|
|
});
|
||
|
|
});
|
||
|
|
it("should parse header with quotes correctly", function () {
|
||
|
|
var testData = __dirname + "/data/csvWithUnclosedHeader";
|
||
|
|
return src_1.default({
|
||
|
|
headers: ["exam_date", "sample_no", "status", "sample_type", "patient_id", "last_name", "first_name", "gender_of_patient", "patient_birth_date", "patient_note", "patient_department", "accession_number", "sample_site", "physician", "operator", "department", "note", "test_order_code", "draw_time", "approval_status", "approval_time", "report_layout", "patient_account_number", "none_1", "errors_detected_during_measurement", "age", "error_code_01", "weight", "error_code_02", "height", "error_code_03", "hcg_beta_p", "error_code_04", "troponin_i_p", "error_code_05", "ck_mb_p", "error_code_06", "d_dimer_p", "error_code_07", "hscrp_p", "error_code_08", "myoglobin_p", "error_code_09", "nt_probnp", "error_code_10", "crp", "error_code_11", "bnp", "error_code_12", "tnt", "error_code_13", "demo_p", "error_code_14", "pct", "error_code_15"]
|
||
|
|
})
|
||
|
|
.fromFile(testData)
|
||
|
|
.then(function (d) {
|
||
|
|
assert.equal(d.length, 2);
|
||
|
|
assert.equal(d[0].sample_no, "12669");
|
||
|
|
});
|
||
|
|
});
|
||
|
|
});
|
||
|
|
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiL1VzZXJzL2t4aWFuZy93b3JrL3Byb2plY3RzL2NzdjJqc29uL3Rlc3QvdGVzdENTVkNvbnZlcnRlcjMudHMiLCJzb3VyY2VzIjpbIi9Vc2Vycy9reGlhbmcvd29yay9wcm9qZWN0cy9jc3YyanNvbi90ZXN0L3Rlc3RDU1ZDb252ZXJ0ZXIzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7Ozs7O0FBQUEsK0NBQXlCO0FBQ3pCLElBQUksTUFBTSxHQUFHLE9BQU8sQ0FBQyxRQUFRLENBQUMsQ0FBQztBQUMvQixJQUFJLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7QUFDdkIsK0JBQWdDO0FBRWhDLElBQU0sRUFBRSxHQUFHLGVBQU8sQ0FBQyxNQUFNLEVBQUUsQ0FBQztBQUM1QixRQUFRLENBQUMsbUJBQW1CLEVBQUU7SUFDNUIsU0FBUyxDQUFDO1FBQ1IsRUFBRSxDQUFDLE9BQU8sRUFBRSxDQUFDO0lBQ2YsQ0FBQyxDQUFDLENBQUM7SUFDSCxFQUFFLENBQUMsb0VBQW9FLEVBQUUsVUFBVSxJQUFJO1FBQ3JGLElBQUksUUFBUSxHQUFHLFNBQVMsR0FBRyxzQkFBc0IsQ0FBQztRQUNsRCxJQUFJLEVBQUUsR0FBRyxFQUFFLENBQUMsZ0JBQWdCLENBQUMsUUFBUSxDQUFDLENBQUM7UUFDdkMsSUFBSSxZQUFZLEdBQUcsYUFBRyxDQUFDLEVBQ3RCLENBQUMsQ0FBQztRQUNILElBQUksS0FBSyxHQUFHLENBQUMsQ0FBQztRQUNkLFlBQVksQ0FBQyxVQUFVLENBQUMsVUFBVSxVQUFVO1lBQzFDLE1BQU0sQ0FBQyxVQUFVLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQyxHQUFHLElBQUksQ0FBQyxDQUFDO1lBQ3hDLE9BQU8sVUFBVSxDQUFDO1FBQ3BCLENBQUMsQ0FBQyxDQUFBO1FBQ0YsWUFBWSxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUU7WUFDdEIsS0FBSyxFQUFFLENBQUM7UUFDVixDQUFDLENBQUMsQ0FBQztRQUNILFlBQVksQ0FBQyxJQUFJLENBQUM7WUFDaEIsTUFBTSxDQUFDLEtBQUssS0FBSyxJQUFJLENBQUMsQ0FBQztZQUN2QixJQUFJLEVBQUUsQ0FBQztRQUNULENBQUMsQ0FBQyxDQUFDO1FBQ0gsRUFBRSxDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsQ0FBQztJQUN4QixDQUFDLENBQUMsQ0FBQztJQUNILEVBQUUsQ0FBQyw4Q0FBOEMsRUFBRSxVQUFVLElBQUk7UUFDL0QsYUFBRyxDQUFDO1lBQ0YsU0FBUyxFQUFFLElBQUk7WUFDZixTQUFTLEVBQUU7Z0JBQ1QsU0FBUyxFQUFFLFFBQVE7Z0JBQ25CLFNBQVMsRUFBRSxVQUFVLElBQUksRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLEdBQUcsRUFBRSxDQUFDO29CQUNoRCxNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxtQkFBbUIsQ0FBQyxDQUFDO29CQUN4QyxNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxTQUFTLENBQUM7d0JBQzNCLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztvQkFDcEIsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDO29CQUNaLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO29CQUNuQixPQUFPLGFBQWEsQ0FBQztnQkFDdkIsQ0FBQzthQUNGO1NBQ0YsQ0FBQzthQUNDLFFBQVEsQ0FBQyxTQUFTLEdBQUcsb0JBQW9CLENBQUM7YUFDMUMsU0FBUyxDQUFDLFVBQVUsSUFBSTtZQUN2QixNQUFNLENBQUMsS0FBSyxDQUFDLE9BQU8sSUFBSSxDQUFDLE9BQU8sRUFBRSxRQUFRLENBQUMsQ0FBQztZQUM1QyxNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsYUFBYSxDQUFDLENBQUM7WUFDMUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDMUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDNUMsQ0FBQyxDQUFDO2FBQ0QsRUFBRSxDQUFDLE1BQU0sRUFBRTtZQUNWLElBQUksRUFBRSxDQUFBO1FBQ1IsQ0FBQyxDQUFDLENBQUM7SUFDUCxDQUFDLENBQUMsQ0FBQTtJQUNGLEVBQUUsQ0FBQyw2QkFBNkIsRUFBRSxVQUFVLElBQUk7UUFDOUMsYUFBRyxDQUFDO1lBQ0YsS0FBSyxFQUFFLEdBQUc7WUFDVixNQUFNLEVBQUUsS0FBSztTQUNkLENBQUM7YUFDQyxRQUFRLENBQUMsU0FBUyxHQUFHLG1CQUFtQixDQUFDO2FBQ3pDLFNBQVMsQ0FBQyxVQUFVLEdBQUc7WUFDdEIsTUFBTSxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsY0FBYyxDQUFDLENBQUM7UUFDdkMsQ0FBQyxDQUFDO2FBQ0QsRUFBRSxDQUFDLE1BQU0sRUFBRTtZQUNWLElBQUksRUFBRSxDQUFBO1FBQ1IsQ0FBQyxDQUFDLENBQUM7SUFDUCxDQUFDLENBQUMsQ0FBQTtJQUNGLEVBQUUsQ0FBQywrREFBK0QsRUFBRTtRQUNsRSxJQUFJLE1BQU0sR0FBRyxLQUFLLENBQUM7UUFDbkIsSUFBTSxFQUFFLEdBQUcsRUFBRSxDQUFDLEdBQUcsQ0FBQyxVQUFDLEdBQUc7WUFDcEIsTUFBTSxDQUFDLEdBQUcsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxPQUFPLENBQUMscUJBQXFCLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQzdELENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxhQUFHLEVBQUU7YUFDVCxRQUFRLENBQUMsVUFBVSxDQUFDO2FBQ3BCLFNBQVMsQ0FBQyxVQUFVLEdBQUc7UUFFeEIsQ0FBQyxDQUFDO2FBQ0QsRUFBRSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUM7YUFDZixJQUFJLENBQUM7WUFDSixNQUFNLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDaEIsQ0FBQyxFQUFFLFVBQUMsR0FBRztZQUNMLE1BQU0sQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxDQUFDLENBQUMsQ0FBQztRQUNoQyxDQUFDLENBQUMsQ0FBQTtJQUVOLENBQUMsQ0FBQyxDQUFBO0lBQ0YsRUFBRSxDQUFDLDBEQUEwRCxFQUFFO1FBQzdELE9BQU8sYUFBRyxDQUFDO1lBQ1QsY0FBYyxFQUFFLEdBQUc7WUFDbkIsYUFBYSxFQUFFLEtBQUs7U0FDckIsQ0FBQzthQUNDLFVBQVUsQ0FBQyxxQkFFWixDQUFDO2FBQ0EsU0FBUyxDQUFDLFVBQUMsQ0FBQztZQUNYLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDWixNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDZixDQUFDLENBQUMsQ0FBQTtJQUNOLENBQUM
|