310 lines
33 KiB
JavaScript
310 lines
33 KiB
JavaScript
|
"use strict";
|
||
|
|
var __extends = (this && this.__extends) || (function () {
|
||
|
|
var extendStatics = Object.setPrototypeOf ||
|
||
|
|
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
||
|
|
function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
|
||
|
|
return function (d, b) {
|
||
|
|
extendStatics(d, b);
|
||
|
|
function __() { this.constructor = d; }
|
||
|
|
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
||
|
|
};
|
||
|
|
})();
|
||
|
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||
|
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||
|
|
};
|
||
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
||
|
|
var Processor_1 = require("./Processor");
|
||
|
|
var bluebird_1 = __importDefault(require("bluebird"));
|
||
|
|
var dataClean_1 = require("./dataClean");
|
||
|
|
var getEol_1 = __importDefault(require("./getEol"));
|
||
|
|
var fileline_1 = require("./fileline");
|
||
|
|
var util_1 = require("./util");
|
||
|
|
var rowSplit_1 = require("./rowSplit");
|
||
|
|
var lineToJson_1 = __importDefault(require("./lineToJson"));
|
||
|
|
var CSVError_1 = __importDefault(require("./CSVError"));
|
||
|
|
var ProcessorLocal = /** @class */ (function (_super) {
|
||
|
|
__extends(ProcessorLocal, _super);
|
||
|
|
function ProcessorLocal() {
|
||
|
|
var _this = _super !== null && _super.apply(this, arguments) || this;
|
||
|
|
_this.rowSplit = new rowSplit_1.RowSplit(_this.converter);
|
||
|
|
_this.eolEmitted = false;
|
||
|
|
_this._needEmitEol = undefined;
|
||
|
|
_this.headEmitted = false;
|
||
|
|
_this._needEmitHead = undefined;
|
||
|
|
return _this;
|
||
|
|
}
|
||
|
|
ProcessorLocal.prototype.flush = function () {
|
||
|
|
var _this = this;
|
||
|
|
if (this.runtime.csvLineBuffer && this.runtime.csvLineBuffer.length > 0) {
|
||
|
|
var buf = this.runtime.csvLineBuffer;
|
||
|
|
this.runtime.csvLineBuffer = undefined;
|
||
|
|
return this.process(buf, true)
|
||
|
|
.then(function (res) {
|
||
|
|
if (_this.runtime.csvLineBuffer && _this.runtime.csvLineBuffer.length > 0) {
|
||
|
|
return bluebird_1.default.reject(CSVError_1.default.unclosed_quote(_this.runtime.parsedLineNumber, _this.runtime.csvLineBuffer.toString()));
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return bluebird_1.default.resolve(res);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return bluebird_1.default.resolve([]);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.destroy = function () {
|
||
|
|
return bluebird_1.default.resolve();
|
||
|
|
};
|
||
|
|
Object.defineProperty(ProcessorLocal.prototype, "needEmitEol", {
|
||
|
|
get: function () {
|
||
|
|
if (this._needEmitEol === undefined) {
|
||
|
|
this._needEmitEol = this.converter.listeners("eol").length > 0;
|
||
|
|
}
|
||
|
|
return this._needEmitEol;
|
||
|
|
},
|
||
|
|
enumerable: true,
|
||
|
|
configurable: true
|
||
|
|
});
|
||
|
|
Object.defineProperty(ProcessorLocal.prototype, "needEmitHead", {
|
||
|
|
get: function () {
|
||
|
|
if (this._needEmitHead === undefined) {
|
||
|
|
this._needEmitHead = this.converter.listeners("header").length > 0;
|
||
|
|
}
|
||
|
|
return this._needEmitHead;
|
||
|
|
},
|
||
|
|
enumerable: true,
|
||
|
|
configurable: true
|
||
|
|
});
|
||
|
|
ProcessorLocal.prototype.process = function (chunk, finalChunk) {
|
||
|
|
var _this = this;
|
||
|
|
if (finalChunk === void 0) { finalChunk = false; }
|
||
|
|
var csvString;
|
||
|
|
if (finalChunk) {
|
||
|
|
csvString = chunk.toString();
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
csvString = dataClean_1.prepareData(chunk, this.converter.parseRuntime);
|
||
|
|
}
|
||
|
|
return bluebird_1.default.resolve()
|
||
|
|
.then(function () {
|
||
|
|
if (_this.runtime.preRawDataHook) {
|
||
|
|
return _this.runtime.preRawDataHook(csvString);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return csvString;
|
||
|
|
}
|
||
|
|
})
|
||
|
|
.then(function (csv) {
|
||
|
|
if (csv && csv.length > 0) {
|
||
|
|
return _this.processCSV(csv, finalChunk);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return bluebird_1.default.resolve([]);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.processCSV = function (csv, finalChunk) {
|
||
|
|
var _this = this;
|
||
|
|
var params = this.params;
|
||
|
|
var runtime = this.runtime;
|
||
|
|
if (!runtime.eol) {
|
||
|
|
getEol_1.default(csv, runtime);
|
||
|
|
}
|
||
|
|
if (this.needEmitEol && !this.eolEmitted && runtime.eol) {
|
||
|
|
this.converter.emit("eol", runtime.eol);
|
||
|
|
this.eolEmitted = true;
|
||
|
|
}
|
||
|
|
// trim csv file has initial blank lines.
|
||
|
|
if (params.ignoreEmpty && !runtime.started) {
|
||
|
|
csv = util_1.trimLeft(csv);
|
||
|
|
}
|
||
|
|
var stringToLineResult = fileline_1.stringToLines(csv, runtime);
|
||
|
|
if (!finalChunk) {
|
||
|
|
this.prependLeftBuf(util_1.bufFromString(stringToLineResult.partial));
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
stringToLineResult.lines.push(stringToLineResult.partial);
|
||
|
|
stringToLineResult.partial = "";
|
||
|
|
}
|
||
|
|
if (stringToLineResult.lines.length > 0) {
|
||
|
|
var prom = void 0;
|
||
|
|
if (runtime.preFileLineHook) {
|
||
|
|
prom = this.runPreLineHook(stringToLineResult.lines);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
prom = bluebird_1.default.resolve(stringToLineResult.lines);
|
||
|
|
}
|
||
|
|
return prom.then(function (lines) {
|
||
|
|
if (!runtime.started
|
||
|
|
&& !_this.runtime.headers) {
|
||
|
|
return _this.processDataWithHead(lines);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return _this.processCSVBody(lines);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return bluebird_1.default.resolve([]);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.processDataWithHead = function (lines) {
|
||
|
|
if (this.params.noheader) {
|
||
|
|
if (this.params.headers) {
|
||
|
|
this.runtime.headers = this.params.headers;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.runtime.headers = [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
var left = "";
|
||
|
|
var headerRow = [];
|
||
|
|
while (lines.length) {
|
||
|
|
var line = left + lines.shift();
|
||
|
|
var row = this.rowSplit.parse(line);
|
||
|
|
if (row.closed) {
|
||
|
|
headerRow = row.cells;
|
||
|
|
left = "";
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
left = line + getEol_1.default(line, this.runtime);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
this.prependLeftBuf(util_1.bufFromString(left));
|
||
|
|
if (headerRow.length === 0) {
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
if (this.params.headers) {
|
||
|
|
this.runtime.headers = this.params.headers;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.runtime.headers = headerRow;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (this.runtime.needProcessIgnoreColumn || this.runtime.needProcessIncludeColumn) {
|
||
|
|
this.filterHeader();
|
||
|
|
}
|
||
|
|
if (this.needEmitHead && !this.headEmitted) {
|
||
|
|
this.converter.emit("header", this.runtime.headers);
|
||
|
|
this.headEmitted = true;
|
||
|
|
}
|
||
|
|
return this.processCSVBody(lines);
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.filterHeader = function () {
|
||
|
|
this.runtime.selectedColumns = [];
|
||
|
|
if (this.runtime.headers) {
|
||
|
|
var headers = this.runtime.headers;
|
||
|
|
for (var i = 0; i < headers.length; i++) {
|
||
|
|
if (this.params.ignoreColumns) {
|
||
|
|
if (this.params.ignoreColumns.test(headers[i])) {
|
||
|
|
if (this.params.includeColumns && this.params.includeColumns.test(headers[i])) {
|
||
|
|
this.runtime.selectedColumns.push(i);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.runtime.selectedColumns.push(i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if (this.params.includeColumns) {
|
||
|
|
if (this.params.includeColumns.test(headers[i])) {
|
||
|
|
this.runtime.selectedColumns.push(i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.runtime.selectedColumns.push(i);
|
||
|
|
}
|
||
|
|
// if (this.params.includeColumns && this.params.includeColumns.test(headers[i])){
|
||
|
|
// this.runtime.selectedColumns.push(i);
|
||
|
|
// }else{
|
||
|
|
// if (this.params.ignoreColumns && this.params.ignoreColumns.test(headers[i])){
|
||
|
|
// continue;
|
||
|
|
// }else{
|
||
|
|
// if (this.params.ignoreColumns && !this.params.includeColumns){
|
||
|
|
// this.runtime.selectedColumns.push(i);
|
||
|
|
// }
|
||
|
|
// }
|
||
|
|
// }
|
||
|
|
}
|
||
|
|
this.runtime.headers = util_1.filterArray(this.runtime.headers, this.runtime.selectedColumns);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.processCSVBody = function (lines) {
|
||
|
|
if (this.params.output === "line") {
|
||
|
|
return lines;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
var result = this.rowSplit.parseMultiLines(lines);
|
||
|
|
this.prependLeftBuf(util_1.bufFromString(result.partial));
|
||
|
|
if (this.params.output === "csv") {
|
||
|
|
return result.rowsCells;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return lineToJson_1.default(result.rowsCells, this.converter);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// var jsonArr = linesToJson(lines.lines, params, this.recordNum);
|
||
|
|
// this.processResult(jsonArr);
|
||
|
|
// this.lastIndex += jsonArr.length;
|
||
|
|
// this.recordNum += jsonArr.length;
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.prependLeftBuf = function (buf) {
|
||
|
|
if (buf) {
|
||
|
|
if (this.runtime.csvLineBuffer) {
|
||
|
|
this.runtime.csvLineBuffer = Buffer.concat([buf, this.runtime.csvLineBuffer]);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
this.runtime.csvLineBuffer = buf;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
};
|
||
|
|
ProcessorLocal.prototype.runPreLineHook = function (lines) {
|
||
|
|
var _this = this;
|
||
|
|
return new bluebird_1.default(function (resolve, reject) {
|
||
|
|
processLineHook(lines, _this.runtime, 0, function (err) {
|
||
|
|
if (err) {
|
||
|
|
reject(err);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
resolve(lines);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
};
|
||
|
|
return ProcessorLocal;
|
||
|
|
}(Processor_1.Processor));
|
||
|
|
exports.ProcessorLocal = ProcessorLocal;
|
||
|
|
function processLineHook(lines, runtime, offset, cb) {
|
||
|
|
if (offset >= lines.length) {
|
||
|
|
cb();
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
if (runtime.preFileLineHook) {
|
||
|
|
var line = lines[offset];
|
||
|
|
var res = runtime.preFileLineHook(line, runtime.parsedLineNumber + offset);
|
||
|
|
offset++;
|
||
|
|
if (res && res.then) {
|
||
|
|
res.then(function (value) {
|
||
|
|
lines[offset - 1] = value;
|
||
|
|
processLineHook(lines, runtime, offset, cb);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
lines[offset - 1] = res;
|
||
|
|
while (offset < lines.length) {
|
||
|
|
lines[offset] = runtime.preFileLineHook(lines[offset], runtime.parsedLineNumber + offset);
|
||
|
|
offset++;
|
||
|
|
}
|
||
|
|
cb();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
cb();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiL1VzZXJzL2t4aWFuZy93b3JrL3Byb2plY3RzL2NzdjJqc29uL3NyYy9Qcm9jZXNzb3JMb2NhbC50cyIsInNvdXJjZXMiOlsiL1VzZXJzL2t4aWFuZy93b3JrL3Byb2plY3RzL2NzdjJqc29uL3NyYy9Qcm9jZXNzb3JMb2NhbC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7QUFBQSx5Q0FBMkQ7QUFDM0Qsc0RBQXlCO0FBQ3pCLHlDQUEwQztBQUMxQyxvREFBOEI7QUFDOUIsdUNBQTJDO0FBQzNDLCtCQUE2RDtBQUM3RCx1Q0FBc0M7QUFDdEMsNERBQXNDO0FBRXRDLHdEQUFrQztBQUlsQztJQUFvQyxrQ0FBUztJQUE3QztRQUFBLHFFQW1PQztRQS9NUyxjQUFRLEdBQWEsSUFBSSxtQkFBUSxDQUFDLEtBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUNsRCxnQkFBVSxHQUFHLEtBQUssQ0FBQztRQUNuQixrQkFBWSxHQUFhLFNBQVMsQ0FBQztRQU9uQyxpQkFBVyxHQUFHLEtBQUssQ0FBQztRQUNwQixtQkFBYSxHQUFhLFNBQVMsQ0FBQzs7SUFxTTlDLENBQUM7SUFsT0MsOEJBQUssR0FBTDtRQUFBLGlCQWVDO1FBZEMsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxFQUFFO1lBQ3ZFLElBQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDO1lBQ3ZDLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxHQUFHLFNBQVMsQ0FBQztZQUN2QyxPQUFPLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQztpQkFDM0IsSUFBSSxDQUFDLFVBQUMsR0FBRztnQkFDUixJQUFJLEtBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxJQUFJLEtBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUU7b0JBQ3ZFLE9BQU8sa0JBQUMsQ0FBQyxNQUFNLENBQUMsa0JBQVEsQ0FBQyxjQUFjLENBQUMsS0FBSSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsRUFBRSxLQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxDQUFDLENBQUE7aUJBQy9HO3FCQUFNO29CQUNMLE9BQU8sa0JBQUMsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUM7aUJBQ3ZCO1lBQ0gsQ0FBQyxDQUFDLENBQUE7U0FDTDthQUFNO1lBQ0wsT0FBTyxrQkFBQyxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUMsQ0FBQztTQUN0QjtJQUNILENBQUM7SUFDRCxnQ0FBTyxHQUFQO1FBQ0UsT0FBTyxrQkFBQyxDQUFDLE9BQU8sRUFBRSxDQUFDO0lBQ3JCLENBQUM7SUFJRCxzQkFBWSx1Q0FBVzthQUF2QjtZQUNFLElBQUksSUFBSSxDQUFDLFlBQVksS0FBSyxTQUFTLEVBQUU7Z0JBQ25DLElBQUksQ0FBQyxZQUFZLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxTQUFTLENBQUMsS0FBSyxDQUFDLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQzthQUNoRTtZQUNELE9BQU8sSUFBSSxDQUFDLFlBQVksQ0FBQztRQUMzQixDQUFDOzs7T0FBQTtJQUdELHNCQUFZLHdDQUFZO2FBQXhCO1lBQ0UsSUFBSSxJQUFJLENBQUMsYUFBYSxLQUFLLFNBQVMsRUFBRTtnQkFDcEMsSUFBSSxDQUFDLGFBQWEsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDLFNBQVMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDO2FBQ3BFO1lBQ0QsT0FBTyxJQUFJLENBQUMsYUFBYSxDQUFDO1FBRTVCLENBQUM7OztPQUFBO0lBQ0QsZ0NBQU8sR0FBUCxVQUFRLEtBQWEsRUFBRSxVQUFrQjtRQUF6QyxpQkF1QkM7UUF2QnNCLDJCQUFBLEVBQUEsa0JBQWtCO1FBQ3ZDLElBQUksU0FBaUIsQ0FBQztRQUN0QixJQUFJLFVBQVUsRUFBRTtZQUNkLFNBQVMsR0FBRyxLQUFLLENBQUMsUUFBUSxFQUFFLENBQUM7U0FDOUI7YUFBTTtZQUNMLFNBQVMsR0FBRyx1QkFBVyxDQUFDLEtBQUssRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLFlBQVksQ0FBQyxDQUFDO1NBRTdEO1FBQ0QsT0FBTyxrQkFBQyxDQUFDLE9BQU8sRUFBRTthQUNmLElBQUksQ0FBQztZQUNKLElBQUksS0FBSSxDQUFDLE9BQU8sQ0FBQyxjQUFjLEVBQUU7Z0JBQy9CLE9BQU8sS0FBSSxDQUFDLE9BQU8sQ0FBQyxjQUFjLENBQUMsU0FBUyxDQUFDLENBQUM7YUFDL0M7aUJBQU07Z0JBQ0wsT0FBTyxTQUFTLENBQUM7YUFDbEI7UUFDSCxDQUFDLENBQUM7YUFDRCxJQUFJLENBQUMsVUFBQyxHQUFHO1lBQ1IsSUFBSSxHQUFHLElBQUksR0FBRyxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUU7Z0JBQ3pCLE9BQU8sS0FBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLEVBQUUsVUFBVSxDQUFDLENBQUM7YUFDekM7aUJBQU07Z0JBQ0wsT0FBTyxrQkFBQyxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUMsQ0FBQzthQUN0QjtRQUNILENBQUMsQ0FBQyxDQUFBO0lBQ04sQ0FBQztJQUNPLG1DQUFVLEdBQWxCLFVBQW1CLEdBQVcsRUFBRSxVQUFtQjtRQUFuRCxpQkE0Q0M7UUEzQ0MsSUFBTSxNQUFNLEdBQUcsSUFBSSxDQUFDLE1BQU0sQ0FBQztRQUMzQixJQUFNLE9BQU8sR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDO1FBQzdCLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFO1lBQ2hCLGdCQUFNLENBQUMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFDO1NBQ3RCO1FBQ0QsSUFBSSxJQUFJLENBQUMsV0FBVyxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsSUFBSSxPQUFPLENBQUMsR0FBRyxFQUFFO1lBQ3ZELElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUM7WUFDeEMsSUFBSSxDQUFDLFVBQVUsR0FBRyxJQUFJLENBQUM7U0FDeEI7UUFDRCx5Q0FBeUM7UUFDekMsSUFBSSxNQUFNLENBQUMsV0FBVyxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUMxQyxHQUFHLEdBQUcsZUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1NBQ3JCO1FBQ0QsSUFBTSxrQkFBa0IsR0FBRyx3QkFBYSxDQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQztRQUN2RCxJQUFJLENBQUMsVUFBVSxFQUFFO1lBQ2YsSUFBSSxDQUFDLGNBQWMsQ0FBQyxvQkFBYSxDQUFDLGtCQUFrQixDQUFDLE9BQU8sQ0FBQyxDQUF
|