diff --git a/build.js b/build.js index ce6fe8f..7d0b91f 100644 --- a/build.js +++ b/build.js @@ -3,7 +3,8 @@ const fs = require('fs'); const os = require('os'); const build = require('extra-build'); const csv = require('csv-parse'); -const descriptions = require('@ifct2017/descriptions') +const columns = require('@ifct2017/columns'); +const descriptions = require('@ifct2017/descriptions'); const groups = require('@ifct2017/groups'); const owner = 'ifct2017'; @@ -150,7 +151,8 @@ async function generateIndexCsv() { async function main() { - grupCorpus = groups.load(); + await columns.load(); + grupCorpus = await groups.load(); descCorpus = await descriptions.load(); factors = await readCsv('configs/factors.csv', (acc, r) => acc.set(r.code, r.factor), new Map()); renames = await readCsv('configs/renames.csv', (acc, r) => acc.set(r.code, r.actual), new Map()); @@ -166,7 +168,7 @@ async function generateIndexCsv() { sumAll(dat); dat = orderAll(dat); var ks = Object.keys(dat); - var a = ks.join() + os.EOL; + var a = ks.map(k => `"${columns(k.replace(/_e$/, ''))[0].name}; ${k}"`).join() + os.EOL; for (var i=0; i { stream.on('error', reject); stream.on('data', (r) => { - if (++i===0) { cols = r; a = createTable(tab, cols, opt, a); a = insertIntoBegin(tab, cols, a); } - a = insertIntoMid(r, a); + var x = fixColumns(r); + if (++i===0) { cols = x; a = createTable(tab, cols, opt, a); a = insertIntoBegin(tab, cols, a); } + a = insertIntoMid(x, a); }); stream.on('end', () => { a = insertIntoEnd(a); @@ -104,11 +105,24 @@ function sql(tab='compositions', opt={}) { -function loadRow(row) { +// Fix column names of a row in the CSV file. +function fixColumns(row) { var a = {}; for (var k in row) { - if (TEXTCOLS.has(k)) a[k] = row[k]; - else a[k] = parseFloat(row[k]); + // Name of column is after the last semicolon. + var l = k.substring(k.lastIndexOf(';')+1).trim(); + a[l] = row[k]; + } + return a; +} + + +// Parse a row from the CSV file. +function parseRow(row) { + var a = {}; + for (var k in row) { + var l = k.substring(k.lastIndexOf(';')+1).trim(); + a[l] = TEXTCOLS.has(l)? row[k] : parseFloat(row[k]); } return a; } @@ -116,7 +130,7 @@ function loadRow(row) { function loadCorpus() { return new Promise((fres) => { var s = fs.createReadStream(csv()).pipe(csvx.parse({columns: true, comment: '#'})); - s.on('data', (r) => corpus.set(r.code, loadRow(r))); + s.on('data', (r) => { var x = parseRow(r); corpus.set(x.code, x); }); s.on('end', fres); }); } diff --git a/package-lock.json b/package-lock.json index a302b95..c60f4e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,11 +14,22 @@ "sql-extra": "^0.1.13" }, "devDependencies": { + "@ifct2017/columns": "^2.0.13", "@ifct2017/descriptions": "^2.0.7", "@ifct2017/groups": "^2.0.11", "extra-build": "^2.2.29" } }, + "node_modules/@ifct2017/columns": { + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/@ifct2017/columns/-/columns-2.0.13.tgz", + "integrity": "sha512-59J9wJlnfm77BUTitVIqFy7UE+qGRwIyjGD061VWHNok4vkQsJGcPydRs6FVBNANhWV8QMnKR3fHSHcyemEIzw==", + "dev": true, + "dependencies": { + "lunr": "^2.3.9", + "sql-extra": "^0.1.13" + } + }, "node_modules/@ifct2017/descriptions": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@ifct2017/descriptions/-/descriptions-2.0.7.tgz", @@ -478,6 +489,16 @@ } }, "dependencies": { + "@ifct2017/columns": { + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/@ifct2017/columns/-/columns-2.0.13.tgz", + "integrity": "sha512-59J9wJlnfm77BUTitVIqFy7UE+qGRwIyjGD061VWHNok4vkQsJGcPydRs6FVBNANhWV8QMnKR3fHSHcyemEIzw==", + "dev": true, + "requires": { + "lunr": "^2.3.9", + "sql-extra": "^0.1.13" + } + }, "@ifct2017/descriptions": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@ifct2017/descriptions/-/descriptions-2.0.7.tgz", diff --git a/package.json b/package.json index fe7088b..756588b 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ }, "homepage": "https://github.com/ifct2017/compositions#readme", "devDependencies": { + "@ifct2017/columns": "^2.0.13", "@ifct2017/descriptions": "^2.0.7", "@ifct2017/groups": "^2.0.11", "extra-build": "^2.2.29"