Skip to content

Commit

Permalink
🥭 decorate code
Browse files Browse the repository at this point in the history
  • Loading branch information
wolfram77 committed Sep 17, 2022
1 parent e2020f2 commit 5ead4d7
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 79 deletions.
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MIT License

Copyright (c) 2018-21 Subhajit Sahu
Copyright (c) 2018-22 Subhajit Sahu

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Expand Down
7 changes: 5 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
Detailed [nutrient composition] of 528 key foods in India.
Detailed [nutrient composition] of 528 key foods in India.<br>
📦 [Node.js](https://www.npmjs.com/package/@ifct2017/compositions),
📜 [Files](https://unpkg.com/@ifct2017/compositions/).

> This is part of package [ifct2017].<br>
> Online database: [ifct2017.github.io].
Expand Down Expand Up @@ -41,7 +43,8 @@ compositions.csv() // → path of csv file
<br>
<br>

[![](https://i.imgur.com/D5UYmbD.jpg)](http://ifct2017.com/)
[![](https://i.imgur.com/D5UYmbD.jpg)](http://ifct2017.com/)<br>
[![ORG](https://img.shields.io/badge/org-ifct2017-green?logo=Org)](https://ifct2017.github.io)

> Data was obtained from the book [Indian Food Composition Tables 2017].<br>
> Food composition values were measured by [National Institute of Nutrition, Hyderabad].<br>
Expand Down
99 changes: 50 additions & 49 deletions build.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
const descriptions = require('@ifct2017/descriptions')
const groups = require('@ifct2017/groups');
const path = require('path');
const fs = require('fs');
const os = require('os');
const parse = require('csv-parse');
const path = require('path');
const fs = require('fs');
const os = require('os');
const descriptions = require('@ifct2017/descriptions')
const groups = require('@ifct2017/groups');

const BASE = ['code', 'name', 'scie', 'lang', 'grup', 'regn', 'tags'];

Expand Down Expand Up @@ -39,12 +39,12 @@ var dat = {
regn: [],
tags: [],
};
var di = 0;
var di = 0;
var map = new Map();
var factors = null;
var renames = null;
var sums = null;
var orders = null;
var sums = null;
var orders = null;
var descCorpus;
var grupCorpus;
groups.load();
Expand All @@ -54,15 +54,15 @@ function valParse(val, code, dat, i) {
var f = factors.get(code);
var fn = parseFloat(f.replace(/\*.*/, ''));
var fi = f.indexOf('*'), fk = fi>=0? f.substring(fi+1):null;
var z = (parseFloat(val)||0)*fn*(fk? parseFloat(dat[fk][i])||0:1);
// z = parseFloat(z.toExponential((significantDigits(parseFloat(val))||1)-1));
return round(z);
var a = (parseFloat(val)||0)*fn*(fk? parseFloat(dat[fk][i])||0:1);
// a = parseFloat(a.toExponential((significantDigits(parseFloat(val))||1)-1));
return round(a);
}


function nameSci(str) {
var bgn = str.lastIndexOf('(');
if(bgn<0) return '';
if (bgn<0) return '';
var end = str.lastIndexOf(')');
end = end<0? str.length:end;
var sci = str.substring(bgn+1, end).trim();
Expand All @@ -72,7 +72,7 @@ function nameSci(str) {

function nameBas(str) {
var sci = nameSci(str);
if(!sci) return str.trim();
if (!sci) return str.trim();
return str.replace(new RegExp(`\\(\\s*${sci}\\s*\\)`), '').trim();
}

Expand All @@ -89,20 +89,20 @@ function readAssetRow(row) {
dat.grup[i] = grupCorpus.get(cod[0]).group;
dat.regn[i] = parseInt(row.regn.trim(), 10);
dat.tags[i] = grupCorpus.get(cod[0]).tags.trim();
for(var k in row) {
if(BASE.includes(k)) continue;
for (var k in row) {
if (BASE.includes(k)) continue;
var val = row[k].trim().split('±'), kt = renames.get(k)||k;
if(!dat[kt]) { dat[kt] = []; dat[kt+'_e'] = []; }
if (!dat[kt]) { dat[kt] = []; dat[kt+'_e'] = []; }
dat[kt][i] = valParse(val[0]||'0', k, dat, i);
dat[kt+'_e'][i] = valParse(val[1]||'0', k, dat, i);
}
}

function readAsset(pth) {
return new Promise((fres) => {
var stm = fs.createReadStream(pth).pipe(parse({columns: true, comment: '#'}));
stm.on('data', readAssetRow);
stm.on('end', () => fres());
return new Promise(resolve => {
var stream = fs.createReadStream(pth).pipe(parse({columns: true, comment: '#'}));
stream.on('data', readAssetRow);
stream.on('end', () => resolve());
});
}

Expand All @@ -111,39 +111,39 @@ function readAsset(pth) {

function nullToZero(d) {
var K = Object.keys(d);
for(var k of K) {
for(var i=0; i<di; i++)
for (var k of K) {
for (var i=0; i<di; i++)
d[k][i] = d[k][i]!=null? d[k][i]:0;
}
}

function sumColumns(d, i, ks) {
var z = 0;
for(var k of ks)
z += d[k][i];
return z;
var a = 0;
for (var k of ks)
a += d[k][i];
return a;
}

function sumAll(d) {
for(var [k, exp] of sums) {
for (var [k, exp] of sums) {
var sumk = exp.replace(/\s/g, '').split('+'); d[k] = d[k]||[];
for(var i=0; i<di; i++)
d[k][i] = d[k][i]||round(sumColumns(d, i, sumk));
for (var i=0; i<di; i++)
d[k][i] = d[k][i] || round(sumColumns(d, i, sumk));
}
}

function orderAll(d) {
var z = {};
for(var k in d) {
if(k in z) continue;
for(var ak of orders.get(k)||[]) {
z[ak] = d[ak];
var a = {};
for (var k in d) {
if (k in a) continue;
for (var ak of orders.get(k) || []) {
a[ak] = d[ak];
var ake = ak+'_e';
if(ake in d) z[ake] = d[ake];
if (ake in d) a[ake] = d[ake];
}
z[k] = d[k];
a[k] = d[k];
}
return z;
return a;
}


Expand All @@ -152,27 +152,28 @@ function orderAll(d) {
async function build() {
grupCorpus = groups.load();
descCorpus = await descriptions.load();
factors = await readCsv('configs/factors.csv', (acc, r) => acc.set(r.code, r.factor), new Map());
renames = await readCsv('configs/renames.csv', (acc, r) => acc.set(r.code, r.actual), new Map());
sums = await readCsv('configs/sums.csv', (acc, r) => acc.set(r.code, r.expression), new Map());
orders = await readCsv('configs/orders.csv', (acc, r) => {
var arr = acc.get(r.before)||[];
factors = await readCsv('configs/factors.csv', (acc, r) => acc.set(r.code, r.factor), new Map());
renames = await readCsv('configs/renames.csv', (acc, r) => acc.set(r.code, r.actual), new Map());
sums = await readCsv('configs/sums.csv', (acc, r) => acc.set(r.code, r.expression), new Map());
orders = await readCsv('configs/orders.csv', (acc, r) => {
var arr = acc.get(r.before) || [];
acc.set(r.before, arr);
arr.push(r.code);
}, new Map());
for(var file of fs.readdirSync('assets'))
for (var file of fs.readdirSync('assets'))
await readAsset(path.join('assets', file));
nullToZero(dat);
sumAll(dat);
dat = orderAll(dat);
var ks = Object.keys(dat), z = ks.join()+os.EOL;
for(var i=0; i<di; i++) {
for(var k of ks) {
var ks = Object.keys(dat);
var a = ks.join() + os.EOL;
for (var i=0; i<di; i++) {
for (var k of ks) {
var v = dat[k][i];
z += JSON.stringify(v==null? 0:v)+',';
a += JSON.stringify(v==null? 0:v)+',';
}
z = z.substring(0, z.length-1)+os.EOL;
a = a.substring(0, a.length-1)+os.EOL;
}
fs.writeFileSync('index.csv', z);
fs.writeFileSync('index.csv', a);
}
build();
49 changes: 25 additions & 24 deletions index.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const lunr = require('lunr');
const parse = require('csv-parse');
const csvx = require('csv-parse');
const esql = require('sql-extra');

const TEXTCOLS = new Set(['code', 'name', 'scie', 'lang', 'grup', 'tags']);

var corpus = new Map();
var index = null;
var ready = null;
var index = null;
var ready = null;



Expand Down Expand Up @@ -41,31 +42,31 @@ function createFunctionLangTags(tab) {
function createTable(tab, cols, opt={}, a='') {
var pre = ['code', 'name', 'scie', 'lang', 'grup', 'regn', 'tags'];
a += `CREATE TABLE IF NOT EXISTS "${tab}" (`;
for(var c of pre) {
for (var c of pre) {
var typ = c==='regn'? 'INT':'TEXT';
a += ` "${c}" ${typ} NOT NULL,`;
}
for(var c in cols) {
if(pre.includes(c)) continue;
for (var c in cols) {
if (pre.includes(c)) continue;
a += ` "${c}" REAL NOT NULL,`;
}
if(opt.pk) a += ` PRIMARY KEY ("code"), `;
if (opt.pk) a += ` PRIMARY KEY ("code"), `;
a = a.endsWith(', ')? a.substring(0, a.length-2) : a;
a += `);\n`;
return a;
}

function insertIntoBegin(tab, cols, a='') {
a += `INSERT INTO "${tab}" (`;
for(var c in cols)
for (var c in cols)
a += `"${c}", `;
a = a.endsWith(', ')? a.substring(0, a.length-2) : a;
a += ') VALUES\n(';
return a;
}

function insertIntoMid(val, a='') {
for(var k in val)
for (var k in val)
a += `'${val[k]}', `;
a = a.endsWith(', ')? a.substring(0, a.length-2) : a;
a += `),\n(`;
Expand All @@ -82,20 +83,20 @@ function sql(tab='compositions', opt={}) {
var i = -1, cols = null, a = '';
var opt = Object.assign({pk: 'code', index: true}, opt);
var tsv = tsvector(tab, {code: 'A', name: 'B', scie: 'B', lang: 'B', grup: 'C', tags: 'C'});
var stream = fs.createReadStream(csv()).pipe(parse({columns: true, comment: '#'}));
return new Promise((fres, frej) => {
stream.on('error', frej);
var stream = fs.createReadStream(csv()).pipe(csvx.parse({columns: true, comment: '#'}));
return new Promise((resolve, reject) => {
stream.on('error', reject);
stream.on('data', (r) => {
if(++i===0) { cols = r; a = createTable(tab, cols, opt, a); a = insertIntoBegin(tab, cols, a); }
if (++i===0) { cols = r; a = createTable(tab, cols, opt, a); a = insertIntoBegin(tab, cols, a); }
a = insertIntoMid(r, a);
});
stream.on('end', () => {
a = insertIntoEnd(a);
a = insertIntoEnd(a);
a += createFunctionLangTags(tab);
a += esql.createView(`${tab}_tsvector`, `SELECT *, ${tsv} AS "tsvector" FROM "${tab}"`);
a += esql.createIndex(`${tab}_tsvector_idx`, tab, `(${tsv})`, {method: 'GIN'});
a = esql.setupTable.index(tab, cols, opt, a);
fres(a);
a = esql.setupTable.index(tab, cols, opt, a);
resolve(a);
});
});
}
Expand All @@ -105,16 +106,16 @@ function sql(tab='compositions', opt={}) {

function loadRow(row) {
var a = {};
for(var k in row) {
if(TEXTCOLS.has(k)) a[k] = row[k];
for (var k in row) {
if (TEXTCOLS.has(k)) a[k] = row[k];
else a[k] = parseFloat(row[k]);
}
return a;
}

function loadCorpus() {
return new Promise((fres) => {
var s = fs.createReadStream(csv()).pipe(parse({columns: true, comment: '#'}));
var s = fs.createReadStream(csv()).pipe(csvx.parse({columns: true, comment: '#'}));
s.on('data', (r) => corpus.set(r.code, loadRow(r)));
s.on('end', fres);
});
Expand All @@ -129,7 +130,7 @@ function createIndex() {
this.field('lang');
this.field('grup');
this.field('tags');
for(var r of corpus.values()) {
for (var r of corpus.values()) {
var {code, name, scie, lang, grup, tags} = r;
name = name.replace(/^(\w+),/g, '$1 $1 $1 $1,');
lang = lang.replace(/\[.*?\]/g, '');
Expand Down Expand Up @@ -158,7 +159,7 @@ function matchRate(m) {

function compositions(txt) {
if (!index) { load(); return []; }
var a = [], txt = txt.replace(/\W/g, ' ');
var a = [], txt = txt.replace(/\W/g, ' ');
var ms = index.search(txt), max = 0;
for (var m of ms)
max = Math.max(max, matchRate(m));
Expand All @@ -167,6 +168,6 @@ function compositions(txt) {
return a;
}
compositions.load = load;
compositions.csv = csv;
compositions.sql = sql;
compositions.csv = csv;
compositions.sql = sql;
module.exports = compositions;
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@ifct2017/compositions",
"version": "2.0.5",
"version": "2.0.6",
"description": "Detailed nutrient composition of 528 key foods in India.",
"main": "index.js",
"scripts": {
Expand Down

0 comments on commit 5ead4d7

Please sign in to comment.