Merge pull request #95 from linkedconnections/development

v1.4.1

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x, 20.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+      - run: npm ci
+      - run: npm run build --if-present
+      - run: npm run test-ci

.gitignore

@@ -1,6 +1,5 @@
 node_modules/
 coverage
-package-lock.json
 datasets_config.json
 server_config.json
 .coveralls.yml

README.md

@@ -1,6 +1,6 @@
 # Linked Connections Server
 
-[![Coverage Status](https://coveralls.io/repos/github/linkedconnections/linked-connections-server/badge.svg?branch=master)](https://coveralls.io/github/linkedconnections/linked-connections-server?branch=master)
+[![Node.js CI](https://github.com/linkedconnections/linked-connections-server/actions/workflows/ci.yml/badge.svg)](https://github.com/linkedconnections/linked-connections-server/actions/workflows/ci.yml)
 
 Express based Web Server that exposes [Linked Connections](http://linkedconnections.org/) data fragments using [JSON-LD](https://json-ld.org/) serialization format. It also provides a built-in tool to parse [GTFS](https://developers.google.com/tansit/gtfs/reference/) and [GTFS Realtime](https://developers.google.com/transit/gtfs-realtime/) transport dataset feeds into a Linked Connections Directed Acyclic Graph using [GTFS2LC](https://github.com/linkedconnections/gtfs2lc) and fragment it following a configurable predefined size.
 

lib/manager/dataset_manager.js

@@ -1,5 +1,6 @@
 import fs from 'fs';
 import util from 'util';
+import os from 'os';
 import { deleteAsync as del } from 'del';
 import child_process from 'child_process';
 import { CronJob } from 'cron';
@@ -558,8 +559,18 @@ export class DatasetManager {
             const k = conn.split('"').indexOf('departureTime') + 2;
 
             // Sort using UNIX sort command
-            const sortCommand = child_process.exec(`sort -S ${mem} -T ${this.storage}/tmp/ -t '\"' -k ${k} --compress-program=gzip < zcat ${unsorted}`);
-            return sortCommand.stdout;
+            const zcat = child_process.spawn('zcat', [`${unsorted}`]);
+            const sort = child_process.spawn('sort', [
+                '-S', mem,
+                '-T', `${this.storage}/tmp/`,
+                '-t', '\"',
+                '-k', k,
+                '--compress-program=gzip',
+                `--parallel=${os.cpus().length}`
+            ]);
+            zcat.stdout.pipe(sort.stdin);
+            // Return readable stream of sorted connections
+            return sort.stdout;
         } catch (err) {
             throw err;
         }

lib/routes/catalog.js

@@ -9,6 +9,7 @@ const writeFile = util.promisify(fs.writeFile);
 
 export class Catalog {
     constructor() {
+        this._organization = utils.datasetsConfig.organization;
         this._storage = utils.datasetsConfig.storage;
         this._datasets = utils.datasetsConfig.datasets;
         this._serverConfig = utils.serverConfig;
@@ -17,23 +18,25 @@ export class Catalog {
     async getCatalog(req, res) {
         const agency = req.params.agency;
         try {
-            if (this.datasets.find(d => d.companyName === agency)) {
+            if (this.datasets && this.datasets.find(d => d.companyName === agency)) {
                 if (fs.existsSync(`${this.storage}/catalog/${agency}/catalog.json`)) {
                     res.set({
                         'Access-Control-Allow-Origin': '*',
                         'Access-Control-Allow-Headers': '*',
                         'Content-Type': 'application/ld+json'
                     });
                     res.send(await readFile(`${this.storage}/catalog/${agency}/catalog.json`, 'utf8'));
+                    res.status(200);
                 } else {
                     const catalog = await this.createCatalog(agency);
-                    writeFile(`${this.storage}/catalog/${agency}/catalog.json`, JSON.stringify(catalog), 'utf8');
+                    await writeFile(`${this.storage}/catalog/${agency}/catalog.json`, JSON.stringify(catalog), 'utf8');
                     res.set({
                         'Access-Control-Allow-Origin': '*',
                         'Access-Control-Allow-Headers': '*',
                         'Content-Type': 'application/ld+json'
                     });
                     res.send(catalog);
+                    res.status(200);
                 }
             } else {
                 res.set({ 'Cache-Control': 'no-cache' });
@@ -97,9 +100,9 @@ export class Catalog {
             "license": "http://creativecommons.org/publicdomain/zero/1.0/",
             "accessRights": "access:PUBLIC",
             "publisher": {
-                "@id": utils.datasetsConfig.organization.id,
+                "@id": this.organization.id,
                 "@type": "Organization",
-                "name": utils.datasetsConfig.organization.name
+                "name": this.organization.name
             },
             "dataset": []
         };
@@ -218,6 +221,10 @@ export class Catalog {
         return dist;
     }
 
+    get organization() {
+        return this._organization;
+    }
+
     get storage() {
         return this._storage;
     }