Merge pull request #103 from terascope/misc-improvements

added extra csv parsing options
terascope · Aug 5, 2019 · 812ce05 · 812ce05
2 parents ceea577 + 4c70bba
commit 812ce05
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 4 deletions.
diff --git a/asset/asset.json b/asset/asset.json
@@ -1,5 +1,5 @@
 {
     "name": "file-assets",
-    "version": "0.7.2",
+    "version": "0.7.3",
     "description": "A set of processors for exporting data to files"
 }
diff --git a/asset/compressed_file_reader/index.js b/asset/compressed_file_reader/index.js
@@ -247,6 +247,21 @@ function schema() {
             doc: 'Determines slice size in bytes',
             default: 100000,
             format: Number,
+        },
+        ignore_empty: {
+            doc: 'Only used with CSV parsing. Ignores any columns not specified in field list. '
+                + 'Since the field list is applied to columns sequentially, this will ignore '
+                + 'any additional columns past the number specified.\n'
+                + 'i.e. If 5 fields are specified, but there are 7 columns in the file, columns'
+                + ' 6 and 7 will be dropped if this is true. Otherwise, the parser will give '
+                + 'the fields generic names.',
+            default: true,
+            format: Boolean
+        },
+        extra_args: {
+            doc: 'An object used to pass in any extra csv parsing arguments',
+            default: {},
+            format: Object
         }
     };
 }

diff --git a/asset/file_reader/schema.js b/asset/file_reader/schema.js
@@ -42,6 +42,21 @@ class Schema extends ConvictSchema {
                 doc: 'Checks for the header row and removes it',
                 default: true,
                 format: 'Boolean'
+            },
+            ignore_empty: {
+                doc: 'Only used with CSV parsing. Ignores any columns not specified in field list. '
+                    + 'Since the field list is applied to columns sequentially, this will ignore '
+                    + 'any additional columns past the number specified.\n'
+                    + 'i.e. If 5 fields are specified, but there are 7 columns in the file, columns'
+                    + ' 6 and 7 will be dropped if this is true. Otherwise, the parser will give '
+                    + 'the fields generic names.',
+                default: true,
+                format: Boolean
+            },
+            extra_args: {
+                doc: 'An object used to pass in any extra csv parsing arguments',
+                default: {},
+                format: Object
             }
         };
     }

diff --git a/asset/package.json b/asset/package.json
@@ -1,9 +1,9 @@
 {
     "name": "file-assets",
-    "version": "0.7.2",
+    "version": "0.7.3",
     "description": "A set of processors for working with files",
     "dependencies": {
-        "@terascope/chunked-file-reader": "^2.1.1",
+        "@terascope/chunked-file-reader": "^2.1.2",
         "@terascope/job-components": "^0.20.7",
         "@terascope/queue": "^1.1.6",
         "@terascope/utils": "^0.14.1",

diff --git a/asset/s3_exporter/schema.js b/asset/s3_exporter/schema.js
@@ -53,6 +53,21 @@ class Schema extends ConvictSchema {
                     + 'capability will be included in a future improvement',
                 default: 'false',
                 format: [false]
+            },
+            ignore_empty: {
+                doc: 'Only used with CSV parsing. Ignores any columns not specified in field list. '
+                    + 'Since the field list is applied to columns sequentially, this will ignore '
+                    + 'any additional columns past the number specified.\n'
+                    + 'i.e. If 5 fields are specified, but there are 7 columns in the file, columns'
+                    + ' 6 and 7 will be dropped if this is true. Otherwise, the parser will give '
+                    + 'the fields generic names.',
+                default: true,
+                format: Boolean
+            },
+            extra_args: {
+                doc: 'An object used to pass in any extra csv parsing arguments',
+                default: {},
+                format: Object
             }
         };
     }

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "file-assets-bundle",
-    "version": "0.7.2",
+    "version": "0.7.3",
     "description": "Teraslice processors for working with data stored in files on disk",
     "repository": "https://github.com/terascope/file-assets.git",
     "author": "Terascope, LLC <[email protected]>",