Refactors to optimize IO (#6)

* Better handle inputs option
* Release 0.1.1

.gitignore

@@ -2,7 +2,7 @@
 *.snap
 /.shards/
 /bin/
-/doc/
+/docs/
 /lib/
 
 # Libraries don't need dependency lock

README.md

@@ -42,7 +42,9 @@ The built binary will be available as `./bin/oq`.  This can be relocated elsewhe
 
 ## Usage
 
-Use the `oq` binary, with a few custom arguments.  All other arguments get passed to `jq`.
+### CLI
+
+Use the `oq` binary, with a few optional custom arguments.  All other arguments get passed to `jq`.
 
 ```bash
 Usage: oq [--help] [oq-arguments] [jq-arguments] jq_filter [file [files...]]
@@ -52,6 +54,22 @@ Usage: oq [--help] [oq-arguments] [jq-arguments] jq_filter [file [files...]]
     --xml-root ROOT                 Name of the root XML element if converting to XML.
 ```
 
+### Serialization
+
+Crystal applications can `require "oq/to_xml"` in order to use the `#to_xml` method to serialize objects to XML.
+
+```crystal
+require "oq/to_xml"
+
+hash = {"name": "Jim"}
+
+puts hash.to_xml
+# <?xml version="1.0" encoding="UTF-8"?>
+# <root>
+#   <name>Jim</name>
+# </root>
+```
+
 ## Roadmap
 
 Plans for `1.0.0`:

shard.yml

@@ -3,7 +3,7 @@ name: oq
 description: |
   A performant, and portable jq wrapper thats facilitates the consumption and output of formats other than JSON; using jq filters to transform the data.
 
-version: 0.1.0
+version: 0.1.1
 
 authors:
   - Blacksmoke16 <[email protected]>

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: oq
-version: '0.1.0'
+version: '0.1.1'
 summary: A performant, and portable jq wrapper to support formats other than JSON
 description: |
   A performant, and portable jq wrapper thats facilitates the consumption and output of formats other than JSON; using jq filters to transform the data.

spec/assets/stream-data.json

@@ -0,0 +1,6 @@
+{"machine": "possible_victim01", "domain": "evil.com", "timestamp":1435071870}
+{"machine": "possible_victim01", "domain": "evil.com", "timestamp":1435071875}
+{"machine": "possible_victim01", "domain": "soevil.com", "timestamp":1435071877}
+{"machine": "possible_victim02", "domain": "bad.com", "timestamp":1435071877}
+{"machine": "possible_victim03", "domain": "soevil.com", "timestamp":1435071879}
+

spec/assets/stream-filter

@@ -0,0 +1,8 @@
+reduce inputs as $line
+  ({};
+   $line.machine as $machine
+   | $line.domain as $domain
+   | .[$machine].total as $total
+   | .[$machine].evildoers as $evildoers
+   | . + { ($machine): {"total": (1 + $total),
+                        "evildoers": ($evildoers | (.[$domain] += 1)) }} )

spec/oq_spec.cr

@@ -123,7 +123,7 @@ describe Oq do
     end
   end
 
-  describe "with raw output" do
+  describe "with YAML raw output" do
     it "should return the correct output" do
       run_binary(input: "", args: ["-R", "-o", "yaml", ".", "spec/assets/data1.json"]) do |output|
         output.should eq %(--- '{"name": "Jim"}'\n)
@@ -173,7 +173,7 @@ describe Oq do
     describe "with input from STDIN" do
       it "should return the correct output" do
         run_binary(input: "foo", args: ["-n"]) do |output|
-          output.should eq "null\n"
+          output.should be_empty
         end
       end
     end
@@ -219,11 +219,10 @@ describe Oq do
     end
   end
 
-  describe "with XML input" do
-    it "should return not implemented" do
-      run_binary(input: "", args: ["-i", "xml", "."]) do |output, status|
-        output.should eq "Not Implemented\n"
-        status.exit_code.should eq 1
+  describe "when using 'input'" do
+    it "should return the correct output" do
+      run_binary(args: ["-cnf", "spec/assets/stream-filter", "spec/assets/stream-data.json"]) do |output|
+        output.should eq %({"possible_victim01":{"total":3,"evildoers":{"evil.com":2,"soevil.com":1}},"possible_victim02":{"total":1,"evildoers":{"bad.com":1}},"possible_victim03":{"total":1,"evildoers":{"soevil.com":1}}}\n)
       end
     end
   end

spec/spec_helper.cr

@@ -11,10 +11,10 @@ def assert_builder_output(expected : String, &block : XML::Builder -> Nil) : Nil
 end
 
 # Runs the the binary with the given *name* and *args*.
-def run_binary(input : String?, name : String = "bin/oq", args : Array(String) = [] of String, &block : String, Process::Status -> Nil)
+def run_binary(input : String? = nil, name : String = "bin/oq", args : Array(String) = [] of String, &block : String, Process::Status -> Nil)
   buffer = IO::Memory.new
   in = IO::Memory.new
   in << input if input
-  status = Process.run(name, args, error: buffer, output: buffer, input: in.rewind)
+  status = Process.run(name, args, output: buffer, input: in.rewind)
   yield buffer.to_s, status
 end

src/oq.cr

@@ -36,59 +36,53 @@ module Oq
     # :nodoc:
     property null_input : Bool = false
 
-    # Consume the input, convert the input to JSON if needed, pass the input/args to `jq`, then convert the output if needed.
-    def process
-      input = IO::Memory.new
-      output = IO::Memory.new
-      error = IO::Memory.new
+    @output : IO = IO::Memory.new
 
+    # Consume the input, convert the input to JSON if needed, pass the input/args to `jq`, then convert the output if needed.
+    def process : Nil
       ARGV.replace ARGV - @args
 
       # Shift off the filter from ARGV
       @args << ARGV.shift unless ARGV.empty?
 
-      if !@null_input
-        case @input_format
-        when .json? then IO.copy(ARGF, input)
-        when .yaml?
-          ARGV.empty? ? (input << YAML.parse(STDIN).to_json) : (ARGV.join('\n', input) { |f, io| io << YAML.parse(File.open(f)).to_json })
-        else
-          STDERR.puts "Not Implemented"
-          exit(1)
-        end
-
-        input.rewind
-      else
-        @args = @args | ARGV
-      end
-
-      run = parallel(
-        Process.run("jq", args, input: input, output: output, error: error)
-      )
+      run_jq input: get_input, output: get_output
 
-      unless run[0].success?
-        if output.empty? && @null_input
-          puts "null"
-          exit
-        end
-
-        STDERR.puts error.to_s
-        exit(1)
-      end
-
-      format_output output
+      format_output
     rescue ex
       puts "oq error: #{ex.message}"
       exit(1)
     end
 
-    private def format_output(io : IO)
-      io.rewind
+    private def format_output
+      @output.rewind
       case @output_format
-      when .json? then print io
-      when .yaml? then print JSON.parse(io).to_yaml
-      when .xml?  then print JSON.parse(io).to_xml root: @xml_root, indent: (@tab ? "\t" : " ")*@indent
+      when .yaml? then print JSON.parse(@output).to_yaml
+      when .xml?  then print JSON.parse(@output).to_xml root: @xml_root, indent: (@tab ? "\t" : " ")*@indent
+      end
+    end
+
+    private def run_jq(input : Process::Stdio, output : Process::Stdio, error = STDERR) : Nil
+      run = Process.run("jq", args, input: input, output: output, error: error)
+      exit(1) unless run.success?
+      exit if @input_format.json? && @output_format.json?
+    end
+
+    private def get_input : Process::Stdio
+      if @null_input
+        @args = @args + ARGV
+        return Process::Redirect::Close
       end
+      return ARGF if @input_format.json?
+      input = IO::Memory.new
+
+      ARGV.empty? ? YAML.parse(ARGF).to_json(input) : (ARGV.each { |f| YAML.parse(File.open(f)).to_json(input << '\n') })
+
+      input.rewind
+    end
+
+    private def get_output : Process::Stdio
+      return STDOUT if @output_format.json?
+      @output
     end
   end
 end

src/oq_cli.cr

@@ -8,7 +8,7 @@ module Oq
   OptionParser.parse! do |parser|
     parser.banner = "Usage: oq [--help] [oq-arguments] [jq-arguments] jq_filter [file [files...]]"
     parser.on("--help", "Show this help message.") { puts parser; exit }
-    parser.on("-i FORMAT", "--input FORMAT", "Format of the input data. Supported formats: json, yaml.") { |format| (f = Format.parse?(format)) ? processor.input_format = f : (puts "Invalid input format: '#{format}'"; exit(1)) }
+    parser.on("-i FORMAT", "--input FORMAT", "Format of the input data. Supported formats: json, yaml.") { |format| (f = Format.parse?(format)) && !f.xml? ? processor.input_format = f : (puts "Invalid input format: '#{format}'"; exit(1)) }
     parser.on("-o FORMAT", "--output FORMAT", "Format of the output data. Supported formats: json, yaml, xml.") { |format| (f = Format.parse?(format)) ? processor.output_format = f : (puts "Invalid output format: '#{format}'"; exit(1)) }
     parser.on("--xml-root ROOT", "Name of the root XML element if converting to XML.") { |r| processor.xml_root = r }
     parser.on("--indent NUMBER", "Use the given number of spaces for indentation (JSON/XML only).") { |n| processor.indent = n.to_i; processor.args << "--indent"; processor.args << n }

src/to_xml.cr

@@ -2,6 +2,7 @@ require "json"
 require "yaml"
 require "xml"
 
+# :nodoc:
 class Object
   def to_xml(*, root : String = "root", indent : String = "  ") : String
     String.build do |str|
@@ -22,6 +23,7 @@ class Object
   end
 end
 
+# :nodoc:
 class Array
   def to_xml(builder : XML::Builder, key : String? = nil) : Nil
     each do |v|
@@ -32,6 +34,7 @@ class Array
   end
 end
 
+# :nodoc:
 class Hash
   def to_xml(builder : XML::Builder) : Nil
     each do |key, value|
@@ -55,6 +58,7 @@ class Hash
   end
 end
 
+# :nodoc:
 struct Set
   def to_xml(builder : XML::Builder, key : String? = nil) : Nil
     each do |v|
@@ -65,6 +69,7 @@ struct Set
   end
 end
 
+# :nodoc:
 struct Tuple
   def to_xml(builder : XML::Builder, key : String? = nil) : Nil
     {% for i in 0...T.size %}
@@ -80,18 +85,21 @@ struct Tuple
   end
 end
 
+# :nodoc:
 struct NamedTuple
   def to_xml(builder : XML::Builder) : Nil
     to_h.transform_keys(&.to_s).to_xml builder
   end
 end
 
+# :nodoc:
 struct JSON::Any
   def to_xml(builder : XML::Builder) : Nil
     raw.to_xml(builder)
   end
 end
 
+# :nodoc:
 struct YAML::Any
   def to_xml(builder : XML::Builder) : Nil
     raw.to_xml(builder)