Can now take hex encoding

lang_ast.py

@@ -12,29 +12,32 @@
 
 
 class AST(object):
-    END_CHARS = ")("
+    END_CHARS = b")("
     run = True
+
+    def __repr__(self):
+        return " ".join("<{}>".format(i) for i in self.nodes) or "EmptyAST"
 
     def setup(self, code, first=False):
         self.first = first
         self.nodes = []
         self.uses_i = False
         self.uses_j = False
-        while code != "" and (self.first or code[0] not in AST.END_CHARS):
+        while code and (self.first or code[:1] not in AST.END_CHARS):
             code, node = self.add_node(code)
             if node.uses_i:
                 self.uses_i = True
-                if node.char == "i":
+                if node.char == b"i":
                     self.i_node = node.__class__
                 else:
                     self.i_node = node.ast.i_node
             if node.uses_j:
                 self.uses_j = True
-                if node.char == "j":
+                if node.char == b"j":
                     self.j_node = node.__class__
                 else:
                     self.j_node = node.ast.j_node
-        if code != "" and code[0] != "(":
+        if code and code[0] != b"(":
             code = code[1:]
         if len(self.nodes) > 1:
             if isinstance(self.nodes[-1], nodes.nodes[nodes.Node.StringLiteral]):
@@ -89,7 +92,7 @@ def _add_node(code):
         accepting = []
         for name in nodes.nodes:
             node = nodes.nodes[name]
-            new_code, new_node = node.accepts(code)
+            new_code, new_node = node.accepts(code[:])
             if new_code is not None:
                 assert(new_node is not None)
                 accepting.append((node.char, new_code, new_node))

literal_gen.py

@@ -1,7 +1,6 @@
 from node.create_list import List
 from node.numeric_literal import NumericLiteral
 from node.string_literal import StringLiteral
-from node.string_single import StringSingle
 from type.type_time import TypeTime
 
 nodes = {int: NumericLiteral,
@@ -10,13 +9,15 @@
          list: List,
          tuple: List}
 
+
 def int_literal(number):
     if number == 0:
         return "0"
     abs_num = abs(number)
-    is_neg = number<0
+    is_neg = number < 0
     return str(abs_num)+"_"*is_neg
 
+
 def float_literal(number):
     if number == 0:
         return ".0"
@@ -25,22 +26,25 @@ def float_literal(number):
     less_one = abs_num < 1
     return str(abs_num)[less_one:]+"_"*is_neg
 
+
 def string_literal(string):
     if len(string) == 1:
         return "\\"+string
     return '"'+string+'"'
 
+
 def list_literal(seq):
-    end = List.char
+    end = List.char[:]
     if len(seq) != List.default_arg:
-        end += int_literal(len(seq))
+        end += bytearray(int_literal(len(seq)).encode("ascii"))
+    #print("LIST", seq, end)
     return stack_literal(seq)+end
 
+
 def tuple_literal(seq):
     return list_literal(seq)
 
 def time_literal(time): raise NotImplementedError()
-
 def set_literal(seq): raise NotImplementedError()
 def dict_literal(dic): raise NotImplementedError()
 
@@ -51,37 +55,46 @@ def dict_literal(dic): raise NotImplementedError()
            tuple: tuple_literal,
            set: set_literal,
            dict: dict_literal,
-           TypeTime:time_literal}
+           TypeTime: time_literal}
+
 
 def parse_item(item):
-    return parsers[type(item)](item)
+    rtn = parsers[type(item)](item)
+    if isinstance(rtn, str):
+        return bytearray(rtn.encode("ascii"))
+    return rtn
+
 
 def stack_literal(stack):
+    #print("STACK LITERAL", stack)
     node_list = list(map(parse_item, stack))
-    rtn = ""
+    #print("NODE LIST", node_list)
+    rtn = bytearray()
     for i in range(len(node_list)-1):
         rtn += node_list[i]
         node_type = nodes[type(stack[i])]
         node_type_next = nodes[type(stack[i+1])]
         is_list = node_type_next is List
         assert(node_type.accepts(node_list[i]))
         code, node = node_type.accepts(node_list[i]+node_list[i+1])
-        if code == "" or code is None or code == "_":
-            rtn += " "
+        #print("underscore", code)
+        if not code or code == b"_":
+            rtn += b" "
         elif node_type is List and node_type_next is NumericLiteral:
-            rtn += " "
+            rtn += b" "
         elif is_list:
             next_test_parsed = node_list[i+1]
             next_test_str = stack[i+1]
             while node_type_next is List:
-                next_test_parsed = next_test_parsed[0]
+                next_test_parsed = next_test_parsed[:1]
                 next_test_str = next_test_str[0]
                 node_type_next = nodes[type(next_test_str)]
             code, node = node_type.accepts(node_list[i]+next_test_parsed)
-            if code == "":
-                rtn += " "
+            #print("empty", code)
+            if code == b"":
+                rtn += b" "
             elif node_type is List and node_type_next is NumericLiteral:
-                rtn += " "
+                rtn += b" "
     if node_list:
-        rtn += node_list[-1]
-    return rtn
+        rtn += bytearray(node_list[-1])
+    return rtn

main.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
+import codecs
 import sys
 from io import StringIO
 
@@ -27,24 +28,20 @@ def print_nodes():
     sys.exit()
 
 
-def run(code):  
-    try:
-        sys.stderr.write("RUNNING: {} ({} bytes)\n".format(repr(code), len(code.encode("utf-8"))))
-    except UnicodeEncodeError:
-        sys.stderr.write("RUNNING BAD UNICODE\n")
+def run(code):
+    if args.hex:
+        code = codecs.decode(code.replace(b" ", b""), 'hex_codec')
+    sys.stderr.write("RUNNING: {} ({} bytes)\n".format(repr(code), len(code)))
 
     ast = lang_ast.AST()
-    ast.setup(code, first=True)
+    ast.setup(bytearray(code), first=True)
     stack = ast.run()
     return stack
 
 
 def run_file(filename):
     with open(filename, "rb") as f_obj:
-        string = ""
-        for char in f_obj.read():
-            string += chr(char)
-        return run(string)
+        return run(f_obj.read())
 
 
 class Writer(type(sys.stdout)):
@@ -113,6 +110,12 @@ def append(self, data):
                     help='Print out all nodes and debug conflicts')
 parser.add_argument('-f', '--file', dest="file", nargs=1,
                     help='The file to look in')
+parser.add_argument('-n', '--no-high', dest="high", action='store_const',
+                    const=True, default=False,
+                    help='Disable reading the high bit')
+parser.add_argument('-x', '--hex', dest="hex", action='store_const',
+                    const=True, default=False,
+                    help='First decode as hexadecimal codepoints')
 parser.add_argument('code', nargs="*",
                     help='The code to run')
 args = parser.parse_args()
@@ -127,7 +130,7 @@ def append(self, data):
         f_name = args.file[0]
         run_string = 'stack = run_file(f_name)'
     else:
-        run_string = 'stack = run(args.code[0])'
+        run_string = 'stack = run(args.code[0].encode("utf-8"))'
     run_func = exec
     if args.profile:
         import cProfile

node/base10_single.py

@@ -2,6 +2,7 @@
 
 from nodes import Node
 
+
 class Base10Single(Node):
     char = ""
     args = 0
@@ -15,14 +16,16 @@ def func(self):
         return self.value
 
     def __repr__(self):
-        return "%s: %d"%(self.__class__.__name__, self.func())
+        return "{}: {}".format(self.__class__.__name__, self.func())
 
     @classmethod
-    def accepts(cls, code, accept = False):
-        if not accept: return None, None
-        if code == "": return None, None
+    def accepts(cls, code, accept=False):
+        if not accept:
+            return None, None
+        if not code:
+            return None, None
         try:
-            value = int(code[0])
+            value = int(chr(code[0]))
         except ValueError:
             return None, None
-        return code[1:], cls(value)
+        return code[1:], cls(value)

node/base96_single.py

@@ -24,8 +24,18 @@ def __repr__(self):
     @classmethod
     def accepts(cls, code, accept=False):
         if accept:
-            code = "w"+code
-        if code == "" or (code[0] != cls.char):
+            code = b"w"+code
+        if not code:
             return None, None
-        value = ord(code[1])-32
-        return code[2:], cls(value)
+        if code[0] != cls.char[0]:
+            return None, None
+        value = 0
+        new = code[1]
+        code = code[2:]
+        while new & 0x80:
+            value |= (new & 0x7F)
+            value <<= 7
+            new = code[0]
+        value |= new
+        value -= 32
+        return code, cls(value)

node/create_list.py

@@ -2,6 +2,7 @@
 
 from nodes import Node
 
+
 class List(Node):
     char = "]"
     args = 0
@@ -16,10 +17,9 @@ def prepare(self, stack):
         if self.args == 0:
             self.args = len(stack)
 
-    @Node.test_func([5,6,1,2], [[5,6,1,2]], "4")
+    @Node.test_func([5, 6, 1, 2], [[5, 6, 1, 2]], "4")
     @Node.test_func(["Hello", "World"], [["Hello", "World"]])
     def func(self, *inp):
         """Take the top `amount` items from the stack and turn them into a list.
 Defaults to the whole stack"""
         return [list(inp)]
-

node/deep_apply.py

@@ -1,7 +1,8 @@
-import lang_ast
-from nodes import Node
 import copy
 
+from nodes import Node
+
+
 class DeepApply(Node):
     char = "A"
     args = 1
@@ -45,16 +46,16 @@ def recurse(self, seq):
             return rtn
 
     @Node.test_func([30], ["30"], '"')
-    @Node.test_func([3,4], [3,3,3,3], 'D')
+    @Node.test_func([3, 4], [3, 3, 3, 3], 'D')
     @Node.is_func
     def splat_args(self, *args):
         """Splat a node with a static suffix and run it"""
-        arg = str(args[-1])
+        arg = bytearray(str(args[-1]).encode("ascii"))
         code, node = self.node.accepts(self.node.char+arg)
         #Warn if code empty?
         stack = list(args[:-1])
         node.prepare(stack[::-1])
         no_args = node.args
         stack, args = stack[no_args:], stack[:no_args]
         stack = node(args[::-1]) + stack
-        return stack
+        return stack

node/deep_for.py

@@ -1,8 +1,8 @@
-import lang_ast
-from nodes import Node
-from node.numeric_literal import NumericLiteral
 import copy
 
+from nodes import Node
+
+
 class DeepFor(Node):
     char = ".F"
     args = None
@@ -13,16 +13,16 @@ def __init__(self, args: Node.NumericLiteral, ast:Node.EvalLiteral):
         self.args = args
         self.ast = ast
         if self.ast.nodes == []:
-            self.ast.add_node("\n")
+            self.ast.add_node(b"\n")
 
-    @Node.test_func([[[[0],1,2,3],[4,5,6,7]]], [[[[2], 4, 6, 8], [10, 12, 14, 16]]], "h}")
-    @Node.test_func([[1,[[2,3,[4],5],6],7]], [[2, [[2, 4, [4], 6], 6], 8]], "D 2%+")
+    @Node.test_func([[[[0], 1, 2, 3], [4, 5, 6, 7]]], [[[[2], 4, 6, 8], [10, 12, 14, 16]]], "h}")
+    @Node.test_func([[1, [[2, 3, [4], 5], 6], 7]], [[2, [[2, 4, [4], 6], 6], 8]], "D 2%+")
     def func(self, *args):
         """Deeply run a for loop across a nD tree.
 Takes a list or tuple with a varying depth.
 Returns a list with the same depth all round with the function applied."""
         seq, *args = copy.deepcopy(args)
-        assert(isinstance(seq,Node.sequence))
+        assert(isinstance(seq, Node.sequence))
         self.type = None
         self.shared_type = False
         rtn = self.recurse(seq, args)
@@ -62,15 +62,16 @@ def cleanup(self, obj, args):
     def get_type(self, obj):
         if obj:
             rtn_type = {str: "",
-			int: 0,
-			list: [],
-			dict: {},
-			tuple: (),
-			set: set(),
-			bool: False}.get(type(obj), None)
+                        int: 0,
+                        list: [],
+                        dict: {},
+                        tuple: (),
+                        set: set(),
+                        bool: False}.get(type(obj), None)
             if self.type is None:
                 self.type = rtn_type
-            elif self.type == rtn_type: pass
+            elif self.type == rtn_type:
+                pass
             else:
                 self.shared_type = True
         return obj

node/deep_for_apply.py

@@ -13,7 +13,7 @@ class DeepForApply(Node):
     def __init__(self, ast: Node.EvalLiteral):
         self.ast = ast
         if self.ast.nodes == []:
-            self.ast.add_node("\n")
+            self.ast.add_node(b"\n")
 
     @Node.test_func([[[(0, 0), (0, 1)], [(1, 0), (1, 1)]]], [[[1, 2], [2, 3]]], "+h")
     def func(self, seq: Node.sequence):