Merge pull request #286 from delph-in/develop

v1.2.4

CHANGELOG.md

@@ -1,11 +1,17 @@
 # Change Log
 
-**Note**: some releases may have changes that may break backward compatibility;
-these changes are prefixed with "**BREAKING**"
-
 ## [Unreleased][unreleased]
 
-(no unreleased changes)
+(no unreleased changes yet)
+
+
+## [v1.2.4]
+
+**Release date: 2020-05-12**
+
+### Fixed
+
+* `delphin.tsdb.open()` does not use universal newlines ([#285])
 
 
 ## [v1.2.3]
@@ -1211,6 +1217,7 @@ information about changes, except for
 [commit messages](../../commits/v0.2).
 
 [unreleased]: ../../tree/develop
+[v1.2.4]: ../../releases/tag/v1.2.4
 [v1.2.3]: ../../releases/tag/v1.2.3
 [v1.2.2]: ../../releases/tag/v1.2.2
 [v1.2.1]: ../../releases/tag/v1.2.1
@@ -1358,3 +1365,4 @@ information about changes, except for
 [#281]: https://github.com/delph-in/pydelphin/issues/281
 [#282]: https://github.com/delph-in/pydelphin/issues/282
 [#283]: https://github.com/delph-in/pydelphin/issues/283
+[#285]: https://github.com/delph-in/pydelphin/issues/285

delphin/__about__.py

@@ -3,7 +3,7 @@
 # the warehouse project:
 #   https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py
 
-__version__ = '1.2.3'
+__version__ = '1.2.4'
 __version_info__ = __version__.replace('.', ' ').replace('-', ' ').split()
 
 __title__ = 'PyDelphin'

delphin/tdl.py

@@ -915,60 +915,6 @@ def _shift(tokens):
     return tok[0], tok[1], tok[2], after[0]
 
 
-def _accumulate(lexitems):
-    """
-    Yield lists of tokens based on very simple parsing that checks the
-    level of nesting within a structure. This is probably much faster
-    than the LookaheadIterator method, but it is less safe; an unclosed
-    list or AVM may cause it to build a list including the rest of the
-    file, or it may return a list that doesn't span a full definition.
-
-    As PyDelphin's goals for TDL parsing do not include speed, this
-    method is not currently used, although it is retained in the source
-    code as an example if future priorities change.
-    """
-    data = []
-    stack = []
-    break_on = 10
-    in_def = False
-    for item in lexitems:
-        gid = item[0]
-        # only yield comments outside of definitions
-        if gid in (2, 3):
-            if len(data) == 0:
-                yield [item]
-            else:
-                continue
-        elif gid == 20:
-            assert len(data) == 0
-            yield [item]
-        # the following just checks if the previous definition was not
-        # terminated when the next one is read in
-        elif gid in (7, 8):
-            if in_def:
-                yield data[:-1]
-                data = data[-1:] + [item]
-                stack = []
-                break_on = 10
-            else:
-                data.append(item)
-                in_def = True
-        else:
-            data.append(item)
-            if gid == break_on:
-                if len(stack) == 0:
-                    yield data
-                    data = []
-                    in_def = False
-                else:
-                    break_on = stack.pop()
-            elif gid in (13, 14, 15):
-                stack.append(break_on)
-                break_on = gid + 3
-    if data:
-        yield data
-
-
 def _lex(stream):
     """
     Lex the input stream according to _tdl_lex_re.

delphin/tsdb.py

@@ -766,9 +766,9 @@ def open(dir: util.PathLike,
     """
     path = get_path(dir, name)
     if path.suffix.lower() == '.gz':
-        return gzopen(path, mode='rt', encoding=encoding)
+        return gzopen(path, mode='rt', encoding=encoding, newline='\n')
     else:
-        return path.open(encoding=encoding)
+        return path.open(encoding=encoding, newline='\n')
 
 
 def write(dir: util.PathLike,

tests/tdl_test.py

@@ -462,6 +462,42 @@ def test_parse_type_features():
     assert t['ATTR'].terms == ['val1', 'val2']
 
 
+def test_parse_cons_list():
+    t = tdlparse('a := b & [ ATTR < > ].')
+    assert isinstance(t['ATTR'], ConsList)
+    assert t['ATTR'].terminated
+
+    t = tdlparse('a := b & [ ATTR < ... > ].')
+    assert isinstance(t['ATTR'], ConsList)
+    assert not t['ATTR'].terminated
+
+    t = tdlparse('a := b & [ ATTR < [] . #x > ].')
+    assert isinstance(t['ATTR'], ConsList)
+    assert t['ATTR'].terminated
+
+    t = tdlparse('a := b & [ ATTR < [] , ... > ].')
+    assert isinstance(t['ATTR'], ConsList)
+    assert not t['ATTR'].terminated
+
+    with pytest.raises(TDLSyntaxError):
+        tdlparse('a := b & [ ATTR < [] , > ].')
+    with pytest.raises(TDLSyntaxError):
+        tdlparse('a := b & [ ATTR < , [] > ].')
+    with pytest.raises(TDLSyntaxError):
+        tdlparse('a := b & [ ATTR < [] [] > ].')
+
+
+def test_parse_diff_list():
+    t = tdlparse('a := b & [ ATTR <! !> ].')
+    assert isinstance(t['ATTR'], DiffList)
+
+    t = tdlparse('a := b & [ ATTR <! [] !> ].')
+    assert isinstance(t['ATTR'], DiffList)
+
+    t = tdlparse('a := b & [ ATTR <! [], [] !> ].')
+    assert isinstance(t['ATTR'], DiffList)
+
+
 def test_parse_multiple_features():
     t = tdlparse('a := b & [ ATTR1 1, ATTR2 2].')
     assert len(t.features()) == 2

tests/tsdb_test.py

@@ -265,6 +265,16 @@ def test_write(single_item_skeleton):
     assert path.with_suffix('').exists()
 
 
+def test_issue_285(empty_testsuite):
+    fields = tsdb.read_schema(empty_testsuite)['item']
+    tsdb.write(empty_testsuite, 'item', [(0, 'The cat meows.\r')], fields)
+    fh = tsdb.open(empty_testsuite, 'item')
+    assert not fh.closed
+    with fh:
+        assert list(fh) == ['0@The cat meows.\r\n']
+    assert fh.closed
+
+
 def test_write_database(tmp_path, mini_testsuite, empty_alt_testsuite):
     tmp_ts = tmp_path.joinpath('test_write_database')
     db = tsdb.Database(mini_testsuite)

tests/vpm_test.py

@@ -6,7 +6,20 @@
 from delphin import vpm
 
 
+def test_load(tmp_path):
+    vpm.load(S('; test vpm\n'
+               'a : b\n'
+               ' 1 >> 2'))
+    vpmfile = tmp_path / 'test.vpm'
+    vpmfile.write_text('; test vpm\n'
+                       'a : b\n'
+                       ' 1 >> 2')
+    vpm.load(vpmfile)
+
+
 def test_invalid():
+    with pytest.raises(vpm.VPMSyntaxError):
+        vpm.load(S('~~'))
     with pytest.raises(vpm.VPMSyntaxError):
         vpm.load(S('<>'))
     with pytest.raises(vpm.VPMSyntaxError):