Merge branch 'develop' for v0.9.2

Author: goodmami

.gitignore

@@ -7,6 +7,7 @@ dist/
 PyDelphin.egg-info/
 docs/_build/
 docs/_templates/
+docs/env/
 tmp/
 # Ignore the compiled python files
 *.pyc

CHANGELOG.md

@@ -5,6 +5,58 @@ these changes are prefixed with "**BREAKING**"
 
 ## [Unreleased][unreleased]
 
+## [v0.9.2][]
+
+### Python Versions
+
+* Added Python 3.7 support
+
+### Added
+
+* `delphin.interfaces.ace.AceProcessError` for unrecoverable ACE crashes (#181)
+* ACE command-line options are allowed with `delphin process --options=...`
+  and in `delphin.commands.process(..., options=...)` (#180)
+* `delphin.itsdb.Record.from_dict()`
+* `delphin.itsdb.Table` (#186)
+  - `write()`
+  - `attach()`
+  - `detach()`
+  - `commit()`
+  - `is_attached()`
+  - `list_changes()`
+  - Various methods to replace `list` functionality: `append()`, `extend()`,
+	`__len__()`, `__getitem__()`, `__setitem__()`, and `__iter__()`
+* `delphin.itsdb.TestSuite.process()` the `gzip` and `buffer_size` parameters
+* `-z` / `--gzip` option to the `delphin process` command
+
+### Fixed
+
+* `delphin.mrs.eds` parsing of predicates and empty property lists (#203)
+* `delphin.commands.convert` wrap the inner step of conversion in a try-except
+  block to more gracefully handle crashes on a single input. (#200)
+* `delphin.itsdb.TestSuite.write()` re-enable the `append` parameter
+
+### Removed
+
+* **BREAKING** `name` parameter on `delphin.itsdb.Table`. The `name` attribute
+  is still available as it is taken from the table's schema.
+
+### Changed
+
+* **BREAKING** `delphin.itsdb.Table` is no longer a subtype of `list`, meaning
+  that some list-like behavior is gone. Most relevant functionality is
+  recreated (e.g., `append()`, `extend()`, `__getitem__()`, `__len__()`, etc.)
+* **BREAKING** `delphin.itsdb.Record` can no longer be instantiated with a dict
+  with column data; use `Record.from_dict()`
+* **BREAKING** `delphin.itsdb.Record` stores data as strings, always retrieves
+  by default as cast types. Use `Record.get(..., cast=False)` to get the raw
+  string value.
+
+### Deprecated
+
+* `delphin.mrs.convert()` use `delphin.commands.convert` instead
+
+
 ## [v0.9.1][]
 
 ### Fixed

CONTRIBUTING.md

@@ -23,7 +23,7 @@ For bug requests, please provide the following, if possible:
   ```python
   >>> from delphin.__about__ import __version__
   >>> __version__
-  '0.9.0'
+  '0.9.2'
   ```
 * Python version (e.g. 2.7, 3.4, etc.)
 

delphin/__about__.py

@@ -3,7 +3,7 @@
 # the warehouse project:
 #   https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py
 
-__version__ = '0.9.1'
+__version__ = '0.9.2'
 __version_info__ = __version__.replace('.', ' ').replace('-', ' ').split()
 
 __title__ = 'PyDelphin'
@@ -16,4 +16,4 @@
 __maintainer__ = 'Michael Wayne Goodman and Angie McMillan-Major'
 
 __license__ = 'MIT'
-__copyright__ = '2013--2018 %s <%s> and contributors' % (__author__, __email__)
+__copyright__ = '2013--2019 %s <%s> and contributors' % (__author__, __email__)

delphin/commands.py

@@ -13,12 +13,16 @@
 import io
 import json
 from functools import partial
+import logging
 
 from delphin import itsdb, tsql
 from delphin.mrs import xmrs
 from delphin.util import safe_int, SExpr
+from delphin.exceptions import PyDelphinException
 
 
+logging.basicConfig()
+
 ###############################################################################
 ### CONVERT ###################################################################
 
@@ -91,7 +95,36 @@ def convert(path, source_fmt, target_fmt, select='result:mrs',
         kwargs['predicate_modifiers'] = predicate_modifiers
     kwargs['properties'] = properties
 
-    return dumps(xs, **kwargs)
+    # this is not a great way to improve robustness when converting
+    # many representations, but it'll do until v1.0.0. Also, it only
+    # improves robustness on the output, not the input.
+    # Note that all the code below is to replace the following:
+    #     return dumps(xs, **kwargs)
+    head, joiner, tail = _get_output_details(target_fmt)
+    parts = []
+    if pretty_print:
+        joiner = joiner.strip() + '\n'
+    def _trim(s):
+        if head and s.startswith(head):
+            s = s[len(head):].lstrip('\n')
+        if tail and s.endswith(tail):
+            s = s[:-len(tail)].rstrip('\n')
+        return s
+    for x in xs:
+        try:
+            s = dumps([x], **kwargs)
+        except (PyDelphinException, KeyError, IndexError):
+            logging.exception('could not convert representation')
+        else:
+            s = _trim(s)
+            parts.append(s)
+    # set these after so head and tail are used correctly in _trim
+    if pretty_print:
+        if head:
+            head += '\n'
+        if tail:
+            tail = '\n' + tail
+    return head + joiner.join(parts) + tail
 
 
 def _get_codec(codec, load=True):
@@ -146,6 +179,21 @@ def _get_codec(codec, load=True):
         raise ValueError('invalid target format: ' + codec)
 
 
+def _get_output_details(codec):
+    if codec == 'mrx':
+        return ('<mrs-list', '', '</mrs-list>')
+
+    elif codec == 'dmrx':
+        from delphin.mrs import dmrx
+        return ('<dmrs-list>', '', '</dmrs-list>')
+
+    elif codec in ('mrs-json', 'dmrs-json', 'eds-json'):
+        return ('[', ',', ']')
+
+    else:
+        return ('', ' ', '')
+
+
 # simulate json codecs for MRS and DMRS
 
 class _MRS_JSON(object):
@@ -329,11 +377,13 @@ def mkprof(destination, source=None, relations=None, where=None,
     dts = itsdb.TestSuite(path=destination, relations=relations)
     # input is sentences on stdin
     if source is None:
-        dts.write({'item': _lines_to_rows(sys.stdin)}, gzip=gzip)
+        dts.write({'item': _lines_to_rows(sys.stdin, dts.relations)},
+                  gzip=gzip)
     # input is sentence file
     elif os.path.isfile(source):
         with open(source) as fh:
-            dts.write({'item': _lines_to_rows(fh)}, gzip=gzip)
+            dts.write({'item': _lines_to_rows(fh, dts.relations)},
+                      gzip=gzip)
     # input is source testsuite
     elif os.path.isdir(source):
         sts = itsdb.TestSuite(source)
@@ -372,20 +422,32 @@ def mkprof(destination, source=None, relations=None, where=None,
             print(fmt.format(stat.st_size, _red(filename + '.gz')))
 
 
-def _lines_to_rows(lines):
+def _lines_to_rows(lines, relations):
+    # field indices only need to be computed once, so don't use
+    # itsdb.Record.from_dict()
+    i_id_idx = relations['item'].index('i-id')
+    i_wf_idx = relations['item'].index('i-wf')
+    i_input_idx = relations['item'].index('i-input')
+    num_fields = len(relations['item'])
+
+    def make_row(i_id, i_wf, i_input):
+        row = [None] * num_fields
+        row[i_id_idx] = i_id
+        row[i_wf_idx] = i_wf
+        row[i_input_idx] = i_input
+        return itsdb.Record(relations['item'], row)
+
     for i, line in enumerate(lines):
-        i_id = i * 10
-        i_wf = 0 if line.startswith('*') else 1
-        i_input = line[1:].strip() if line.startswith('*') else line.strip()
-        yield {'i-id': i_id, 'i-wf': i_wf, 'i-input': i_input}
+        i_wf, i_input = (0, line[1:]) if line.startswith('*') else (1, line)
+        yield make_row(i * 10, i_wf, i_input.strip())
 
 
 ###############################################################################
 ### PROCESS ###################################################################
 
 def process(grammar, testsuite, source=None, select=None,
-            generate=False, transfer=False,
-            all_items=False, result_id=None):
+            generate=False, transfer=False, options=None,
+            all_items=False, result_id=None, gzip=False):
     """
     Process (e.g., parse) a [incr tsdb()] profile.
 
@@ -413,10 +475,15 @@ def process(grammar, testsuite, source=None, select=None,
             (default: `False`)
         transfer (bool): if `True`, transfer instead of parse
             (default: `False`)
+        options (list): list of ACE command-line options to use when
+            invoking the ACE subprocess; unsupported options will
+            give an error message
         all_items (bool): if `True`, don't exclude ignored items
             (those with `i-wf==2`) when parsing
         result_id (int): if given, only keep items with the specified
             `result-id`
+        gzip (bool): if `True`, non-empty tables will be compressed
+            with gzip
     """
     from delphin.interfaces import ace
 
@@ -441,18 +508,14 @@ def process(grammar, testsuite, source=None, select=None,
     target = itsdb.TestSuite(testsuite)
     column, tablename, condition = _interpret_selection(select, source)
     table = itsdb.Table(
-        tablename,
         source[tablename].fields,
         tsql.select(
             '* from {} {}'.format(tablename, condition),
             source,
             cast=False))
 
-    with processor(grammar) as cpu:
-        target.process(cpu, tablename + ':' + column, source=table)
-
-    target.write()
-
+    with processor(grammar, cmdargs=options) as cpu:
+        target.process(cpu, ':' + column, source=table, gzip=gzip)
 
 def _interpret_selection(select, source):
     queryobj = tsql.inspect_query('select ' + select)

delphin/exceptions.py

@@ -19,27 +19,27 @@ class ItsdbError(PyDelphinException):
 
 
 class XmrsError(PyDelphinException):
-    """Raised when there is an error processing *MRS objects."""
+    """Raised when there is an error processing \*MRS objects."""
     pass
 
 
 class XmrsSerializationError(XmrsError):
-    """Raised when serializing *MRS objects fails."""
+    """Raised when serializing \*MRS objects fails."""
     pass
 
 
 class XmrsDeserializationError(XmrsError):
-    """Raised when deserializing *MRS objects fails."""
+    """Raised when deserializing \*MRS objects fails."""
     pass
 
 
 class XmrsStructureError(XmrsError):
-    """Raised when a *MRS object is structurally ill-formed."""
+    """Raised when a \*MRS object is structurally ill-formed."""
     pass
 
 
 class XmrsWarning(PyDelphinWarning):
-    """Warning class for *MRS processing."""
+    """Warning class for \*MRS processing."""
     pass
 
 

delphin/interfaces/ace.py

@@ -80,6 +80,11 @@
 from delphin.interfaces.base import ParseResponse, Processor
 from delphin.util import SExpr, stringtypes
 from delphin.__about__ import __version__ as pydelphin_version
+from delphin.exceptions import PyDelphinException
+
+
+class AceProcessError(PyDelphinException):
+    """Raised when the ACE process has crashed and cannot be recovered."""
 
 
 class AceProcess(Processor):
@@ -166,6 +171,8 @@ def _open(self):
             'os': platform(),
             'start': datetime.now()
         })
+        if self._p.poll() is not None and self._p.returncode != 0:
+            raise AceProcessError('Process closed on startup; see <stderr>.')
 
     def __enter__(self):
         return self
@@ -587,7 +594,7 @@ def error(self, message):
 _ace_argparser.add_argument('--ubertagging', nargs='?', type=float)
 _ace_argparser.add_argument('--pcfg', type=argparse.FileType())
 _ace_argparser.add_argument('--rooted-derivations', action='store_true')
-_ace_argparser.add_argument('--udx', nargs='?', choices=('all'))
+_ace_argparser.add_argument('--udx', nargs='?', choices=('all',))
 _ace_argparser.add_argument('--yy-rules', action='store_true')
 _ace_argparser.add_argument('--max-words', type=int)
 

delphin/interfaces/base.py

@@ -176,7 +176,7 @@ def tokens(self, tokenset='internal'):
 
 class FieldMapper(object):
     """
-    A class for mapping respsonses to [incr tsdb()] fields.
+    A class for mapping responses to [incr tsdb()] fields.
 
     This class provides two methods for mapping responses to fields:
 
@@ -186,8 +186,19 @@ class FieldMapper(object):
     * cleanup() - returns any (table, data) tuples resulting from
         aggregated data over all runs, then clears this data
 
+    In addition, the :attr:`affected_tables` attribute should list
+    the names of tables that become invalidated by using this
+    FieldMapper to process a profile. Generally this is the list of
+    tables that :meth:`map` and :meth:`cleanup` create records for,
+    but it may also include those that rely on the previous set
+    (e.g., treebanking preferences, etc.).
+
     Alternative [incr tsdb()] schema can be handled by overriding
     these two methods and the __init__() method.
+
+    Attributes:
+        affected_tables: list of tables that are affected by the
+            processing
     """
     def __init__(self):
         # the parse keys exclude some that are handled specially
@@ -211,6 +222,11 @@ def __init__(self):
         self._runs = {}
         self._last_run_id = -1
 
+        self.affected_tables = '''
+            run parse result rule output edge tree decision preference
+            update fold score
+        '''.split()
+
     def map(self, response):
         """
         Process *response* and return a list of (table, rowdata) tuples.

delphin/interfaces/rest.py

@@ -122,8 +122,7 @@ def parse(self, sentence, params=None, headers=None):
             A ParseResponse containing the results, if the request was
             successful.
         Raises:
-            A requests.HTTPError is raised if the status code was not
-            200.
+            requests.HTTPError: if the status code was not 200
         """
         if params is None:
             params = {}
@@ -163,7 +162,7 @@ def parse(input, server=default_erg_server, params=None, headers=None):
         A ParseResponse containing the results, if the request was
         successful.
     Raises:
-        A requests.HTTPError is raised if the status code was not 200.
+        requests.HTTPError: if the status code was not 200
     """
     return next(parse_from_iterable([input], server, params, headers), None)
 
@@ -184,8 +183,8 @@ def parse_from_iterable(
     Yields:
         ParseResponse objects for each successful response.
     Raises:
-        A requests.HTTPError is raised for the first response with
-        a status code that is not 200.
+        requests.HTTPError: for the first response with a status code
+            that is not 200
     """
     client = DelphinRestClient(server)
     for input in inputs: