Merge pull request #31 from mluis7/feature_20_21

feature_20_21: Allow reading XML from stdin, provide raw mode ouput as json

Author: mluis7

README.md

@@ -49,21 +49,21 @@ It can be used as a [command line utility](#command-line-usage) or as a [module]
 ## Installation
 Installing from PyPi
 
-`pip3.9 install pyxml2xpath`
+`pip3.12 install pyxml2xpath`
 
 Or building from source repo
 
 ```bash
 git clone https://github.com/mluis7/pyxml2xpath.git
 cd pyxml2xpath
-python3.9 -m build
-python3.9 -m pip install dist/pyxml2xpath-0.2.0-py3-none-any.whl --upgrade
+python3.12 -m build
+python3.12 -m pip install dist/pyxml2xpath-0.2.0-py3-none-any.whl --upgrade
 ```
 
 Alternative without cloning the repo yourself
 
 ```
-pip3.9 install git+https://github.com/mluis7/pyxml2xpath.git
+pip3.12 install git+https://github.com/mluis7/pyxml2xpath.git
 ```
 
 ## Command line usage
@@ -84,7 +84,13 @@ pyxml2xpath tests/resources/HL7.xml 'values' '//*[local-name()= "act"]'
 
 pyxml2xpath ~/tmp/test.html all none none 11 true
 ```
+Reading from stdin pass `-` as file
 
+```bash
+cat tests/resources/simple-ns.xml | pyxml2xpath - xpath none false 100 false
+
+pyxml2xpath - xpath none false 100 false < tests/resources/simple-ns.xml
+```
 
 ## Module usage
 
@@ -175,12 +181,16 @@ If there are more than 1 default namespace, prefix will be incremental:
 ## Print result modes
 Print xpath expressions and validate by count of elements found with it.  
 
-`mode` argument values (optional):
+**mode** argument values (optional):
 
-- `path`  : print elements xpath expressions (default)  
-- `all`   : also print attribute xpath expressions  
-- `raw`   : print unqualified xpath and found values (tuple)  
-- `values`: print tuple of found values only  
+| Mode | Description |
+| ---- | ----------- |
+| `xpath` | print elements xpath expressions (default) |
+| `all` | also print attribute xpath expressions |
+| `raw` | print unqualified xpath and found values (tuple) |
+| `values` | print tuple of found values only |
+| `raw_json` | json string with internal xpath and namespace maps. |
+| `raw_json_pretty` | same as raw_json with pretty print |
 
 `pyxml2xpath ~/tmp/soap-ws-oasis.xml 'all'`
 

VERSION

@@ -1 +1 @@
-0.3.4
+0.3.5

pyproject.toml

@@ -16,7 +16,7 @@ authors = [
 
 description = "Generate xpath expressions from XML document."
 readme = "README.md"
-license = {text = "GPL-3.0"}
+license = "GPL-3.0"
 keywords = ["xpath", "xml"]
 classifiers = [
   "Development Status :: 5 - Production/Stable",

src/xml2xpath/xml2xpath.py

@@ -1,12 +1,13 @@
-'''Find all xpath expressions on XML document'''
+'''Generate xpath expressions from an XML document'''
 
 
 from collections import OrderedDict
-from io import StringIO
+from io import StringIO, BytesIO
 from os import path, devnull, strerror
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 import errno
 import sys
+import json
 
 from lxml import etree
 
@@ -15,17 +16,22 @@
 WITH_COUNT = False
 MAX_ITEMS = 100000
 OUT_FD = sys.stdout
-modes = ['xpath', 'all', 'raw', 'values']
+modes = ['xpath', 'all', 'raw', 'values', 'raw_json', 'raw_json_pretty']
 
 def usage():
     helpstr='''
     pyxml2xpath <file path> [mode] [initial xpath expression] [with element count: yes|true] [max elements: int] [no banner: yes|true]
     
+    File path: pass '-' to read from stdin
+    
     mode: str
-        path  : print elements xpath expressions (default)
+        xpath  : print elements xpath expressions (default)
         all   : also print attribute xpath expressions
         raw   : print unqualified xpath and found values (list)
         values: print list of found values only
+        raw_json: json string with internal xpath and namespace maps.
+                {"xpath": { ... }, "namespaces": { ... }}
+        raw_json_pretty: same as raw_json with pretty print
     
     Initial xpath expression: str
         Start at some element defined by an xpath expression.
@@ -54,6 +60,16 @@ def _get_qualified_name(qname, revns):
         lname = f"{revns.get(qname.namespace)}:{qname.localname}"
     return lname
 
+def get_encoding_from_fragment(xml_frag):
+    '''
+    Get encoding attribute value from xml declaration if present.
+    :param xml_frag: first 100 characters from XML string.
+    '''
+    declstr = xml_frag.split(b'?>')[0]
+    # Fake a minimal doc to parse declaration ensuring encoding is valid for lxml
+    frdoc= etree.parse(BytesIO(declstr + b'?><root/>'))
+    return frdoc.docinfo.encoding
+    
 def _get_dict_list_value(value, element):
     '''Initialize tuple for xpath dictionary values.
     Items:
@@ -110,24 +126,23 @@ def parse_mixed_ns(tree: etree._ElementTree,
     '''Parse XML document that may contain anonymous namespace.
     Returns a dict with original xpath as keys, xpath with qualified names and
     count of elements found with the latter or None if an error occurred.
-        xmap = {
+        xmap = ``{
             "/some/xpath/*[1]": ("/some/xpath/ns:ele1", 1, ["id", "class"])
-        }
+        }``
     To get the qualified xpath:
-        xmap['/some/xpath/*[1]'][0]
-        
-    Parameters
-    ----------
-    tree: lxml.etree._ElementTree
+        ``xmap['/some/xpath/*[1]'][0]``
+    
+    :param tree: lxml.etree._ElementTree
         ElementTree from current document
-    nsmap: dict
+    :param nsmap: dict
         namespaces dictionary from current document
-    xpath_base: str
+    :param xpath_base: str
         Xpath expression to start from
-    with_count: bool
+    :param with_count: bool
         add count of found elements (performance cost on large documents).
-    max_items: int
-        max number of elements to parse. Default: 100000'''
+    :param max_items: int
+        max number of elements to parse. Default: 100000
+    '''
 
     revns = {v:k or 'ns' for k,v in nsmap.items()}
     elements = tree.xpath(xpath_base, namespaces=nsmap)
@@ -203,36 +218,68 @@ def parse_mixed_ns(tree: etree._ElementTree,
             xmap[xp] = xmap[xp][0], xcount, xmap[xp][2]
     return xmap
 
+
+def get_json_response(xmap, nsmap, mode):
+    '''
+    Return a json string as
+        {"xpath": { ... }.
+        "namespaces": { ... }}
+    :param xmap:
+    :param nsmap:
+    :param mode:
+    '''
+    indent = None
+    raw_json = {}
+    raw_json['xpath'] = xmap
+    if nsmap:
+        raw_json['namespaces'] = nsmap
+    if mode == 'raw_json_pretty':
+        indent = 4
+    return json.dumps(raw_json, indent=indent)
+
 def print_xpaths(xmap: Dict,
-                 mode: str ="path",
+                 mode: str ="xpath",
                  *,
-                 out_fd = OUT_FD):
-    '''Print xpath expressions and validate by count of elements found with it.
+                 out_fd = OUT_FD, nsmap= None):
+    '''
+    Print xpath expressions according to 'mode' argument.
+    
+    :param xmap: map of xpath expressions
+    :param mode: output mode
+    :param out_fd: output file descriptor
+    :param nsmap: namespaces map
+    
     mode: str
-        path  : print elements xpath expressions (default)
+        xpath  : print elements xpath expressions (default)
         all   : also print attribute xpath expressions
         raw   : print unqualified xpath and found values (list)
+        raw_json: json string with internal xpath and namespace maps.
+                {"xpath": { ... }.
+                 "namespaces": { ... }}
+        raw_json_pretty: same as raw_json with pretty print
         values: print tuple of found values only
     '''
 
     acount=0
     acountmsg=''
 
-    for unq_xpath, qual_xpath_lst in xmap.items():
-        if mode not in ['raw', 'values']:
-            print(qual_xpath_lst[0])
-    
-        if mode == "all":
-            #Print xpath for attributes
-            if qual_xpath_lst[2] is not []:
-                for a in qual_xpath_lst[2]:
-                    print(f"{qual_xpath_lst[0]}/@{a}")
-                    acount += 1
-                acountmsg = f"Found {acount:3} xpath expressions for attributes\n"
-        if mode == "raw":
-            print(unq_xpath, qual_xpath_lst)
-        elif mode == "values":
-            print(qual_xpath_lst)
+    if mode in ['raw_json', 'raw_json_pretty']:
+        print(get_json_response(xmap, nsmap, mode))
+    else:
+        for unq_xpath, qual_xpath_lst in xmap.items():
+            if mode in ['xpath', 'all']:
+                print(qual_xpath_lst[0])
+                if mode == "all":
+                    #Print xpath for attributes
+                    if qual_xpath_lst[2] is not []:
+                        for a in qual_xpath_lst[2]:
+                            print(f"{qual_xpath_lst[0]}/@{a}")
+                            acount += 1
+                        acountmsg = f"Found {acount:3} xpath expressions for attributes\n"
+            elif mode == "raw":
+                print(unq_xpath, qual_xpath_lst)
+            elif mode == "values":
+                print(qual_xpath_lst)
 
     print(f"\nFound {len(xmap.keys()):3} xpath expressions for elements\n{acountmsg}", file=out_fd)
 
@@ -259,11 +306,25 @@ def build_namespace_dict(tree: etree._ElementTree) ->  Dict[str, str]:
         nsmap[ns] = v
     return nsmap
 
-def fromstring(xmlstr: str, *,
+def fromstring(xmlstr: Union[str,bytes], *,
                xpath_base: str = '//*',
                with_count: bool = WITH_COUNT,
-               max_items: int = MAX_ITEMS) -> (etree._ElementTree, Dict[str, str], OrderedDict[str, Tuple[str, int, List[str]]]):
-    doc = etree.parse(StringIO(xmlstr))
+               max_items: int = MAX_ITEMS,
+               encoding='utf-8') -> (etree._ElementTree, Dict[str, str], OrderedDict[str, Tuple[str, int, List[str]]]):
+    '''
+    Parse from string. If the document contains an encoding in XML declaration, use it.
+    If it starts with an XML declaration but has no ``encoding`` attribute, ``utf-8`` will be used be default.
+    
+    :param xmlstr: document or fragment as string or bytes.
+    :param xpath_base: xpath expression to start searching xpaths for.
+    :param with_count: Include count of elements found with each expression. Default: False
+    :param max_items: limit the number of parsed elements. Default: 100000
+    :param encoding: xml string encoding, ``utf-8`` by default. Use :func:`get_encoding_from_fragment` to get encoding.
+    '''
+    xmlbytes = xmlstr
+    if isinstance(xmlstr, str):
+        xmlbytes = xmlstr.encode(encoding)
+    doc = etree.parse(BytesIO(xmlbytes))
     return parse(file=None, itree=doc, xpath_base=xpath_base, with_count=with_count, max_items=max_items)
 
 def parse(file: str, *,
@@ -283,20 +344,20 @@ def parse(file: str, *,
     
     Parameters
     ----------
-        file: file path string
-        itree: lxml.etree._ElementTree
+        :param file: file path string
+        :param itree: lxml.etree._ElementTree
                 ElementTree object
-        xpath_base: xpath expression to start searching xpaths for.
-        with_count: Include count of elements found with each expression. Default: False
-        max_items: limit the number of parsed elements. Default: 100000
+        :param xpath_base: xpath expression to start searching xpaths for.
+        :param with_count: Include count of elements found with each expression. Default: False
+        :param max_items: limit the number of parsed elements. Default: 100000
     '''
 
     try:
         tree = itree
         if tree is None:
             if not path.isfile(file):
                 raise FileNotFoundError(errno.ENOENT, strerror(errno.ENOENT), file)
-            with open(file, "r") as fin:
+            with open(file, "rb") as fin:
                 tree = etree.parse(fin)
 
         nsmap = build_namespace_dict(tree)
@@ -308,20 +369,24 @@ def parse(file: str, *,
         raise(e)
 
 def main():
+    if len(sys.argv) == 1:
+        print(f"ERROR: No arguments passed.\nRun with -h for help.", file=sys.stderr)
+        sys.exit(1)
+    
     if sys.argv[1] in ["-h", "--help"]:
         usage()
         sys.exit()
 
     file = sys.argv[1]
-    mode = "path"
+    mode = "xpath"
     xpath_base = XPATH_ALL
     with_count = WITH_COUNT
     max_items = MAX_ITEMS
     out_fd = OUT_FD
     no_banners = False
     warns = None
 
-    if not path.isfile(file):
+    if file != '-' and not path.isfile(file):
         print(f"[Errno {errno.ENOENT}] {strerror(errno.ENOENT)}", file=sys.stderr)
         sys.exit(errno.ENOENT)
 
@@ -357,10 +422,14 @@ def main():
     print(f"{'no_banners':10}: {no_banners}", file=out_fd, flush=True)
     if warns is not None:
         print(f"\n{warns}\n", file=sys.stderr)
-    nsmap, xmap = parse(file,  xpath_base=xpath_base, with_count=with_count, max_items=max_items)[1:]
+    if file != '-':
+        nsmap, xmap = parse(file,  xpath_base=xpath_base, with_count=with_count, max_items=max_items)[1:]
+    else:
+        xmlstr = sys.stdin.buffer.read()
+        nsmap, xmap = fromstring(xmlstr,  xpath_base=xpath_base, with_count=with_count, max_items=max_items)[1:]
     if xmap is not None:
         print(f"namespaces: {nsmap}\n", file=out_fd, flush=True)
-        print_xpaths(xmap, mode, out_fd=out_fd)
+        print_xpaths(xmap, mode, out_fd=out_fd, nsmap=nsmap)
     else:
         sys.exit(1)
 

tests/resources/iso-8859-1.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<doc>
+     <más>más</más>
+    <a>a</a>
+</doc>

tests/resources/shift-jis.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="shift_jis"?>
+<doc>
+     <‚±‚ê>&#x005C;&#x007E;&#x2015;&#x005C;&#x301C;
+    &#x2016;&#x2212;&#x00A2;&#x00A3;&#x00AC;</‚±‚ê>
+    <a>&#x005C;&#x007E;&#x2015;&#x005C;&#x301C;
+&#x2016;&#x2212;&#x00A2;&#x00A3;&#x00AC;</a>
+</doc>