diff --git a/.vscode/launch.json b/.vscode/launch.json index f746b8e77547d350ee84ba48347de706d4b19165..1c6175a2c381b42d2b8d13c0e119ead6a845c166 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -25,6 +25,16 @@ "worker", "-l" , "debug", "-A", "nomad.processing" ] }, + { + "name": "Python: test/test_parsing.py", + "type": "python", + "request": "launch", + "cwd": "${workspaceFolder}", + "program": "${workspaceFolder}/.pyenv/bin/pytest", + "args": [ + "-v", "tests/test_parsing.py" + ] + }, { "name": "Python: tests/test_dependencies.py", "type": "python", diff --git a/nomad/parsing.py b/nomad/parsing.py index 40bce0100d1245a731f9857b01b8670336f7d1a7..5cdd576968b643bb7e964f28cc721d885fabc437 100644 --- a/nomad/parsing.py +++ b/nomad/parsing.py @@ -1,19 +1,33 @@ import json -class JSONStreamGenerator(): +class JSONStreamWriter(): + START = 0 + OBJECT = 1 + ARRAY = 2 + KEY_VALUE = 3 + """ A generator that allows to output JSON based on calling 'event' functions. Its pure python and could be replaced by some faster implementation, e.g. yajl-py. - It does not do anychecks. Expect random exceptions when events are out of order or - imcomplete. + It uses standard json decode to write values. This allows to mix streaming with + normal encoding. + + Arguments: + file: A file like to write to. + pretty: True to indent and use separators. + + Raises: + AssertionError: If methods were called in a non JSON fashion. Call :func:`close` + to make sure everything was closed properly. """ - def __init__(self, fp, pretty=False): - self._fp = fp + def __init__(self, file, pretty=False): + self._fp = file self._pretty = pretty - self._indent = '' - self._separators = [''] + self._indent = '' # the current indent + self._separators = [''] # a stack of the next necessary separator + self._states = [JSONStreamWriter.START] # a stack of what is currenty open def _write(self, str): self._fp.write(str) @@ -42,15 +56,25 @@ class JSONStreamGenerator(): self._separators.append(self._seperator_with_newline(',')) def open_object(self): + assert self._states[-1] != JSONStreamWriter.OBJECT, "Cannot open object in object." + if self._states[-1] == JSONStreamWriter.KEY_VALUE: + self._states.pop() self._open('{') + self._states.append(JSONStreamWriter.OBJECT) def close_object(self): + assert self._states.pop() == JSONStreamWriter.OBJECT, "Can only close object in object." self._close('}') def open_array(self): + assert self._states[-1] != JSONStreamWriter.OBJECT, "Cannot open array in object." + if self._states[-1] == JSONStreamWriter.KEY_VALUE: + self._states.pop() self._open('[') + self._states.append(JSONStreamWriter.ARRAY) def close_array(self): + assert self._states.pop() == JSONStreamWriter.ARRAY, "Can only close array in array." self._close(']') def key_value(self, key, value): @@ -58,11 +82,20 @@ class JSONStreamGenerator(): self.value(value) def key(self, key): + assert self._states[-1] == JSONStreamWriter.OBJECT, "Key can only be in objects." self._write_seperator() json.dump(key, self._fp) self._separators.append(': ' if self._pretty else ':') + self._states.append(JSONStreamWriter.KEY_VALUE) def value(self, value): + assert self._states[-1] != JSONStreamWriter.OBJECT, "Values can not be in objects." + if self._states[-1] == JSONStreamWriter.KEY_VALUE: + self._states.pop() + self._write_seperator() json.dump(value, self._fp) self._separators.append(self._seperator_with_newline(',')) + + def close(self): + assert self._states[-1] == JSONStreamWriter.START, "Something was not closed." diff --git a/requirements-dev.txt b/requirements-dev.txt index 27caaf5c7bbcf624cdb1c2a282cc20b82c511e12..613631a40ec8e624a68826c37e322a3e159ab26e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,5 @@ mypy pylint pycodestyle pytest -pytest-timeout \ No newline at end of file +pytest-timeout +rope \ No newline at end of file diff --git a/tests/test_parsing.py b/tests/test_parsing.py index f8c1187328ebe7eb2ad8aba9558e98c723c8ae07..ee3f163c71e83112163b825aa333904299b5f6f0 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -1,4 +1,4 @@ -from nomad.parsing import JSONStreamGenerator +from nomad.parsing import JSONStreamWriter from io import StringIO import json import pytest @@ -26,22 +26,23 @@ def test_stream_generator(pretty): ] out = StringIO() - generator = JSONStreamGenerator(out, pretty=pretty) - generator.open_array() - generator.open_object() - generator.key('key1') - generator.value('value') - generator.key('key2') - generator.value(1) - generator.close_object() - generator.open_object() - generator.key('key') - generator.open_object() - generator.key('key') - generator.value('value') - generator.close_object() - generator.close_object() - generator.close_array() + writer = JSONStreamWriter(out, pretty=pretty) + writer.open_array() + writer.open_object() + writer.key('key1') + writer.value('value') + writer.key('key2') + writer.value(1) + writer.close_object() + writer.open_object() + writer.key('key') + writer.open_object() + writer.key('key') + writer.value('value') + writer.close_object() + writer.close_object() + writer.close_array() + writer.close() assert create_reference(example_data, pretty) == out.getvalue()