Some entries cannot be retrieved with ArchiveQuery
Error in kibana:
{
"_index": "logstash-2020.05.11-000001",
"_type": "_doc",
"_id": "_7e2GHIBo6FkA2KLrfT4",
"_version": 1,
"_score": null,
"_source": {
"nomad.app.endpoint": "gui.static",
"nomad.app.args": {},
"port": 39981,
"path": "/app/nomad/app/api/archive.py",
"exception": "Traceback (most recent call last):\n File \"/app/nomad/app/api/archive.py\", line 317, in post\n with upload_files.read_archive(calc_id, access) as archive:\n File \"/app/nomad/files.py\", line 870, in read_archive\n if calc_id in archive:\n File \"/usr/local/lib/python3.6/_collections_abc.py\", line 666, in __contains__\n self[key]\n File \"/app/nomad/archive.py\", line 404, in __getitem__\n first, last = self._load_toc_block(i_block)\n File \"/app/nomad/archive.py\", line 369, in _load_toc_block\n entry_uuid = unpackb(block_data[offset:offset + _toc_uuid_size])\n File \"/app/nomad/archive.py\", line 36, in unpackb\n return msgpack.unpackb(o, raw=False)\n File \"msgpack/_unpacker.pyx\", line 208, in msgpack._unpacker.unpackb\nmsgpack.exceptions.ExtraData: unpack(b) received extra data.",
"nomad.app.json": null,
"nomad.upload_id": "Czf5OvsYR6idvpve2Wa03Q",
"@version": "1",
"logger_name": "nomad.app",
"digest": "ffset:offset + _toc_uuid_size])\n File \"/app/nomad/archive.py\", line 36, in unpackb\n return msgpack.unpackb(o, raw=False)\n File \"msgpack/_unpacker.pyx\", line 208, in msgpack._unpacker.unpackb\nmsgpack.exceptions.ExtraData: unpack(b) received extra data.",
"nomad.app.blueprint": "gui",
"nomad.app.url": "http://10.244.0.37/fairdi/nomad/testing-major/gui/index.html",
"stack_info": null,
"nomad.app.filename": "index.html",
"host": "nomad-testing-major-app-848d6bc49f-fxlrg",
"nomad.calc_id": "8fHX3EVeb0G0S0fz31o2ZfzoR72M",
"level": "ERROR",
"event": "unpack(b) received extra data.",
"nomad.release": "nomad-testing-major",
"type": "Logstash",
"nomad.app.method": "GET",
"tags": [
"nomad",
"nomad-testing-major"
],
"@timestamp": "2020-05-15T14:22:03.796Z",
"nomad.service": "app"
},
"fields": {
"@timestamp": [
"2020-05-15T14:22:03.796Z"
]
},
"highlight": {
"level.keyword": [
"@kibana-highlighted-field@ERROR@/kibana-highlighted-field@"
],
"nomad.release.keyword": [
"@kibana-highlighted-field@nomad-testing-major@/kibana-highlighted-field@"
]
},
"sort": [
1589552523796
]
}
Exception on server side:
Traceback (most recent call last):
File "/app/nomad/app/api/archive.py", line 317, in post
with upload_files.read_archive(calc_id, access) as archive:
File "/app/nomad/files.py", line 870, in read_archive
if calc_id in archive:
File "/usr/local/lib/python3.6/_collections_abc.py", line 666, in __contains__
self[key]
File "/app/nomad/archive.py", line 404, in __getitem__
first, last = self._load_toc_block(i_block)
File "/app/nomad/archive.py", line 369, in _load_toc_block
entry_uuid = unpackb(block_data[offset:offset + _toc_uuid_size])
File "/app/nomad/archive.py", line 36, in unpackb
return msgpack.unpackb(o, raw=False)
File "msgpack/_unpacker.pyx", line 208, in msgpack._unpacker.unpackb
msgpack.exceptions.ExtraData: unpack(b) received extra data.
Query used:
from nomad import client, config
config.client.url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api'
query = client.query_archive(
query={
'dft.compound_type': 'ternary',
'atoms': ['O'],
'dft.crystal_system':'cubic'
},
required={
'section_run[0]': {
'section_single_configuration_calculation[-2]': {
'energy_total': '*'
},
'section_system[-2]': '*'
}
},
scroll = True,
per_page=1000,
max=50000)
i = 0
for _ in query:
i += 1
if i % 1000 == 0:
print(i)
pass
It is possible, yet unlikely, that this file was re-processed during or after this occured.