Mongo documents get 'too large' for big uploads (100k+ calcs)
This happened while publishing /nomad/repository/data/extracted/bda3ef9f8dd1f2eadcfdcdcdaa496e3
(package id 81I7ZF14SBKYNvKk6NXEoQ
).
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/site-packages/flask_restplus/api.py", line 325, in wrapper
resp = resource(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask/views.py", line 88, in view
return self.dispatch_request(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_restplus/resource.py", line 44, in dispatch_request
resp = meth(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_restplus/marshalling.py", line 136, in wrapper
resp = f(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_httpauth.py", line 104, in decorated
return f(*args, **kwargs)
File "/app/nomad/api/auth.py", line 106, in wrapper
return func(*args, **kwargs)
File "/app/nomad/api/auth.py", line 125, in wrapper
return func(*args, **kwargs)
File "/app/nomad/api/upload.py", line 373, in post
upload.publish_upload()
File "/app/nomad/processing/base.py", line 506, in wrapper
self.save()
File "/usr/local/lib/python3.6/site-packages/mongoengine/document.py", line 392, in save
write_concern)
File "/usr/local/lib/python3.6/site-packages/mongoengine/document.py", line 510, in _save_update
upsert=upsert, **write_concern)
File "/usr/local/lib/python3.6/site-packages/pymongo/collection.py", line 3194, in update
write_concern, collation=collation)
File "/usr/local/lib/python3.6/site-packages/pymongo/collection.py", line 851, in _update_retryable
_update, session)
File "/usr/local/lib/python3.6/site-packages/pymongo/mongo_client.py", line 1248, in _retryable_write
return self._retry_with_session(retryable, func, s, None)
File "/usr/local/lib/python3.6/site-packages/pymongo/mongo_client.py", line 1201, in _retry_with_session
return func(session, sock_info, retryable)
File "/usr/local/lib/python3.6/site-packages/pymongo/collection.py", line 847, in _update
retryable_write=retryable_write)
File "/usr/local/lib/python3.6/site-packages/pymongo/collection.py", line 817, in _update
retryable_write=retryable_write).copy()
File "/usr/local/lib/python3.6/site-packages/pymongo/pool.py", line 584, in command
self._raise_connection_failure(error)
File "/usr/local/lib/python3.6/site-packages/pymongo/pool.py", line 745, in _raise_connection_failure
raise error
File "/usr/local/lib/python3.6/site-packages/pymongo/pool.py", line 579, in command
unacknowledged=unacknowledged)
File "/usr/local/lib/python3.6/site-packages/pymongo/network.py", line 128, in command
name, size, max_bson_size + message._COMMAND_OVERHEAD)
File "/usr/local/lib/python3.6/site-packages/pymongo/message.py", line 965, in _raise_document_too_large
raise DocumentTooLarge("%r command document too large" % (operation,))
pymongo.errors.DocumentTooLarge: 'update' command document too large