Commit 16daa234 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixed restriction bug on non embargoed files. Added api tutorial and example.

parent 80b4ab5c
Pipeline #43981 passed with stages
in 23 minutes and 50 seconds
......@@ -157,6 +157,41 @@ This time we needed some exception handling, since the upload will be removed fr
staging area, and you will get a 404 on the `uploads` endpoint.
### Searching for data
## Searching for data
The *repo* part of the API contains a *search* endpoint that support many different
quantities to search for. These include `formula` (e.g. *AcAg*), `system` (e.g. *bulk/2D/atom*), `spacegroup`, `authors`, `code` (e.g. *VASP*), etc.
In the following example, we search for the specific path segment `AcAg`.
### Downloading data
```python
result = client.repo.search(paths='AcAg').response().result
if result.pagination.total == 0:
print('not found')
elif result.pagination.total > 1:
print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
calc = result.results[0]
print(calc)
```
The result of a search always contains the key `pagination` with pagination data (`total`, `page`, `per_page`) and `results` with an array of the search result. The search results depend on
the type of search and their is no formal swagger model for it, therefore you get plain
dictionaries.
## Downloading data
The *raw* api allows to download data. You can do that either via bravado:
```python
client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
```
In case of published data, you can also create plain URLs and use a tool like *curl*:
```python
print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))
```
There are different options to download individual files, or zips with multiple files.
## Conclusions
This was just a small glimpse into the nomad API. You should checkout our swagger documentation
for more details on all the API endpoints and their parameters. You can explore the
API via swagger-ui and even try it in your browser. Just visit the API url.
......@@ -10,6 +10,7 @@ and infrastructure with a simplyfied architecture and consolidated code base.
introduction
setup
dev_guidelines
api_tutorial
api
ops
parser_tutorial
......
......@@ -48,7 +48,7 @@ client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
# 'coauthors': ['sheldon.cooper@ucla.edu'], this does not yet work with emails
# 'external_id': 'external_id' this does also not work, but we could implement something like this
}
})
}).response().result
while upload.process_running:
try:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
......@@ -68,10 +68,8 @@ elif result.pagination.total > 1:
calc = result.results[0]
# download data
# via api
client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
# via download
# just the 'mainfile'
url = '%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile'])
# all files
url = '%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile']))
# download urls, e.g. for curl
print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))
......@@ -448,7 +448,7 @@ class StagingUploadFiles(UploadFiles):
# move public data .restricted -> .public
for calc in self.metadata:
if not calc.get('restricted', True):
if not calc.get('with_embargo', False):
mainfile: str = calc['mainfile']
assert mainfile is not None
for filepath in self.calc_files(mainfile):
......@@ -483,7 +483,7 @@ class StagingUploadFiles(UploadFiles):
archive_restricted_zip = create_zipfile('restricted')
for calc in self.metadata:
archive_zip = archive_restricted_zip if calc.get('restricted', False) else archive_public_zip
archive_zip = archive_restricted_zip if calc.get('with_embargo', False) else archive_public_zip
archive_filename = '%s.%s' % (calc['calc_id'], self._archive_ext)
archive_zip.write(self._archive_dir.join_file(archive_filename).os_path, archive_filename)
......
......@@ -395,6 +395,10 @@ class Upload(Proc):
with utils.timer(
logger, 'staged upload files packed', step='publish',
upload_size=self.upload_files.size):
for calc_metadata in upload_with_metadata.calcs:
calc_metadata.published = True
self.upload_files.metadata.update(
calc_id=calc_metadata.calc_id, updates=calc_metadata.to_dict())
self.upload_files.pack()
with utils.timer(
......
......@@ -22,7 +22,7 @@ import inspect
from passlib.hash import bcrypt
from datetime import datetime
from nomad import config, coe_repo, search, parsing
from nomad import config, coe_repo, search, parsing, files
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
......@@ -253,6 +253,20 @@ class TestUploads:
assert_coe_upload(upload_id, user_metadata=metadata)
assert_search_upload(upload_id, published=True)
upload_files = files.UploadFiles.get(upload_id=upload_id)
assert isinstance(upload_files, files.PublicUploadFiles)
for calc_metadata in upload_files.metadata:
assert calc_metadata.get('published', False)
assert 'with_embargo' in calc_metadata
assert calc_metadata['with_embargo'] == metadata.get('with_embargo', False)
try:
with upload_files.raw_file(calc_metadata['mainfile']) as f:
assert f.read() is not None
except files.Restricted:
assert calc_metadata['with_embargo']
else:
assert not calc_metadata['with_embargo']
def assert_upload_does_not_exist(self, client, upload_id: str, test_user_auth):
# poll until publish/delete completed
while True:
......
......@@ -212,10 +212,10 @@ class UploadFilesContract(UploadFilesFixtures):
with test_upload.raw_file(example_file_mainfile) as f:
assert len(f.read()) > 0
if not test_upload._is_authorized():
assert not test_upload.metadata.get(example_calc_id).get('restricted', False)
assert not test_upload.metadata.get(example_calc_id).get('with_embargo', False)
except Restricted:
assert not test_upload._is_authorized()
assert test_upload.metadata.get(example_calc_id).get('restricted', False)
assert test_upload.metadata.get(example_calc_id).get('with_embargo', False)
@pytest.mark.parametrize('prefix', [None, 'examples'])
def test_raw_file_manifest(self, test_upload: StagingUploadFiles, prefix: str):
......@@ -233,10 +233,10 @@ class UploadFilesContract(UploadFilesFixtures):
assert json.load(f) == 'archive'
if not test_upload._is_authorized():
assert not test_upload.metadata.get(example_calc_id).get('restricted', False)
assert not test_upload.metadata.get(example_calc_id).get('with_embargo', False)
except Restricted:
assert not test_upload._is_authorized()
assert test_upload.metadata.get(example_calc_id).get('restricted', False)
assert test_upload.metadata.get(example_calc_id).get('with_embargo', False)
def test_metadata(self, test_upload):
assert_example_calc(test_upload.metadata.get(example_calc_id))
......@@ -269,9 +269,9 @@ def create_staging_upload(upload_id: str, calc_specs: str) -> StagingUploadFiles
if prefix > 0:
calc['mainfile'] = os.path.join(str(prefix), calc['mainfile'])
if calc_spec == 'r':
calc['restricted'] = True
calc['with_embargo'] = True
elif calc_spec == 'p':
calc['restricted'] = False
calc['with_embargo'] = False
upload.metadata.insert(calc)
prefix += 1
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment