Commit 5bd89b57 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'bugfixes' into 'v0.10.2'

Bugfixes

See merge request !310
parents afccfbfa 7d1a3c6f
Pipeline #98253 passed with stages
in 23 minutes and 31 seconds
...@@ -46,6 +46,11 @@ contributing, and API reference. ...@@ -46,6 +46,11 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes. Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.10.2
- fixes small parser and normalizer issues
- fixes broken embargo lifting
- fixes default keycloak configuration for authenticated access via ArchiveQuery
### v0.10.0 ### v0.10.0
- The entries page shows visualizations for key properties of the underlying data - The entries page shows visualizations for key properties of the underlying data
- A new more consistent API (/api/v1) alongside the old API (/api) - A new more consistent API (/api/v1) alongside the old API (/api)
......
...@@ -137,6 +137,15 @@ via your preferred tools. Just make sure to use the right ports. ...@@ -137,6 +137,15 @@ via your preferred tools. Just make sure to use the right ports.
## Running NOMAD ## Running NOMAD
Before you run NOMAD for development purposes, you should configure it to use the `test`
realm of our user management system. By default, NOMAD will use the `prod` realm.
Create a `nomad.yaml` file in the root folder:
```
keycloak:
realm: fairdi_nomad_test
```
NOMAD consist of the NOMAD app/api, a worker, and the GUI. You can run app and worker with NOMAD consist of the NOMAD app/api, a worker, and the GUI. You can run app and worker with
the NOMAD cli. These commands will run the services and show their logout put. You should open the NOMAD cli. These commands will run the services and show their logout put. You should open
them in separate shells as they run continuously. They will not watch code changes and them in separate shells as they run continuously. They will not watch code changes and
......
...@@ -787,7 +787,9 @@ class EditUserMetadataDialogUnstyled extends React.Component { ...@@ -787,7 +787,9 @@ class EditUserMetadataDialogUnstyled extends React.Component {
coauthors: [], coauthors: [],
shared_with: [], shared_with: [],
datasets: [], datasets: [],
with_embargo: 'lift' with_embargo: {
value: 'lift'
}
} }
this.unmounted = false this.unmounted = false
} }
...@@ -983,11 +985,12 @@ class EditUserMetadataDialogUnstyled extends React.Component { ...@@ -983,11 +985,12 @@ class EditUserMetadataDialogUnstyled extends React.Component {
} }
} }
const metadataFieldProps = (key, verify) => ({ const metadataFieldProps = (key, verify, defaultValue) => ({
modified: Boolean(actions[key]), modified: Boolean(actions[key]),
defaultValue: defaultValue,
onChange: checked => { onChange: checked => {
if (checked) { if (checked) {
this.setState({actions: {...actions, [key]: editDataToActions(this.editData[key])}}, () => { this.setState({actions: {...actions, [key]: editDataToActions(this.editData[key] || defaultValue)}}, () => {
if (verify) { if (verify) {
this.verify() this.verify()
} }
...@@ -1060,7 +1063,7 @@ class EditUserMetadataDialogUnstyled extends React.Component { ...@@ -1060,7 +1063,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
label="Datasets" label="Datasets"
/> />
</UserMetadataField> </UserMetadataField>
<UserMetadataField classes={{container: classes.liftEmbargoLabel}} {...metadataFieldProps('with_embargo', true)}> <UserMetadataField classes={{container: classes.liftEmbargoLabel}} {...metadataFieldProps('with_embargo', true, 'lift')}>
<FormLabel>Lift embargo</FormLabel> <FormLabel>Lift embargo</FormLabel>
</UserMetadataField> </UserMetadataField>
</DialogContent> </DialogContent>
......
...@@ -579,6 +579,7 @@ class EditRepoCalcsResource(Resource): ...@@ -579,6 +579,7 @@ class EditRepoCalcsResource(Resource):
mongo_update = {} mongo_update = {}
uploader_ids = None uploader_ids = None
lift_embargo = False lift_embargo = False
has_error = False
removed_datasets = None removed_datasets = None
with utils.timer(common.logger, 'edit verified'): with utils.timer(common.logger, 'edit verified'):
...@@ -607,7 +608,22 @@ class EditRepoCalcsResource(Resource): ...@@ -607,7 +608,22 @@ class EditRepoCalcsResource(Resource):
action_value = action.get('value') action_value = action.get('value')
action_value = action_value if action_value is None else action_value.strip() action_value = action_value if action_value is None else action_value.strip()
if action_value is None: if action_quantity_name == 'with_embargo':
# ignore the actual value ... just lift the embargo
mongo_value = False
lift_embargo = True
# check if necessary
search_request = search.SearchRequest()
apply_search_parameters(search_request, parsed_query)
search_request.q = search_request.q & Q('term', with_embargo=True)
if search_request.execute()['total'] == 0:
action['success'] = False
has_error = True
action['message'] = 'There is no embargo to lift'
continue
elif action_value is None:
mongo_value = None mongo_value = None
elif action_value == '': elif action_value == '':
...@@ -644,20 +660,6 @@ class EditRepoCalcsResource(Resource): ...@@ -644,20 +660,6 @@ class EditRepoCalcsResource(Resource):
dataset.a_mongo.create() dataset.a_mongo.create()
mongo_value = dataset.dataset_id mongo_value = dataset.dataset_id
elif action_quantity_name == 'with_embargo':
# ignore the actual value ... just lift the embargo
mongo_value = False
lift_embargo = True
# check if necessary
search_request = search.SearchRequest()
apply_search_parameters(search_request, parsed_query)
search_request.q = search_request.q & Q('term', with_embargo=True)
if search_request.execute()['total'] == 0:
action['success'] = False
has_error = True
action['message'] = 'There is no embargo to lift'
continue
else: else:
mongo_value = action_value mongo_value = action_value
......
...@@ -101,7 +101,7 @@ def __run_processing( ...@@ -101,7 +101,7 @@ def __run_processing(
else: else:
upload.reset(force=True) upload.reset(force=True)
process(upload) process(upload)
upload.block_until_complete(interval=.5) upload.block_until_process_complete(interval=.5)
if upload.tasks_status == proc.FAILURE: if upload.tasks_status == proc.FAILURE:
logger.info('%s with failure' % label, upload_id=upload.upload_id) logger.info('%s with failure' % label, upload_id=upload.upload_id)
......
...@@ -126,9 +126,9 @@ elastic = NomadConfig( ...@@ -126,9 +126,9 @@ elastic = NomadConfig(
keycloak = NomadConfig( keycloak = NomadConfig(
server_url='https://nomad-lab.eu/fairdi/keycloak/auth/', server_url='https://nomad-lab.eu/fairdi/keycloak/auth/',
realm_name='fairdi_nomad_test', realm_name='fairdi_nomad_prod',
username='admin', username='admin',
password='password', password='*',
client_id='nomad_public', client_id='nomad_public',
client_secret=None, client_secret=None,
oasis=False) oasis=False)
......
...@@ -360,11 +360,21 @@ class Proc(Document, metaclass=ProcMetaclass): ...@@ -360,11 +360,21 @@ class Proc(Document, metaclass=ProcMetaclass):
pass pass
def block_until_complete(self, interval=0.01): def block_until_complete(self, interval=0.01):
'''
Reloads the process constantly until it sees a completed process with finished tasks.
Should be used with care as it can block indefinitely. Just intended for testing
purposes.
'''
while self.tasks_running or self.process_running:
time.sleep(interval)
self.reload()
def block_until_process_complete(self, interval=0.01):
''' '''
Reloads the process constantly until it sees a completed process. Should be Reloads the process constantly until it sees a completed process. Should be
used with care as it can block indefinitely. Just intended for testing purposes. used with care as it can block indefinitely. Just intended for testing purposes.
''' '''
while self.tasks_running or self.process_running: while self.process_running:
time.sleep(interval) time.sleep(interval)
self.reload() self.reload()
......
...@@ -1092,16 +1092,7 @@ class Upload(Proc): ...@@ -1092,16 +1092,7 @@ class Upload(Proc):
def re_pack(self): def re_pack(self):
''' A *process* that repacks the raw and archive data based on the current embargo data. ''' ''' A *process* that repacks the raw and archive data based on the current embargo data. '''
assert self.published assert self.published
# mock the steps of actual processing
self._continue_with('uploading')
self._continue_with('extracting')
self._continue_with('parse_all')
self._continue_with('cleanup')
self.upload_files.re_pack(self.user_metadata()) self.upload_files.re_pack(self.user_metadata())
self.joined = True
self._complete()
@process @process
def process_upload(self): def process_upload(self):
......
...@@ -1747,22 +1747,33 @@ class TestEditRepo(): ...@@ -1747,22 +1747,33 @@ class TestEditRepo():
@pytest.mark.timeout(config.tests.default_timeout) @pytest.mark.timeout(config.tests.default_timeout)
def test_edit_lift_embargo(api, published, other_test_user_auth): def test_edit_lift_embargo(api, published, other_test_user_auth, no_warn):
example_calc = Calc.objects(upload_id=published.upload_id).first() example_calc = Calc.objects(upload_id=published.upload_id).first()
assert example_calc.metadata['with_embargo'] assert example_calc.metadata['with_embargo']
elastic_calc = next(
search.SearchRequest().search_parameters(calc_id=example_calc.calc_id).execute_scan())
assert elastic_calc['with_embargo'] is True
with pytest.raises(files.Restricted):
with files.UploadFiles.get(published.upload_id).read_archive(example_calc.calc_id) as archive:
archive[example_calc.calc_id].to_dict()
rv = api.post( rv = api.post(
'/repo/edit', headers=other_test_user_auth, content_type='application/json', '/repo/edit', headers=other_test_user_auth, content_type='application/json',
data=json.dumps({ data=json.dumps({
'actions': { 'actions': {
'with_embargo': { 'with_embargo': {}
'value': 'lift'
}
} }
})) }))
assert rv.status_code == 200, rv.data assert rv.status_code == 200, rv.data
assert not Calc.objects(calc_id=example_calc.calc_id).first().metadata['with_embargo'] assert not Calc.objects(calc_id=example_calc.calc_id).first().metadata['with_embargo']
Upload.get(published.upload_id).block_until_complete() Upload.get(published.upload_id).block_until_complete()
elastic_calc = next(
search.SearchRequest().search_parameters(calc_id=example_calc.calc_id).execute_scan())
assert elastic_calc['with_embargo'] is False
# should not raise Restricted anymore # should not raise Restricted anymore
with files.UploadFiles.get(published.upload_id).read_archive(example_calc.calc_id) as archive: with files.UploadFiles.get(published.upload_id).read_archive(example_calc.calc_id) as archive:
archive[example_calc.calc_id].to_dict() archive[example_calc.calc_id].to_dict()
......
...@@ -324,6 +324,9 @@ class KeycloakMock: ...@@ -324,6 +324,9 @@ class KeycloakMock:
return g.oidc_access_token return g.oidc_access_token
config.keycloak.realm_name = 'fairdi_nomad_test'
config.keycloak.password = 'password'
_keycloak = infrastructure.keycloak _keycloak = infrastructure.keycloak
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment