Skip to content

GitLab

Commit 01ca37ce authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge v0.6.4 and edit dialog with checkmarks and lifting embargo.

parent 350e43b2
Pipeline #64654 passed with stages
in 13 minutes and 39 seconds
Hide whitespace changes
Inline Side-by-side
examples/external_project_parallel_upload/example-2.tar.gz 0 → 120000
View file @ 01ca37ce
example-1.tar.gz
\ No newline at end of file
examples/external_project_parallel_upload/example-3.tar.gz 0 → 120000
View file @ 01ca37ce
example-1.tar.gz
\ No newline at end of file
examples/external_project_parallel_upload/upload.py
View file @ 01ca37ce
......@@ -16,6 +16,7 @@ import tarfile
import io
import zipfile
import zipstream
import uuid
# config
nomad_url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/mp/api'
......@@ -126,7 +127,7 @@ def upload_next_data(sources: Iterator[Tuple[str, str, str]], upload_name='next
yield chunk
# stream .zip to nomad
response = requests.put(url=url, headers={'X-Token': token}, data=content())
response = requests.put(url=url, headers={'X-Token': token, 'Content-type': 'application/octet-stream'}, data=content())
if response.status_code != 200:
raise Exception('nomad return status %d' % response.status_code)
......
examples/files.py 0 → 100644
View file @ 01ca37ce
from nomad import infrastructure, files, processing as proc
infrastructure.setup_logging()
infrastructure.setup_mongo()
upload_id = 'NvVyk3gATxCJW6dWS4cRWw'
upload = proc.Upload.get(upload_id)
upload_with_metadata = upload.to_upload_with_metadata()
upload_files = files.PublicUploadFiles(upload_id)
upload_files.repack(upload_with_metadata)
# try:
# public_upload_files = files.PublicUploadFiles(upload_id)
# public_upload_files.delete()
# except Exception:
# pass
# staging_upload_files = files.StagingUploadFiles(upload_id)
# staging_upload_files.pack(upload_with_metadata)
gui/src/components/ApiDialogButton.js
View file @ 01ca37ce
......@@ -56,10 +56,10 @@ class ApiDialogUnstyled extends React.Component {
}
</DialogContent>
<DialogActions>
<Button onClick={this.handleToggleRaw} color="primary">
<Button onClick={this.handleToggleRaw}>
{showRaw ? 'show tree' : 'show raw JSON'}
</Button>
<Button onClick={onClose} color="primary">
<Button onClick={onClose}>
Close
</Button>
</DialogActions>
......
gui/src/components/EditUserMetadataDialog.js
View file @ 01ca37ce
......@@ -7,7 +7,7 @@ import DialogContent from '@material-ui/core/DialogContent'
import DialogContentText from '@material-ui/core/DialogContentText'
import DialogTitle from '@material-ui/core/DialogTitle'
import PropTypes from 'prop-types'
import { IconButton, Tooltip, withStyles, Paper, MenuItem, Popper, CircularProgress } from '@material-ui/core'
import { IconButton, Tooltip, withStyles, Paper, MenuItem, Popper, CircularProgress, FormGroup, Checkbox, FormLabel } from '@material-ui/core'
import EditIcon from '@material-ui/icons/Edit'
import AddIcon from '@material-ui/icons/Add'
import RemoveIcon from '@material-ui/icons/Delete'
......@@ -348,7 +348,7 @@ class DatasetInputUnstyled extends React.Component {
shouldRenderSuggestions={() => true}
margin={margin}
label={usedLabel}
placeholder={`Type the dataset's name`}
placeholder="Type the dataset's name"
/>
}
}
......@@ -398,6 +398,7 @@ class ReferenceInput extends React.Component {
onChange={this.handleChange.bind(this)}
error={value === undefined}
label={value === undefined ? 'A reference must be a valid url' : label}
placeholder="Enter a URL to a related resource"
/>
}
}
......@@ -648,7 +649,7 @@ class InviteUserDialogUnstyled extends React.Component {
{input('affiliation', 'Affiliation')}
</DialogContent>
<DialogActions>
<Button onClick={this.handleClose.bind(this)} color="primary" disabled={submitting}>
<Button onClick={this.handleClose.bind(this)} disabled={submitting}>
Cancel
</Button>
<div className={classes.submitWrapper}>
......@@ -665,6 +666,47 @@ class InviteUserDialogUnstyled extends React.Component {
const InviteUserDialog = compose(withApi(true, false), withStyles(InviteUserDialogUnstyled.styles))(InviteUserDialogUnstyled)
class UserMetadataFieldUnstyled extends React.PureComponent {
static propTypes = {
classes: PropTypes.object.isRequired,
children: PropTypes.node,
modified: PropTypes.bool,
onChange: PropTypes.func.isRequired
}
static styles = theme => ({
root: {
flexWrap: 'nowrap',
alignItems: 'flex-start',
marginTop: theme.spacing.unit * 2
},
container: {
width: '100%'
},
checkbox: {
marginLeft: -theme.spacing.unit * 2,
marginRight: theme.spacing.unit,
marginTop: theme.spacing.unit
}
})
render() {
const {children, classes, modified, onChange} = this.props
return <FormGroup row className={classes.root}>
<Checkbox
classes={{root: classes.checkbox}}
checked={modified}
onChange={(event, checked) => onChange(checked)}
/>
<div className={classes.container}>
{children}
</div>
</FormGroup>
}
}
const UserMetadataField = withStyles(UserMetadataFieldUnstyled.styles)(UserMetadataFieldUnstyled)
class EditUserMetadataDialogUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
......@@ -694,6 +736,9 @@ class EditUserMetadataDialogUnstyled extends React.Component {
left: '50%',
marginTop: -12,
marginLeft: -12
},
liftEmbargoLabel: {
marginTop: theme.spacing.unit * 3
}
})
......@@ -710,7 +755,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
coauthors: [],
shared_with: [],
datasets: [],
with_embargo: true
with_embargo: 'lift'
}
this.unmounted = false
}
......@@ -720,13 +765,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
actions: {},
isVerifying: false,
verified: true,
submitting: false,
testCoauthors: [],
testUser: {
message: null,
success: true,
value: null
}
submitting: false
}
componentWillUnmount() {
......@@ -746,8 +785,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
shared_with: (example.owners || [])
.filter(user => user.user_id !== example.uploader.user_id)
.map(user => user.user_id),
datasets: (example.datasets || []).map(ds => ds.name),
with_embargo: example.with_embargo
datasets: (example.datasets || []).map(ds => ds.name)
}
}
......@@ -882,8 +920,16 @@ class EditUserMetadataDialogUnstyled extends React.Component {
const dialogEnabled = user && example.uploader && example.uploader.user_id === user.sub && !disabled
const submitEnabled = Object.keys(actions).length && !submitting && verified
const editDataToActions = editData => {
if (Array.isArray(editData)) {
return editData.map(value => ({value: value}))
} else {
return {value: editData}
}
}
const listTextInputProps = (key, verify) => {
const values = actions[key] ? actions[key] : this.editData[key].map(value => ({value: value}))
const values = actions[key] ? actions[key] : editDataToActions(this.editData[key])
return {
values: values,
......@@ -897,6 +943,21 @@ class EditUserMetadataDialogUnstyled extends React.Component {
}
}
const metadataFieldProps = (key, verify) => ({
modified: Boolean(actions[key]),
onChange: checked => {
if (checked) {
this.setState({actions: {...actions, [key]: editDataToActions(this.editData[key])}}, () => {
if (verify) {
this.verify()
}
})
} else {
this.setState({actions: {...actions, [key]: undefined}})
}
}
})
return (
<React.Fragment>
<IconButton {...(buttonProps || {})} onClick={this.handleButtonClick} disabled={!dialogEnabled}>
......@@ -911,58 +972,63 @@ class EditUserMetadataDialogUnstyled extends React.Component {
<DialogContentText>
You are editing {total} {total === 1 ? 'entry' : 'entries'}. {total > 1
? 'The fields are pre-filled with data from the first entry for.' : ''
} Only the fields that you change will be updated.
Be aware that all references, co-authors, shared_with, or datasets count as
one field.
} Only the checked fields will be updated.
The fields references, co-authors, shared with users,
and datasets can have many values. Changing one value, will apply all values.
</DialogContentText>
<ActionInput component={TextField}
label="Comment"
value={actions.comment !== undefined ? actions.comment : {value: this.editData.comment}}
onChange={value => this.setState({actions: {...actions, comment: value}})}
margin="normal"
multiline rows="4"
fullWidth
placeholder="Add a comment"
InputLabelProps={{ shrink: true }}
/>
<ListTextInput
component={ReferenceInput}
{...listTextInputProps('references', true)}
label="References"
/>
<ListTextInput
component={UserInput}
{...listTextInputProps('coauthors', true)}
label="Co-author"
/>
<ListTextInput
component={UserInput}
{...listTextInputProps('shared_with', true)}
label="Shared with"
/>
<ListTextInput
component={DatasetInput}
{...listTextInputProps('datasets', true)}
label="Datasets"
/>
<UserMetadataField {...metadataFieldProps('comment')}>
<ActionInput component={TextField}
label="Comment"
value={actions.comment !== undefined ? actions.comment : {value: this.editData.comment}}
onChange={value => this.setState({actions: {...actions, comment: value}})}
margin="normal"
multiline
rowsMax="10"
fullWidth
placeholder="Add a comment"
InputLabelProps={{ shrink: true }}
/>
</UserMetadataField>
<UserMetadataField {...metadataFieldProps('references', true)}>
<ListTextInput
component={ReferenceInput}
{...listTextInputProps('references', true)}
label="References"
/>
</UserMetadataField>
<UserMetadataField {...metadataFieldProps('coauthors', true)}>
<ListTextInput
component={UserInput}
{...listTextInputProps('coauthors', true)}
label="Co-author"
/>
</UserMetadataField>
<UserMetadataField {...metadataFieldProps('shared_with', true)}>
<ListTextInput
component={UserInput}
{...listTextInputProps('shared_with', true)}
label="Shared with"
/>
</UserMetadataField>
<UserMetadataField {...metadataFieldProps('datasets', true)}>
<ListTextInput
component={DatasetInput}
{...listTextInputProps('datasets', true)}
label="Datasets"
/>
</UserMetadataField>
<UserMetadataField classes={{container: classes.liftEmbargoLabel}} {...metadataFieldProps('with_embargo', true)}>
<FormLabel>Lift embargo</FormLabel>
</UserMetadataField>
</DialogContent>
{Object.keys(actions).length
? <DialogContent>
<DialogContentText>
The following fields will be updated with the given values: <i>
{Object.keys(actions).map(action => action).join(', ')}</i>.
Updating many entries might take a few seconds.
</DialogContentText>
</DialogContent>
: ''}
<DialogActions>
<InviteUserDialog />
<span style={{flexGrow: 1}} />
<Button onClick={this.handleClose} color="primary" disabled={submitting}>
<Button onClick={this.handleClose} disabled={submitting}>
Cancel
</Button>
<div className={classes.submitWrapper}>
<Button onClick={this.handleSubmit} color="primary" disabled={!submitEnabled}>
<Button onClick={this.handleSubmit} disabled={!submitEnabled} color="primary">
Submit
</Button>
{submitting && <CircularProgress size={24} className={classes.submitProgress} />}
......
gui/src/components/entry/ArchiveEntryView.js
View file @ 01ca37ce
......@@ -66,7 +66,7 @@ class MetainfoDialogUnstyled extends React.PureComponent {
)}
data={metaInfoData ? metaInfoData.miJson : {}} title="Metainfo JSON"
/>
<Button color="primary" onClick={onClose}>
<Button onClick={onClose}>
Close
</Button>
</DialogActions>
......
gui/src/components/entry/ArchiveLogView.js
View file @ 01ca37ce
......@@ -85,7 +85,7 @@ class ArchiveLogView extends React.Component {
<Download
classes={{root: classes.downloadFab}} tooltip="download logfile"
component={Fab} className={classes.downloadFab} color="primary" size="medium"
component={Fab} className={classes.downloadFab} color="secondary" size="medium"
url={`archive/logs/${uploadId}/${calcId}`} fileName={`${calcId}.log`}
>
<DownloadIcon />
......
gui/src/components/uploads/ConfirmDialog.js
View file @ 01ca37ce
......@@ -51,7 +51,7 @@ class ConfirmDialog extends React.Component {
</FormGroup>
</DialogContent>
<DialogActions>
<Button onClick={onClose} color="primary">
<Button onClick={onClose}>
Cancel
</Button>
<Button onClick={() => onPublish(withEmbargo)} color="primary" autoFocus>
......
nomad/app/api/repo.py
View file @ 01ca37ce
......@@ -348,11 +348,15 @@ repo_edit_model = api.model('RepoEdit', {
})
def edit(parsed_query: Dict[str, Any], logger, mongo_update: Dict[str, Any] = None, re_index=True):
def edit(parsed_query: Dict[str, Any], logger, mongo_update: Dict[str, Any] = None, re_index=True) -> List[str]:
# get all calculations that have to change
search_request = search.SearchRequest()
add_query(search_request, parsed_query)
calc_ids = list(hit['calc_id'] for hit in search_request.execute_scan())
upload_ids = set()
calc_ids = []
for hit in search_request.execute_scan():
calc_ids.append(hit['calc_id'])
upload_ids.add(hit['upload_id'])
# perform the update on the mongo db
if mongo_update is not None:
......@@ -378,6 +382,8 @@ def edit(parsed_query: Dict[str, Any], logger, mongo_update: Dict[str, Any] = No
'edit repo with failed elastic updates',
payload=mongo_update, nfailed=len(failed))
return list(upload_ids)
def get_uploader_ids(query):
""" Get all the uploader from the query, to check coauthers and shared_with for uploaders. """
......@@ -427,6 +433,7 @@ class EditRepoCalcsResource(Resource):
# checking the edit actions and preparing a mongo update on the fly
mongo_update = {}
uploader_ids = None
lift_embargo = False
for action_quantity_name, quantity_actions in actions.items():
quantity = UserMetadata.m_def.all_quantities.get(action_quantity_name)
if quantity is None:
......@@ -437,9 +444,6 @@ class EditRepoCalcsResource(Resource):
if not g.user.is_admin():
abort(404, 'Only the admin user can set %s' % quantity.name)
if quantity.name == 'Embargo':
abort(400, 'Cannot raise an embargo, you can only lift the embargo')
if isinstance(quantity_actions, list) == quantity.is_scalar:
abort(400, 'Wrong shape for quantity %s' % action_quantity_name)
......@@ -491,7 +495,10 @@ class EditRepoCalcsResource(Resource):
name=action_value)
dataset.m_x('me').create()
mongo_value = dataset.dataset_id
elif action_quantity_name == 'with_embargo':
# ignore the actual value ... just lift the embargo
mongo_value = False
lift_embargo = True
else:
mongo_value = action_value
......@@ -519,7 +526,13 @@ class EditRepoCalcsResource(Resource):
return json_data, 400
# perform the change
edit(parsed_query, logger, mongo_update, True)
upload_ids = edit(parsed_query, logger, mongo_update, True)
# lift embargo
if lift_embargo:
for upload_id in upload_ids:
upload = proc.Upload.get(upload_id)
upload.re_pack()
return json_data, 200
......@@ -643,7 +656,6 @@ class RepoQuantitiesResource(Resource):
quantities = args.get('quantities', [])
size = args.get('size', 5)
print('A ', quantities)
try:
assert size >= 0
except AssertionError:
......
nomad/cli/admin/uploads.py
View file @ 01ca37ce
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List
from typing import List, Callable
import click
from tabulate import tabulate
from mongoengine import Q
......@@ -204,11 +204,8 @@ def rm(ctx, uploads, skip_es, skip_mongo, skip_files):
upload.delete()
@uploads.command(help='Reprocess selected uploads.')
@click.argument('UPLOADS', nargs=-1)
@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes. Default is 1.')
@click.pass_context
def re_process(ctx, uploads, parallel: int):
def __run_processing(
ctx, uploads, parallel: int, process: Callable[[proc.Upload], None], label: str):
_, uploads = query_uploads(ctx, uploads)
uploads_count = uploads.count()
uploads = list(uploads) # copy the whole mongo query set to avoid cursor timeouts
......@@ -223,29 +220,29 @@ def re_process(ctx, uploads, parallel: int):
logger = utils.get_logger(__name__)
print('%d uploads selected, re-processing ...' % uploads_count)
print('%d uploads selected, %s ...' % (uploads_count, label))
def re_process_upload(upload: proc.Upload):
logger.info('re-processing started', upload_id=upload.upload_id)
def process_upload(upload: proc.Upload):
logger.info('%s started' % label, upload_id=upload.upload_id)
completed = False
if upload.process_running:
logger.warn(
'cannot trigger re-process, since the upload is already/still processing',
'cannot trigger %s, since the upload is already/still processing' % label,
current_process=upload.current_process,
current_task=upload.current_task, upload_id=upload.upload_id)
else:
upload.reset()
upload.re_process_upload()
process(upload)
upload.block_until_complete(interval=.5)
if upload.tasks_status == proc.FAILURE:
logger.info('re-processing with failure', upload_id=upload.upload_id)
logger.info('%s with failure' % label, upload_id=upload.upload_id)
completed = True
logger.info('re-processing complete', upload_id=upload.upload_id)
logger.info('%s complete' % label, upload_id=upload.upload_id)
with cv:
state['completed_count'] += 1 if completed else 0
......@@ -253,8 +250,8 @@ def re_process(ctx, uploads, parallel: int):
state['available_threads_count'] += 1
print(
' re-processed %s and skipped %s of %s uploads' %
(state['completed_count'], state['skipped_count'], uploads_count))
' %s %s and skipped %s of %s uploads' %
(label, state['completed_count'], state['skipped_count'], uploads_count))
cv.notify()
......@@ -262,7 +259,7 @@ def re_process(ctx, uploads, parallel: int):
with cv:
cv.wait_for(lambda: state['available_threads_count'] > 0)
state['available_threads_count'] -= 1
thread = threading.Thread(target=lambda: re_process_upload(upload))
thread = threading.Thread(target=lambda: process_upload(upload))
threads.append(thread)
thread.start()
......@@ -270,6 +267,22 @@ def re_process(ctx, uploads, parallel: int):
thread.join()
@uploads.command(help='Reprocess selected uploads.')
@click.argument('UPLOADS', nargs=-1)
@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes. Default is 1.')
@click.pass_context
def re_process(ctx, uploads, parallel: int):
__run_processing(ctx, uploads, parallel, lambda upload: upload.re_process_upload(), 're-processing')
@uploads.command(help='Repack selected uploads.')
@click.argument('UPLOADS', nargs=-1)
@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes. Default is 1.')
@click.pass_context
def re_pack(ctx, uploads, parallel: int):
__run_processing(ctx, uploads, parallel, lambda upload: upload.re_pack(), 're-packing')
@uploads.command(help='Attempt to abort the processing of uploads.')
@click.argument('UPLOADS', nargs=-1)
@click.option('--calcs', is_flag=True, help='Only stop calculation processing.')
......
nomad/files.py
View file @ 01ca37ce
......@@ -368,7 +368,9 @@ class StagingUploadFiles(UploadFiles):
def archive_log_file_object(self, calc_id: str) -> PathObject:
return self._archive_dir.join_file('%s.log' % calc_id)
def add_rawfiles(self, path: str, move: bool = False, prefix: str = None, force_archive: bool = False) -> None:
def add_rawfiles(
self, path: str, move: bool = False, prefix: str = None,
force_archive: bool = False, target_dir: DirectoryObject = None) -> None:
"""
Add rawfiles to the upload. The given file will be copied, moved, or extracted.
......@@ -378,11 +380,12 @@ class StagingUploadFiles(UploadFiles):
prefix: Optional path prefix for the added files.
force_archive: Expect the file to be a zip or other support archive file.
Usually those files are only extracted if they can be extracted and copied instead.
target_dir: Overwrite the used directory to extract to. Default is the raw directory of this upload.
"""
assert not self.is_frozen
assert os.path.exists(path)
self._size += os.stat(path).st_size
target_dir = self._raw_dir
target_dir = self._raw_dir if target_dir is None else target_dir
if prefix is not None:
target_dir = target_dir.join_dir(prefix, create=True)
ext = os.path.splitext(path)[1]
......@@ -424,7 +427,7 @@ class StagingUploadFiles(UploadFiles):
def pack(
self, upload: UploadWithMetadata, target_dir: DirectoryObject = None,
skip_raw: bool = False) -> None:
skip_raw: bool = False, skip_archive: bool = False) -> None:
"""
Replaces the staging upload data with a public upload record by packing all
data into files. It is only available if upload *is_bag*.
......@@ -435,6 +438,7 @@ class StagingUploadFiles(UploadFiles):
target_dir: optional DirectoryObject to override where to put the files. Default
is the corresponding public upload files directory.
skip_raw: determine to not pack the raw data, only archive and user metadata
skip_raw: determine to not pack the archive data, only raw and user metadata
"""
self.logger.info('started to pack upload')
......@@ -464,6 +468,17 @@ class StagingUploadFiles(UploadFiles):
return zipfile.ZipFile(file.os_path, mode='w')
# zip archives
if not skip_archive:
self._pack_archive_files(upload, create_zipfile)
self.logger.info('packed archives')