Commit 2dbf8799 authored by Mohammad Nakhaee's avatar Mohammad Nakhaee Committed by Markus Scheidgen
Browse files

Upload dataset fixes ( #709)

parent fbd66d32
...@@ -96,11 +96,10 @@ function EditComments() { ...@@ -96,11 +96,10 @@ function EditComments() {
} }
function EditReferences() { function EditReferences() {
const {data, api, setIsReferencesChanged, setReferences} = useContext(editMetaDataDialogContext) const {data, api, setIsReferencesChanged, setReferences, defaultReferences, setDefaultReferences} = useContext(editMetaDataDialogContext)
const [newReference, setNewReference] = useState('') const [newReference, setNewReference] = useState('')
const [validation, setValidation] = useState('') const [validation, setValidation] = useState('')
const [newReferences, setNewReferences] = useState([]) const [newReferences, setNewReferences] = useState([])
const [defaultReferences, setDefaultReferences] = useState([])
const [edit, setEdit] = useState({index: -1, value: '', validation: ''}) const [edit, setEdit] = useState({index: -1, value: '', validation: ''})
const columns = [ const columns = [
...@@ -128,13 +127,16 @@ function EditReferences() { ...@@ -128,13 +127,16 @@ function EditReferences() {
}, [defaultReferences, setIsReferencesChanged]) }, [defaultReferences, setIsReferencesChanged])
const validateAPI = useCallback((value) => { const validateAPI = useCallback((value) => {
let query = {metadata: {references: value}, verify_only: true} return new Promise(async (resolve, reject) => {
let error = '' try {
api.post(`uploads/${data.upload.upload_id}/edit`, query) let query = {metadata: {references: value}, verify_only: true}
.catch(err => { let response = await api.post(`uploads/${data.upload.upload_id}/edit`, query)
error = err.apiMessage[0].msg if (response) {}
}) resolve('')
return error } catch (error) {
reject(error.apiMessage[0].msg)
}
})
}, [api, data]) }, [api, data])
const validate = useCallback((value, index) => { const validate = useCallback((value, index) => {
...@@ -158,7 +160,7 @@ function EditReferences() { ...@@ -158,7 +160,7 @@ function EditReferences() {
setDefaultReferences(_references) setDefaultReferences(_references)
} }
} }
}, [data]) }, [data, setDefaultReferences])
const handleTextFieldChange = (event) => { const handleTextFieldChange = (event) => {
let _newReference = event.target.value let _newReference = event.target.value
...@@ -169,12 +171,19 @@ function EditReferences() { ...@@ -169,12 +171,19 @@ function EditReferences() {
const handleAdd = () => { const handleAdd = () => {
if (newReference) { if (newReference) {
let _validation = validate(newReference, '') let _validation = validate(newReference, '')
if (_validation === '') _validation = validateAPI(newReference)
if (_validation === '') { if (_validation === '') {
let _newReferences = [...newReferences, newReference] validateAPI(newReference)
setNewReferences(_newReferences) .then(_api_validation => {
setReferences(_newReferences) if (_api_validation === '') {
checkChanges(_newReferences) let _newReferences = [...newReferences, newReference]
setNewReferences(_newReferences)
setReferences(_newReferences)
checkChanges(_newReferences)
}
})
.catch(_api_validation => {
setValidation(_api_validation)
})
} else { } else {
setValidation(_validation) setValidation(_validation)
} }
...@@ -288,7 +297,7 @@ ReferencesActions.propTypes = { ...@@ -288,7 +297,7 @@ ReferencesActions.propTypes = {
} }
function EditDatasets() { function EditDatasets() {
const {data, api, raiseError, setIsDatasetChanged, setDatasets} = useContext(editMetaDataDialogContext) const {data, api, raiseError, setIsDatasetChanged, setDatasets, defaultDatasets, setDefaultDatasets} = useContext(editMetaDataDialogContext)
const [suggestions, setSuggestions] = useState([]) const [suggestions, setSuggestions] = useState([])
const [validation, setValidation] = useState('') const [validation, setValidation] = useState('')
const [allDatasets, setAllDatasets] = useState([]) const [allDatasets, setAllDatasets] = useState([])
...@@ -296,7 +305,7 @@ function EditDatasets() { ...@@ -296,7 +305,7 @@ function EditDatasets() {
const [addDataset, setAddDataset] = useState('') const [addDataset, setAddDataset] = useState('')
const [newDatasets, setNewDatasets] = useState([]) const [newDatasets, setNewDatasets] = useState([])
const [isDuplicated, setIsDuplicated] = useState(false) const [isDuplicated, setIsDuplicated] = useState(false)
const [defaultDatasets, setDefaultDatasets] = useState([]) const [apiValidation, setApiValidation] = useState('')
const columns = useMemo(() => ([ const columns = useMemo(() => ([
{key: '', align: 'left', render: dataset => (dataset.doi ? <span> {`${dataset.dataset_name}, DOI:`} <DOI doi={dataset.doi} /></span> : dataset.dataset_name)} {key: '', align: 'left', render: dataset => (dataset.doi ? <span> {`${dataset.dataset_name}, DOI:`} <DOI doi={dataset.doi} /></span> : dataset.dataset_name)}
...@@ -334,7 +343,20 @@ function EditDatasets() { ...@@ -334,7 +343,20 @@ function EditDatasets() {
setDefaultDatasets(__datasets) setDefaultDatasets(__datasets)
} }
} }
}, [data, allDatasets]) }, [data, allDatasets, setDefaultDatasets])
const validateAPI = useCallback((value) => {
return new Promise(async (resolve, reject) => {
try {
let query = {metadata: {datasets: value}, verify_only: true}
let response = await api.post(`uploads/${data.upload.upload_id}/edit`, query)
if (response) {}
resolve('')
} catch (error) {
reject(error.apiMessage[0].msg)
}
})
}, [api, data])
const validate = useCallback((value) => { const validate = useCallback((value) => {
if (allDatasets.map(dataset => dataset.dataset_name).includes(value.dataset_name)) return `There is already a dataset with name ${value.dataset_name}` if (allDatasets.map(dataset => dataset.dataset_name).includes(value.dataset_name)) return `There is already a dataset with name ${value.dataset_name}`
...@@ -345,8 +367,13 @@ function EditDatasets() { ...@@ -345,8 +367,13 @@ function EditDatasets() {
const handleAutoCompleteChange = (event, value) => { const handleAutoCompleteChange = (event, value) => {
if (value && value?.dataset_id) { if (value && value?.dataset_id) {
setAddDataset(value) validateAPI([value.dataset_id]).then(_validation => {
setIsDuplicated(newDatasets.map(dataset => dataset.dataset_id).includes(value.dataset_id)) setApiValidation(_validation)
if (_validation === '') {
setAddDataset(value)
setIsDuplicated(newDatasets.map(dataset => dataset.dataset_id).includes(value.dataset_id))
}
}).catch(_validation => setApiValidation(_validation))
} else { } else {
setAddDataset('') setAddDataset('')
} }
...@@ -398,7 +425,7 @@ function EditDatasets() { ...@@ -398,7 +425,7 @@ function EditDatasets() {
addDatasetButton = <Button color="primary" variant="contained" onClick={handleCreate}> addDatasetButton = <Button color="primary" variant="contained" onClick={handleCreate}>
add entry to new dataset add entry to new dataset
</Button> </Button>
} else if (!isDuplicated && addDataset !== '') { } else if (!isDuplicated && !apiValidation && addDataset !== '') {
addDatasetButton = <Button variant="contained" color="primary" onClick={handleAdd}> addDatasetButton = <Button variant="contained" color="primary" onClick={handleAdd}>
add entry to existing dataset add entry to existing dataset
</Button> </Button>
...@@ -429,7 +456,7 @@ function EditDatasets() { ...@@ -429,7 +456,7 @@ function EditDatasets() {
<TextField <TextField
{...params} {...params}
variant='filled' label='Search for an existing dataset' placeholder='Dataset name' argin='normal' fullWidth size='small' variant='filled' label='Search for an existing dataset' placeholder='Dataset name' argin='normal' fullWidth size='small'
error={isDuplicated} helperText={isDuplicated && 'The data is already in the selected dataset'} error={isDuplicated || apiValidation} helperText={(isDuplicated ? 'The data is already in the selected dataset' : apiValidation)}
/> />
)} )}
/> />
...@@ -491,7 +518,9 @@ function EditMetaDataDialog({...props}) { ...@@ -491,7 +518,9 @@ function EditMetaDataDialog({...props}) {
const [comment, setComment] = useState('') const [comment, setComment] = useState('')
const [references, setReferences] = useState([]) const [references, setReferences] = useState([])
const [defaultReferences, setDefaultReferences] = useState([])
const [datasets, setDatasets] = useState([]) const [datasets, setDatasets] = useState([])
const [defaultDatasets, setDefaultDatasets] = useState([])
const [isCommentChanged, setIsCommentChanged] = useState(false) const [isCommentChanged, setIsCommentChanged] = useState(false)
const [isReferencesChanged, setIsReferencesChanged] = useState(false) const [isReferencesChanged, setIsReferencesChanged] = useState(false)
const [isDatasetChanged, setIsDatasetChanged] = useState(false) const [isDatasetChanged, setIsDatasetChanged] = useState(false)
...@@ -533,12 +562,24 @@ function EditMetaDataDialog({...props}) { ...@@ -533,12 +562,24 @@ function EditMetaDataDialog({...props}) {
if (isCommentChanged || isReferencesChanged || isDatasetChanged) { if (isCommentChanged || isReferencesChanged || isDatasetChanged) {
let metadata = {} let metadata = {}
if (isCommentChanged) metadata.comment = comment if (isCommentChanged) metadata.comment = comment
if (isReferencesChanged) metadata.references = references if (isReferencesChanged) {
metadata.references = {}
let referencesToAdd = references.filter(dataset => !defaultReferences.includes(dataset))
let referencesToRemove = defaultReferences.filter(dataset => !references.includes(dataset))
if (referencesToAdd && referencesToAdd.length !== 0) metadata.references.add = referencesToAdd
if (referencesToRemove && referencesToRemove.length !== 0) metadata.references.remove = referencesToRemove
}
if (isDatasetChanged) { if (isDatasetChanged) {
metadata.datasets = {}
createNewDatasets().then(newDatasets => { createNewDatasets().then(newDatasets => {
metadata.datasets = datasets.filter(_dataset => _dataset.dataset_id !== _dataset.dataset_name) let newDatasetsIDs = datasets.filter(_dataset => _dataset.dataset_id !== _dataset.dataset_name)
.map(_dataset => _dataset.dataset_id) .map(_dataset => _dataset.dataset_id)
.concat(newDatasets.map(_dataset => _dataset.dataset_id)) .concat(newDatasets.map(_dataset => _dataset.dataset_id))
let defaultDatasetsIDs = defaultDatasets.map(_dataset => _dataset.dataset_id)
let datasetsToAdd = newDatasetsIDs.filter(dataset => !defaultDatasetsIDs.includes(dataset))
let datasetsToRemove = defaultDatasetsIDs.filter(dataset => !newDatasetsIDs.includes(dataset))
if (datasetsToAdd && datasetsToAdd.length !== 0) metadata.datasets.add = datasetsToAdd
if (datasetsToRemove && datasetsToRemove.length !== 0) metadata.datasets.remove = datasetsToRemove
submitChanges(metadata) submitChanges(metadata)
}) })
} else { } else {
...@@ -567,8 +608,13 @@ function EditMetaDataDialog({...props}) { ...@@ -567,8 +608,13 @@ function EditMetaDataDialog({...props}) {
data: data, data: data,
setComment: setComment, setComment: setComment,
setReferences: setReferences, setReferences: setReferences,
setDatasets: setDatasets defaultReferences: defaultReferences,
}), [api, raiseError, setIsCommentChanged, setIsReferencesChanged, setIsDatasetChanged, upload, data, setComment, setReferences, setDatasets]) setDefaultReferences: setDefaultReferences,
setDatasets: setDatasets,
defaultDatasets: defaultDatasets,
setDefaultDatasets: setDefaultDatasets
}), [api, raiseError, setIsCommentChanged, setIsReferencesChanged, setIsDatasetChanged, upload,
data, setComment, setReferences, defaultReferences, setDefaultReferences, setDatasets, defaultDatasets, setDefaultDatasets])
return <editMetaDataDialogContext.Provider value={contextValue}> return <editMetaDataDialogContext.Provider value={contextValue}>
<React.Fragment> <React.Fragment>
......
...@@ -28,6 +28,7 @@ import EntryDownloadButton from '../entry/EntryDownloadButton' ...@@ -28,6 +28,7 @@ import EntryDownloadButton from '../entry/EntryDownloadButton'
import Quantity from '../Quantity' import Quantity from '../Quantity'
import {uploadPageContext} from './UploadPage' import {uploadPageContext} from './UploadPage'
import EditMetaDataDialog from './EditMetaDataDialog' import EditMetaDataDialog from './EditMetaDataDialog'
import {pluralize} from '../../utils'
const columns = [ const columns = [
{ {
...@@ -86,7 +87,7 @@ const defaultSelectedColumns = [ ...@@ -86,7 +87,7 @@ const defaultSelectedColumns = [
export default function ProcessingTable(props) { export default function ProcessingTable(props) {
const [selected, setSelected] = useState([]) const [selected, setSelected] = useState([])
const {pagination} = props const {pagination, customTitle} = props
const {upload, isWriter} = useContext(uploadPageContext) const {upload, isWriter} = useContext(uploadPageContext)
const selectedQuery = useMemo(() => { const selectedQuery = useMemo(() => {
...@@ -102,7 +103,7 @@ export default function ProcessingTable(props) { ...@@ -102,7 +103,7 @@ export default function ProcessingTable(props) {
columns={columns} shownColumns={defaultSelectedColumns} {...props} columns={columns} shownColumns={defaultSelectedColumns} {...props}
selected={selected} onSelectedChanged={setSelected} selected={selected} onSelectedChanged={setSelected}
> >
<DatatableToolbar title={`${pagination.total} search results`}> <DatatableToolbar title={pluralize((customTitle || 'search result'), pagination.total, true)}>
<DatatableToolbarActions selection> <DatatableToolbarActions selection>
<EntryDownloadButton tooltip="Download files" query={selectedQuery} /> <EntryDownloadButton tooltip="Download files" query={selectedQuery} />
{isWriter && <EditMetaDataDialog isIcon selectedEntries={selectedQuery}/>} {isWriter && <EditMetaDataDialog isIcon selectedEntries={selectedQuery}/>}
...@@ -117,5 +118,6 @@ export default function ProcessingTable(props) { ...@@ -117,5 +118,6 @@ export default function ProcessingTable(props) {
ProcessingTable.propTypes = { ProcessingTable.propTypes = {
data: PropTypes.arrayOf(PropTypes.object).isRequired, data: PropTypes.arrayOf(PropTypes.object).isRequired,
pagination: PropTypes.object.isRequired, pagination: PropTypes.object.isRequired,
onPaginationChanged: PropTypes.func.isRequired onPaginationChanged: PropTypes.func.isRequired,
customTitle: PropTypes.string
} }
...@@ -536,6 +536,7 @@ function UploadPage() { ...@@ -536,6 +536,7 @@ function UploadPage() {
<ProcessingTable <ProcessingTable
data={data.data.map(entry => ({...entry.entry_metadata, ...entry}))} data={data.data.map(entry => ({...entry.entry_metadata, ...entry}))}
pagination={combinePagination(pagination, data.pagination)} pagination={combinePagination(pagination, data.pagination)}
customTitle='entry'
onPaginationChanged={setPagination}/> onPaginationChanged={setPagination}/>
</StepContent> </StepContent>
</Step> </Step>
......
...@@ -670,6 +670,7 @@ export function pluralize(word, count, inclusive, format = true, prefix) { ...@@ -670,6 +670,7 @@ export function pluralize(word, count, inclusive, format = true, prefix) {
// these words, the pluralize-library should be used instead. // these words, the pluralize-library should be used instead.
const dictionary = { const dictionary = {
'result': 'results', 'result': 'results',
'search result': 'search results',
'entry': 'entries', 'entry': 'entries',
'material': 'materials', 'material': 'materials',
'dataset': 'datasets' 'dataset': 'datasets'
......
...@@ -61,6 +61,7 @@ from nomad.archive import ( ...@@ -61,6 +61,7 @@ from nomad.archive import (
from nomad.app.v1.models import ( from nomad.app.v1.models import (
MetadataEditRequest, And, Aggregation, TermsAggregation, MetadataPagination, MetadataRequired) MetadataEditRequest, And, Aggregation, TermsAggregation, MetadataPagination, MetadataRequired)
from nomad.search import update_metadata as es_update_metadata from nomad.search import update_metadata as es_update_metadata
import validators
section_metadata = datamodel.EntryArchive.metadata.name section_metadata = datamodel.EntryArchive.metadata.name
section_workflow = datamodel.EntryArchive.workflow.name section_workflow = datamodel.EntryArchive.workflow.name
...@@ -465,6 +466,8 @@ class MetadataEditRequestHandler: ...@@ -465,6 +466,8 @@ class MetadataEditRequestHandler:
assert value is None or type(value) == definition.type, f'Expected a {definition.type.__name__}' assert value is None or type(value) == definition.type, f'Expected a {definition.type.__name__}'
if definition.name == 'embargo_length': if definition.name == 'embargo_length':
assert 0 <= value <= 36, 'Value should be between 0 and 36' assert 0 <= value <= 36, 'Value should be between 0 and 36'
if definition.name == 'references':
assert validators.url(value), 'Please enter a valid URL ...'
return None if value == '' else value return None if value == '' else value
elif definition.type == metainfo.Datetime: elif definition.type == metainfo.Datetime:
if value is not None: if value is not None:
......
...@@ -86,6 +86,7 @@ fastapi==0.63.0 ...@@ -86,6 +86,7 @@ fastapi==0.63.0
uvicorn[standard]==0.13.4 uvicorn[standard]==0.13.4
a2wsgi==1.4.0 a2wsgi==1.4.0
python-multipart==0.0.5 python-multipart==0.0.5
validators==0.18.2
# [dev] # [dev]
setuptools==57.5.0 setuptools==57.5.0
......
...@@ -697,22 +697,22 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user, tmp): ...@@ -697,22 +697,22 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user, tmp):
zf.write('tests/data/proc/templates/template.json', 'examples/template.json') zf.write('tests/data/proc/templates/template.json', 'examples/template.json')
entry_1 = dict( entry_1 = dict(
comment='Entry 1 of 3', comment='Entry 1 of 3',
references='http://test1', references='http://test1.com',
external_id='external_id_1') external_id='external_id_1')
with zf.open('examples/entry_1/nomad.yaml', 'w') as f: f.write(yaml.dump(entry_1).encode()) with zf.open('examples/entry_1/nomad.yaml', 'w') as f: f.write(yaml.dump(entry_1).encode())
entry_2 = dict( entry_2 = dict(
comment='Entry 2 of 3', comment='Entry 2 of 3',
references=['http://test2'], references=['http://test2.com'],
external_id='external_id_2') external_id='external_id_2')
with zf.open('examples/entry_2/nomad.json', 'w') as f: f.write(json.dumps(entry_2).encode()) with zf.open('examples/entry_2/nomad.json', 'w') as f: f.write(json.dumps(entry_2).encode())
metadata = { metadata = {
'upload_name': 'my name', 'upload_name': 'my name',
'coauthors': other_test_user.user_id, 'coauthors': other_test_user.user_id,
'references': ['http://test0'], 'references': ['http://test0.com'],
'entries': { 'entries': {
'examples/entry_3/template.json': { 'examples/entry_3/template.json': {
'comment': 'Entry 3 of 3', 'comment': 'Entry 3 of 3',
'references': 'http://test3', 'references': 'http://test3.com',
'external_id': 'external_id_3' 'external_id': 'external_id_3'
}, },
'examples/entry_1/template.json': { 'examples/entry_1/template.json': {
...@@ -729,7 +729,7 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user, tmp): ...@@ -729,7 +729,7 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user, tmp):
comment = ['root entries comment 1', 'Entry 2 of 3', 'Entry 3 of 3', None] comment = ['root entries comment 1', 'Entry 2 of 3', 'Entry 3 of 3', None]
external_ids = ['external_id_1', 'external_id_2', 'external_id_3', None] external_ids = ['external_id_1', 'external_id_2', 'external_id_3', None]
references = [['http://test1'], ['http://test2'], ['http://test3'], ['http://test0']] references = [['http://test1.com'], ['http://test2.com'], ['http://test3.com'], ['http://test0.com']]
expected_coauthors = [other_test_user] expected_coauthors = [other_test_user]
for i in range(len(entries)): for i in range(len(entries)):
......
...@@ -33,7 +33,7 @@ all_coauthor_metadata = dict( ...@@ -33,7 +33,7 @@ all_coauthor_metadata = dict(
coauthors=['lhofstadter'], coauthors=['lhofstadter'],
external_id='31415926536', external_id='31415926536',
comment='a humble comment', comment='a humble comment',
references=['a reference', 'another reference'], references=['http://reference1.com', 'http://reference2.com'],
external_db='AFLOW', external_db='AFLOW',
reviewers=['lhofstadter'], reviewers=['lhofstadter'],
datasets=['test_dataset_1']) datasets=['test_dataset_1'])
...@@ -359,25 +359,32 @@ def test_set_and_clear_all(proc_infra, example_data_writeable, example_datasets, ...@@ -359,25 +359,32 @@ def test_set_and_clear_all(proc_infra, example_data_writeable, example_datasets,
id='reviewers-remove+set'), id='reviewers-remove+set'),
pytest.param( pytest.param(
dict( dict(
metadata_1=dict(references='ref1'), metadata_1=dict(references='http://ref1.com'),
expected_metadata_1=dict(references=['ref1']), expected_metadata_1=dict(references=['http://ref1.com']),
metadata_2=dict(references={'add': ['ref2']}), metadata_2=dict(references={'add': ['http://ref2.com']}),
expected_metadata_2=dict(references=['ref1', 'ref2'])), expected_metadata_2=dict(references=['http://ref1.com', 'http://ref2.com'])),
id='references-add'), id='references-add'),
pytest.param( pytest.param(
dict( dict(
metadata_1=dict(references=['ref1', 'ref2']), metadata_1=dict(references=['http://ref1.com', 'http://ref2.com']),
expected_metadata_1=dict(references=['ref1', 'ref2']), expected_metadata_1=dict(references=['http://ref1.com', 'http://ref2.com']),
metadata_2=dict(references={'add': 'ref3', 'remove': ['ref1']}), metadata_2=dict(references={'add': 'http://ref3.com', 'remove': ['http://ref1.com']}),
expected_metadata_2=dict(references=['ref2', 'ref3'])), expected_metadata_2=dict(references=['http://ref2.com', 'http://ref3.com'])),
id='references-add+remove'), id='references-add+remove'),
pytest.param( pytest.param(
dict( dict(
metadata_1=dict(references='ref1'), metadata_1=dict(references='http://ref1.com'),
expected_metadata_1=dict(references=['ref1']), expected_metadata_1=dict(references=['http://ref1.com']),
metadata_2=dict(references={'remove': 'ref4', 'add': ['ref2', 'ref3', 'ref4']}), metadata_2=dict(references={'remove': 'http://ref4.com', 'add': ['http://ref2.com', 'http://ref3.com', 'http://ref4.com']}),
expected_error_loc_2=('metadata', 'references')), expected_error_loc_2=('metadata', 'references')),
id='references-add+remove-incoherent')]) id='references-add+remove-incoherent'),
pytest.param(
dict(
metadata_1=dict(references='http://ref1.com'),
expected_metadata_1=dict(references=['http://ref1.com']),
metadata_2=dict(references={'add': ['http://ref2', 'http://ref3.com']}),
expected_error_loc_2=('metadata', 'references')),
id='references-not-valid-URL')])
def test_list_quantities(proc_infra, purged_app, example_data_writeable, example_datasets, test_users_dict, kwargs): def test_list_quantities(proc_infra, purged_app, example_data_writeable, example_datasets, test_users_dict, kwargs):
def replace_dataset_ref(dataset_ref): def replace_dataset_ref(dataset_ref):
if dataset_ref.startswith('ref:'): if dataset_ref.startswith('ref:'):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment