Commit 7dc8cdbf authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.4.6-bugfixes' into 'master'

V0.4.6 bugfixes

See merge request !47
parents 7fae3069 c7fb69e5
Pipeline #52258 canceled with stages
in 15 seconds
......@@ -51,6 +51,7 @@ buildgui:
stage: build
script:
- cd gui
- ./version.sh
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
......
......@@ -57,6 +57,12 @@ your browser.
## Change log
### v0.4.6
- admin commands to directly manipulate upload data
- additional migration scripts
- fixed system normalizer to understand indexed atom labels correctly
- many minor bugfixes
### v0.4.5
- improved uploads view with published uploads
- support for publishing to the existing nomad CoE repository
......
File added
......@@ -5,17 +5,16 @@ that might be necessary to integrate external project data.
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from bravado.exception import HTTPNotFound
from urllib.parse import urlparse
import time
import os.path
import sys
nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/api'
nomad_url = 'https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing/api'
user = 'leonard.hofstadter@nomad-fairdi.tests.de'
password = 'password'
upload_file = 'external_project_example.zip'
upload_file = os.path.join(os.path.dirname(__file__), 'external_project_example.zip')
# create the bravado client
host = urlparse(nomad_url).netloc.split(':')[0]
......@@ -24,14 +23,17 @@ http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
# upload data
print('uploading a file with "external_id/AcAg/vasp.xml" inside ...')
with open(upload_file, 'rb') as f:
upload = client.uploads.upload(file=f).response().result
print('processing ...')
while upload.tasks_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(5)
print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
# check if processing was a success
# check if processing was a success
if upload.tasks_status != 'SUCCESS':
print('something went wrong')
print('errors: %s' % str(upload.errors))
......@@ -40,22 +42,44 @@ if upload.tasks_status != 'SUCCESS':
sys.exit(1)
# publish data
print('publishing ...')
client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
'operation': 'publish',
'metadata': {
# these metadata are applied to all calcs in the upload
'comment': 'Data from a cool external project',
'references': ['http://external.project.eu'],
# 'coauthors': ['sheldon.cooper@ucla.edu'], this does not yet work with emails
# 'external_id': 'external_id' this does also not work, but we could implement something like this
'calculations': [
{
# these metadata are only applied to the calc identified by its 'mainfile'
'mainfile': 'external_id/AcAg/vasp.xml',
# 'coauthors': ['sheldon.cooper@ucla.edu'], this does not YET work with emails,
# Currently you have to use user_ids: leonard (the uploader, who is automatically an author) is 2 and sheldon is 1.
# Ask NOMAD developers about how to find out about user_ids.
'coauthors': [1],
# If users demand, we can implement a specific metadata keys (e.g. 'external_id', 'external_url') for external projects.
# This could allow to directly search for, or even have API endpoints that work with external_ids
# 'external_id': 'external_id',
# 'external_url': 'http://external.project.eu/data/calc/external_id/'
}
]
}
}).response().result
while upload.process_running:
try:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(1)
except HTTPNotFound:
# upload gets deleted from the upload staging area once published
break
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(1)
if upload.tasks_status != 'SUCCESS' or len(upload.errors) > 0:
print('something went wrong')
print('errors: %s' % str(upload.errors))
# delete the unsuccessful upload
client.uploads.delete_upload(upload_id=upload.upload_id).response().result
sys.exit(1)
# search for data
result = client.repo.search(paths='external_id').response().result
......@@ -65,11 +89,15 @@ if result.pagination.total == 0:
elif result.pagination.total > 1:
print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
# The results key holds an array with the current page data
calc = result.results[0]
print('Found the following calcs for my "external_id".')
print(', '.join(calc['calc_id'] for calc in result.results))
# download data
calc = result.results[0]
client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
print('Download of first calc works.')
# download urls, e.g. for curl
print('Possible download URLs are:')
print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))
"""
This is a brief example demonstrating the public nomad@FAIRDI API for doing operations
that might be necessary to integrate external project data.
"""
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
import math
from urllib.parse import urlparse
from concurrent.futures import ThreadPoolExecutor
# nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api'
nomad_url = 'http://localhost:8000/nomad/api/'
user = 'admin'
password = 'password'
upload_file = 'external_project_example.zip'
# create the bravado client
host = urlparse(nomad_url).netloc.split(':')[0]
http_client = RequestsClient()
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
uploads = [upload.upload_id for upload in client.uploads.get_uploads().response().result]
executor = ThreadPoolExecutor(max_workers=10)
def run(upload_id):
upload = client.uploads.get_upload(upload_id=upload_id).response().result
upload_total_calcs = upload.calcs.pagination.total
per_page = 200
for page in range(1, math.ceil(upload_total_calcs / per_page) + 1):
search = client.repo.search(
page=page, per_page=per_page, order_by='mainfile',
upload_id=upload_id).response().result
print(search.pagination.page)
for upload in uploads:
executor.submit(lambda: run(upload))
#!/bin/sh
echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe)\", \"$(git rev-parse --verify HEAD)\" > nomad/gitinfo.py
\ No newline at end of file
echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify HEAD)\" > nomad/gitinfo.py
\ No newline at end of file
{
"name": "nomad-fair-gui",
"version": "0.4.5",
"version": "nomad-gui-version-placeholder",
"private": true,
"dependencies": {
"@material-ui/core": "^3.9.0",
......
......@@ -41,6 +41,7 @@ class LoginLogout extends React.Component {
super(props)
this.handleLogout = this.handleLogout.bind(this)
this.handleChange = this.handleChange.bind(this)
this.handleKeyPress = this.handleKeyPress.bind(this)
}
state = {
......@@ -82,7 +83,7 @@ class LoginLogout extends React.Component {
handleChange = name => event => {
this.setState({
[name]: event.target.value
[name]: event.target.value, failure: false
})
}
......@@ -93,6 +94,13 @@ class LoginLogout extends React.Component {
}
}
handleKeyPress(ev) {
if (ev.key === 'Enter') {
ev.preventDefault()
this.handleLoginDialogClosed(true)
}
}
render() {
const { classes, user, variant, color, isLoggingIn } = this.props
const { failure } = this.state
......@@ -141,6 +149,7 @@ class LoginLogout extends React.Component {
fullWidth
value={this.state.userName}
onChange={this.handleChange('userName')}
onKeyPress={this.handleKeyPress}
/>
<TextField
autoComplete="current-password"
......@@ -152,6 +161,7 @@ class LoginLogout extends React.Component {
fullWidth
value={this.state.password}
onChange={this.handleChange('password')}
onKeyPress={this.handleKeyPress}
/>
</FormGroup>
</form>
......
......@@ -4,7 +4,7 @@ import { withErrors } from './errors'
import { UploadRequest } from '@navjobs/upload'
import Swagger from 'swagger-client'
import { apiBase } from '../config'
import { Typography, withStyles } from '@material-ui/core'
import { Typography, withStyles, Link } from '@material-ui/core'
import LoginLogout from './LoginLogout'
import { Cookies, withCookies } from 'react-cookie'
import { compose } from 'recompose'
......@@ -455,12 +455,18 @@ class LoginRequiredUnstyled extends React.Component {
})
render() {
const {classes, isLoggingIn, onLoggedIn} = this.props
const {classes, isLoggingIn, onLoggedIn, message} = this.props
let loginMessage = ''
if (message) {
loginMessage = <Typography>
{this.props.message} Register for a Nomad Repository account <Link href='http://nomad-repository.eu:8080/NomadRepository-1.1/register/'>here</Link>.
</Typography>
}
return (
<div className={classes.root}>
<Typography>
{this.props.message || ''}
</Typography>
{loginMessage}
<LoginLogout onLoggedIn={onLoggedIn} variant="outlined" color="primary" isLoggingIn={isLoggingIn}/>
</div>
)
......
......@@ -61,7 +61,7 @@ class DomainProviderBase extends React.Component {
Keep in mind that there are limitations:
* You can only login with users that already exist in the Nomad Repository. If you
are new to Nomad, visit the [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1)
and create a user account there.
or register for a user account [here](http://nomad-repository.eu:8080/NomadRepository-1.1/register/).
* When you published your data here, it will still take a day to index. Therefore,
your data will not appear in the Nomad Repository immediately.
* The existing entries from the original Nomad do not appear in the search. We
......
......@@ -96,7 +96,7 @@ class ElementUnstyled extends React.Component {
? <Typography
classes={{root: classes.count}} variant="caption"
style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
{count}
{count.toLocaleString()}
</Typography> : ''
}
</div>
......
......@@ -237,7 +237,7 @@ class SearchBar extends React.Component {
getSuggestionValue={getSuggestionValue}
renderSuggestion={renderSuggestion}
onSuggestionSelected={(e, { suggestionValue }) => { this.handleAddChip(suggestionValue); e.preventDefault() }}
focusInputOnSuggestionClick={false}
focusInputOnSuggestionClick={true}
inputProps={{
classes,
chips: this.getChips(),
......
......@@ -17,6 +17,7 @@ import { Help } from '../help'
class SearchPage extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
match: PropTypes.any,
api: PropTypes.object.isRequired,
user: PropTypes.object,
raiseError: PropTypes.func.isRequired,
......@@ -69,7 +70,7 @@ class SearchPage extends React.Component {
state = {
data: SearchPage.emptySearchData,
owner: 'migrated',
owner: 'all',
searchState: {
...SearchAggregations.defaultState
},
......@@ -85,6 +86,8 @@ class SearchPage extends React.Component {
this.updateSearchResultList = this.updateSearchResultList.bind(this)
this.updateSearch = this.updateSearch.bind(this)
this.handleClickExpand = this.handleClickExpand.bind(this)
this._mounted = false
}
updateSearchResultList(changes) {
......@@ -102,6 +105,10 @@ class SearchPage extends React.Component {
}
update(changes) {
if (!this._mounted) {
return
}
changes = changes || {}
const { owner, searchResultListState, searchState } = {...this.state, ...changes}
const { searchValues, ...searchStateRest } = searchState
......@@ -123,12 +130,22 @@ class SearchPage extends React.Component {
}
componentDidMount() {
this._mounted = true
this.update()
}
componentWillUnmount() {
this._mounted = false
}
componentDidUpdate(prevProps) {
if (prevProps.api !== this.props.api) {
if (prevProps.api !== this.props.api) { // login/logout case, reload results
this.update()
} else if (prevProps.match.path !== this.props.match.path) { // navigation case
// update if we went back to the search
if (this.props.match.path === '/search' && this.props.match.isExact) {
this.update()
}
}
}
......@@ -147,7 +164,6 @@ class SearchPage extends React.Component {
const { pagination: { total }, metrics } = data
const ownerLabel = {
migrated: 'With PID',
all: 'All entries',
public: 'Only public entries',
user: 'Only your entries',
......@@ -155,14 +171,13 @@ class SearchPage extends React.Component {
}
const ownerTooltips = {
migrated: 'Only show entries with established provenance in the original Nomad repository.',
all: 'This will show all entries in the database, even those that might be duplicates.',
public: 'Do not show entries that are only visible to you.',
user: 'Do only show entries visible to you.',
staging: 'Will only show entries that you uploaded, but not yet published.'
}
const withoutLogin = ['migrated', 'all']
const withoutLogin = ['all']
const useMetric = Object.keys(metrics).find(metric => metric !== 'code_runs') || 'code_runs'
const helperText = <span>
......@@ -214,7 +229,7 @@ class SearchPage extends React.Component {
<FormControl>
<FormLabel>Filter entries and show: </FormLabel>
<FormGroup row>
{['migrated', 'all', 'public', 'user', 'staging']
{['all', 'public', 'user', 'staging']
.filter(key => user || withoutLogin.indexOf(key) !== -1)
.map(owner => (
<Tooltip key={owner} title={ownerTooltips[owner]}>
......@@ -258,7 +273,7 @@ class SearchPage extends React.Component {
<div className={classes.searchResults}>
<Typography variant="caption" style={{margin: 12}}>
About {total} results:
About {total.toLocaleString()} results:
</Typography>
<SearchResultList
......
......@@ -234,23 +234,25 @@ class Uploads extends React.Component {
const { selectedUnpublishedUploads, showPublishDialog } = this.state
const unpublishedUploads = this.state.unpublishedUploads || []
if (unpublishedUploads.length === 0) {
return ''
}
const reloadButton = <Tooltip title="reload uploads" >
<IconButton onClick={() => this.update()}><ReloadIcon /></IconButton>
</Tooltip>
return (<div className={classes.uploadsContainer}>
<div style={{width: '100%'}}>
<FormLabel className={classes.uploadsLabel}>Your unpublished uploads: </FormLabel>
<FormGroup className={classes.selectFormGroup} row>
<FormControlLabel label="all" style={{flexGrow: 1}} control={(
<Checkbox
checked={selectedUnpublishedUploads.length === unpublishedUploads.length && unpublishedUploads.length !== 0}
onChange={(_, checked) => this.onSelectionAllChanged(checked)}
/>
)} />
<Tooltip title="reload uploads" >
<IconButton onClick={() => this.update()}><ReloadIcon /></IconButton>
</Tooltip>
{(unpublishedUploads.length === 0) ? ''
: <FormLabel className={classes.uploadsLabel}>Your unpublished uploads: </FormLabel>
}
<FormGroup className={classes.selectFormGroup} style={{alignItems: 'center'}}row>
{(unpublishedUploads.length === 0) ? <FormLabel label="all" style={{flexGrow: 1}}>You have currently no unpublished uploads</FormLabel>
: <FormControlLabel label="all" style={{flexGrow: 1}} control={(
<Checkbox
checked={selectedUnpublishedUploads.length === unpublishedUploads.length && unpublishedUploads.length !== 0}
onChange={(_, checked) => this.onSelectionAllChanged(checked)}
/>
)} />
}
{reloadButton}
<FormLabel classes={{root: classes.selectLabel}}>
{`selected uploads ${selectedUnpublishedUploads.length}/${unpublishedUploads.length}`}
</FormLabel>
......@@ -283,31 +285,31 @@ class Uploads extends React.Component {
</FormGroup>
</div>
<div className={classes.uploads}>
<div>
<Help cookie="uploadList">{`
These are all your uploads in the *staging area*. You can see the
progress on data progresses and review your uploads before publishing
them to the *nomad repository*.
Select uploads to delete or publish them. Click on uploads to see individual
calculations. Click on calculations to see more details on each calculation.
When you select and click publish, you will be ask if you want to publish
with or without the optional *embargo period*.
`}</Help>
{
this.sortedUnpublishedUploads().map(upload => (
{
(unpublishedUploads.length === 0)
? ''
: <div className={classes.uploads}>
<Help cookie="uploadList">{`
These are all your uploads in the *staging area*. You can see the
progress on data progresses and review your uploads before publishing
them to the *nomad repository*.
Select uploads to delete or publish them. Click on uploads to see individual
calculations. Click on calculations to see more details on each calculation.
When you select and click publish, you will be ask if you want to publish
with or without the optional *embargo period*.
`}</Help>
{ this.sortedUnpublishedUploads().map(upload => (
<Upload key={upload.gui_upload_id} upload={upload}
checked={selectedUnpublishedUploads.indexOf(upload) !== -1}
onDoesNotExist={() => this.handleDoesNotExist(upload)}
onPublished={() => this.handlePublished(upload)}
onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
))
}
</div>
</div>
}
</div>
}
</div>)
}
......@@ -385,4 +387,4 @@ class Uploads extends React.Component {
}
}
export default compose(withApi(true, false, 'To upload data, you must have a nomad account and you must be logged in.'), withCookies, withStyles(Uploads.styles))(Uploads)
export default compose(withApi(true, false, 'To upload data, you must have a Nomad Repository account and you must be logged in.'), withCookies, withStyles(Uploads.styles))(Uploads)
#!/bin/sh
version=`git describe --tags`
sed -i -e "s/nomad-gui-version-placeholder/$version/g" package.json
rm -f package.json-e
\ No newline at end of file
......@@ -16,5 +16,11 @@
Swagger/bravado based python client library for the API and various usefull shell commands.
"""
from nomad.utils import POPO
from . import upload, run
from .__main__ import cli
from .__main__ import cli as cli_main
def cli():
cli_main(obj=POPO()) # pylint: disable=E1120,E1123
......@@ -29,7 +29,8 @@ from nomad.search import Search
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
def cli(verbose: bool, debug: bool, config: str):
@click.pass_context
def cli(ctx, verbose: bool, debug: bool, config: str):
if config is not None:
nomad_config.load_config(config_file=config)
......@@ -64,13 +65,37 @@ def qa(skip_tests: bool):
@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
@click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
@click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
@click.option('--skip-es', is_flag=True, help='Skip cleaning the es index.')
def clean(dry, skip_fs, skip_es):
def clean(dry, skip_calcs, skip_fs, skip_es):
infrastructure.setup_logging()
infrastructure.setup_mongo()
mongo_client = infrastructure.setup_mongo()
infrastructure.setup_elastic()
if not skip_calcs:
uploads_for_calcs = mongo_client[nomad_config.mongo.db_name]['calc'].distinct('upload_id')
uploads = {}
for upload in mongo_client[nomad_config.mongo.db_name]['upload'].distinct('_id'):
uploads[upload] = True
missing_uploads = []
for upload_for_calc in uploads_for_calcs:
if upload_for_calc not in uploads:
missing_uploads.append(upload_for_calc)
if not dry and len(missing_uploads) > 0:
input('Will delete calcs (mongo + es) for %d missing uploads. Press any key to continue ...' % len(missing_uploads))
for upload in missing_uploads:
mongo_client[nomad_config.mongo.db_name]['calc'].remove(dict(upload_id=upload))
Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
else:
print('Found %s uploads that have calcs in mongo, but there is no upload entry.' % len(missing_uploads))
print('List first 10:')
for upload in missing_uploads[:10]:
print(upload)
if not skip_fs:
upload_dirs = []
for bucket in [nomad_config.fs.public, nomad_config.fs.staging]:
......@@ -120,4 +145,4 @@ def clean(dry, skip_fs, skip_es):
if __name__ == '__main__':
cli() # pylint: disable=E1120
cli(obj={}) # pylint: disable=E1120,E1123
......@@ -15,28 +15,22 @@
import click
from tabulate import tabulate
from mongoengine import Q
from pymongo import UpdateOne
from nomad import processing as proc, infrastructure, utils, search, files
from nomad import processing as proc, infrastructure, utils, search, files, coe_repo
from .__main__ import cli
uploads = None
query = None
@cli.group(help='Upload related commands')
@click.option('--upload', help='Select upload of with given id', type=str)
@click.option('--user', help='Select uploads of user with given id', type=str)
@click.option('--staging', help='Select only uploads in staging', is_flag=True)
@click.option('--processing', help='Select only processing uploads', is_flag=True)
def upload(upload: str, user: str, staging: bool, processing: bool):
@click.pass_context
def upload(ctx, user: str, staging: bool, processing: bool):