Scheduled maintenance on Monday 2019-06-24 between 10:00-11:00 CEST

Commit 78bb3949 authored by Scheidgen, Markus (mscheidg)'s avatar Scheidgen, Markus (mscheidg)

Merge branch 'migration' into 'master'

Migration, merge 0.4.3 with parsers

Closes #126

See merge request !36
parents ad663ee6 f0091c44
Pipeline #45854 passed with stages
in 3 minutes and 23 seconds
......@@ -15,3 +15,4 @@ local/
target/
*.swp
*.vscode
nomad.yaml
......@@ -104,7 +104,7 @@ tests:
RABBITMQ_DEFAULT_USER: rabbitmq
RABBITMQ_DEFAULT_PASS: rabbitmq
RABBITMQ_DEFAULT_VHOST: /
NOMAD_COE_REPO_DB_HOST: postgres
NOMAD_REPOSITORY_DB_HOST: postgres
NOMAD_RABBITMQ_HOST: rabbitmq
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
......@@ -147,7 +147,7 @@ tests:
# RABBITMQ_DEFAULT_USER: rabbitmq
# RABBITMQ_DEFAULT_PASS: rabbitmq
# RABBITMQ_DEFAULT_VHOST: /
# NOMAD_COE_REPO_DB_HOST: postgres
# NOMAD_REPOSITORY_DB_HOST: postgres
# NOMAD_RABBITMQ_HOST: rabbitmq
# NOMAD_ELASTIC_HOST: elastic
# NOMAD_MONGO_HOST: mongo
......@@ -240,7 +240,9 @@ deploy:
--set dbname=${DBNAME}
--set worker.replicas=1
--set worker.memrequest=32
--set volumes.files=${FILES_PATH};
--set volumes.public=${FILES_PATH}/public
--set volumes.staging=${FILES_PATH}/staging
--set volumes.tmp=${FILES_PATH}/tmp;
else
helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
--set images.nomad.tag=${CI_COMMIT_REF_NAME}
......
......@@ -70,3 +70,33 @@
[submodule "dependencies/parsers/octopus"]
path = dependencies/parsers/octopus
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-octopus
[submodule "dependencies/parsers/phonopy"]
path = dependencies/parsers/phonopy
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-phonopy
[submodule "dependencies/parsers/phonopy-library"]
path = dependencies/parsers/phonopy-library
url = https://gitlab.mpcdf.mpg.de/nomad-lab/phonopy
[submodule "dependencies/parsers/gpaw"]
path = dependencies/parsers/gpaw
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gpaw
[submodule "dependencies/parsers/atk"]
path = dependencies/parsers/atk
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-atk
[submodule "dependencies/parsers/gulp"]
path = dependencies/parsers/gulp
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gulp
[submodule "dependencies/parsers/siesta"]
path = dependencies/parsers/siesta
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-siesta
[submodule "dependencies/parsers/elk"]
path = dependencies/parsers/elk
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-elk
[submodule "dependencies/parsers/elastic"]
path = dependencies/parsers/elastic
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-elastic
[submodule "dependencies/parsers/gamess"]
path = dependencies/parsers/gamess
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gamess
[submodule "dependencies/parsers/turbomole"]
path = dependencies/parsers/turbomole
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-turbomole
.ropeproject/
*.sql
\ No newline at end of file
*.sql
launch.json
\ No newline at end of file
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "chrome",
"request": "launch",
"name": "Launch Chrome against localhost",
"url": "http://localhost:3000",
"webRoot": "${workspaceFolder}/gui"
},
{
"name": "Python: API Flask (0.11.x or later)",
"type": "python",
"request": "launch",
"module": "flask",
"env": {
"FLASK_APP": "nomad/api.py"
},
"args": [
"run",
"--port", "8000",
"--no-debugger",
"--no-reload"
]
},
{
"name": "Python: worker nomad.processing",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/celery",
"args": [
"worker", "-l" , "debug", "-A", "nomad.processing"
]
},
{
"name": "Python: some test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
]
},
{
"name": "Python: crystal normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/crystal-tests/data/parsers/crystal/si.out]"
]
},
{
"name": "Python: cp2k normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/cp2k-tests/data/parsers/cp2k/si_bulk8.out]"
]
},
{
"name": "Python: cpmd normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/cpmd-tests/data/parsers/cpmd/geo_output.out]"
]
},
{
"name": "Python: wien2k normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/wien2k-tests/data/parsers/wien2k/AlN/AlN_ZB.scf]"
]
},
{
"name": "Python: gaussian normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/Al.out]"
]
},
{
"name": "Python: aniline gaussian normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/al-1.out]"
]
},
{
"name": "Python: test_parsing match test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_match"
]
},
{
"name": "Python: nwchem normalizer test h2o sp test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/nwchem-tests/data/parsers/nwchem/sp_output.out]"
]
},
{
"name": "Python: Vasp XML test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/vasp-tests/data/parsers/vasp/vasp.xml]"
]
},
{
"name": "Quantum Espresso Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
]
},
{
"name": "Abinit Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/abinit-tests/data/parsers/abinit/Fe.out]"
]
},
{
"name": "Castep Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/castep-tests/data/parsers/castep/BC2N-Pmm2-Raman.castep]"
]
},
{
"name": "DL-Poly Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/dl-poly-tests/data/parsers/dl-poly/OUTPUT]"
]
},
{
"name": "Lib Atoms Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/lib-atoms-tests/data/parsers/lib-atoms/gp.xml]"
]
},
{
"name": "Octopus Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/octopus-tests/data/parsers/octopus/stdout.txt]"
]
},
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}"
},
{
"name": "Python: Attach",
"type": "python",
"request": "attach",
"localRoot": "${workspaceFolder}",
"remoteRoot": "${workspaceFolder}",
"port": 3000,
"secret": "my_secret",
"host": "localhost"
}
]
}
\ No newline at end of file
......@@ -55,6 +55,13 @@ your browser.
## Change log
### v0.4.3
- more flexible celery routing
- config via nomad.yml
- repo_db can be disabled
- publishing of calculations with failed processing
- cli for managing running processing tasks
### v0.4.2
- bugfixes regarding the migration
- better migration configurability and reproducibility
......
Subproject commit b232a5fdd11c344656ff1e6ec7a2ed6f787cf49f
Subproject commit 8c917126279cbbecb7adb7e30fc7d641d9a31a78
Subproject commit 18f8ccde48f39a9237882064e20e5a1e7baff5c0
Subproject commit 428877e7795cc1760d681b71519b5ba7ec61336c
Subproject commit b03240e3d78b7e4f58974252e7d845229ec81188
Subproject commit 5c7cf8add659c51d940960daa430927923720f7e
Subproject commit f9979f05e6a26e0512c26d5a316f4b0749528640
Subproject commit 523a9c202aa97612ed616f6545e4f84882a4687f
Subproject commit d08b1f7f6273c9e8b857a43cb76956cba2040df9
Subproject commit cb54c37d0c1b85cd57486f572de7bc2dbc294123
Subproject commit 67e781f7288b7e87da0a25ec02bf33d12de436f5
Subproject commit 92005ec9ff4b8e13bd86373d14bd5fafe2b52cd1
Subproject commit f5be159e49f643d220497726da9801138ed615c3
Subproject commit 3b199d1f17f9cdf94e08fc5a3b46e2895147463d
Subproject commit d7bfc62deebfd40d3033817c529d1f6f89c901e5
Subproject commit 22bc82b9d41354cfb571b2659e317ec376c9eb08
Subproject commit 83a4de4efa40731f7e192c795fc1e82f00f27fc8
Subproject commit 9b5f343563313ca1ad8fc84f8fc127eeba64e42b
Subproject commit a2f03d731496565e065c6f56769bc9135c0fe17f
Subproject commit dcc29f7418dd193c53e8815bdbe20a32ccaf37ea
Subproject commit 7505b8b16035ec22025875bb6dfbe22f812ddcdd
Subproject commit 4802d6d8ab942d2431a39d37e2ffefaf5c6ef478
Subproject commit aa7414dad899c0ff568802e80e030c13f97afe3c
Subproject commit 190b1dc12fe09a9ab1ad08d317fa08205c6f75ae
Subproject commit 7547643f594f612752e0d96d77fd3705c2a809b2
Subproject commit 628e4b333004ffa7636b70a99b13efea606119e1
Subproject commit fcda3a9ab4a23ea0c7363955999943d9be6fa0fb
Subproject commit 0e0c20ab4adea61ad31f8a7e96876434008fb490
......@@ -6,13 +6,17 @@
"@material-ui/core": "^3.9.0",
"@material-ui/icons": "^3.0.2",
"@navjobs/upload": "^3.1.3",
"autosuggest-highlight": "^3.1.1",
"base-64": "^0.1.0",
"chroma-js": "^2.0.3",
"d3": "^5.9.1",
"fetch": "^1.1.0",
"file-saver": "^2.0.0",
"html-to-react": "^1.3.3",
"marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14",
"react": "^16.4.2",
"react-autosuggest": "^9.4.3",
"react-cookie": "^3.0.8",
"react-copy-to-clipboard": "^5.0.1",
"react-dom": "^16.4.2",
......
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles } from '@material-ui/core/styles'
import ChipInput from 'material-ui-chip-input'
import Autosuggest from 'react-autosuggest'
import match from 'autosuggest-highlight/match'
import parse from 'autosuggest-highlight/parse'
import Paper from '@material-ui/core/Paper'
import MenuItem from '@material-ui/core/MenuItem'
function renderInput(inputProps) {
const { classes, autoFocus, value, onChange, onAdd, onDelete, chips, ref, ...other } = inputProps
return (
<ChipInput
clearInputValueOnChange
onUpdateInput={onChange}
onAdd={onAdd}
onDelete={onDelete}
value={chips}
inputRef={ref}
{...other}
/>
)
}
function renderSuggestion(suggestion, { query, isHighlighted }) {
const matches = match(getSuggestionValue(suggestion), query)
const parts = parse(getSuggestionValue(suggestion), matches)
return (
<MenuItem
selected={isHighlighted}
component='div'
onMouseDown={(e) => e.preventDefault()} // prevent the click causing the input to be blurred
>
<div>
{parts.map((part, index) => {
return part.highlight ? (
<span key={String(index)} style={{ fontWeight: 300 }}>
{part.text}
</span>
) : (
<strong key={String(index)} style={{ fontWeight: 500 }}>
{part.text}
</strong>
)
})}
</div>
</MenuItem>
)
}
function renderSuggestionsContainer(options) {
const { containerProps, children } = options
return (
<Paper {...containerProps} square>
{children}
</Paper>
)
}
function getSuggestionValue(suggestion) {
return `${suggestion.key}=${suggestion.value}`
}
class SearchBar extends React.Component {
static styles = theme => ({
root: {
height: 250,
flexGrow: 1
},
container: {
position: 'relative'
},
suggestionsContainerOpen: {
position: 'absolute',
zIndex: 100,
marginTop: theme.spacing.unit,
left: 0,
right: 0
},
suggestion: {
display: 'block'
},
suggestionsList: {
margin: 0,
padding: 0,
listStyleType: 'none'
},
divider: {
height: theme.spacing.unit * 2
},
textField: {
width: '100%'
}
})
static propTypes = {
classes: PropTypes.object.isRequired,
aggregations: PropTypes.object.isRequired,
metric: PropTypes.string.isRequired,
values: PropTypes.object.isRequired,
onChanged: PropTypes.func.isRequired
}
state = {
suggestions: [],
textFieldInput: ''
}
getSuggestions(value) {
value = value.toLowerCase()
const { aggregations } = this.props
const suggestions = []
Object.keys(aggregations).forEach(aggKey => {
Object.keys(aggregations[aggKey]).forEach(aggValue => {
if (aggValue.toLowerCase().startsWith(value)) {
suggestions.push({
key: aggKey,
value: aggValue
})
}
})
})
return suggestions
}
handleSuggestionsFetchRequested = ({ value }) => {
this.setState({
suggestions: this.getSuggestions(value)
})
};
handleSuggestionsClearRequested = () => {
this.setState({
suggestions: []
})
};
handleTextFieldInputChange = (event, { newValue }) => {
this.setState({
textFieldInput: newValue
})
};
handleAddChip(chip) {
const values = {...this.props.values}
let key, value
if (chip.includes('=')) {
const parts = chip.split('=')
key = parts[0]
value = parts[1]
} else {
const suggestion = this.getSuggestions(chip)[0]
key = suggestion.key
value = suggestion.value
}
if (values[key]) {
values[key] = key === 'atoms' ? [...values[key], value] : value
} else {
values[key] = key === 'atoms' ? [value] : value
}
this.setState({
textFieldInput: ''
})
this.props.onChanged(values)
}
handleBeforeAddChip(chip) {
const suggestions = this.getSuggestions(chip)
if (suggestions.length > 0) {
return true
} else {
return false
}
}
handleDeleteChip(chip) {
if (!chip) {
return
}
const parts = chip.split('=')
const key = parts[0]
const values = {...this.props.values}
delete values[key]
this.props.onChanged(values)
}
getChips() {
const values = {...this.props.values}
return Object.keys(values).filter(key => values[key]).map(key => {
if (key === 'atoms') {
return `atoms=[${values[key].join(',')}]`
} else {
return `${key}=${values[key]}`
}
})
}
render() {
const { classes, values, onChanged, ...rest } = this.props
return (
<Autosuggest
theme={{
container: classes.container,
suggestionsContainerOpen: classes.suggestionsContainerOpen,
suggestionsList: classes.suggestionsList,
suggestion: classes.suggestion
}}
renderInputComponent={renderInput}
suggestions={this.state.suggestions}
onSuggestionsFetchRequested={this.handleSuggestionsFetchRequested}
onSuggestionsClearRequested={this.handleSuggestionsClearRequested}
renderSuggestionsContainer={renderSuggestionsContainer}
getSuggestionValue={getSuggestionValue}
renderSuggestion={renderSuggestion}
onSuggestionSelected={(e, { suggestionValue }) => { this.handleAddChip(suggestionValue); e.preventDefault() }}
focusInputOnSuggestionClick={false}
inputProps={{
classes,
chips: this.getChips(),
onChange: this.handleTextFieldInputChange,
value: this.state.textFieldInput,
onAdd: (chip) => this.handleAddChip(chip),
onBeforeAdd: (chip) => this.handleBeforeAddChip(chip),
onDelete: (chip, index) => this.handleDeleteChip(chip, index),
...rest
}}
/>
)
}
}
export default withStyles(SearchBar.styles)(SearchBar)
......@@ -12,12 +12,12 @@ import ListItemIcon from '@material-ui/core/ListItemIcon'
import ListItemText from '@material-ui/core/ListItemText'
import BackupIcon from '@material-ui/icons/Backup'
import SearchIcon from '@material-ui/icons/Search'
import AccountIcon from '@material-ui/icons/AccountCircle'
// import AccountIcon from '@material-ui/icons/AccountCircle'
import DocumentationIcon from '@material-ui/icons/Help'
import HomeIcon from '@material-ui/icons/Home'
import ArchiveIcon from '@material-ui/icons/Storage'
import EncIcon from '@material-ui/icons/Assessment'
import AnalyticsIcon from '@material-ui/icons/Settings'
// import ArchiveIcon from '@material-ui/icons/Storage'
// import EncIcon from '@material-ui/icons/Assessment'
// import AnalyticsIcon from '@material-ui/icons/Settings'
import DevelIcon from '@material-ui/icons/ReportProblem'
import ChevronLeftIcon from '@material-ui/icons/ChevronLeft'
import MenuIcon from '@material-ui/icons/Menu'
......@@ -33,7 +33,7 @@ const drawerWidth = 200
const toolbarTitles = {
'/': 'Welcome',
'/repo': 'Raw Code Outputs',
'/repo': 'Search',
'/upload': 'Upload Your Own Data',
'/profile': 'Your Profile',
'/docs': 'Documentation',
......@@ -210,7 +210,7 @@ class Navigation extends React.Component {
<ListItemIcon>
<SearchIcon style={{fill: repoTheme.palette.primary.main}}/>
</ListItemIcon>
<ListItemText inset primary="Repository"/>
<ListItemText inset primary="Search"/>
</MenuItem>
<MenuItem className={classes.menuItem} component={Link} to="/upload" selected={ pathname === '/upload' }>
<ListItemIcon>
......@@ -218,7 +218,7 @@ class Navigation extends React.Component {
</ListItemIcon>
<ListItemText inset primary="Upload"/>
</MenuItem>
<MenuItem className={classes.menuItem} component={Link} to="/archive" selected={ pathname.startsWith('/archive') }>
{/* <MenuItem className={classes.menuItem} component={Link} to="/archive" selected={ pathname.startsWith('/archive') }>