Commit 025bcfa7 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merged latest 0.5.x bugfix release. Documentation and version number update.

parents 86f7ab57 9a46c6eb
......@@ -16,4 +16,5 @@ target/
*.swp
*.vscode
.vscode/
vscode/
nomad.yaml
......@@ -76,6 +76,12 @@ Open [http://localhost:8888/html/setup.html](http://localhost:8888/html/setup.ht
your browser.
## Change log
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.5.2
- allows to download large files over longer time period
- streamlined deployment without API+GUI proxy
- minor bugfixes
### v0.5.1
- integrated parsers Dmol3, qbox, molcas, fleur, and onetep
......
Subproject commit a824f0f72303bb8e39c8ae7ee710210edc287331
Subproject commit 7863212e4bde3443fae8fbb57caf3b0d95fb28f3
......@@ -32,10 +32,14 @@ class ConfirmDialog extends React.Component {
<DialogContent>
<Markdown>{`
If you agree the selected uploads will move out of your private staging
area into the public NOMAD Repository. If you wish to put an embargo
on your data it will last upto 36 month. Afterwards, your data will
area into the public [NOMAD Repository](https://repository.nomad-coe.eu/NomadRepository-1.1/).
If you wish to put an embargo on your data it will last upto 36 month. Afterwards, your data will
be made public. All public data will be made available under the Creative
Commons Attribution license ([CC BY 3.0](https://creativecommons.org/licenses/by/3.0/)).
The published data will be added to the NOMAD Repository's index overnight.
Therefore, it will take until tomorrow before your data appears in the
[NOMAD Repository](https://repository.nomad-coe.eu/NomadRepository-1.1/).
`}</Markdown>
<FormGroup row style={{alignItems: 'center'}}>
......
......@@ -49,13 +49,18 @@ class Upload extends React.Component {
width: 350,
overflowX: 'hidden'
},
title: {
shortTitle: {
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
overflowX: 'inherit',
direction: 'rtl',
textAlign: 'left'
},
title: {
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
overflowX: 'inherit'
},
checkbox: {
marginRight: theme.spacing.unit * 2
},
......@@ -110,15 +115,15 @@ class Upload extends React.Component {
}
const {page, perPage, orderBy, order} = this.state.params
const wasPublished = this.state.published
this.state.upload.get(page, perPage, orderBy, order === 'asc' ? 1 : -1)
.then(upload => {
const {tasks_running, process_running, current_task, published} = upload
if (!this._unmounted) {
if (published) {
if (published && !wasPublished) {
if (this.props.onPublished) {
this.props.onPublished()
}
return
}
const continueUpdating = tasks_running || process_running || current_task === 'uploading'
this.setState({upload: upload, updating: continueUpdating})
......@@ -186,7 +191,7 @@ class Upload extends React.Component {
return (
<div className={classes.titleContainer}>
<Typography variant="h6" className={classes.title}>
<Typography variant="h6" className={name ? classes.shortTitle : classes.title}>
{name || new Date(Date.parse(create_time)).toLocaleString()}
</Typography>
{name
......
......@@ -248,12 +248,17 @@ class Uploads extends React.Component {
this.update()
}
onDrop(files) {
files.forEach(file => {
onDrop(files, rejectedFiles) {
const upload = file => {
const upload = this.props.api.createUpload(file.name)
this.setState({unpublishedUploads: [...this.state.unpublishedUploads, upload]})
upload.uploadFile(file).catch(this.props.raiseError)
})
}
files.forEach(upload)
rejectedFiles
.filter(file => file.name.match(/(\.zip)|(\.bz)|(\.tgz)|(\.gz)|(\.bz2)$/i))
.forEach(upload)
}
onSelectionChanged(upload, checked) {
......@@ -391,9 +396,19 @@ class Uploads extends React.Component {
<Paper className={classes.dropzoneContainer}>
<Dropzone
accept={[
'application/zip', 'application/gzip', 'application/bz2', 'application/x-gzip',
'application/x-bz2', 'application/x-gtar', 'application/x-tgz', 'application/tar+gzip',
'application/x-tar', 'application/tar+bz2']}
'application/zip',
'application/gzip',
'application/bz2',
'application/x-gzip',
'application/x-bz2',
'application/x-gtar',
'application/x-tgz',
'application/tar+gzip',
'application/x-tar',
'application/tar+bz2',
'application/x-zip-compressed',
'application/x-compressed',
'application/x-zip']}
className={classes.dropzone}
activeClassName={classes.dropzoneAccept}
rejectClassName={classes.dropzoneReject}
......
......@@ -77,16 +77,47 @@ def query_uploads(ctx, uploads):
@uploads.command(help='List selected uploads')
@click.argument('UPLOADS', nargs=-1)
@click.option('-c', '--calculations', is_flag=True, help='Show details about calculations.')
@click.option('--ids', is_flag=True, help='Only show a list of ids.')
@click.option('--json', is_flag=True, help='Output a JSON array of ids.')
@click.pass_context
def ls(ctx, uploads):
def ls(ctx, uploads, calculations, ids, json):
_, uploads = query_uploads(ctx, uploads)
def row(upload):
row = [
upload.upload_id,
upload.name,
upload.user_id,
upload.process_status,
upload.tasks_status,
upload.published]
if calculations:
row += [
upload.total_calcs,
upload.failed_calcs,
upload.total_calcs - upload.processed_calcs]
return row
headers = ['id', 'name', 'user', 'process', 'tasks', 'published']
if calculations:
headers += ['calcs', 'failed', 'processing']
if ids:
for upload in uploads:
print(upload.upload_id)
return
if json:
print('[%s]' % ','.join(['"%s"' % upload.upload_id for upload in uploads]))
return
print('%d uploads selected, showing no more than first 10' % uploads.count())
print(tabulate(
[
[upload.upload_id, upload.name, upload.user_id, upload.process_status, upload.published]
for upload in uploads[:10]],
headers=['id', 'name', 'user', 'status', 'published']))
[row(upload) for upload in uploads[:10]],
headers=headers))
@uploads.command(help='Change the owner of the upload and all its calcs.')
......
......@@ -200,7 +200,7 @@ def bar_plot(client, retrieve, metric1, metric2=None, title=None):
@client.command(help='Generate various matplotlib charts')
@click.option('--errors', is_flag=True, help='Two charts with relative and absolute parser/normalizer errors per code.')
@click.option('--x-axis', type=str, help='Aggregation used for x-axis, values are "code" and "time".')
@click.option('--y-axis', multiple=True, type=str, help='Metrics used for y-axis, values are "entries", "energies", "users".')
@click.option('--y-axis', multiple=True, type=str, help='Metrics used for y-axis, values are "entries", "energies", "calculations", "users".')
@click.option('--cumulate', is_flag=True, help='Cumulate over x-axis.')
@click.option('--title', type=str, help='Override chart title with given value.')
@click.option('--total', is_flag=True, help='Provide total sums of key metrics.')
......@@ -258,6 +258,11 @@ def statistics(errors, title, x_axis, y_axis, cumulate, total):
'total_energies',
label='total energy calculations',
cumulate=cumulate,
power=0.25 if not cumulate else 1, multiplier=1e-6, format='{x:,.1f}M'),
'calculations': Metric(
'calculations',
label='single configuration calculations',
cumulate=cumulate,
power=0.25 if not cumulate else 1, multiplier=1e-6, format='{x:,.1f}M')
}
......@@ -279,5 +284,5 @@ def statistics(errors, title, x_axis, y_axis, cumulate, total):
bar_plot(client, x_axis, *y_axis, title=title)
if total:
data = client.repo.search(per_page=1, owner='admin', metrics=['total_energies', 'users', 'datasets']).response().result
data = client.repo.search(per_page=1, owner='admin', metrics=['total_energies', 'calculations', 'users', 'datasets']).response().result
print(data.quantities['total'])
......@@ -30,7 +30,7 @@ def parse(
else:
parser_name = parser.__class__.__name__
assert parser is not None, 'there is not parser matching %s' % mainfile
assert parser is not None, 'there is no parser matching %s' % mainfile
logger = logger.bind(parser=parser.name) # type: ignore
logger.info('identified parser')
......
......@@ -151,11 +151,10 @@ tests = NomadConfig(
)
def api_url():
return '%s://%s%s%s' % (
'https' if services.https else 'http',
def api_url(ssl: bool = True):
return '%s://%s%s' % (
'https' if services.https and ssl else 'http',
services.api_host,
':%s' % services.api_port if int(services.api_port) != 80 else '',
services.api_base_path)
......@@ -195,6 +194,7 @@ release = 'devel'
domain = 'DFT'
service = 'unknown nomad service'
auxfile_cutoff = 100
parser_matching_size = 9128
console_log_level = logging.WARNING
......
......@@ -116,7 +116,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile
compression, open_compressed = _compressions.get(f.read(3), (None, open))
with open_compressed(mainfile_path, 'rb') as cf:
buffer = cf.read(2048)
buffer = cf.read(config.parser_matching_size)
mime_type = magic.from_buffer(buffer, mime=True)
for parser in parsers:
......@@ -263,13 +263,10 @@ parsers = [
LegacyParser(
name='parsers/gaussian', code_name='Gaussian',
parser_class_name='gaussianparser.GaussianParser',
# This previous file matching string was too far down the line.
# r'\s*Cite this work as:'
# r'\s*Gaussian [0-9]+, Revision [A-Za-z0-9.]*,'
# r'\s\*\*\*\*\*\*\*\*\*\*\*\**'
# r'\s*Gaussian\s*([0-9]+):\s*([A-Za-z0-9-.]+)\s*([0-9][0-9]?\-[A-Z][a-z][a-z]\-[0-9]+)'
# r'\s*([0-9][0-9]?\-[A-Z][a-z][a-z]\-[0-9]+)')
mainfile_contents_re=r'Gaussian, Inc'),
mainfile_contents_re=(
r'\s*Cite this work as:'
r'\s*Gaussian [0-9]+, Revision [A-Za-z0-9\.]*,')
),
LegacyParser(
name='parsers/quantumespresso', code_name='Quantum Espresso',
parser_class_name='quantumespressoparser.QuantumEspressoParserPWSCF',
......
......@@ -836,7 +836,7 @@ class Upload(Proc):
base = config.api_url()[:-3]
if base.endswith('/'):
base = base[:-1]
return '%s/uploads/' % base
return '%s/gui/uploads/' % base
def _cleanup_after_processing(self):
# send email about process finish
......
images.nomad.tag: "stable"
images.frontend.tag: "stable"
services:
apiSecret: 'nomad-keycloak-prod-api-secret'
proxy:
nodePort: 30011
external:
host: "repository.nomad-coe.eu"
path: "/uploads"
api:
adminPasswordSecret: 'nomad-production-repository-password'
worker:
replicas: 1
routing: "queue"
......
images.nomad.tag: "stable"
images.frontend.tag: "stable"
services:
apiSecret: 'nomad-keycloak-prod-api-secret'
proxy:
nodePort: 30012
external:
host: "repository.nomad-coe.eu"
path: "/uploads"
api:
adminPasswordSecret: 'nomad-production-repository-password'
worker:
replicas: 1
routing: "queue"
......
services:
apiSecret: 'nomad-keycloak-prod-api-secret'
proxy:
nodePort: 30014
external:
host: "labdev-nomad.esc.rzg.mpg.de"
path: "/fairdi/nomad/prod-test"
api:
adminPasswordSecret: 'nomad-production-repository-password'
gui:
debug: true
......
services:
apiSecret: 'nomad-keycloak-prod-api-secret'
proxy:
nodePort: 30013
external:
......
......@@ -64,6 +64,10 @@ metadata:
data:
gunicorn.conf: |
secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}
worker_class = '{{ .Values.api.workerClass }}'
threads = {{ .Values.api.threads }}
worker_connections = 1000
worker = {{ .Values.api.worker }}
---
apiVersion: apps/v1
kind: Deployment
......@@ -112,11 +116,18 @@ spec:
value: "{{ .Values.api.console_loglevel }}"
- name: NOMAD_LOGSTASH_LEVEL
value: "{{ .Values.api.logstash_loglevel }}"
{{ if .Values.api.adminPasswordSecret }}
- name: NOMAD_SERVICES_ADMIN_PASSWORD
{{ if .Values.services.apiSecret }}
- name: NOMAD_SERVICES_API_SECRET
valueFrom:
secretKeyRef:
name: {{ .Values.api.adminPasswordSecret }}
name: {{ .Values.services.apiSecret}}
key: password
{{ end }}
{{ if .Values.keycloak.clientSecret }}
- name: NOMAD_KEYCLOAK_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: {{ .Values.keycloak.clientSecret }}
key: password
{{ end }}
{{ if .Values.keycloak.passwordSecret }}
......@@ -126,7 +137,7 @@ spec:
name: {{ .Values.keycloak.passwordSecret }}
key: password
{{ end }}
command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"]
command: ["python", "-m", "gunicorn.app.wsgiapp", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-b 0.0.0.0:8000", "nomad.api:app"]
livenessProbe:
httpGet:
path: "{{ .Values.proxy.external.path }}/api/alive"
......
......@@ -10,8 +10,8 @@ metadata:
spec:
type: ClusterIP
ports:
- port: {{ .Values.api.port }}
targetPort: {{ .Values.api.port }}
- port: 8000
targetPort: 8000
protocol: TCP
name: http
selector:
......
......@@ -10,13 +10,19 @@ metadata:
data:
nginx.conf: |
server {
listen 8080;
listen 80;
server_name www.example.com;
location {{ .Values.proxy.external.path }} {
return 301 {{ .Values.proxy.external.path }}/gui;
}
location {{ .Values.proxy.external.path }}/gui {
root /app/;
rewrite ^{{ .Values.proxy.external.path }}/gui/(.*)$ /nomad/$1 break;
try_files $uri {{ .Values.proxy.external.path }}/gui/index.html;
}
location {{ .Values.proxy.external.path }}/gui/service-worker.js {
add_header Last-Modified $date_gmt;
add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
......@@ -26,6 +32,33 @@ data:
root /app/;
rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break;
}
location {{ .Values.proxy.external.path }}/api {
proxy_set_header Host $host;
proxy_pass_request_headers on;
proxy_pass http://{{ include "nomad.fullname" . }}-api:8000;
proxy_connect_timeout {{ .Values.proxy.timeout }};
proxy_send_timeout {{ .Values.proxy.timeout }};
proxy_read_timeout {{ .Values.proxy.timeout }};
send_timeout {{ .Values.proxy.timeout }};
}
location {{ .Values.proxy.external.path }}/api/uploads {
client_max_body_size 35g;
proxy_request_buffering off;
proxy_set_header Host $host;
proxy_pass_request_headers on;
proxy_pass http://{{ include "nomad.fullname" . }}-api:8000;
proxy_connect_timeout {{ .Values.proxy.timeout }};
}
location {{ .Values.proxy.external.path }}/api/raw {
proxy_buffering off;
proxy_set_header Host $host;
proxy_pass_request_headers on;
proxy_pass http://{{ include "nomad.fullname" . }}-api:8000;
proxy_connect_timeout {{ .Values.proxy.timeout }};
}
}
env.js: |
window.nomadEnv = {
......@@ -66,7 +99,7 @@ spec:
image: "{{ .Values.images.frontend.name }}:{{ .Values.images.frontend.tag }}"
command: ["./run.sh", "{{ .Values.proxy.external.path }}"]
ports:
- containerPort: 8080
- containerPort: 80
volumeMounts:
- mountPath: /etc/nginx/conf.d
readOnly: true
......@@ -80,13 +113,13 @@ spec:
livenessProbe:
httpGet:
path: "{{ .Values.proxy.external.path }}/gui/index.html"
port: 8080
port: 80
initialDelaySeconds: 15
periodSeconds: 15
readinessProbe:
httpGet:
path: "{{ .Values.proxy.external.path }}/gui/index.html"
port: 8080
port: 80
initialDelaySeconds: 3
periodSeconds: 3
nodeSelector:
......
......@@ -8,12 +8,16 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
type: ClusterIP
type: NodePort
externalIPs:
- {{ .Values.proxy.nodeIP }}
ports:
- port: {{ .Values.gui.port }}
targetPort: {{ .Values.gui.port }}
- nodePort: {{ .Values.proxy.nodePort }}
port: 80
targetPort: 80
protocol: TCP
name: http
selector:
app.kubernetes.io/name: {{ include "nomad.name" . }}-gui
app.kubernetes.io/instance: {{ .Release.Name }}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment