Commit 009758ce authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merged fixes from migration. Continued to use domains at all necessary places.

parents c7cb86e0 8b4d6d9a
......@@ -8,7 +8,7 @@ import TablePagination from '@material-ui/core/TablePagination'
import TableRow from '@material-ui/core/TableRow'
import Paper from '@material-ui/core/Paper'
import { TableHead, LinearProgress, FormControl, FormControlLabel, Checkbox, FormGroup,
FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel, Grid, CircularProgress } from '@material-ui/core'
FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel, Grid } from '@material-ui/core'
import { compose } from 'recompose'
import { withErrors } from './errors'
import AnalyticsIcon from '@material-ui/icons/Settings'
......
......@@ -16,7 +16,7 @@
All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths.
"""
from flask import Flask, jsonify
from flask import Flask, jsonify, url_for
from flask_restplus import Api, fields
from flask_cors import CORS
from werkzeug.exceptions import HTTPException
......@@ -31,6 +31,21 @@ from nomad import config, utils
base_path = config.services.api_base_path
""" Provides the root path of the nomad APIs. """
@property # type: ignore
def specs_url(self):
"""
Fixes issue where swagger-ui makes a call to swagger.json over HTTP.
This can ONLY be used on servers that actually use HTTPS. On servers that use HTTP,
this code should not be used at all.
"""
return url_for(self.endpoint('specs'), _external=True, _scheme='https')
if config.services.https:
Api.specs_url = specs_url
app = Flask(
__name__,
static_url_path='/docs',
......
......@@ -392,10 +392,7 @@ class UploadCommandResource(Resource):
@login_really_required
def get(self):
""" Get url and example command for shell based uploads. """
upload_url = 'http://%s:%s%s/uploads/' % (
config.services.api_host,
config.services.api_port,
config.services.api_base_path)
upload_url = '%s/uploads' % config.api_url()
upload_command = 'curl -X PUT -H "X-Token: %s" "%s" -F file=@<local_file>' % (
g.user.get_auth_token().decode('utf-8'), upload_url)
......
......@@ -110,7 +110,8 @@ services = NomadConfig(
admin_password='password',
disable_reset=True,
not_processed_value='not processed',
unavailable_value='unavailable'
unavailable_value='unavailable',
https=False
)
tests = NomadConfig(
......@@ -118,8 +119,12 @@ tests = NomadConfig(
)
def upload_url():
return 'http://%s:%s/%s/uploads' % (services.api_host, services.api_port, services.api_base_path[:-3])
def api_url():
return '%s://%s%s/%s' % (
'https' if services.https else 'http',
services.api_host,
':%s' % services.api_port if services.api_port != 80 else '',
services.api_base_path)
migration_source_db = NomadConfig(
......
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Iterable, List, Dict, Type
from typing import Iterable, List, Dict, Type, Tuple
import datetime
from elasticsearch_dsl import Keyword
......@@ -95,9 +95,7 @@ class CalcWithMetadata():
self.references: List[utils.POPO] = []
self.datasets: List[utils.POPO] = []
# temporary reference to the backend after successful processing
self.backend = None
# parser related general (not domain specific) metadata
self.parser_name = None
self.update(**kwargs)
......@@ -110,7 +108,13 @@ class CalcWithMetadata():
def update(self, **kwargs):
for key, value in kwargs.items():
if value is None:
continue
if isinstance(value, list):
if len(value) == 0:
continue
if len(value) > 0 and isinstance(value[0], dict) and not isinstance(value[0], utils.POPO):
value = list(utils.POPO(**item) for item in value)
if isinstance(value, dict) and not isinstance(value, utils.POPO):
......@@ -143,15 +147,36 @@ class CalcWithMetadata():
class DomainQuantity:
"""
This class can be used to define further details about a domain specific metadata
quantity.
Attributes:
name: The name of the quantity, also the key used to store values in
:class:`CalcWithMetadata`
description: A human friendly description. The description is used to define
the swagger documentation on the relevant API endpoints.
multi: Indicates a list of values. This is important for the elastic mapping.
order_default: Indicates that this metric should be used for the default order of
search results.
aggregations: Indicates that search aggregations (and how many) should be provided.
0 (the default) means no aggregations.
metric: Indicates that this quantity should be used as search metric. Values need
to be tuples with metric name and elastic aggregation (e.g. sum, cardinality)
elastic_mapping: An optional elasticsearch_dsl mapping. Default is ``Keyword``.
"""
def __init__(
self, description: str = None, multi: bool = False, aggregate: bool = False,
self, description: str = None, multi: bool = False, aggregations: int = 0,
order_default: bool = False, metric: Tuple[str, str] = None,
elastic_mapping=None):
self.name: str = None
self.description = description
self.multi = multi
self.aggregate = aggregate
self.order_default = order_default
self.aggregations = aggregations
self.metric = metric
self.elastic_mapping = elastic_mapping
if self.elastic_mapping is None:
......@@ -159,6 +184,21 @@ class DomainQuantity:
class Domain:
"""
A domain defines all metadata quantities that are specific to a certain scientific
domain, e.g. DFT calculations, or experimental material science.
For each domain there needs to define a subclass of :class:`CalcWithMetadata`. This
class has to define the necessary domain specific metadata quantities and how these
are filled from parser results (usually an instance of :class:LocalBackend).
Furthermore, the class method :func:`register_domain` of this ``Domain`` class has
to be used to register a domain with ``domain_nam``. This also allows to provide
further descriptions on each domain specific quantity via instance of :class:`DomainQuantity`.
While there can be multiple domains registered. Currently, only one domain can be
active. This active domain is define in the configuration using the ``domain_name``.
"""
domain_class: Type[CalcWithMetadata] = None
quantities: List[DomainQuantity] = []
......@@ -188,5 +228,5 @@ class Domain:
assert hasattr(reference_domain_calc, name) and not hasattr(reference_general_calc, name), \
'quantity does not exist or overrides general non domain quantity'
# utils.get_logger(__name__).info(
# 'configured domain', domain=domain_name, domain_quantities=len(cls.quantities))
assert any(quantity.order_default for quantity in Domain.quantities), \
'you need to define a order default quantity'
......@@ -646,7 +646,7 @@ class Upload(Proc):
'',
'your data %suploaded %s has completed processing.' % (
self.name if self.name else '', self.upload_time.isoformat()),
'You can review your data on your upload page: %s' % config.upload_url()
'You can review your data on your upload page: %s/uploads' % config.api_url()[:-3]
])
try:
infrastructure.send_mail(
......
......@@ -153,8 +153,8 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
for key, value in structlog.items():
if key in ('event', 'stack_info', 'id', 'timestamp'):
continue
elif key in ['exception']:
pass
elif key == 'exception':
message['digest'] = str(value)[-256:]
elif key in (
'upload_id', 'calc_id', 'mainfile',
'service', 'release'):
......@@ -189,8 +189,8 @@ class ConsoleFormatter(LogstashFormatter):
level = message_dict.pop('level', None)
exception = message_dict.pop('exception', None)
time = message_dict.pop('@timestamp', None)
for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version']:
message_dict.pop(key)
for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version', 'digest']:
message_dict.pop(key, None)
keys = list(message_dict.keys())
keys.sort()
......
......@@ -15,15 +15,25 @@
version: '3.4'
services:
postgres:
ports:
- 5432:5432
volumes:
- /nomad/fairdi/db/postgres:/var/lib/postgresql/data
# the search engine
elastic:
ports:
- 19200:9200
- 9200:9200
volumes:
- /nomad/fairdi/db/elastic:/usr/share/elasticsearch/data
# the user data db
mongo:
ports:
- 37017:27017
- 27017:27017
volumes:
- /nomad/fairdi/db/mongo:/data/db
# used for centralized logging
elk:
......@@ -34,11 +44,11 @@ services:
expose:
- 5000 # logstash beats
volumes:
- nomad_elk:/var/lib/elasticsearch
- /nomad/fairdi/db/elk:/var/lib/elasticsearch
ports:
- 15601:5601 # kibana web
- 15000:5000
- 29200:9200 # allows metricbeat config to access es
- 5601:5601 # kibana web
- 5000:5000
- 9201:9200 # allows metricbeat config to access es
api:
environment:
......@@ -58,4 +68,4 @@ services:
proxy:
ports:
- 10080:80
- 8080:80
......@@ -29,7 +29,6 @@ services:
environment:
POSTGRES_PASSWORD: 'nomad'
POSTGRES_USER: 'postgres'
POSTGRES_DB: 'nomad'
volumes:
- nomad_postgres:/var/lib/postgresql/data
......
......@@ -11,47 +11,60 @@ data:
gunicorn.log.conf: |
[loggers]
keys=root, gunicorn.error, gunicorn.access
[handlers]
keys=console, access, error
[formatters]
keys=generic
[logger_root]
level=INFO
handlers=console
[logger_gunicorn.error]
level=INFO
handlers=error
qualname=gunicorn.error
[logger_gunicorn.access]
level=INFO
handlers=access
qualname=gunicorn.access
[handler_console]
class=StreamHandler
formatter=generic
args=(sys.stdout, )
[handler_access]
class=StreamHandler
formatter=generic
args=(sys.stdout, )
[handler_error]
class=StreamHandler
formatter=generic
args=(sys.stdout, )
[formatter_generic]
format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
datefmt=%Y-%m-%d %H:%M:%S
class=logging.Formatter
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "nomad.fullname" . }}-api-gunicorn-config
labels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-api-gunicorn-config
helm.sh/chart: {{ include "nomad.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
gunicorn.conf: |
secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}
---
apiVersion: apps/v1
kind: Deployment
metadata:
......@@ -83,6 +96,9 @@ spec:
- mountPath: /app/gunicorn.log.conf
name: gunicorn-log-conf
subPath: gunicorn.log.conf
- mountPath: /app/gunicorn.conf
name: gunicorn-conf
subPath: gunicorn.conf
- mountPath: /app/.volumes/fs/public
name: public-volume
- mountPath: /app/.volumes/fs/staging
......@@ -96,7 +112,7 @@ spec:
value: "{{ .Values.api.console_loglevel }}"
- name: NOMAD_LOGSTASH_LEVEL
value: "{{ .Values.api.logstash_loglevel }}"
command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"]
command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"]
livenessProbe:
httpGet:
path: "{{ .Values.proxy.external.path }}/api/alive"
......@@ -118,6 +134,9 @@ spec:
- name: gunicorn-log-conf
configMap:
name: {{ include "nomad.fullname" . }}-api-gunicorn-log-config
- name: gunicorn-conf
configMap:
name: {{ include "nomad.fullname" . }}-api-gunicorn-config
- name: nomad-conf
configMap:
name: {{ include "nomad.fullname" . }}-configmap
......
......@@ -24,6 +24,7 @@ data:
api_secret: "{{ .Values.api.secret }}"
admin_password: "{{ .Values.api.adminPassword }}"
disable_reset: {{ .Values.api.disableReset }}
https: {{ .Values.api.https }}
rabbitmq:
host: "{{ .Release.Name }}-rabbitmq"
elastic:
......
......@@ -40,7 +40,7 @@ spec:
env:
- name: NOMAD_SERVICE
value: "worker"
- name: NOMAD_CONSOLE_LOGLEVEL
- name: NOMAD_CONSOLE_LOG_LEVEL
value: "{{ .Values.worker.console_loglevel }}"
- name: NOMAD_LOGSTASH_LEVEL
value: "{{ .Values.worker.logstash_loglevel }}"
......@@ -50,7 +50,7 @@ spec:
command:
- bash
- -c
- NOMAD_LOGSTASH_LEVEL=30 python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK"
- NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK"
initialDelaySeconds: 30
periodSeconds: 30
readinessProbe:
......@@ -58,7 +58,7 @@ spec:
command:
- bash
- -c
- NOMAD_LOGSTASH_LEVEL=30 python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK"
- NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK"
initialDelaySeconds: 5
periodSeconds: 120
nodeSelector:
......
......@@ -27,8 +27,9 @@ images:
## Everthing concerning the nomad api
api:
replicas: 1
https: true
## Number of gunicorn worker. Recommendation is 2xnum_cores+1
worker: 21
worker: 10
port: 8000
console_loglevel: INFO
logstash_loglevel: INFO
......@@ -41,7 +42,7 @@ api:
## Everthing concerning the nomad worker
worker:
replicas: 2
replicas: 1
# request and limit in GB
memrequest: 64
memlimit: 420
......@@ -59,14 +60,14 @@ gui:
# It is run via NodePort service
proxy:
port: 80
nodePort: 30003
nodeIP: 130.183.207.116
nodePort: 30001
nodeIP: 130.183.207.104
timeout: 600
datatimeout: 3600
external:
host: "enc-staging-nomad.esc.rzg.mpg.de"
host: "labdev-nomad.esc.rzg.mpg.de"
port: 80
path: "/fairdi/nomad"
path: "/fairdi/nomad/latest"
kibanaPath: "/fairdi/kibana"
## configuration of the chart dependency for rabbitmq
......@@ -80,34 +81,34 @@ rabbitmq:
erlangCookie: SWQOKODSQALRPCLNMEQG
## A common name/prefix for all dbs and indices.
dbname: fairdi_nomad
dbname: fairdi_nomad_latest
## The url for the upload page, used in emails to forward the user
uploadurl: 'http://nomad.fairdi.eu/uploads'
uploadurl: 'https://labdev-nomad.esc.mpg.rzg.de/fairdi/nomad/latest/uploads'
## Databases that are not run within the cluster.
# To run databases in the cluster, use the nomad-full helm chart.
mongo:
host: enc-preprocessing-nomad.esc
port: 37017
host: nomad-flink-01.esc
port: 27017
elastic:
host: enc-preprocessing-nomad.esc
port: 19200
host: nomad-flink-01.esc
port: 9200
postgres:
sequential_publish: false
publish_enabled: true
host: enc-preprocessing-nomad.esc
host: nomad-flink-01.esc
port: 5432
logstash:
port: 15000
host: enc-preprocessing-nomad.esc
port: 5000
host: nomad-flink-01.esc
kibana:
port: 15601
host: enc-preprocessing-nomad.esc
port: 5601
host: nomad-flink-01.esc
mail:
host: ''
......@@ -119,7 +120,7 @@ mail:
## Everything concerning the data that is used by the service
volumes:
prefixSize: 2
public: /nomad/fairdi/fs/public
staging: /nomad/fairdi/fs/staging
tmp: /nomad/fairdi/fs/tmp
public: /nomad/fairdi/latest/fs/public
staging: /nomad/fairdi/latest/fs/staging
tmp: /nomad/fairdi/latest/fs/tmp
nomad: /nomad
......@@ -44,4 +44,7 @@ sysctl --system
systemctl daemon-reload
systemctl restart kubelet
echo "Still have to use kubeadm init/join"
\ No newline at end of file
echo "Still have to use kubeadm init/join"
echo "Run on master:"
echo "kubeadm token create --print-join-command"
echo "Run output here"
\ No newline at end of file
......@@ -46,7 +46,15 @@ sysctl --system
systemctl daemon-reload
systemctl restart kubelet
echo "Still have to use kubeadm init/join"
# init master node with flannel
kubeadm init --pod-network-cidr=10.244.0.0/16
export KUBECONFIG=/etc/kubernetes/admin.conf
sysctl net.bridge.bridge-nf-call-iptables=1
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml
# allow to schedule nodes on master
kubectl taint nodes --all node-role.kubernetes.io/master-
echo "Run on master node to create join command:"
echo "kubeadm token create --print-join-command"
echo "Run join command here"
\ No newline at end of file
echo "kubeadm token create --print-join-command"
\ No newline at end of file
kubeadm reset
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
ipvsadm --clear
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig docker0 down
ip link delete cni0
ip link delete flannel.1
systemctl start docker
\ No newline at end of file
export NOMAD_CLIENT_URL=http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api
export NOMAD_CLIENT_USER=admin
export NOMAD_LOGSTASH_TCP_PORT=15000
export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages
export NOMAD_FS_STAGING=/nomad/fairdi/migration/fs/staging
export NOMAD_FS_PUBLIC=/nomad/fairdi/migration/fs/public
export NOMAD_FS_TMP=/nomad/fairdi/migration/fs/tmp
export NOMAD_FS_LOCAL_TMP=/scratch/fairdi/tmp
......@@ -26,6 +26,36 @@ content-type: application/json
DELETE http://localhost:9200/calcs HTTP/1.1
###
# Get errors from ELK
GET http://localhost:29200/logstash-2019.03.27/_search HTTP/1.1
content-type: application/json
{
"size": 0,
"query": {
"match": {
"level.keyword": "ERROR"
}
},
"aggs": {
"event": {
"terms": {
"field": "digest.keyword",
"size": 100
},
"aggs": {
"upload_path": {
"terms": {
"field": "nomad.migration.upload_path.keyword",
"size": 100
}
}
}
}
}
}
###
POST http://localhost:9200/calcs/_update_by_query HTTP/1.1
......
export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/latest/api
export NOMAD_CLIENT_USER=admin
export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages
export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api
export NOMAD_CLIENT_USER=admin
export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages
export NOMAD_MONGO_DB_NAME=coe_migration
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment