From c6bd4b75a8a6099e100468a3825f54ee5c731b9b Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Wed, 27 Mar 2019 09:23:31 +0100 Subject: [PATCH 01/14] Added exception digest to elk logging. [skip ci] --- nomad/utils.py | 5 ++++- ops/scripts/misc.http | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/nomad/utils.py b/nomad/utils.py index ec0eeb5cb..da79e0199 100644 --- a/nomad/utils.py +++ b/nomad/utils.py @@ -154,7 +154,10 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): if key in ('event', 'stack_info', 'id', 'timestamp'): continue elif key in ['exception']: - pass + digest_value = str(value) + if len(digest_value) > 256: + digest_value = digest_value[:256] + message['exception.digest'] = digest_value elif key in ( 'upload_id', 'calc_id', 'mainfile', 'service', 'release'): diff --git a/ops/scripts/misc.http b/ops/scripts/misc.http index f6f988ff7..37d1f5277 100644 --- a/ops/scripts/misc.http +++ b/ops/scripts/misc.http @@ -26,6 +26,36 @@ content-type: application/json DELETE http://localhost:9200/calcs HTTP/1.1 +### +# Get errors from ELK +GET http://localhost:29200/logstash-2019.03.27,logstash-2019.03.26/_search HTTP/1.1 +content-type: application/json + +{ + "size": 10, + "query": { + "match": { + "level.keyword": "ERROR" + } + }, + "aggs": { + "event": { + "terms": { + "field": "event.keyword", + "size": 100 + }, + "aggs": { + "upload_path": { + "terms": { + "field": "nomad.migration.upload_path.keyword", + "size": 100 + } + } + } + } + } +} + ### POST http://localhost:9200/calcs/_update_by_query HTTP/1.1 -- GitLab From 94aabdbb96edb92c57e21caf1b4f3ed37f37caa4 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Wed, 27 Mar 2019 09:33:30 +0100 Subject: [PATCH 02/14] Improved exception digest in logging. [skip ci] --- nomad/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nomad/utils.py b/nomad/utils.py index da79e0199..96a01e225 100644 --- a/nomad/utils.py +++ b/nomad/utils.py @@ -153,11 +153,8 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): for key, value in structlog.items(): if key in ('event', 'stack_info', 'id', 'timestamp'): continue - elif key in ['exception']: - digest_value = str(value) - if len(digest_value) > 256: - digest_value = digest_value[:256] - message['exception.digest'] = digest_value + elif key == 'exception': + message['exception.digest'] = str(value)[-256:] elif key in ( 'upload_id', 'calc_id', 'mainfile', 'service', 'release'): -- GitLab From c7e6b5939ea96bd933dcbefce6d861b8a5e93dd7 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Wed, 27 Mar 2019 09:39:42 +0100 Subject: [PATCH 03/14] Improved exception digest in logging. [skip ci] --- nomad/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomad/utils.py b/nomad/utils.py index 96a01e225..333fc2999 100644 --- a/nomad/utils.py +++ b/nomad/utils.py @@ -154,7 +154,7 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): if key in ('event', 'stack_info', 'id', 'timestamp'): continue elif key == 'exception': - message['exception.digest'] = str(value)[-256:] + message['digest'] = str(value)[-256:] elif key in ( 'upload_id', 'calc_id', 'mainfile', 'service', 'release'): @@ -189,7 +189,7 @@ class ConsoleFormatter(LogstashFormatter): level = message_dict.pop('level', None) exception = message_dict.pop('exception', None) time = message_dict.pop('@timestamp', None) - for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version']: + for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version', 'digest']: message_dict.pop(key) keys = list(message_dict.keys()) keys.sort() -- GitLab From 84002c2de16b8b82bc1254cd87797398e86179c1 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Wed, 27 Mar 2019 09:44:05 +0100 Subject: [PATCH 04/14] Improved es query for retrieving package errors from ELK. [skip ci] --- ops/scripts/misc.http | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ops/scripts/misc.http b/ops/scripts/misc.http index 37d1f5277..a908047ef 100644 --- a/ops/scripts/misc.http +++ b/ops/scripts/misc.http @@ -28,11 +28,11 @@ DELETE http://localhost:9200/calcs HTTP/1.1 ### # Get errors from ELK -GET http://localhost:29200/logstash-2019.03.27,logstash-2019.03.26/_search HTTP/1.1 +GET http://localhost:29200/logstash-2019.03.27/_search HTTP/1.1 content-type: application/json { - "size": 10, + "size": 0, "query": { "match": { "level.keyword": "ERROR" @@ -41,7 +41,7 @@ content-type: application/json "aggs": { "event": { "terms": { - "field": "event.keyword", + "field": "digest.keyword", "size": 100 }, "aggs": { -- GitLab From 5e96c927719f35d365c39884fab1989fecff5561 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 09:24:11 +0100 Subject: [PATCH 05/14] Minor fixes. --- dependencies/parsers/vasp | 2 +- ops/scripts/kubernetes_install_master.sh | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/dependencies/parsers/vasp b/dependencies/parsers/vasp index 190b1dc12..039ed6cb5 160000 --- a/dependencies/parsers/vasp +++ b/dependencies/parsers/vasp @@ -1 +1 @@ -Subproject commit 190b1dc12fe09a9ab1ad08d317fa08205c6f75ae +Subproject commit 039ed6cb532b26926f8e0d7dc2027403e965e67c diff --git a/ops/scripts/kubernetes_install_master.sh b/ops/scripts/kubernetes_install_master.sh index 8e07b9871..5d9363d6e 100644 --- a/ops/scripts/kubernetes_install_master.sh +++ b/ops/scripts/kubernetes_install_master.sh @@ -46,7 +46,13 @@ sysctl --system systemctl daemon-reload systemctl restart kubelet -echo "Still have to use kubeadm init/join" +# init master node with flannel +kubeadm init --pod-network-cidr=10.244.0.0/16 +kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml + +# allow to schedule nodes on master +kubectl taint nodes --all node-role.kubernetes.io/master- + + echo "Run on master node to create join command:" -echo "kubeadm token create --print-join-command" -echo "Run join command here" \ No newline at end of file +echo "kubeadm token create --print-join-command" \ No newline at end of file -- GitLab From 071fe5d5afc14a502a0d86a9d5df1bc055d19a94 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 09:26:11 +0100 Subject: [PATCH 06/14] Minor fixes. [skip ci] --- ops/scripts/kubernetes_install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ops/scripts/kubernetes_install.sh b/ops/scripts/kubernetes_install.sh index 76dca66cc..0cf4744e4 100644 --- a/ops/scripts/kubernetes_install.sh +++ b/ops/scripts/kubernetes_install.sh @@ -44,4 +44,7 @@ sysctl --system systemctl daemon-reload systemctl restart kubelet -echo "Still have to use kubeadm init/join" \ No newline at end of file +echo "Still have to use kubeadm init/join" +echo "Run on master:" +echo "kubeadm token create --print-join-command" +echo "Run output here" \ No newline at end of file -- GitLab From c781a8a49ee7a85f2a0b3d1adea981b34e918455 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 13:08:10 +0100 Subject: [PATCH 07/14] Lots of small changes to ops scripts. --- .../nomad/docker-compose.prod.yml | 24 +++++++---- ops/docker-compose/nomad/docker-compose.yml | 1 - .../nomad/templates/worker-deployment.yaml | 6 +-- ops/helm/nomad/values.yaml | 40 +++++++++---------- ops/scripts/migration.env.sh | 7 +--- ops/scripts/migration.values.yaml | 8 ++-- ops/scripts/nomad.latest.env.sh | 3 ++ ops/scripts/tiller-rbac-config.yaml | 18 +++++++++ 8 files changed, 66 insertions(+), 41 deletions(-) create mode 100644 ops/scripts/nomad.latest.env.sh create mode 100644 ops/scripts/tiller-rbac-config.yaml diff --git a/ops/docker-compose/nomad/docker-compose.prod.yml b/ops/docker-compose/nomad/docker-compose.prod.yml index 013c4ae57..df4f778cd 100644 --- a/ops/docker-compose/nomad/docker-compose.prod.yml +++ b/ops/docker-compose/nomad/docker-compose.prod.yml @@ -15,15 +15,25 @@ version: '3.4' services: + + postgres: + ports: + - 5432:5432 + volumes: + - /nomad/fairdi/db/postgres:/var/lib/postgresql/data # the search engine elastic: ports: - - 19200:9200 + - 9200:9200 + volumes: + - /nomad/fairdi/db/elastic:/usr/share/elasticsearch/data # the user data db mongo: ports: - - 37017:27017 + - 27017:27017 + volumes: + - /nomad/fairdi/db/mongo:/data/db # used for centralized logging elk: @@ -34,11 +44,11 @@ services: expose: - 5000 # logstash beats volumes: - - nomad_elk:/var/lib/elasticsearch + - /nomad/fairdi/db/elk:/var/lib/elasticsearch ports: - - 15601:5601 # kibana web - - 15000:5000 - - 29200:9200 # allows metricbeat config to access es + - 5601:5601 # kibana web + - 5000:5000 + - 9201:9200 # allows metricbeat config to access es api: environment: @@ -58,4 +68,4 @@ services: proxy: ports: - - 10080:80 + - 8080:80 diff --git a/ops/docker-compose/nomad/docker-compose.yml b/ops/docker-compose/nomad/docker-compose.yml index ae68974fd..0f01aa7ff 100644 --- a/ops/docker-compose/nomad/docker-compose.yml +++ b/ops/docker-compose/nomad/docker-compose.yml @@ -29,7 +29,6 @@ services: environment: POSTGRES_PASSWORD: 'nomad' POSTGRES_USER: 'postgres' - POSTGRES_DB: 'nomad' volumes: - nomad_postgres:/var/lib/postgresql/data diff --git a/ops/helm/nomad/templates/worker-deployment.yaml b/ops/helm/nomad/templates/worker-deployment.yaml index 972536412..aea1e985d 100644 --- a/ops/helm/nomad/templates/worker-deployment.yaml +++ b/ops/helm/nomad/templates/worker-deployment.yaml @@ -40,7 +40,7 @@ spec: env: - name: NOMAD_SERVICE value: "worker" - - name: NOMAD_CONSOLE_LOGLEVEL + - name: NOMAD_CONSOLE_LOG_LEVEL value: "{{ .Values.worker.console_loglevel }}" - name: NOMAD_LOGSTASH_LEVEL value: "{{ .Values.worker.logstash_loglevel }}" @@ -50,7 +50,7 @@ spec: command: - bash - -c - - NOMAD_LOGSTASH_LEVEL=30 python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" + - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" initialDelaySeconds: 30 periodSeconds: 30 readinessProbe: @@ -58,7 +58,7 @@ spec: command: - bash - -c - - NOMAD_LOGSTASH_LEVEL=30 python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" + - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" initialDelaySeconds: 5 periodSeconds: 120 nodeSelector: diff --git a/ops/helm/nomad/values.yaml b/ops/helm/nomad/values.yaml index beb3a99e7..9c3c9a802 100644 --- a/ops/helm/nomad/values.yaml +++ b/ops/helm/nomad/values.yaml @@ -28,7 +28,7 @@ images: api: replicas: 1 ## Number of gunicorn worker. Recommendation is 2xnum_cores+1 - worker: 21 + worker: 10 port: 8000 console_loglevel: INFO logstash_loglevel: INFO @@ -41,7 +41,7 @@ api: ## Everthing concerning the nomad worker worker: - replicas: 2 + replicas: 1 # request and limit in GB memrequest: 64 memlimit: 420 @@ -59,14 +59,14 @@ gui: # It is run via NodePort service proxy: port: 80 - nodePort: 30003 - nodeIP: 130.183.207.116 + nodePort: 30001 + nodeIP: 130.183.207.104 timeout: 600 datatimeout: 3600 external: - host: "enc-staging-nomad.esc.rzg.mpg.de" + host: "labdev-nomad.esc.rzg.mpg.de" port: 80 - path: "/fairdi/nomad" + path: "/fairdi/nomad/latest" kibanaPath: "/fairdi/kibana" ## configuration of the chart dependency for rabbitmq @@ -80,34 +80,34 @@ rabbitmq: erlangCookie: SWQOKODSQALRPCLNMEQG ## A common name/prefix for all dbs and indices. -dbname: fairdi_nomad +dbname: fairdi_nomad_latest ## The url for the upload page, used in emails to forward the user -uploadurl: 'http://nomad.fairdi.eu/uploads' +uploadurl: 'https://labdev-nomad.esc.mpg.rzg.de/fairdi/nomad/latest/uploads' ## Databases that are not run within the cluster. # To run databases in the cluster, use the nomad-full helm chart. mongo: - host: enc-preprocessing-nomad.esc - port: 37017 + host: nomad-flink-01.esc + port: 27017 elastic: - host: enc-preprocessing-nomad.esc - port: 19200 + host: nomad-flink-01.esc + port: 9200 postgres: sequential_publish: false publish_enabled: true - host: enc-preprocessing-nomad.esc + host: nomad-flink-01.esc port: 5432 logstash: - port: 15000 - host: enc-preprocessing-nomad.esc + port: 5000 + host: nomad-flink-01.esc kibana: - port: 15601 - host: enc-preprocessing-nomad.esc + port: 5601 + host: nomad-flink-01.esc mail: host: '' @@ -119,7 +119,7 @@ mail: ## Everything concerning the data that is used by the service volumes: prefixSize: 2 - public: /nomad/fairdi/fs/public - staging: /nomad/fairdi/fs/staging - tmp: /nomad/fairdi/fs/tmp + public: /nomad/fairdi/latest/fs/public + staging: /nomad/fairdi/latest/fs/staging + tmp: /nomad/fairdi/latest/fs/tmp nomad: /nomad diff --git a/ops/scripts/migration.env.sh b/ops/scripts/migration.env.sh index c9b3faeac..154d998c0 100644 --- a/ops/scripts/migration.env.sh +++ b/ops/scripts/migration.env.sh @@ -1,8 +1,3 @@ -export NOMAD_CLIENT_URL=http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api +export NOMAD_CLIENT_URL=http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api export NOMAD_CLIENT_USER=admin -export NOMAD_LOGSTASH_TCP_PORT=15000 export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages -export NOMAD_FS_STAGING=/nomad/fairdi/migration/fs/staging -export NOMAD_FS_PUBLIC=/nomad/fairdi/migration/fs/public -export NOMAD_FS_TMP=/nomad/fairdi/migration/fs/tmp -export NOMAD_FS_LOCAL_TMP=/scratch/fairdi/tmp diff --git a/ops/scripts/migration.values.yaml b/ops/scripts/migration.values.yaml index 2d6d10787..184c5a1ba 100644 --- a/ops/scripts/migration.values.yaml +++ b/ops/scripts/migration.values.yaml @@ -1,15 +1,15 @@ proxy: - nodePort: 30001 + nodePort: 30002 external: - host: "enc-staging-nomad.esc.rzg.mpg.de" + host: "labdev-nomad.esc.rzg.mpg.de" path: "/fairdi/nomad/migration" worker: - replicas: 2 + replicas: 3 dbname: fairdi_nomad_migration -uploadurl: 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/upload' +uploadurl: 'https://labdev-nomad.rzg.mpg.de/fairdi/nomad/migration/upload' volumes: prefixSize: 2 diff --git a/ops/scripts/nomad.latest.env.sh b/ops/scripts/nomad.latest.env.sh new file mode 100644 index 000000000..bc6abb4ea --- /dev/null +++ b/ops/scripts/nomad.latest.env.sh @@ -0,0 +1,3 @@ +export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/latest/api +export NOMAD_CLIENT_USER=admin +export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages diff --git a/ops/scripts/tiller-rbac-config.yaml b/ops/scripts/tiller-rbac-config.yaml new file mode 100644 index 000000000..1fcf47dca --- /dev/null +++ b/ops/scripts/tiller-rbac-config.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tiller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tiller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: tiller + namespace: kube-system -- GitLab From d5baa8bd1b8c9e9ac1db2f08f546174b9ee7c630 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 13:10:32 +0100 Subject: [PATCH 08/14] Lots of small changes to ops scripts. [skip ci] --- ...igration.env.sh => nomad.migration.env.sh} | 0 ...alues.yaml => nomad.migration.values.yaml} | 0 ops/scripts/{prm.py => parallel_rm.py} | 0 ops/scripts/services_firewall.sh | 22 ------------------- 4 files changed, 22 deletions(-) rename ops/scripts/{migration.env.sh => nomad.migration.env.sh} (100%) rename ops/scripts/{migration.values.yaml => nomad.migration.values.yaml} (100%) rename ops/scripts/{prm.py => parallel_rm.py} (100%) delete mode 100644 ops/scripts/services_firewall.sh diff --git a/ops/scripts/migration.env.sh b/ops/scripts/nomad.migration.env.sh similarity index 100% rename from ops/scripts/migration.env.sh rename to ops/scripts/nomad.migration.env.sh diff --git a/ops/scripts/migration.values.yaml b/ops/scripts/nomad.migration.values.yaml similarity index 100% rename from ops/scripts/migration.values.yaml rename to ops/scripts/nomad.migration.values.yaml diff --git a/ops/scripts/prm.py b/ops/scripts/parallel_rm.py similarity index 100% rename from ops/scripts/prm.py rename to ops/scripts/parallel_rm.py diff --git a/ops/scripts/services_firewall.sh b/ops/scripts/services_firewall.sh deleted file mode 100644 index 63f7ce902..000000000 --- a/ops/scripts/services_firewall.sh +++ /dev/null @@ -1,22 +0,0 @@ -# rabbit -firewall-cmd --permanent --add-port=5672/tcp - -# mongo -firewall-cmd --permanent --add-port=27017/tcp -firewall-cmd --permanent --add-port=37017/tcp - -# raw api -firewall-cmd --permanent --add-port=18001/tcp - -# es -firewall-cmd --permanent --add-port=19200/tcp - -# postgres -firewall-cmd --permanent --add-port=5432/tcp - -# elk -firewall-cmd --permanent --add-port=29200/tcp -firewall-cmd --permanent --add-port=15601/tcp -firewall-cmd --permanent --add-port=15000/tcp - -firewall-cmd --reload \ No newline at end of file -- GitLab From ab25dc2a79eeb49fc55ec7997e876b5da77964f9 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 13:12:52 +0100 Subject: [PATCH 09/14] Add kubeadm reset script. --- ops/scripts/kubernetes_install_master.sh | 2 ++ ops/scripts/kubernetes_reset.sh | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 ops/scripts/kubernetes_reset.sh diff --git a/ops/scripts/kubernetes_install_master.sh b/ops/scripts/kubernetes_install_master.sh index 5d9363d6e..86a3cdc9c 100644 --- a/ops/scripts/kubernetes_install_master.sh +++ b/ops/scripts/kubernetes_install_master.sh @@ -48,6 +48,8 @@ systemctl restart kubelet # init master node with flannel kubeadm init --pod-network-cidr=10.244.0.0/16 +export KUBECONFIG=/etc/kubernetes/admin.conf +sysctl net.bridge.bridge-nf-call-iptables=1 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml # allow to schedule nodes on master diff --git a/ops/scripts/kubernetes_reset.sh b/ops/scripts/kubernetes_reset.sh new file mode 100644 index 000000000..dbb10b419 --- /dev/null +++ b/ops/scripts/kubernetes_reset.sh @@ -0,0 +1,14 @@ +kubeadm reset +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X +ipvsadm --clear +systemctl stop kubelet +systemctl stop docker +rm -rf /var/lib/cni/ +rm -rf /var/lib/kubelet/* +rm -rf /etc/cni/ +ifconfig cni0 down +ifconfig flannel.1 down +ifconfig docker0 down +ip link delete cni0 +ip link delete flannel.1 +systemctl start docker \ No newline at end of file -- GitLab From a41d69fc6c318db43686338bdf7528cf6d5b6ee1 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 13:42:57 +0100 Subject: [PATCH 10/14] Fixed keyerror on missing digest in log entries. --- nomad/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/utils.py b/nomad/utils.py index 333fc2999..9473f20af 100644 --- a/nomad/utils.py +++ b/nomad/utils.py @@ -190,7 +190,7 @@ class ConsoleFormatter(LogstashFormatter): exception = message_dict.pop('exception', None) time = message_dict.pop('@timestamp', None) for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version', 'digest']: - message_dict.pop(key) + message_dict.pop(key, None) keys = list(message_dict.keys()) keys.sort() -- GitLab From cb6eb6723b60c08dbedb3632c0f2055cd3b0ee0f Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 13:43:17 +0100 Subject: [PATCH 11/14] Added support for https. --- nomad/api/app.py | 15 +++++++++++ nomad/api/upload.py | 5 +--- nomad/config.py | 11 +++++--- nomad/processing/data.py | 2 +- ops/helm/nomad/templates/api-deployment.yaml | 28 +++++++++++++------- ops/helm/nomad/templates/nomad-configmap.yml | 1 + ops/helm/nomad/values.yaml | 4 ++- 7 files changed, 47 insertions(+), 19 deletions(-) diff --git a/nomad/api/app.py b/nomad/api/app.py index 8eb1fe28d..b23bc69a1 100644 --- a/nomad/api/app.py +++ b/nomad/api/app.py @@ -31,6 +31,21 @@ from nomad import config, utils base_path = config.services.api_base_path """ Provides the root path of the nomad APIs. """ + +@property +def specs_url(self): + """ + Fixes issue where swagger-ui makes a call to swagger.json over HTTP. + This can ONLY be used on servers that actually use HTTPS. On servers that use HTTP, + this code should not be used at all. + """ + return flask.url_for(self.endpoint('specs'), _external=True, _scheme='https') + + +if config.services.https: + Api.specs_url = specs_url + + app = Flask( __name__, static_url_path='/docs', diff --git a/nomad/api/upload.py b/nomad/api/upload.py index acfa3a8c5..5348ab3fb 100644 --- a/nomad/api/upload.py +++ b/nomad/api/upload.py @@ -392,10 +392,7 @@ class UploadCommandResource(Resource): @login_really_required def get(self): """ Get url and example command for shell based uploads. """ - upload_url = 'http://%s:%s%s/uploads/' % ( - config.services.api_host, - config.services.api_port, - config.services.api_base_path) + upload_url = '%s/uploads' % config.api_url() upload_command = 'curl -X PUT -H "X-Token: %s" "%s" -F file=@' % ( g.user.get_auth_token().decode('utf-8'), upload_url) diff --git a/nomad/config.py b/nomad/config.py index 2ff76cee4..6422bba10 100644 --- a/nomad/config.py +++ b/nomad/config.py @@ -108,7 +108,8 @@ services = NomadConfig( api_secret='defaultApiSecret', admin_password='password', disable_reset=True, - not_processed_value='not processed' + not_processed_value='not processed', + https=False ) tests = NomadConfig( @@ -116,8 +117,12 @@ tests = NomadConfig( ) -def upload_url(): - return 'http://%s:%s/%s/uploads' % (services.api_host, services.api_port, services.api_base_path[:-3]) +def api_url(): + return '%s://%s%s/%s' % ( + 'https' if services.https else 'http', + services.api_host, + ':%s' % services.api_port if services.api_port != 80 else '', + services.api_base_path) migration_source_db = NomadConfig( diff --git a/nomad/processing/data.py b/nomad/processing/data.py index 933c9b93d..e3aafb994 100644 --- a/nomad/processing/data.py +++ b/nomad/processing/data.py @@ -645,7 +645,7 @@ class Upload(Proc): '', 'your data %suploaded %s has completed processing.' % ( self.name if self.name else '', self.upload_time.isoformat()), - 'You can review your data on your upload page: %s' % config.upload_url() + 'You can review your data on your upload page: %s/uploads' % config.api_url()[:-3] ]) try: infrastructure.send_mail( diff --git a/ops/helm/nomad/templates/api-deployment.yaml b/ops/helm/nomad/templates/api-deployment.yaml index 1864cef8e..8d0b59d3e 100644 --- a/ops/helm/nomad/templates/api-deployment.yaml +++ b/ops/helm/nomad/templates/api-deployment.yaml @@ -8,45 +8,47 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + gunicorn.conf: | + secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'} gunicorn.log.conf: | [loggers] keys=root, gunicorn.error, gunicorn.access - + [handlers] keys=console, access, error - + [formatters] keys=generic - + [logger_root] level=INFO handlers=console - + [logger_gunicorn.error] level=INFO handlers=error qualname=gunicorn.error - + [logger_gunicorn.access] level=INFO handlers=access qualname=gunicorn.access - + [handler_console] class=StreamHandler formatter=generic args=(sys.stdout, ) - + [handler_access] class=StreamHandler formatter=generic args=(sys.stdout, ) - + [handler_error] class=StreamHandler formatter=generic args=(sys.stdout, ) - + [formatter_generic] format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s datefmt=%Y-%m-%d %H:%M:%S @@ -83,6 +85,9 @@ spec: - mountPath: /app/gunicorn.log.conf name: gunicorn-log-conf subPath: gunicorn.log.conf + - mountPath: /app/gunicorn.conf + name: gunicorn-conf + subPath: gunicorn.conf - mountPath: /app/.volumes/fs/public name: public-volume - mountPath: /app/.volumes/fs/staging @@ -96,7 +101,7 @@ spec: value: "{{ .Values.api.console_loglevel }}" - name: NOMAD_LOGSTASH_LEVEL value: "{{ .Values.api.logstash_loglevel }}" - command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"] + command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"] livenessProbe: httpGet: path: "{{ .Values.proxy.external.path }}/api/alive" @@ -118,6 +123,9 @@ spec: - name: gunicorn-log-conf configMap: name: {{ include "nomad.fullname" . }}-api-gunicorn-log-config + - name: gunicorn-conf + configMap: + name: {{ include "nomad.fullname" . }}-api-gunicorn-config - name: nomad-conf configMap: name: {{ include "nomad.fullname" . }}-configmap diff --git a/ops/helm/nomad/templates/nomad-configmap.yml b/ops/helm/nomad/templates/nomad-configmap.yml index 08775d10c..445db2e3e 100644 --- a/ops/helm/nomad/templates/nomad-configmap.yml +++ b/ops/helm/nomad/templates/nomad-configmap.yml @@ -24,6 +24,7 @@ data: api_secret: "{{ .Values.api.secret }}" admin_password: "{{ .Values.api.adminPassword }}" disable_reset: {{ .Values.api.disableReset }} + https: {{ .Values.api.https }} rabbitmq: host: "{{ .Release.Name }}-rabbitmq" elastic: diff --git a/ops/helm/nomad/values.yaml b/ops/helm/nomad/values.yaml index 9c3c9a802..50ff57690 100644 --- a/ops/helm/nomad/values.yaml +++ b/ops/helm/nomad/values.yaml @@ -27,9 +27,11 @@ images: ## Everthing concerning the nomad api api: replicas: 1 + https: true ## Number of gunicorn worker. Recommendation is 2xnum_cores+1 worker: 10 - port: 8000 + ## The external port to connect to the api + port: 80 console_loglevel: INFO logstash_loglevel: INFO ## Secret used as cryptographic seed -- GitLab From b41006f62e6854188e0df1c7fbf0d1656648be1a Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 13:52:17 +0100 Subject: [PATCH 12/14] Minor fixes related to https support. --- nomad/api/app.py | 6 +++--- ops/scripts/nomad.migration.env.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nomad/api/app.py b/nomad/api/app.py index b23bc69a1..3630a045b 100644 --- a/nomad/api/app.py +++ b/nomad/api/app.py @@ -16,7 +16,7 @@ All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths. """ -from flask import Flask, jsonify +from flask import Flask, jsonify, url_for from flask_restplus import Api, fields from flask_cors import CORS from werkzeug.exceptions import HTTPException @@ -32,14 +32,14 @@ base_path = config.services.api_base_path """ Provides the root path of the nomad APIs. """ -@property +@property # type: ignore def specs_url(self): """ Fixes issue where swagger-ui makes a call to swagger.json over HTTP. This can ONLY be used on servers that actually use HTTPS. On servers that use HTTP, this code should not be used at all. """ - return flask.url_for(self.endpoint('specs'), _external=True, _scheme='https') + return url_for(self.endpoint('specs'), _external=True, _scheme='https') if config.services.https: diff --git a/ops/scripts/nomad.migration.env.sh b/ops/scripts/nomad.migration.env.sh index 154d998c0..ce80b9dad 100644 --- a/ops/scripts/nomad.migration.env.sh +++ b/ops/scripts/nomad.migration.env.sh @@ -1,3 +1,3 @@ -export NOMAD_CLIENT_URL=http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api +export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api export NOMAD_CLIENT_USER=admin export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages -- GitLab From b6e67221933c6a02e36b289ffa00ca98c73e8c0f Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 14:47:19 +0100 Subject: [PATCH 13/14] Fixed bug with missing atoms in search index. --- gui/src/components/Repo.js | 2 +- nomad/datamodel.py | 6 ++++++ tests/test_api.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gui/src/components/Repo.js b/gui/src/components/Repo.js index 3725ca2ad..a68526181 100644 --- a/gui/src/components/Repo.js +++ b/gui/src/components/Repo.js @@ -8,7 +8,7 @@ import TablePagination from '@material-ui/core/TablePagination' import TableRow from '@material-ui/core/TableRow' import Paper from '@material-ui/core/Paper' import { TableHead, LinearProgress, FormControl, FormControlLabel, Checkbox, FormGroup, - FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel, Grid, CircularProgress } from '@material-ui/core' + FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel, Grid } from '@material-ui/core' import { compose } from 'recompose' import { withErrors } from './errors' import AnalyticsIcon from '@material-ui/icons/Settings' diff --git a/nomad/datamodel.py b/nomad/datamodel.py index 3b51af46e..44cb84382 100644 --- a/nomad/datamodel.py +++ b/nomad/datamodel.py @@ -148,7 +148,13 @@ class CalcWithMetadata(): def update(self, **kwargs): for key, value in kwargs.items(): + if value is None: + continue + if isinstance(value, list): + if len(value) == 0: + continue + if len(value) > 0 and isinstance(value[0], dict) and not isinstance(value[0], utils.POPO): value = list(utils.POPO(**item) for item in value) if isinstance(value, dict) and not isinstance(value, utils.POPO): diff --git a/tests/test_api.py b/tests/test_api.py index 052da1111..a3d9d7790 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -226,7 +226,7 @@ class TestUploads: upload_with_metadata = get_upload_with_metadata(upload) assert_upload_files(upload_with_metadata, files.StagingUploadFiles) - assert_search_upload(upload_with_metadata) + assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system']) def assert_published(self, client, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}): rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth) -- GitLab From 635f95bda3d5484e655eb10e4c1426e9428dd704 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen Date: Thu, 28 Mar 2019 14:50:09 +0100 Subject: [PATCH 14/14] Small fixes to ops scripts. [skip ci] --- ops/helm/nomad/templates/api-deployment.yaml | 15 +++++++++++++-- ops/helm/nomad/templates/nomad-configmap.yml | 2 +- ops/helm/nomad/values.yaml | 3 +-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ops/helm/nomad/templates/api-deployment.yaml b/ops/helm/nomad/templates/api-deployment.yaml index 8d0b59d3e..02c8becaf 100644 --- a/ops/helm/nomad/templates/api-deployment.yaml +++ b/ops/helm/nomad/templates/api-deployment.yaml @@ -8,8 +8,6 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - gunicorn.conf: | - secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'} gunicorn.log.conf: | [loggers] keys=root, gunicorn.error, gunicorn.access @@ -54,6 +52,19 @@ data: datefmt=%Y-%m-%d %H:%M:%S class=logging.Formatter --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nomad.fullname" . }}-api-gunicorn-config + labels: + app.kubernetes.io/name: {{ include "nomad.name" . }}-api-gunicorn-config + helm.sh/chart: {{ include "nomad.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + gunicorn.conf: | + secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'} +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/ops/helm/nomad/templates/nomad-configmap.yml b/ops/helm/nomad/templates/nomad-configmap.yml index 445db2e3e..88c66dbe1 100644 --- a/ops/helm/nomad/templates/nomad-configmap.yml +++ b/ops/helm/nomad/templates/nomad-configmap.yml @@ -24,7 +24,7 @@ data: api_secret: "{{ .Values.api.secret }}" admin_password: "{{ .Values.api.adminPassword }}" disable_reset: {{ .Values.api.disableReset }} - https: {{ .Values.api.https }} + https: {{ .Values.api.https }} rabbitmq: host: "{{ .Release.Name }}-rabbitmq" elastic: diff --git a/ops/helm/nomad/values.yaml b/ops/helm/nomad/values.yaml index 50ff57690..5af34b654 100644 --- a/ops/helm/nomad/values.yaml +++ b/ops/helm/nomad/values.yaml @@ -30,8 +30,7 @@ api: https: true ## Number of gunicorn worker. Recommendation is 2xnum_cores+1 worker: 10 - ## The external port to connect to the api - port: 80 + port: 8000 console_loglevel: INFO logstash_loglevel: INFO ## Secret used as cryptographic seed -- GitLab