diff --git a/dependencies/parsers/vasp b/dependencies/parsers/vasp index 190b1dc12fe09a9ab1ad08d317fa08205c6f75ae..039ed6cb532b26926f8e0d7dc2027403e965e67c 160000 --- a/dependencies/parsers/vasp +++ b/dependencies/parsers/vasp @@ -1 +1 @@ -Subproject commit 190b1dc12fe09a9ab1ad08d317fa08205c6f75ae +Subproject commit 039ed6cb532b26926f8e0d7dc2027403e965e67c diff --git a/gui/src/components/Repo.js b/gui/src/components/Repo.js index 3725ca2ad55174899a49ec33f0bc904d114cb6aa..a6852618153da77b50c4218e7310cf837cffe5fc 100644 --- a/gui/src/components/Repo.js +++ b/gui/src/components/Repo.js @@ -8,7 +8,7 @@ import TablePagination from '@material-ui/core/TablePagination' import TableRow from '@material-ui/core/TableRow' import Paper from '@material-ui/core/Paper' import { TableHead, LinearProgress, FormControl, FormControlLabel, Checkbox, FormGroup, - FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel, Grid, CircularProgress } from '@material-ui/core' + FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel, Grid } from '@material-ui/core' import { compose } from 'recompose' import { withErrors } from './errors' import AnalyticsIcon from '@material-ui/icons/Settings' diff --git a/nomad/api/app.py b/nomad/api/app.py index 8eb1fe28db7cfb3dee1880915674846021d4bb93..3630a045bfe7008b4d01cb3338f4aa046c82a0ce 100644 --- a/nomad/api/app.py +++ b/nomad/api/app.py @@ -16,7 +16,7 @@ All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths. """ -from flask import Flask, jsonify +from flask import Flask, jsonify, url_for from flask_restplus import Api, fields from flask_cors import CORS from werkzeug.exceptions import HTTPException @@ -31,6 +31,21 @@ from nomad import config, utils base_path = config.services.api_base_path """ Provides the root path of the nomad APIs. """ + +@property # type: ignore +def specs_url(self): + """ + Fixes issue where swagger-ui makes a call to swagger.json over HTTP. + This can ONLY be used on servers that actually use HTTPS. On servers that use HTTP, + this code should not be used at all. + """ + return url_for(self.endpoint('specs'), _external=True, _scheme='https') + + +if config.services.https: + Api.specs_url = specs_url + + app = Flask( __name__, static_url_path='/docs', diff --git a/nomad/api/upload.py b/nomad/api/upload.py index acfa3a8c50ead256e1b9489c422e32b75fc04913..5348ab3fb6baea2b52b1a24f1a4bc01871c7e753 100644 --- a/nomad/api/upload.py +++ b/nomad/api/upload.py @@ -392,10 +392,7 @@ class UploadCommandResource(Resource): @login_really_required def get(self): """ Get url and example command for shell based uploads. """ - upload_url = 'http://%s:%s%s/uploads/' % ( - config.services.api_host, - config.services.api_port, - config.services.api_base_path) + upload_url = '%s/uploads' % config.api_url() upload_command = 'curl -X PUT -H "X-Token: %s" "%s" -F file=@' % ( g.user.get_auth_token().decode('utf-8'), upload_url) diff --git a/nomad/config.py b/nomad/config.py index 2ff76cee40aeaefa4635ccfa03caa9a96915d7dd..6422bba105954bf3d710c734bf8c26645a22f61e 100644 --- a/nomad/config.py +++ b/nomad/config.py @@ -108,7 +108,8 @@ services = NomadConfig( api_secret='defaultApiSecret', admin_password='password', disable_reset=True, - not_processed_value='not processed' + not_processed_value='not processed', + https=False ) tests = NomadConfig( @@ -116,8 +117,12 @@ tests = NomadConfig( ) -def upload_url(): - return 'http://%s:%s/%s/uploads' % (services.api_host, services.api_port, services.api_base_path[:-3]) +def api_url(): + return '%s://%s%s/%s' % ( + 'https' if services.https else 'http', + services.api_host, + ':%s' % services.api_port if services.api_port != 80 else '', + services.api_base_path) migration_source_db = NomadConfig( diff --git a/nomad/datamodel.py b/nomad/datamodel.py index 3b51af46e4671b457dc360ee75091b8b1132016e..44cb843825202bacb97c32f979951b466bea293a 100644 --- a/nomad/datamodel.py +++ b/nomad/datamodel.py @@ -148,7 +148,13 @@ class CalcWithMetadata(): def update(self, **kwargs): for key, value in kwargs.items(): + if value is None: + continue + if isinstance(value, list): + if len(value) == 0: + continue + if len(value) > 0 and isinstance(value[0], dict) and not isinstance(value[0], utils.POPO): value = list(utils.POPO(**item) for item in value) if isinstance(value, dict) and not isinstance(value, utils.POPO): diff --git a/nomad/processing/data.py b/nomad/processing/data.py index 933c9b93d12d29fa0559f57576882d0251d72ef7..e3aafb994e20a32b6286cb1c26078f32965e2249 100644 --- a/nomad/processing/data.py +++ b/nomad/processing/data.py @@ -645,7 +645,7 @@ class Upload(Proc): '', 'your data %suploaded %s has completed processing.' % ( self.name if self.name else '', self.upload_time.isoformat()), - 'You can review your data on your upload page: %s' % config.upload_url() + 'You can review your data on your upload page: %s/uploads' % config.api_url()[:-3] ]) try: infrastructure.send_mail( diff --git a/nomad/utils.py b/nomad/utils.py index ec0eeb5cb93e99eb372469485e344258175c020f..9473f20af4547b6c9516c3fe1383cce29003a2b2 100644 --- a/nomad/utils.py +++ b/nomad/utils.py @@ -153,8 +153,8 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): for key, value in structlog.items(): if key in ('event', 'stack_info', 'id', 'timestamp'): continue - elif key in ['exception']: - pass + elif key == 'exception': + message['digest'] = str(value)[-256:] elif key in ( 'upload_id', 'calc_id', 'mainfile', 'service', 'release'): @@ -189,8 +189,8 @@ class ConsoleFormatter(LogstashFormatter): level = message_dict.pop('level', None) exception = message_dict.pop('exception', None) time = message_dict.pop('@timestamp', None) - for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version']: - message_dict.pop(key) + for key in ['type', 'tags', 'stack_info', 'path', 'message', 'host', '@version', 'digest']: + message_dict.pop(key, None) keys = list(message_dict.keys()) keys.sort() diff --git a/ops/docker-compose/nomad/docker-compose.prod.yml b/ops/docker-compose/nomad/docker-compose.prod.yml index 013c4ae574db319426c49770b814d3586ede1722..df4f778cd67f6eeb9f174d7340a3e7e0fb0183d0 100644 --- a/ops/docker-compose/nomad/docker-compose.prod.yml +++ b/ops/docker-compose/nomad/docker-compose.prod.yml @@ -15,15 +15,25 @@ version: '3.4' services: + + postgres: + ports: + - 5432:5432 + volumes: + - /nomad/fairdi/db/postgres:/var/lib/postgresql/data # the search engine elastic: ports: - - 19200:9200 + - 9200:9200 + volumes: + - /nomad/fairdi/db/elastic:/usr/share/elasticsearch/data # the user data db mongo: ports: - - 37017:27017 + - 27017:27017 + volumes: + - /nomad/fairdi/db/mongo:/data/db # used for centralized logging elk: @@ -34,11 +44,11 @@ services: expose: - 5000 # logstash beats volumes: - - nomad_elk:/var/lib/elasticsearch + - /nomad/fairdi/db/elk:/var/lib/elasticsearch ports: - - 15601:5601 # kibana web - - 15000:5000 - - 29200:9200 # allows metricbeat config to access es + - 5601:5601 # kibana web + - 5000:5000 + - 9201:9200 # allows metricbeat config to access es api: environment: @@ -58,4 +68,4 @@ services: proxy: ports: - - 10080:80 + - 8080:80 diff --git a/ops/docker-compose/nomad/docker-compose.yml b/ops/docker-compose/nomad/docker-compose.yml index ae68974fdc546716edacefff318c1c7f3929372a..0f01aa7fffbbf9ace2d114a95ea794639a89ab9f 100644 --- a/ops/docker-compose/nomad/docker-compose.yml +++ b/ops/docker-compose/nomad/docker-compose.yml @@ -29,7 +29,6 @@ services: environment: POSTGRES_PASSWORD: 'nomad' POSTGRES_USER: 'postgres' - POSTGRES_DB: 'nomad' volumes: - nomad_postgres:/var/lib/postgresql/data diff --git a/ops/helm/nomad/templates/api-deployment.yaml b/ops/helm/nomad/templates/api-deployment.yaml index 1864cef8e3ce0698a1e80eb3dcc3f9bcdc08eb49..02c8becafaad1bce9a5d7163089a51380130067a 100644 --- a/ops/helm/nomad/templates/api-deployment.yaml +++ b/ops/helm/nomad/templates/api-deployment.yaml @@ -11,47 +11,60 @@ data: gunicorn.log.conf: | [loggers] keys=root, gunicorn.error, gunicorn.access - + [handlers] keys=console, access, error - + [formatters] keys=generic - + [logger_root] level=INFO handlers=console - + [logger_gunicorn.error] level=INFO handlers=error qualname=gunicorn.error - + [logger_gunicorn.access] level=INFO handlers=access qualname=gunicorn.access - + [handler_console] class=StreamHandler formatter=generic args=(sys.stdout, ) - + [handler_access] class=StreamHandler formatter=generic args=(sys.stdout, ) - + [handler_error] class=StreamHandler formatter=generic args=(sys.stdout, ) - + [formatter_generic] format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s datefmt=%Y-%m-%d %H:%M:%S class=logging.Formatter --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nomad.fullname" . }}-api-gunicorn-config + labels: + app.kubernetes.io/name: {{ include "nomad.name" . }}-api-gunicorn-config + helm.sh/chart: {{ include "nomad.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + gunicorn.conf: | + secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'} +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -83,6 +96,9 @@ spec: - mountPath: /app/gunicorn.log.conf name: gunicorn-log-conf subPath: gunicorn.log.conf + - mountPath: /app/gunicorn.conf + name: gunicorn-conf + subPath: gunicorn.conf - mountPath: /app/.volumes/fs/public name: public-volume - mountPath: /app/.volumes/fs/staging @@ -96,7 +112,7 @@ spec: value: "{{ .Values.api.console_loglevel }}" - name: NOMAD_LOGSTASH_LEVEL value: "{{ .Values.api.logstash_loglevel }}" - command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"] + command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"] livenessProbe: httpGet: path: "{{ .Values.proxy.external.path }}/api/alive" @@ -118,6 +134,9 @@ spec: - name: gunicorn-log-conf configMap: name: {{ include "nomad.fullname" . }}-api-gunicorn-log-config + - name: gunicorn-conf + configMap: + name: {{ include "nomad.fullname" . }}-api-gunicorn-config - name: nomad-conf configMap: name: {{ include "nomad.fullname" . }}-configmap diff --git a/ops/helm/nomad/templates/nomad-configmap.yml b/ops/helm/nomad/templates/nomad-configmap.yml index 08775d10c4dd6d606902def70bbaac37cf7e8539..88c66dbe140c98cfad78101ba2533196f57a8b26 100644 --- a/ops/helm/nomad/templates/nomad-configmap.yml +++ b/ops/helm/nomad/templates/nomad-configmap.yml @@ -24,6 +24,7 @@ data: api_secret: "{{ .Values.api.secret }}" admin_password: "{{ .Values.api.adminPassword }}" disable_reset: {{ .Values.api.disableReset }} + https: {{ .Values.api.https }} rabbitmq: host: "{{ .Release.Name }}-rabbitmq" elastic: diff --git a/ops/helm/nomad/templates/worker-deployment.yaml b/ops/helm/nomad/templates/worker-deployment.yaml index 97253641261dd56e8258d031418f30f3ebaf813a..aea1e985d21f06737113bec85dc3aadec082aa4e 100644 --- a/ops/helm/nomad/templates/worker-deployment.yaml +++ b/ops/helm/nomad/templates/worker-deployment.yaml @@ -40,7 +40,7 @@ spec: env: - name: NOMAD_SERVICE value: "worker" - - name: NOMAD_CONSOLE_LOGLEVEL + - name: NOMAD_CONSOLE_LOG_LEVEL value: "{{ .Values.worker.console_loglevel }}" - name: NOMAD_LOGSTASH_LEVEL value: "{{ .Values.worker.logstash_loglevel }}" @@ -50,7 +50,7 @@ spec: command: - bash - -c - - NOMAD_LOGSTASH_LEVEL=30 python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" + - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" initialDelaySeconds: 30 periodSeconds: 30 readinessProbe: @@ -58,7 +58,7 @@ spec: command: - bash - -c - - NOMAD_LOGSTASH_LEVEL=30 python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" + - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${HOSTNAME}:.*OK" initialDelaySeconds: 5 periodSeconds: 120 nodeSelector: diff --git a/ops/helm/nomad/values.yaml b/ops/helm/nomad/values.yaml index beb3a99e71683a51542f4f1991291a768c19991f..5af34b6548030ad218472df37705ed42b49dd351 100644 --- a/ops/helm/nomad/values.yaml +++ b/ops/helm/nomad/values.yaml @@ -27,8 +27,9 @@ images: ## Everthing concerning the nomad api api: replicas: 1 + https: true ## Number of gunicorn worker. Recommendation is 2xnum_cores+1 - worker: 21 + worker: 10 port: 8000 console_loglevel: INFO logstash_loglevel: INFO @@ -41,7 +42,7 @@ api: ## Everthing concerning the nomad worker worker: - replicas: 2 + replicas: 1 # request and limit in GB memrequest: 64 memlimit: 420 @@ -59,14 +60,14 @@ gui: # It is run via NodePort service proxy: port: 80 - nodePort: 30003 - nodeIP: 130.183.207.116 + nodePort: 30001 + nodeIP: 130.183.207.104 timeout: 600 datatimeout: 3600 external: - host: "enc-staging-nomad.esc.rzg.mpg.de" + host: "labdev-nomad.esc.rzg.mpg.de" port: 80 - path: "/fairdi/nomad" + path: "/fairdi/nomad/latest" kibanaPath: "/fairdi/kibana" ## configuration of the chart dependency for rabbitmq @@ -80,34 +81,34 @@ rabbitmq: erlangCookie: SWQOKODSQALRPCLNMEQG ## A common name/prefix for all dbs and indices. -dbname: fairdi_nomad +dbname: fairdi_nomad_latest ## The url for the upload page, used in emails to forward the user -uploadurl: 'http://nomad.fairdi.eu/uploads' +uploadurl: 'https://labdev-nomad.esc.mpg.rzg.de/fairdi/nomad/latest/uploads' ## Databases that are not run within the cluster. # To run databases in the cluster, use the nomad-full helm chart. mongo: - host: enc-preprocessing-nomad.esc - port: 37017 + host: nomad-flink-01.esc + port: 27017 elastic: - host: enc-preprocessing-nomad.esc - port: 19200 + host: nomad-flink-01.esc + port: 9200 postgres: sequential_publish: false publish_enabled: true - host: enc-preprocessing-nomad.esc + host: nomad-flink-01.esc port: 5432 logstash: - port: 15000 - host: enc-preprocessing-nomad.esc + port: 5000 + host: nomad-flink-01.esc kibana: - port: 15601 - host: enc-preprocessing-nomad.esc + port: 5601 + host: nomad-flink-01.esc mail: host: '' @@ -119,7 +120,7 @@ mail: ## Everything concerning the data that is used by the service volumes: prefixSize: 2 - public: /nomad/fairdi/fs/public - staging: /nomad/fairdi/fs/staging - tmp: /nomad/fairdi/fs/tmp + public: /nomad/fairdi/latest/fs/public + staging: /nomad/fairdi/latest/fs/staging + tmp: /nomad/fairdi/latest/fs/tmp nomad: /nomad diff --git a/ops/scripts/kubernetes_install.sh b/ops/scripts/kubernetes_install.sh index 76dca66cc1a4fabc49ad70dfac0341240a486cac..0cf4744e49ad33a49e57a6e1efff76618c29a5ae 100644 --- a/ops/scripts/kubernetes_install.sh +++ b/ops/scripts/kubernetes_install.sh @@ -44,4 +44,7 @@ sysctl --system systemctl daemon-reload systemctl restart kubelet -echo "Still have to use kubeadm init/join" \ No newline at end of file +echo "Still have to use kubeadm init/join" +echo "Run on master:" +echo "kubeadm token create --print-join-command" +echo "Run output here" \ No newline at end of file diff --git a/ops/scripts/kubernetes_install_master.sh b/ops/scripts/kubernetes_install_master.sh index 8e07b987197d607b51263b902c776ad0055c0679..86a3cdc9c7e85b2c4ae0465d32c2be592c0c0e4e 100644 --- a/ops/scripts/kubernetes_install_master.sh +++ b/ops/scripts/kubernetes_install_master.sh @@ -46,7 +46,15 @@ sysctl --system systemctl daemon-reload systemctl restart kubelet -echo "Still have to use kubeadm init/join" +# init master node with flannel +kubeadm init --pod-network-cidr=10.244.0.0/16 +export KUBECONFIG=/etc/kubernetes/admin.conf +sysctl net.bridge.bridge-nf-call-iptables=1 +kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml + +# allow to schedule nodes on master +kubectl taint nodes --all node-role.kubernetes.io/master- + + echo "Run on master node to create join command:" -echo "kubeadm token create --print-join-command" -echo "Run join command here" \ No newline at end of file +echo "kubeadm token create --print-join-command" \ No newline at end of file diff --git a/ops/scripts/kubernetes_reset.sh b/ops/scripts/kubernetes_reset.sh new file mode 100644 index 0000000000000000000000000000000000000000..dbb10b41941f96619236dbd52e10fb19aaea8d36 --- /dev/null +++ b/ops/scripts/kubernetes_reset.sh @@ -0,0 +1,14 @@ +kubeadm reset +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X +ipvsadm --clear +systemctl stop kubelet +systemctl stop docker +rm -rf /var/lib/cni/ +rm -rf /var/lib/kubelet/* +rm -rf /etc/cni/ +ifconfig cni0 down +ifconfig flannel.1 down +ifconfig docker0 down +ip link delete cni0 +ip link delete flannel.1 +systemctl start docker \ No newline at end of file diff --git a/ops/scripts/migration.env.sh b/ops/scripts/migration.env.sh deleted file mode 100644 index c9b3faeac98c7b9fe67252b79f6ef135ae047fcf..0000000000000000000000000000000000000000 --- a/ops/scripts/migration.env.sh +++ /dev/null @@ -1,8 +0,0 @@ -export NOMAD_CLIENT_URL=http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api -export NOMAD_CLIENT_USER=admin -export NOMAD_LOGSTASH_TCP_PORT=15000 -export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages -export NOMAD_FS_STAGING=/nomad/fairdi/migration/fs/staging -export NOMAD_FS_PUBLIC=/nomad/fairdi/migration/fs/public -export NOMAD_FS_TMP=/nomad/fairdi/migration/fs/tmp -export NOMAD_FS_LOCAL_TMP=/scratch/fairdi/tmp diff --git a/ops/scripts/misc.http b/ops/scripts/misc.http index f6f988ff7786ab18f919520c4724bfbf66707ef6..a908047efab87358ef727d7ddfb183932c996d0b 100644 --- a/ops/scripts/misc.http +++ b/ops/scripts/misc.http @@ -26,6 +26,36 @@ content-type: application/json DELETE http://localhost:9200/calcs HTTP/1.1 +### +# Get errors from ELK +GET http://localhost:29200/logstash-2019.03.27/_search HTTP/1.1 +content-type: application/json + +{ + "size": 0, + "query": { + "match": { + "level.keyword": "ERROR" + } + }, + "aggs": { + "event": { + "terms": { + "field": "digest.keyword", + "size": 100 + }, + "aggs": { + "upload_path": { + "terms": { + "field": "nomad.migration.upload_path.keyword", + "size": 100 + } + } + } + } + } +} + ### POST http://localhost:9200/calcs/_update_by_query HTTP/1.1 diff --git a/ops/scripts/nomad.latest.env.sh b/ops/scripts/nomad.latest.env.sh new file mode 100644 index 0000000000000000000000000000000000000000..bc6abb4eaaa43257c725c440e9fbb5e81023d345 --- /dev/null +++ b/ops/scripts/nomad.latest.env.sh @@ -0,0 +1,3 @@ +export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/latest/api +export NOMAD_CLIENT_USER=admin +export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages diff --git a/ops/scripts/nomad.migration.env.sh b/ops/scripts/nomad.migration.env.sh new file mode 100644 index 0000000000000000000000000000000000000000..ce80b9dad9f8df2ec47c4232918976048255e58c --- /dev/null +++ b/ops/scripts/nomad.migration.env.sh @@ -0,0 +1,3 @@ +export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api +export NOMAD_CLIENT_USER=admin +export NOMAD_FS_MIGRATION_PACKAGES=/nomad/fairdi/migration/fs/migration_packages diff --git a/ops/scripts/migration.values.yaml b/ops/scripts/nomad.migration.values.yaml similarity index 62% rename from ops/scripts/migration.values.yaml rename to ops/scripts/nomad.migration.values.yaml index 2d6d107876a5dbedf4a59109efe3df87fb8ced2a..184c5a1ba3996c4c51ba6daabf829993e5fe78c0 100644 --- a/ops/scripts/migration.values.yaml +++ b/ops/scripts/nomad.migration.values.yaml @@ -1,15 +1,15 @@ proxy: - nodePort: 30001 + nodePort: 30002 external: - host: "enc-staging-nomad.esc.rzg.mpg.de" + host: "labdev-nomad.esc.rzg.mpg.de" path: "/fairdi/nomad/migration" worker: - replicas: 2 + replicas: 3 dbname: fairdi_nomad_migration -uploadurl: 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/upload' +uploadurl: 'https://labdev-nomad.rzg.mpg.de/fairdi/nomad/migration/upload' volumes: prefixSize: 2 diff --git a/ops/scripts/prm.py b/ops/scripts/parallel_rm.py similarity index 100% rename from ops/scripts/prm.py rename to ops/scripts/parallel_rm.py diff --git a/ops/scripts/services_firewall.sh b/ops/scripts/services_firewall.sh deleted file mode 100644 index 63f7ce90224384c2ed3419caecbc7add090d9055..0000000000000000000000000000000000000000 --- a/ops/scripts/services_firewall.sh +++ /dev/null @@ -1,22 +0,0 @@ -# rabbit -firewall-cmd --permanent --add-port=5672/tcp - -# mongo -firewall-cmd --permanent --add-port=27017/tcp -firewall-cmd --permanent --add-port=37017/tcp - -# raw api -firewall-cmd --permanent --add-port=18001/tcp - -# es -firewall-cmd --permanent --add-port=19200/tcp - -# postgres -firewall-cmd --permanent --add-port=5432/tcp - -# elk -firewall-cmd --permanent --add-port=29200/tcp -firewall-cmd --permanent --add-port=15601/tcp -firewall-cmd --permanent --add-port=15000/tcp - -firewall-cmd --reload \ No newline at end of file diff --git a/ops/scripts/tiller-rbac-config.yaml b/ops/scripts/tiller-rbac-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fcf47dca766e4d2e9700f569011866cf6686c36 --- /dev/null +++ b/ops/scripts/tiller-rbac-config.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tiller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tiller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: tiller + namespace: kube-system diff --git a/tests/test_api.py b/tests/test_api.py index 052da11111ebac946c4f6919f4dd1e735e2600e5..a3d9d7790bb7b3417b52cbc65bcca56147e9807a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -226,7 +226,7 @@ class TestUploads: upload_with_metadata = get_upload_with_metadata(upload) assert_upload_files(upload_with_metadata, files.StagingUploadFiles) - assert_search_upload(upload_with_metadata) + assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system']) def assert_published(self, client, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}): rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)