Merge branch 'bugfixes' into 'v0.10.0'

Bugfixes

See merge request !261

.gitlab-ci.yml

@@ -47,7 +47,7 @@ python linting:
   script:
     - cd /app
     - python -m pycodestyle --ignore=E501,E701,E731 nomad tests
-    - python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
+    - python -m pylint --load-plugins=pylint_mongoengine,nomad.metainfo.pylint_plugin nomad tests
     - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
   except:
     refs:
@@ -176,13 +176,27 @@ deploy prod beta:
     - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
   script:
     - helm dependency update ops/helm/nomad
-    - helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - helm upgrade --install nomad-beta ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
     - docker pull $TEST_IMAGE
     - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
   except:
     - /^dev-.*$/
   when: manual
 
+deploy prod test:
+  stage: release
+  before_script:
+    - mkdir -p /etc/deploy
+    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
+  script:
+    - helm dependency update ops/helm/nomad
+    - helm upgrade --install nomad-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-test-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - docker pull $TEST_IMAGE
+    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
+  except:
+    - /^dev-.*$/
+  when: manual
+
 release latest image:
   stage: release
   script:

materia @ 52e4caf2

-Subproject commit c234b1b184534bbe334acfcd59fc4e0618612814
+Subproject commit 52e4caf2dab39e1ddf3fbff7d9c4adef06851f87

gui/src/components/search/EntryList.js

@@ -104,12 +104,14 @@ export class EntryListUnstyled extends React.Component {
       overflow: 'auto'
     },
     entryDetails: {
-      paddingTop: theme.spacing(3),
-      paddingLeft: theme.spacing(3),
-      paddingRight: theme.spacing(3)
+      paddingTop: theme.spacing(2),
+      paddingLeft: theme.spacing(2),
+      paddingRight: theme.spacing(2)
     },
     entryDetailsContents: {
-      display: 'flex'
+      display: 'flex',
+      maxWidth: 1024,
+      margin: 'auto'
     },
     entryDetailsRow: {
       paddingRight: theme.spacing(3)
@@ -121,9 +123,7 @@ export class EntryListUnstyled extends React.Component {
       flexShrink: 0,
       justifyContent: 'flex-end',
       marginBottom: theme.spacing(1),
-      marginLeft: theme.spacing(0.5),
-      marginRight: theme.spacing(0.5),
-      marginTop: theme.spacing(1)
+      marginTop: theme.spacing(2)
     }
   })
 
@@ -247,17 +247,6 @@ export class EntryListUnstyled extends React.Component {
     }
   }
 
-  selectionQuery() {
-    const { selected } = this.state
-    if (selected) {
-      return {
-        'calc_id': selected.join(',')
-      }
-    } else {
-      return this.props.query
-    }
-  }
-
   renderEntryDetails(row) {
     const { classes } = this.props
     const domain = (row.domain && domains[row.domain]) || domains.dft
@@ -375,7 +364,7 @@ export class EntryListUnstyled extends React.Component {
     />
 
     const example = selected && selected.length > 0 ? results.find(d => d.calc_id === selected[0]) : results[0]
-    const selectQuery = (selected && selected.length > 0) ? {calc_id: selected} : query
+    const selectQuery = (selected && selected.length > 0) ? {calc_id: selected, owner: query['owner']} : query
     const createActions = (props, moreActions) => <React.Fragment>
       {example && editable ? <EditUserMetadataDialog
         example={example} total={selected === null ? totalNumber : selected.length}

nomad/cli/admin/admin.py

@@ -91,7 +91,7 @@ def __run_processing(
                 current_task=upload.current_task, upload_id=upload.upload_id)
             return False
         else:
-            upload.reset()
+            upload.reset(force=True)
             process(upload)
             upload.block_until_complete(interval=.5)
 

nomad/cli/dev.py

@@ -41,7 +41,7 @@ def qa(skip_tests: bool, exitfirst: bool):
     click.echo('Run code style checks ...')
     ret_code += os.system('python -m pycodestyle --ignore=E501,E701,E731 nomad tests')
     click.echo('Run linter ...')
-    ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests')
+    ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine,nomad.metainfo.pylint_plugin nomad tests')
     click.echo('Run static type checks ...')
     ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
 

nomad/processing/base.py

@@ -212,9 +212,9 @@ class Proc(Document, metaclass=ProcMetaclass):
 
         return self
 
-    def reset(self, worker_hostname: str = None):
+    def reset(self, worker_hostname: str = None, force: bool = False):
         ''' Resets the task chain. Assumes there no current running process. '''
-        assert not self.process_running
+        assert not self.process_running or force
 
         self.current_task = None
         self.process_status = None

nomad/processing/data.py

@@ -1302,9 +1302,9 @@ class Upload(Proc):
                 # the join was already done due to a prior call
                 pass
 
-    def reset(self):
+    def reset(self, force=False):
         self.joined = False
-        super().reset()
+        super().reset(force=force)
 
     @classmethod
     def reset_pymongo_update(cls, worker_hostname: str = None):

nomad/search.py

@@ -39,7 +39,7 @@ from nomad.app.v1.models import (
 
 _entry_metadata_defaults = {
     quantity.name: quantity.default
-    for quantity in datamodel.EntryMetadata.m_def.quantities
+    for quantity in datamodel.EntryMetadata.m_def.quantities  # pylint: disable=not-an-iterable
     if quantity.default not in [None, [], False, 0]
 }
 

ops/helm/nomad/deployments/prod-beta-values.yaml

@@ -22,6 +22,9 @@ gui:
   encyclopediaEnabled: true
   aitoolkitEnabled: true
 
+app:
+  replicas: 2
+
 worker:
   replicas: 1
   routing: "queue"

ops/helm/nomad/deployments/prod-test-values.yaml

+version:
+  label: "v0.9.8"
+  isBeta: true
+  usesBetaData: false
+  officialUrl: "https://nomad-lab.eu/prod/rae/gui"
+
+image:
+    tag: "latest"
+    pullPolicy: "Always"
+
+proxy:
+  nodePort: 30014
+  nodeIP: '130.183.207.81'
+  external:
+    host: "nomad-lab.eu"
+    path: "/prod/rae/test"
+
+gui:
+  debug: true
+  matomoEnabled: false
+  matomoSiteId: 2
+  encyclopediaEnabled: true
+  aitoolkitEnabled: true
+
+app:
+  replicas: 1
+
+worker:
+  replicas: 1
+  routing: "queue"
+  processes: 10
+  nomadNodeType: "prod-worker"
+
+elastic:
+  host: elasticsearch.elasticsearch.svc.cluster.local
+  port: 9200
+
+mongo:
+  host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local
+
+logstash:
+  host: logstash.elk.svc.cluster.local
+
+dbname: fairdi_nomad_test_v0_8
+
+uploadurl: 'http://nomad-lab.eu/prod/rae/test/api/uploads'
+
+client:
+  passwordSecret: 'nomad-keycloak-password'
+
+keycloak:
+  serverUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
+  passwordSecret: 'nomad-keycloak-password'
+  realmName: 'fairdi_nomad_prod'
+  clientId: 'nomad_public'
+  admin_user_id: '82efac55-6187-408c-8027-b98580c0e1c5'
+
+volumes:
+  prefixSize: 1
+  public: /nomad/fairdi/test/fs/public
+  staging: /nomad/fairdi/test/fs/staging
+  tmp: /nomad/fairdi/test/fs/tmp
+  nomad: /nomad
+
+mail:
+  enabled: true
+  host: 'mailrelay.mpcdf.mpg.de'
+  port: 25
+  from: 'webmaster@nomad-coe.eu'

ops/helm/nomad/values.yaml

@@ -140,7 +140,7 @@ kibana:
   host: nomad-flink-01.esc
 
 mail:
-  enabled: "false"
+  enabled: false
   host: 'localhost'
   port: 25
   from: 'support@nomad-lab.eu'

requirements.txt

@@ -64,6 +64,7 @@ bagit
 bcrypt
 filelock
 pyjwt[crypto]==1.7.1
+cryptography==3.3.2
 jsonschema[format]
 runstats
 pyyaml

tests/app/v1/routers/test_datasets.py

@@ -104,7 +104,7 @@ def assert_dataset(dataset, query: Query = None, entries: List[str] = None, n_en
 
     mongo_dataset = Dataset.m_def.a_mongo.objects(dataset_id=dataset_id).first()
     assert mongo_dataset is not None
-    for quantity in Dataset.m_def.quantities:
+    for quantity in Dataset.m_def.quantities:  # pylint: disable=not-an-iterable
         if quantity in [Dataset.pid, Dataset.doi]:
             assert quantity.name not in dataset or dataset[quantity.name] is not None
         else:

tests/test_search.py

@@ -339,7 +339,7 @@ def assert_search_upload(
 
 
 if __name__ == '__main__':
-    from test_datamodel import generate_calc
+    from test_datamodel import generate_calc  # pylint: disable=import-error
     from elasticsearch.helpers import bulk
     import sys
     print('Generate index with random example calculation data. First arg is number of items')