Merge branch 'v0.7.0' into 'master'

V0.7.0

See merge request !71

.gitignore

@@ -17,5 +17,6 @@ local/
 target/
 *.swp
 *.vscode
+.vscode/
 vscode/
 nomad.yaml

.gitlab-ci.yml

@@ -76,10 +76,6 @@ tests:
   stage: test
   image: $TEST_IMAGE
   services:
-    - postgres:latest
-      # this will cause a warning, as the gitlab ci runner health check will test the wrong
-      # port on rabbitmq container:
-      # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
     - rabbitmq
     - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
       alias: elastic
@@ -87,17 +83,15 @@ tests:
       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
       command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
   variables:
-    POSTGRES_DB: nomad
-    POSTGRES_USER: postgres
-    POSTGRES_PASSWORD: nomad
     RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
     RABBITMQ_DEFAULT_USER: rabbitmq
     RABBITMQ_DEFAULT_PASS: rabbitmq
     RABBITMQ_DEFAULT_VHOST: /
-    NOMAD_REPOSITORY_DB_HOST: postgres
     NOMAD_RABBITMQ_HOST: rabbitmq
     NOMAD_ELASTIC_HOST: elastic
     NOMAD_MONGO_HOST: mongo
+    NOMAD_KEYCLOAK_CLIENT_SECRET: ${CI_KEYCLOAK_TEST_CLIENT_SECRET}
+    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
   script:
     - cd /app
     - python -m pytest --cov=nomad -sv tests

Dockerfile

@@ -44,6 +44,7 @@ RUN pip install mdanalysis==0.16.2
 
 # Make will be necessary to build the docs with sphynx
 RUN apt-get update && apt-get install -y make
+RUN apt-get update && apt-get install -y vim
 
 # We also install the -dev dependencies, to use this image for test and qa
 COPY requirements.txt /install/requirements.txt

README.md

@@ -78,6 +78,13 @@ your browser.
 ## Change log
 Omitted versions are plain bugfix releases with only minor changes and fixes.
 
+### v0.7.0
+- User metadata editing and datasets with DOIs
+- Revised GUI lists (entries, grouped entries, datasets, uploads)
+- Keycloak based user management
+- Rawfile preview
+- no dependencies with the old NOMAD CoE Repository
+
 ### v0.6.2
 - GUI performance enhancements
 - API /raw/query endpoint takes file pattern to further filter download contents and

docs/introduction.md

@@ -81,13 +81,11 @@ processing of uploaded files and therein contained calculations. We use
 [mongoengine](http://docs.mongoengine.org/) to program with mongodb.
 
 
-### PostgreSQL
+### Keycloak
 
-A relational database is used to store all user provided metadata: users, datasets
-(curated sets of uploaded data), references, comments, DOIs, coauthors, etc.
-Furthermore, it is still used to store some of the calculation metadata derived
-via parsing. *This will most likely move out of Postgres in the future.* We
-use [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) as on ORM framework.
+[Keycloak](https://www.keycloak.org/) is used for user management. It manages users and
+provide functions for registering, password forget, editing user accounts, and single
+sign on of fairdi@nomad and other related services.
 
 
 ### flask, et al.
@@ -124,7 +122,7 @@ The component library [Material-UI](https://material-ui.com/)
 ### docker
 
 To run a **nomad@FAIRDI** instance, many services have to be orchestrated:
-the nomad app, nomad worker, mongodb, Elasticsearch, PostgreSQL, RabbitMQ,
+the nomad app, nomad worker, mongodb, Elasticsearch, Keycloak, RabbitMQ,
 Elasticstack (logging), the nomad GUI, and a reverse proxy to keep everything together.
 Further services might be needed (e.g. JypiterHUB), when nomad grows.
 The container platform [Docker](https://docs.docker.com/) allows us to provide all services
@@ -224,7 +222,7 @@ passed, stored, etc. by the various nomad modules.
 ### Implementation
 The different entities have often multiple implementations for different storage systems.
 For example, aspects of calculations are stored in files (raw files, calc metadata, archive data),
-Postgres (user metadata), Elasticsearch (metadata), and mongodb (processing state).
+Elasticsearch (metadata), and mongodb (metadata, processing state).
 Different transformation between different implementations exist. See
 :py:mod:`nomad.datamodel` for further information.
 

docs/reference.rst

@@ -42,10 +42,6 @@ nomad.search
 .. automodule:: nomad.search
     :members:
 
-nomad.coe_repo
---------------
-.. automodule:: nomad.coe_repo
-
 nomad.app
 ---------
 .. automodule:: nomad.app

docs/setup.md

@@ -281,7 +281,7 @@ The rest should be mocked or provided by the tests. Make sure that you do no run
 worker, as they will fight for tasks in the queue.
 ```
 cd ops/docker-compose
-docker-compose up -d elastic rabbitmq postgres
+docker-compose up -d elastic rabbitmq
 cd ../..
 pytest -svx tests
 ```

examples/external_project_parallel_upload/upload.py

@@ -143,7 +143,7 @@ def publish_upload(upload, calc_metadata):
         'comment': 'Data from a cool external project',
         'references': ['http://external.project.eu'],
         # '_uploader': <nomad_user_id>,  # only works if the admin user is publishing
-        # 'co_authors': [<nomad_user_id>, <nomad_user_id>, <nomad_user_id>]
+        # 'coauthors': [<nomad_user_id>, <nomad_user_id>, <nomad_user_id>]
         # these are calc specific metadata that supercede any upload metadata
         'calculations': calc_metadata}
 

gui/package.json

@@ -3,16 +3,18 @@
   "version": "nomad-gui-version-placeholder",
   "private": true,
   "dependencies": {
-    "@material-ui/core": "^3.9.0",
+    "@material-ui/core": "^3.9.3",
     "@material-ui/icons": "^3.0.2",
     "@navjobs/upload": "^3.1.3",
     "autosuggest-highlight": "^3.1.1",
     "base-64": "^0.1.0",
     "chroma-js": "^2.0.3",
+    "clsx": "^1.0.4",
     "d3": "^5.9.1",
     "fetch": "^1.1.0",
     "file-saver": "^2.0.0",
     "html-to-react": "^1.3.3",
+    "keycloak-js": "^6.0.0",
     "marked": "^0.6.0",
     "material-ui-chip-input": "^1.0.0-beta.14",
     "material-ui-flat-pagination": "^3.2.0",
@@ -28,7 +30,9 @@
     "react-dom": "^16.4.2",
     "react-dropzone": "^5.0.1",
     "react-highlight": "^0.12.0",
+    "react-infinite-scroller": "^1.2.4",
     "react-json-view": "^1.19.1",
+    "react-keycloak": "^6.1.0",
     "react-router-dom": "^4.3.1",
     "react-router-hash-link": "^1.2.0",
     "react-scripts": "1.1.4",

gui/public/env.js

 window.nomadEnv = {
+  'keycloakBase': 'https://labdev-nomad.esc.rzg.mpg.de/fairdi/keycloak/auth/',
+  'keycloakRealm': 'fairdi_nomad_test',
+  'keycloakClientId': 'nomad_gui_dev',
   'appBase': 'http://localhost:8000/fairdi/nomad/latest',
   'kibanaBase': '/fairdi/kibana',
   'debug': false

gui/public/index.html

@@ -35,7 +35,7 @@
     <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
     <title>NOMAD upload</title>
   </head>
-  <body>
+  <body style="overflow-y: hidden;">
     <noscript>
       You need to enable JavaScript to run this app.
     </noscript>

gui/public/pace.css

@@ -36,7 +36,7 @@ img.bg {
 }
 
 .pace .pace-progress {
-  background: #feb60d;
+  background: #ffb300;
   position: fixed;
   z-index: 2000;
   top: 0;