Fixed dataset issues. Introduced new visibility to allow search with embargo data.

README.md

@@ -29,6 +29,10 @@ contributing, and API reference.
 
 Omitted versions are plain bugfix releases with only minor changes and fixes.
 
+### v0.7.7
+- Shows dataset contents with embargo data, but hides the entry details (raw-files, archive)
+- minor bugfixes
+
 ### v0.7.5
 - AFLOWLIB prototypes (archive)
 - primitive label search

gui/package.json

 {
   "name": "nomad-fair-gui",
-  "version": "0.7.6",
+  "version": "0.7.7",
   "commit": "nomad-gui-commit-placeholder",
   "private": true,
   "dependencies": {

gui/src/components/DatasetPage.js

@@ -44,6 +44,7 @@ class DatasetPage extends React.Component {
 
   state = {
     dataset: {},
+    empty: false,
     update: 0
   }
 
@@ -58,14 +59,13 @@ class DatasetPage extends React.Component {
       const entry = data.results[0]
       const dataset = entry && entry.datasets.find(ds => ds.id + '' === datasetId)
       if (!dataset) {
-        this.setState({dataset: {}})
-        raiseError(new DoesNotExist('Dataset does not exist any more or is not visible to you.'))
+        this.setState({dataset: {}, empty: true})
       }
       this.setState({dataset: {
-        ...dataset, example: entry
+        ...dataset, example: entry, empty: false
       }})
     }).catch(error => {
-      this.setState({dataset: {}})
+      this.setState({dataset: {}, empty: false})
       raiseError(error)
     })
   }
@@ -76,7 +76,7 @@ class DatasetPage extends React.Component {
 
   componentDidUpdate(prevProps) {
     if (prevProps.api !== this.props.api || prevProps.datasetId !== this.props.datasetId) {
-      this.setState({dataset: {}}, () => this.update())
+      this.setState({dataset: {}, empty: false}, () => this.update())
     }
   }
 
@@ -90,13 +90,13 @@ class DatasetPage extends React.Component {
 
   render() {
     const { classes, datasetId } = this.props
-    const { dataset, update } = this.state
+    const { dataset, update, empty } = this.state
 
     return (
       <div>
         <div className={classes.header}>
           <div className={classes.description}>
-            <Typography variant="h4">{dataset.name || 'loading ...'}</Typography>
+            <Typography variant="h4">{dataset.name || (empty && 'Empty or non existing dataset') ||'loading ...'}</Typography>
             <Typography>
               dataset{dataset.doi ? <span>, with DOI <DOI doi={dataset.doi} /></span> : ''}
             </Typography>
@@ -111,7 +111,9 @@ class DatasetPage extends React.Component {
         </div>
 
         <SearchContext
-          query={{dataset_id: datasetId}} ownerTypes={['all', 'public']} update={update}
+          initialQuery={{owner: 'all'}}
+          query={{dataset_id: datasetId}}
+          ownerTypes={['all', 'public']} update={update}
         >
           <Search resultTab="entries" tabs={['entries', 'groups', 'datasets']} />
         </SearchContext>

gui/src/components/EditUserMetadataDialog.js

@@ -647,7 +647,7 @@ class InviteUserDialogUnstyled extends React.Component {
             If you want to add a user as co-author or share your data with someone that
             is not already a NOMAD user, you can invite this person here. We need just a few
             details about this person. After your invite, the new user will receive an
-            Email that allows him to set a password and further details. Anyhow, you will
+            Email that allows her to set a password and further details. Anyhow, you will
             be able to add the user as co-author or someone to share with immediately after the
             invite.
           </DialogContentText>

gui/src/components/FAQ.js

@@ -107,12 +107,15 @@ class FAQ extends React.Component {
           publishing anything.
 
           Second, you can publish your uploads with an *embargo* period. This can last up to
-          3 years. You can lift the embargo at anytime. This allows you to privately create
-          datasets and DOIs, share data with selected people, before your work is published, e.g.
-          in a paper.
-
-          Non *published* and *embargoed* data is only visible to you (the uploader) and users
-          that you explicitly share your entries with.
+          3 years. You can lift the embargo at anytime. Embargoed data is
+          visible to and findable by others. This makes only some few metadata (e.g.
+          chemical formula, system type, spacegroup, etc.) public, but the raw-file
+          and archive contents remain hidden (except to you, and users you explicitly
+          share the data with).
+          You can already create datasets and assign DOIs for data with embargo, e.g.
+          to put it into your unpublished paper.
+          The embargo will last up to 36 month. Afterwards, your data will be made publicly
+          available. You can also lift the embargo on entries at any time.
 
           ### How do I cite uploaded data in a paper?
 

gui/src/components/api.js

@@ -308,12 +308,13 @@ class Api {
       .finally(this.onFinishLoading)
   }
 
-  async getRawFile(uploadId, path, kwargs) {
+  async getRawFile(uploadId, calcId, path, kwargs) {
     this.onStartLoading()
     const length = (kwargs && kwargs.length) || 4096
     return this.swagger()
-      .then(client => client.apis.raw.get({
+      .then(client => client.apis.raw.get_file_from_calc({
         upload_id: uploadId,
+        calc_id: calcId,
         path: path,
         decompress: true,
         ...(kwargs || {}),
@@ -746,12 +747,12 @@ class WithApiComponent extends React.Component {
     if (notAuthorized) {
       if (keycloak.authenticated) {
         return (
-          <div>
+          <div style={{marginTop: 24}}>
             <Typography variant="h6">Not Authorized</Typography>
             <Typography>
               You are not authorized to access this information. If someone send
-              you this link, ask him to make his data publicly available or share
-              it with you.
+              you a link to this data, ask the authors to make the data publicly available
+              or share it with you.
             </Typography>
           </div>
         )

gui/src/components/entry/RawFiles.js

@@ -130,9 +130,9 @@ class RawFiles extends React.Component {
   }
 
   handleFileClicked(file) {
-    const {api, uploadId, raiseError} = this.props
+    const {api, uploadId, calcId, raiseError} = this.props
     this.setState({shownFile: file, fileContents: null})
-    api.getRawFile(uploadId, file, {length: 16 * 1024})
+    api.getRawFile(uploadId, calcId, file.split('/').reverse()[0], {length: 16 * 1024})
       .then(contents => this.setState({fileContents: contents}))
       .catch(raiseError)
   }

gui/src/components/search/EntryList.js

@@ -12,6 +12,7 @@ import EditUserMetadataDialog from '../EditUserMetadataDialog'
 import DownloadButton from '../DownloadButton'
 import PublishedIcon from '@material-ui/icons/Public'
 import PrivateIcon from '@material-ui/icons/AccountCircle'
+import { withApi } from '../api'
 
 export function Published(props) {
   const {entry} = props
@@ -271,21 +272,33 @@ export class EntryListUnstyled extends React.Component {
           </Quantity>
         </div>
       </div>
+
       <div className={classes.entryDetailsActions}>
-        <Button color="primary" onClick={event => this.handleViewEntryPage(event, row)}>
-          Show raw files and archive
-        </Button>
+        {this.showEntryActions(row) &&
+          <Button color="primary" onClick={event => this.handleViewEntryPage(event, row)}>
+            Show raw files and archive
+          </Button>
+        }
       </div>
     </div>)
   }
 
+  showEntryActions(row) {
+    const { user } = this.props
+    if (row.with_embargo && !(user && row.owners.find(owner => owner.user_id === user.sub))) {
+      return false
+    } else {
+      return !this.props.showEntryActions || this.props.showEntryActions(row)
+    }
+  }
+
   handleViewEntryPage(event, row) {
     event.stopPropagation()
     this.props.history.push(`/entry/id/${row.upload_id}/${row.calc_id}`)
   }
 
   renderEntryActions(row, selected) {
-    if (!this.props.showEntryActions || this.props.showEntryActions(row)) {
+    if (this.showEntryActions(row)) {
       return <Tooltip title="Show raw files and archive">
         <IconButton style={selected ? {color: 'white'} : null} onClick={event => this.handleViewEntryPage(event, row)}>
           <DetailsIcon />
@@ -311,7 +324,7 @@ export class EntryListUnstyled extends React.Component {
 
     const defaultSelectedColumns = this.props.selectedColumns || [
       ...domain.defaultSearchResultColumns,
-      'authors']
+      'published', 'authors']
 
     const pagination = <TablePagination
       count={totalNumber}
@@ -365,6 +378,6 @@ export class EntryListUnstyled extends React.Component {
   }
 }
 
-const EntryList = compose(withRouter, withDomain, withStyles(EntryListUnstyled.styles))(EntryListUnstyled)
+const EntryList = compose(withRouter, withDomain, withApi(false), withStyles(EntryListUnstyled.styles))(EntryListUnstyled)
 
 export default EntryList

gui/src/components/search/Search.js

@@ -371,6 +371,7 @@ class VisualizationSelect extends React.Component {
 class OwnerSelect extends React.Component {
   static ownerLabel = {
     all: 'All entries',
+    visible: 'Include your private entries',
     public: 'Only public entries',
     user: 'Only your entries',
     staging: 'Staging area only'
@@ -378,7 +379,8 @@ class OwnerSelect extends React.Component {
 
   static ownerTooltips = {
     all: 'This will show all entries in the database.',
-    public: 'Do not show entries that are only visible to you.',
+    visible: 'Do also show entries that are only visible to you.',
+    public: 'Do not entries with embargo.',
     user: 'Do only show entries visible to you.',
     staging: 'Will only show entries that you uploaded, but not yet published.'
   }

gui/src/components/search/SearchPage.js

@@ -75,7 +75,7 @@ class SearchPage extends React.Component {
     const { classes, user, location, update } = this.props
 
     let query = {
-      owner: 'all'
+      owner: 'public'
     }
     if (location && location.search) {
       query = {
@@ -91,7 +91,7 @@ class SearchPage extends React.Component {
         <SearchContext
           update={update}
           initialQuery={query}
-          ownerTypes={['all', 'public'].filter(key => user || withoutLogin.indexOf(key) !== -1)}
+          ownerTypes={['public', 'visible'].filter(key => user || withoutLogin.indexOf(key) !== -1)}
         >
           <Search visualization="elements" tabs={['entries', 'groups', 'datasets']} />
         </SearchContext>

gui/src/components/uploads/Upload.js

@@ -50,12 +50,16 @@ class PublishConfirmDialog extends React.Component {
               area into the public NOMAD. This step is final. All public data will be made available under the Creative
               Commons Attribution license ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)).
 
-              If you wish, you can put an embargo on your data. Embargoed data is not
-              visible to others (unless explicitly shared), but you can already create
-              datasets and assign DOIs for data with embargo, e.g. to put it into your
-              unpublished paper. The embargo will last up to 36 month. Afterwards, your
-              data will be made publicly available. You can also lift the embargo on
-              entries at any time. This functionality is part of editing entries.
+              If you wish, you can put an embargo on your data. Embargoed data is
+              visible to and findable by others. This makes some metadata (e.g.
+              chemical formula, system type, spacegroup, etc.) public, but the raw-file
+              and archive contents remain hidden (except to you, and users you explicitly
+              share the data with).
+              You can already create datasets and assign DOIs for data with embargo, e.g.
+              to put it into your unpublished paper.
+              The embargo will last up to 36 month. Afterwards, your data will be made publicly
+              available. You can also lift the embargo on entries at any time.
+              This functionality is part of editing entries.
             `}</Markdown>
 
             <FormControl style={{width: '100%', marginTop: 24}}>

gui/src/components/uploads/UploadPage.js

@@ -65,8 +65,9 @@ If you press publish, a dialog will appear that allows you to set an
 *embargo* or publish your data as *Open Access* right away. The *embargo* allows you to share
 data with selected users, create a DOI for your data, and later publish the data.
 The *embargo* might last up to 36 month before data becomes public automatically.
-During an *embargo* the data (and datasets created from this data) are only visible to you
-and users you share the data with (i.e. users you added under *share with* when editing entries).
+During an *embargo* the data (and datasets created from this data) are already visible and
+findable, but only you and users you share the data with (i.e. users you added under
+*share with* when editing entries) can view and download the raw-data and archive.
 
 #### Processing errors
 

nomad/app/api/archive.py

@@ -306,14 +306,14 @@ class ArchiveQueryResource(Resource):
                     if upload_files is not None:
                         upload_files.close_zipfile_cache()
 
-                    upload_files = UploadFiles.get(
-                        upload_id, create_authorization_predicate(upload_id))
+                    upload_files = UploadFiles.get(upload_id)
 
                     if upload_files is None:
                         raise KeyError
 
                     upload_files.open_zipfile_cache()
 
+                upload_files._is_authorized =  create_authorization_predicate(upload_id, entry['calc_id'])
                 fo = upload_files.archive_file(calc_id, 'rb')
                 data.append(json.loads(fo.read()))
 

nomad/app/api/auth.py

@@ -314,7 +314,14 @@ def create_authorization_predicate(upload_id, calc_id=None):
         # look in mongo
         try:
             upload = processing.Upload.get(upload_id)
-            return g.user.user_id == upload.user_id
+            if g.user.user_id == upload.user_id:
+                return True
+
+            try:
+                calc = processing.Calc.get(calc_id)
+            except KeyError:
+                return False
+            return g.user.user_id in calc.metadata.get('shared_with', [])
 
         except KeyError as e:
             logger = utils.get_logger(__name__, upload_id=upload_id, calc_id=calc_id)

nomad/app/api/common.py

@@ -98,7 +98,7 @@ def add_search_parameters(request_parser):
     # more search parameters
     request_parser.add_argument(
         'owner', type=str,
-        help='Specify which calcs to return: ``all``, ``public``, ``user``, ``staging``, default is ``all``')
+        help='Specify which calcs to return: ``visible``, ``public``, ``all``, ``user``, ``staging``, default is ``visible``')
     request_parser.add_argument(
         'from_time', type=lambda x: rfc3339DateTime.parse(x),
         help='A yyyy-MM-ddTHH:mm:ss (RFC3339) minimum entry time (e.g. upload time)')
@@ -120,7 +120,7 @@ def apply_search_parameters(search_request: search.SearchRequest, args: Dict[str
     args = {key: value for key, value in args.items() if value is not None}
 
     # owner
-    owner = args.get('owner', 'all')
+    owner = args.get('owner', 'visible')
     try:
         search_request.owner(
             owner,

nomad/app/api/dataset.py

@@ -104,7 +104,7 @@ class DatasetListResource(Resource):
         return Dataset(dataset_id=dataset_id, **data).m_x('me').create(), 200
 
 
-@ns.route('/<string:name>')
+@ns.route('/<path:name>')
 @api.doc(params=dict(name='The name of the requested dataset.'))
 class DatasetResource(Resource):
     @api.doc('get_dataset')

nomad/app/api/raw.py

@@ -272,12 +272,16 @@ class RawFileFromCalcPathResource(Resource):
             path = urllib.parse.unquote(path)
 
         calc_filepath = path if path is not None else ''
-        authorization_predicate = create_authorization_predicate(upload_id)
+        authorization_predicate = create_authorization_predicate(upload_id, calc_id=calc_id)
         upload_files = UploadFiles.get(upload_id, authorization_predicate)
         if upload_files is None:
             abort(404, message='The upload with id %s does not exist.' % upload_id)
 
-        calc = Calc.get(calc_id)
+        try:
+            calc = Calc.get(calc_id)
+        except KeyError:
+            pass
+
         if calc is None:
             abort(404, message='The calc with id %s does not exist.' % calc_id)
         if calc.upload_id != upload_id:
@@ -455,8 +459,7 @@ class RawFileQueryResource(Resource):
                         if upload_files is not None:
                             upload_files.close_zipfile_cache()
 
-                        upload_files = UploadFiles.get(
-                            upload_id, create_authorization_predicate(upload_id))
+                        upload_files = UploadFiles.get(upload_id)
 
                         if upload_files is None:
                             logger.error('upload files do not exist', upload_id=upload_id)
@@ -467,6 +470,8 @@ class RawFileQueryResource(Resource):
                         def open_file(upload_filename):
                             return upload_files.raw_file(upload_filename, 'rb')
 
+                    upload_files._is_authorized = create_authorization_predicate(
+                        upload_id=upload_id, calc_id=entry['calc_id'])
                     directory = os.path.dirname(mainfile)
                     directory_w_upload = os.path.join(upload_files.upload_id, directory)
                     if directory_w_upload not in directories:

nomad/config.py

@@ -208,7 +208,7 @@ datacite = NomadConfig(
     password='*'
 )
 
-version = '0.7.6'
+version = '0.7.7'
 commit = gitinfo.commit
 release = 'devel'
 domain = 'DFT'

nomad/search.py

@@ -275,9 +275,10 @@ class SearchRequest:
         """
         Uses the query part of the search to restrict the results based on the owner.
         The possible types are: ``all`` for all calculations; ``public`` for
-        caclulations visible by everyone, excluding entries only visible to the given user;
-        ``user`` for all calculations of to the given user; ``staging`` for all
-        calculations in staging of the given user.
+        calculations visible by everyone, excluding embargo-ed entries and entries only visible
+        to the given user; ``visible`` all data that is visible by the user, excluding
+        embargo-ed entries from other users; ``user`` for all calculations of to the given
+        user; ``staging`` for all calculations in staging of the given user.
 
         Arguments:
             owner_type: The type of the owner query, see above.
@@ -289,11 +290,15 @@ class SearchRequest:
                 given user is not allowed to use the given owner_type.
         """
         if owner_type == 'all':
-            q = Q('term', published=True) & Q('term', with_embargo=False)
+            q = Q('term', published=True)
             if user_id is not None:
                 q = q | Q('term', owners__user_id=user_id)
         elif owner_type == 'public':
             q = Q('term', published=True) & Q('term', with_embargo=False)
+        elif owner_type == 'visible':
+            q = Q('term', published=True) & Q('term', with_embargo=False)
+            if user_id is not None:
+                q = q | Q('term', owners__user_id=user_id)
         elif owner_type == 'shared':
             if user_id is None:
                 raise ValueError('Authentication required for owner value shared.')

ops/helm/nomad/Chart.yaml

 apiVersion: v1
-appVersion: "0.7.6"
+appVersion: "0.7.7"
 description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
 name: nomad
-version: 0.7.6
+version: 0.7.7