Merge branch 'v0.9.3' into 'master'

Merge for release

See merge request !204

Dockerfile

@@ -29,7 +29,7 @@ RUN mkdir /install
 # Install linux package dependencies
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends libgomp1
-RUN apt-get install -y libmagic-dev curl vim make cmake swig libnetcdf-dev
+RUN apt-get install -y libmagic-dev curl make cmake swig libnetcdf-dev
 
 # Install some specific dependencies necessary for the build process
 RUN pip install --upgrade pip
@@ -104,8 +104,7 @@ RUN npx webpack --mode=production
 
 # Third, create a slim final image
 FROM final
-
-RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl
+RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl vim
 
 # copy the sources for tests, coverage, qa, etc.
 COPY . /app

encyclopedia-gui @ 8ca8b534

-Subproject commit bdffb455f21577435477daa9c871205e6c118efd
+Subproject commit 8ca8b53425e2808a80b69c8397a9cd977bc08f7e

abinit @ 74651abd

-Subproject commit 748673f1fcd370cb998178653b745e69da8b906c
+Subproject commit 74651abd3196ffdd919efc5069053adb3fb16d71

octopus @ 2dba3f35

-Subproject commit f6efba081da47e47d4cfe057bcb96745f607c808
+Subproject commit 2dba3f3531a96a293e8d62db9c05949ff7e591c1

phonopy @ 3a846398

-Subproject commit 59ecd6cddb2bbf75000a1454fe40ceace2d2c207
+Subproject commit 3a846398a4727ce98f1bbad9c59b1aae6e07af74

siesta @ aac537c8

-Subproject commit 6b1c30d50162fca1707914774943a15691b9789a
+Subproject commit aac537c894f9d7b6caeb19d68c8e23e67c0706c7

wien2k @ cf6abfe7

-Subproject commit 6007fb8dc88b225e42baaf9725867c9ed0caf257
+Subproject commit cf6abfe712ff2d0052947f46bdb27131ac6fbaf8

docs/api_tutorial.md

@@ -98,7 +98,7 @@ Here are a few more examples for downloading the raw data of based on DOI or dat
 You will have to encode non URL safe characters in potential dataset names (e.g. with a service like [www.urlencoder.org](https://www.urlencoder.org/)):
 
 ```
-curl "http://nomad-lab.eu/prod/rae/api/raw/query?doi=10.17172/NOMAD/2020.03.18-1" -o download.zip
+curl "http://nomad-lab.eu/prod/rae/api/raw/query?datasets.doi=10.17172/NOMAD/2020.03.18-1" -o download.zip
 curl "http://nomad-lab.eu/prod/rae/api/raw/query?dataset=Full%20ahnarmonic%20stAViC%20approach%3A%20Silicon%20and%20SrTiO3" -o download.zip
 ```
 

gui/package.json

 {
   "name": "nomad-fair-gui",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "commit": "e98694e",
   "private": true,
   "dependencies": {
-    "@lauri-codes/materia": "0.0.6",
+    "@lauri-codes/materia": "0.0.9",
     "@material-ui/core": "^4.0.0",
     "@material-ui/icons": "^4.0.0",
     "@material-ui/lab": "^4.0.0-alpha.49",

gui/public/env.js

@@ -9,7 +9,7 @@ window.nomadEnv = {
   'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
   'matomoSiteId': '2',
   'version': {
-    'label': '0.9.2',
+    'label': '0.9.3',
     'isBeta': false,
     'isTest': true,
     'usesBetaData': true,

gui/src/components/ErrorHandler.js

@@ -4,6 +4,7 @@ import { makeStyles } from '@material-ui/core/styles'
 import PropTypes from 'prop-types'
 import {
   Box,
+  Button,
   Card,
   CardContent,
   Typography
@@ -48,7 +49,7 @@ ErrorHandler.propTypes = ({
   className: PropTypes.string
 })
 
-export function ErrorCard({message, className, classes}) {
+export function ErrorCard({message, className, classes, actions}) {
   const useStyles = makeStyles((theme) => {
     return {
       root: {
@@ -66,9 +67,17 @@ export function ErrorCard({message, className, classes}) {
       pos: {
         marginBottom: 12
       },
-      container: {
+      row: {
         display: 'flex'
       },
+      actions: {
+        display: 'flex',
+        justifyContent: 'flex-end'
+      },
+      column: {
+        display: 'flex',
+        flexDirection: 'column'
+      },
       errorIcon: {
         marginRight: theme.spacing(1)
       }
@@ -76,14 +85,26 @@ export function ErrorCard({message, className, classes}) {
   })
 
   const style = useStyles(classes)
+  console.log(actions)
 
   return <Card className={clsx(style.root, className)}>
     <CardContent className={[style.content, style['content:last-child']].join(' ')}>
-      <Box className={style.container}>
+      <Box className={style.row}>
         <Error className={style.errorIcon}/>
-        <Typography className={style.title} color="error" gutterBottom>
-          {message}
-        </Typography>
+        <Box className={style.column}>
+          <Typography className={style.title} color="error" gutterBottom>
+            {message}
+          </Typography>
+          {actions
+            ? <Box className={style.actions}>
+              {actions.map((action) => <Button key={action.label} onClick={action.onClick}>
+                {action.label}
+              </Button>
+              )}
+            </Box>
+            : ''
+          }
+        </Box>
       </Box>
     </CardContent>
   </Card>
@@ -92,5 +113,6 @@ export function ErrorCard({message, className, classes}) {
 ErrorCard.propTypes = ({
   message: PropTypes.string,
   classes: PropTypes.object,
-  className: PropTypes.string
+  className: PropTypes.string,
+  actions: PropTypes.array
 })

gui/src/components/archive/ArchiveBrowser.js

@@ -46,8 +46,8 @@ export const unitsState = atom({
 })
 
 // Shared instance of the StructureViewer
-const viewer = new StructureViewer()
-const bzViewer = new BrillouinZoneViewer()
+const viewer = new StructureViewer(undefined, {view: {autoResize: false}})
+const bzViewer = new BrillouinZoneViewer(undefined, {view: {autoResize: false}})
 
 // Contains details about the currently visualized system. Used to detect if a
 // reload is needed for the StructureViewer.
@@ -361,6 +361,8 @@ QuantityValue.propTypes = ({
 function Overview({section, def}) {
   // States
   const [mode, setMode] = useState('bs')
+  const [warningIgnored, setWarningIgnored] = useState(false)
+
   // Styles
   const useStyles = makeStyles(
     {
@@ -405,13 +407,6 @@ function Overview({section, def}) {
       return ''
     }
     const nAtoms = section.atom_species.length
-    if (nAtoms >= 300) {
-      return <ErrorCard
-        message='Visualization is disabled due to large system size.'
-        className={style.error}
-      >
-      </ErrorCard>
-    }
     // Loading exact same system, no need to reload visualizer
     if (sectionPath === visualizedSystem.sectionPath && index === visualizedSystem.index) {
     // Loading same system with different positions
@@ -420,11 +415,22 @@ function Overview({section, def}) {
       system = {
         positions: convertSI(section.atom_positions, 'meter', {length: 'angstrom'}, false)
       }
-    // Completely new system
+    // Loading a completely new system. When trying to visualize the system for
+    // the first time, check the system size and for large systems ask the user
+    // for permission.
     } else {
+      const sizeLimit = 300
+      if (nAtoms >= sizeLimit && !warningIgnored) {
+        return <ErrorCard
+          message={`Visualization is by default disabled for systems with more than ${sizeLimit} atoms. Do you wish to enable visualization for this system with ${nAtoms} atoms?`}
+          className={style.error}
+          actions={[{label: 'Yes', onClick: e => setWarningIgnored(true)}]}
+        >
+        </ErrorCard>
+      }
       system = {
         'species': section.atom_species,
-        'cell': convertSI(section.lattice_vectors, 'meter', {length: 'angstrom'}, false),
+        'cell': section.lattice_vectors ? convertSI(section.lattice_vectors, 'meter', {length: 'angstrom'}, false) : undefined,
         'positions': convertSI(section.atom_positions, 'meter', {length: 'angstrom'}, false),
         'pbc': section.configuration_periodic_dimensions
       }
@@ -443,7 +449,7 @@ function Overview({section, def}) {
         positionsOnly={positionsOnly}
       ></Structure>
     </ErrorHandler>
-  // Band structure plot for section_k_band or section_k_band_normalized
+  // Band structure plot for section_k_band
   } else if (def.name === 'section_k_band') {
     return <>
       {mode === 'bs'

gui/src/components/domains.js

@@ -17,9 +17,9 @@ import { amber } from '@material-ui/core/colors'
 export const domains = ({
   dft: {
     name: 'Computational data',
-    label: 'Computational material science data',
+    label: 'Computational materials science data',
     key: 'dft',
-    about: 'This include data from many computational material science codes',
+    about: 'This include data from many computational materials science codes',
     disclaimer: <Typography>
       First time users can find an introduction to NOMAD, tutorials, and videos <Link href="https://nomad-lab.eu/services/repo-arch" target="nomad-lab">here</Link>.
     </Typography>,
@@ -173,7 +173,7 @@ export const domains = ({
     name: 'Experimental data',
     key: 'ems',
     label: 'Experimental data (beta)',
-    about: 'This includes first metadata from material science experiments. This aspect of NOMAD is still in development and mnight change frequently.',
+    about: 'This includes first metadata from materials science experiments. This aspect of NOMAD is still in development and mnight change frequently.',
     disclaimer: <Typography style={{color: amber[700]}}>
       This aspect of NOMAD is still under development. The offered functionality and displayed data
       might change frequently, is not necessarely reviewed by NOMAD, and might contain
@@ -259,8 +259,8 @@ export const domains = ({
   qcms: {
     name: 'Quantum-computer data',
     key: 'qcms',
-    label: 'Quantum-computer material science data (beta)',
-    about: 'This includes first data material science data calculated by quantum-computers. This aspect of NOMAD is still in development and mnight change frequently.',
+    label: 'Quantum-computer data (beta)',
+    about: 'This includes first data materials science data calculated by quantum-computers. This aspect of NOMAD is still in development and mnight change frequently.',
     disclaimer: <Typography style={{color: amber[700]}}>
       This aspect of NOMAD is still under development. The offered functionality and displayed data
       might change frequently, is not necessarely reviewed by NOMAD, and might contain

gui/src/components/visualization/Structure.js

@@ -21,7 +21,7 @@ import {
 import { StructureViewer } from '@lauri-codes/materia'
 import Floatable from './Floatable'
 
-export default function Structure({className, classes, system, options, viewer, captureName, aspectRatio, positionsOnly, sizeLimit}) {
+export default function Structure({className, classes, system, options, viewer, captureName, aspectRatio, positionsOnly}) {
   // States
   const [anchorEl, setAnchorEl] = React.useState(null)
   const [fullscreen, setFullscreen] = useState(false)
@@ -122,7 +122,7 @@ export default function Structure({className, classes, system, options, viewer,
       refViewer.current = new StructureViewer(undefined, viewerOptions)
     } else {
       refViewer.current = viewer
-      refViewer.current.setOptions(options, false, false)
+      refViewer.current.setOptions(viewerOptions, false, false)
     }
     if (refCanvas.current !== null) {
       refViewer.current.changeHostElement(refCanvas.current, false, false)
@@ -157,7 +157,7 @@ export default function Structure({className, classes, system, options, viewer,
             [1, 0, 0, 30]
           ]
         }
-      }})
+      }}, false, false)
     // Systems without cell are centered on the center of positions
     } else {
       refViewer.current.setOptions({layout: {
@@ -168,7 +168,7 @@ export default function Structure({className, classes, system, options, viewer,
             [1, 0, 0, 30]
           ]
         }
-      }})
+      }}, false, false)
     }
     refViewer.current.load(system)
     refViewer.current.fitToCanvas()
@@ -304,11 +304,9 @@ Structure.propTypes = {
   options: PropTypes.object, // Viewer options
   captureName: PropTypes.string, // Name of the file that the user can download
   aspectRatio: PropTypes.number, // Fixed aspect ratio for the viewer canvas
-  sizeLimit: PropTypes.number, // Maximum number of atoms to attempt to display
   positionsOnly: PropTypes.bool // Whether to update only positions. This is much faster than loading the entire structure.
 }
 Structure.defaultProps = {
   aspectRatio: 4 / 3,
-  captureName: 'structure',
-  sizeLimit: 300
+  captureName: 'structure'
 }

gui/yarn.lock

@@ -1287,10 +1287,10 @@
   resolved "https://registry.yarnpkg.com/@kyleshockey/object-assign-deep/-/object-assign-deep-0.4.2.tgz#84900f0eefc372798f4751b5262830b8208922ec"
   integrity sha1-hJAPDu/DcnmPR1G1JigwuCCJIuw=
-"@lauri-codes/materia@0.0.6":
-  version "0.0.6"
-  resolved "https://registry.yarnpkg.com/@lauri-codes/materia/-/materia-0.0.6.tgz#b1a64ef174fc0f4107940f9ca19ec8a2305d7376"
-  integrity sha512-RAC8iRS430f6pNum6zlK/PuFgENO1mOM1C1VSE6v+Hv1u8J9FDOyULvhyVuKdqOB6Gh6bmQikPBWX0+XGeFynw==
+"@lauri-codes/materia@0.0.9":
+  version "0.0.9"
+  resolved "https://registry.yarnpkg.com/@lauri-codes/materia/-/materia-0.0.9.tgz#3ac8cac1b9ce8201a9e881afc0c5c6652c5a692a"
+  integrity sha512-qtY3pOWCKVG6z6iEs88vl/hqQNgK3tSCVNcAkSKFnuhk/Qdk66xHl7ZGVDMAOyqdXukNR2+iuZC/6CdFKc6/ow==
   dependencies:
     three "^0.119.1"
     threejs-meshline "^2.0.11"

nomad/app/api/repo.py

@@ -26,7 +26,6 @@ import elasticsearch.helpers
 from datetime import datetime
 
 from nomad import search, utils, datamodel, processing as proc, infrastructure, files
-from nomad.metainfo import search_extension
 from nomad.datamodel import Dataset, User, EditableUserMetadata
 from nomad.app import common
 from nomad.app.common import RFC3339DateTime, DotKeyNested
@@ -88,13 +87,13 @@ _search_request_parser.add_argument(
 _search_request_parser.add_argument(
     'metrics', type=str, action='append', help=(
         'Metrics to aggregate over all quantities and their values as comma separated list. '
-        'Possible values are %s.' % ', '.join(search_extension.metrics.keys())))
+        'Possible values are %s.' % ', '.join(search.metrics.keys())))
 _search_request_parser.add_argument(
     'statistics', type=str, action='append', help=(
         'Quantities for which to aggregate values and their metrics.'))
 _search_request_parser.add_argument(
     'exclude', type=str, action='split', help='Excludes the given keys in the returned data.')
-for group_name in search_extension.groups:
+for group_name in search.groups:
     _search_request_parser.add_argument(
         group_name, type=bool, help=('Return %s group data.' % group_name))
     _search_request_parser.add_argument(
@@ -106,15 +105,15 @@ _repo_calcs_model_fields = {
         'A dict with all statistics. Each statistic is dictionary with a metrics dict as '
         'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
         'There is a pseudo quantity "total" with a single value "all" that contains the '
-        ' metrics over all results. ' % ', '.join(search_extension.metrics.keys())))}
+        ' metrics over all results. ' % ', '.join(search.metrics.keys())))}
 
-for group_name in search_extension.groups:
+for group_name in search.groups:
     _repo_calcs_model_fields[group_name] = (DotKeyNested if '.' in group_name else fields.Nested)(api.model('RepoGroup', {
         'after': fields.String(description='The after value that can be used to retrieve the next %s.' % group_name),
         'values': fields.Raw(description='A dict with %s as key. The values are dicts with "total" and "examples" keys.' % group_name)
     }), skip_none=True)
 
-for qualified_name, quantity in search_extension.search_quantities.items():
+for qualified_name, quantity in search.search_quantities.items():
     _repo_calcs_model_fields[qualified_name] = fields.Raw(
         description=quantity.description, allow_null=True, skip_none=True)
 
@@ -123,7 +122,7 @@ _repo_calcs_model_fields.update(**{
     'interval': fields.String(description='Interval to use for upload time aggregation.', allow_null=True, skip_none=True),
     'metrics': fields.List(fields.String, description=(
         'Metrics to aggregate over all quantities and their values as comma separated list. '
-        'Possible values are %s.' % ', '.join(search_extension.metrics.keys())), allow_null=True, skip_none=True),
+        'Possible values are %s.' % ', '.join(search.metrics.keys())), allow_null=True, skip_none=True),
     'statistics_required': fields.List(fields.String, description='Quantities for which to aggregate values and their metrics.', allow_null=True, skip_none=True),
     'exclude': fields.List(fields.String, description='Excludes the given keys in the returned data.', allow_null=True, skip_none=True)
 })
@@ -192,7 +191,7 @@ class RepoCalcsResource(Resource):
             abort(400, message='bad parameters: %s' % str(e))
 
         for metric in metrics:
-            if metric not in search_extension.metrics:
+            if metric not in search.metrics:
                 abort(400, message='there is no metric %s' % metric)
 
         search_request = search.SearchRequest()
@@ -214,7 +213,7 @@ class RepoCalcsResource(Resource):
 
         group_metrics = [
             group_quantity.metric_name
-            for group_name, group_quantity in search_extension.groups.items()
+            for group_name, group_quantity in search.groups.items()
             if args.get(group_name, False)]
         total_metrics = metrics + group_metrics
         if len(total_metrics) > 0:
@@ -230,7 +229,7 @@ class RepoCalcsResource(Resource):
                 results = search_request.execute_scrolled(scroll_id=scroll_id, size=per_page)
 
             else:
-                for group_name, group_quantity in search_extension.groups.items():
+                for group_name, group_quantity in search.groups.items():
                     if args.get(group_name, False):
                         kwargs: Dict[str, Any] = {}
                         if group_name == 'uploads_grouped':
@@ -252,7 +251,7 @@ class RepoCalcsResource(Resource):
                 if 'quantities' in results:
                     quantities = results.pop('quantities')
 
-                for group_name, group_quantity in search_extension.groups.items():
+                for group_name, group_quantity in search.groups.items():
                     if args.get(group_name, False):
                         results[group_name] = quantities[group_quantity.qualified_name]
 
@@ -335,7 +334,7 @@ class RepoCalcsResource(Resource):
             abort(400, message='bad parameters: %s' % str(e))
 
         for metric in metrics:
-            if metric not in search_extension.metrics:
+            if metric not in search.metrics:
                 abort(400, message='there is no metric %s' % metric)
 
         search_request = search.SearchRequest()
@@ -363,7 +362,7 @@ class RepoCalcsResource(Resource):
 
         group_metrics = [
             group_quantity.metric_name
-            for group_name, group_quantity in search_extension.groups.items()
+            for group_name, group_quantity in search.groups.items()
             if group_name in data_in]
         total_metrics = metrics + group_metrics
         if len(total_metrics) > 0:
@@ -379,7 +378,7 @@ class RepoCalcsResource(Resource):
                 results = search_request.execute_scrolled(scroll_id=scroll_id, size=per_page)
 
             else:
-                for group_name, group_quantity in search_extension.groups.items():
+                for group_name, group_quantity in search.groups.items():
                     if group_name in data_in:
                         kwargs: Dict[str, Any] = {}
                         if group_name == 'uploads_grouped':
@@ -401,7 +400,7 @@ class RepoCalcsResource(Resource):
                 if 'quantities' in results:
                     quantities = results.pop('quantities')
 
-                for group_name, group_quantity in search_extension.groups.items():
+                for group_name, group_quantity in search.groups.items():
                     if group_name in data_in:
                         results[group_name] = quantities[group_quantity.qualified_name]
 
@@ -865,7 +864,7 @@ _repo_quantities_search_request_parser.add_argument(
 _repo_quantities_model = api.model('RepoQuantitiesResponse', {
     'quantities': fields.Nested(api.model('RepoQuantities', {
         quantity: fields.List(fields.Nested(_repo_quantity_model))
-        for quantity in search_extension.search_quantities
+        for quantity in search.search_quantities
     }))
 })
 

nomad/cli/admin/admin.py

@@ -21,6 +21,10 @@ import threading
 
 from nomad import processing as proc, search, datamodel, infrastructure, utils, config
 from nomad.cli.cli import cli
+from nomad.datamodel.material import Material, Calculation
+from nomad.datamodel.encyclopedia import EncyclopediaMetadata
+from nomad.search import material_document
+from nomad.datamodel.material import Material, Calculation, Method, Properties, IdealizedStructure, Energies, Workflow, Bulk
 
 
 def __run_parallel(
@@ -209,16 +213,367 @@ def index(threads, dry):
     if dry:
         for _ in elastic_updates():
             pass
-    if threads > 1:
-        print('  use %d threads' % threads)
-        for _ in elasticsearch.helpers.parallel_bulk(
-                infrastructure.elastic_client, elastic_updates(), chunk_size=500,
-                thread_count=threads):
+    else:
+        if threads > 1:
+            print('  use %d threads' % threads)
+            for _ in elasticsearch.helpers.parallel_bulk(
+                    infrastructure.elastic_client, elastic_updates(), chunk_size=500,
+                    thread_count=threads):
+                pass
+        else:
+            elasticsearch.helpers.bulk(
+                infrastructure.elastic_client, elastic_updates())
+        search.refresh()
+
+    print('')
+    print('indexing completed')
+
+
+@admin.command()
+@click.option('--threads', type=int, default=1, help='Number of threads to use.')
+@click.option('--code', multiple=True, type=str, help='Index only calculcations of given codes.')
+@click.option('--dry', is_flag=True, help='Do not index, just compute entries.')
+@click.option('--in-place', is_flag=True, default=False, help='Perform indexing in the current elastic search index. Meant only for small reindex operations.')
+@click.option('-n', type=int, default=None, help='Number of calculations to process. Leave undefined to process all calculations.')
+@click.option('--source',
+              type=click.Choice(['mongo', 'es'], case_sensitive=True))
+def index_materials(threads, code, dry, in_place, n, source):
+    """(Re-)index all materials.
+
+    This command will completely rebuild the materials index. The index is
+    built from the material metainfo stored in MongoDB. The materials index can
+    be used normally during the reindexing.
+    """
+    chunk_size = 500
+    infrastructure.setup_mongo()
+    client = infrastructure.setup_elastic()
+
+    # In order to do the reindexing with zero downtime, two different indices
+    # are rotated and an alias is used
+    old_index_name = list(client.indices.get(config.elastic.materials_index_name).keys())[0]
+    if in_place:
+        target_index_name = old_index_name
+    else:
+        if old_index_name == config.elastic.materials_index_name + "_a":
+            target_index_name = config.elastic.materials_index_name + "_b"
+        elif old_index_name == config.elastic.materials_index_name + "_b":
+            target_index_name = config.elastic.materials_index_name + "_a"
+        else:
+            raise ValueError(
+                "Unrecognized index name accociated with the alias {}"
+                .format(config.elastic.materials_index_name)
+            )
+
+    if source == "mongo":
+        all_calcs = proc.Calc.objects().count()
+        print('indexing materials from %d calculations ...' % all_calcs)
+
+        # Bulk update
+        def elastic_updates():
+            with utils.ETA(all_calcs, '   index %10d of %10d calcs, ETA %s') as eta:
+                mongo_db = infrastructure.mongo_client[config.mongo.db_name]
+                mongo_collection = mongo_db['archive']
+                i_calc = 0
+                for mongo_archive in mongo_collection.find():
+                    i_calc += 1
+                    if n is not None:
+                        if i_calc > n:
+                            return
+                    eta.add()
+
+                    # Do not process entries that do not have the material
+                    # information
+                    try:
+                        status = mongo_archive["section_metadata"]["encyclopedia"]["status"]
+                        if status != EncyclopediaMetadata.status.type.success:
+                            raise AttributeError
+                    except (KeyError, AttributeError, IndexError):
+                        continue
+
+                    # Create material information
+                    metadata = mongo_archive["section_metadata"]
+                    encyclopedia = EncyclopediaMetadata.m_from_dict(metadata["encyclopedia"])
+                    dft = metadata["dft"]
+                    material: Material = Material()
+                    material.material_id = encyclopedia.material.material_id
+                    material.material_type = encyclopedia.material.material_type
+                    material.material_name = encyclopedia.material.material_name
+                    material.material_classification = encyclopedia.material.material_classification
+                    material.formula = encyclopedia.material.formula
+                    material.formula_reduced = encyclopedia.material.formula_reduced
+                    material.species_and_counts = encyclopedia.material.species_and_counts
+                    material.species = encyclopedia.material.species
+                    enc_bulk = encyclopedia.material.bulk
+                    if enc_bulk:
+                        bulk = Bulk.m_from_dict(enc_bulk.m_to_dict())
+                        material.m_add_sub_section(Material.bulk, bulk)
+
+                    # Create calculation info for this entry
+                    calc = Calculation()
+                    calc.calc_id = metadata["calc_id"]
+                    calc.upload_id = metadata["upload_id"]
+                    mongo_calc = proc.Calc.get(calc.calc_id)
+                    calc.published = mongo_calc["metadata"]["published"]
+                    calc.with_embargo = mongo_calc["metadata"]["with_embargo"]
+                    calc.owners = [mongo_calc["metadata"]["uploader"]] + mongo_calc["metadata"]["shared_with"]
+                    enc_idealized_structure = encyclopedia.material.idealized_structure
+                    idealized_structure = IdealizedStructure()
+                    cell_volume = enc_idealized_structure.cell_volume
+                    if cell_volume is not None:
+                        idealized_structure.cell_volume = cell_volume
+                    idealized_structure.lattice_parameters = enc_idealized_structure.lattice_parameters
+                    calc.m_add_sub_section(Calculation.idealized_structure, idealized_structure)
+                    enc_method = encyclopedia.method
+                    method = Method.m_from_dict(enc_method.m_to_dict())
+                    method.program_name = dft["code_name"]
+                    method.program_version = dft["code_version"]
+                    method.basis_set = dft["basis_set"]
+                    calc.m_add_sub_section(Calculation.method, method)
+                    enc_props = encyclopedia.properties
+
+                    # Properties may not exist at all
+                    if enc_props is not None:
+                        properties = Properties()
+
+                        # Energies may not be present in all calculations
+                        try:
+                            energies = Energies.m_from_dict(enc_props.energies.m_to_dict())
+                            properties.m_add_sub_section(Properties.energies, energies)
+                        except AttributeError:
+                            pass
+
+                        properties.has_electronic_dos = enc_props.electronic_dos is not None
+                        properties.has_electronic_band_structure = enc_props.electronic_band_structure is not None
+                        properties.has_thermodynamical_properties = enc_props.thermodynamical_properties is not None
+                        atomic_density = enc_props.atomic_density
+                        if atomic_density is not None:
+