diff --git a/gui/src/components/entry/RepoEntryView.js b/gui/src/components/entry/RepoEntryView.js
index 52a5c71a2fb5a4ce8b0d105af0b1a04089f5fdfe..91dc6859d345d3bec876d026719df2b283b94956 100644
--- a/gui/src/components/entry/RepoEntryView.js
+++ b/gui/src/components/entry/RepoEntryView.js
@@ -146,12 +146,12 @@ class RepoEntryView extends React.Component {
                   <Quantity quantity="upload_id" label='upload id' {...quantityProps} noWrap withClipboard />
                   <Quantity quantity="upload_time" label='upload time' noWrap {...quantityProps} >
                     <Typography noWrap>
-                      {new Date(calcData.upload_time * 1000).toLocaleString()}
+                      {new Date(calcData.upload_time).toLocaleString()}
                     </Typography>
                   </Quantity>
                   <Quantity quantity="last_processing" label='last processing' loading={loading} placeholder="not processed" noWrap {...quantityProps}>
                     <Typography noWrap>
-                      {new Date(calcData.last_processing * 1000).toLocaleString()}
+                      {new Date(calcData.last_processing).toLocaleString()}
                     </Typography>
                   </Quantity>
                   <Quantity quantity="last_processing" label='processing version' loading={loading} noWrap placeholder="not processed" {...quantityProps}>
diff --git a/nomad/app/api/archive.py b/nomad/app/api/archive.py
index fd9685720ab8ac3f7da8879cce25dfdde6e09039..2bbce2321673c3feb995eab27050adacc4854d11 100644
--- a/nomad/app/api/archive.py
+++ b/nomad/app/api/archive.py
@@ -26,7 +26,6 @@ import json
 import orjson
 import importlib
 import urllib.parse
-from collections.abc import Mapping
 
 import metainfo
 
@@ -100,9 +99,10 @@ class ArchiveCalcResource(Resource):
         try:
             with upload_files.read_archive(calc_id) as archive:
                 return {
-                    key: value.to_dict()
-                    for key, value in archive[calc_id].items()
-                    if isinstance(value, Mapping)}
+                    key: value
+                    for key, value in archive[calc_id].to_dict().items()
+                    if key != 'processing_logs'}
+
         except Restricted:
             abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))
         except KeyError:
diff --git a/nomad/app/api/repo.py b/nomad/app/api/repo.py
index 6ddfcc689f783df423d892d26d45f11e5593d887..18f5937b022191a4f6b1596262f805a9f08b1463 100644
--- a/nomad/app/api/repo.py
+++ b/nomad/app/api/repo.py
@@ -25,7 +25,7 @@ from elasticsearch.exceptions import NotFoundError
 import elasticsearch.helpers
 from datetime import datetime
 
-from nomad import search, utils, datamodel, processing as proc, infrastructure
+from nomad import search, utils, datamodel, processing as proc, infrastructure, files
 from nomad.metainfo import search_extension
 from nomad.datamodel import Dataset, User, EditableUserMetadata
 from nomad.app import common
@@ -315,6 +315,7 @@ def edit(parsed_query: Dict[str, Any], mongo_update: Dict[str, Any] = None, re_i
         apply_search_parameters(search_request, parsed_query)
         upload_ids = set()
         calc_ids = []
+
         for hit in search_request.execute_scan():
             calc_ids.append(hit['calc_id'])
             upload_ids.add(hit['upload_id'])
@@ -330,12 +331,24 @@ def edit(parsed_query: Dict[str, Any], mongo_update: Dict[str, Any] = None, re_i
     with utils.timer(common.logger, 'edit elastic update executed', size=len(calc_ids)):
         if re_index:
             def elastic_updates():
+                upload_files_cache: Dict[str, files.UploadFiles] = dict()
+
                 for calc in proc.Calc.objects(calc_id__in=calc_ids):
-                    entry_metadata = datamodel.EntryMetadata.m_from_dict(calc['metadata'])
+                    upload_id = calc.upload_id
+                    upload_files = upload_files_cache.get(upload_id)
+                    if upload_files is None:
+                        upload_files = files.UploadFiles.get(upload_id, is_authorized=lambda: True)
+                        upload_files_cache[upload_id] = upload_files
+
+                    entry_metadata = calc.entry_metadata(upload_files)
                     entry = entry_metadata.a_elastic.create_index_entry().to_dict(include_meta=True)
                     entry['_op_type'] = 'index'
+
                     yield entry
 
+                for upload_files in upload_files_cache.values():
+                    upload_files.close()
+
             _, failed = elasticsearch.helpers.bulk(
                 infrastructure.elastic_client, elastic_updates(), stats_only=True)
             search.refresh()
diff --git a/nomad/app/api/upload.py b/nomad/app/api/upload.py
index f84aa6012f616e3436d99699edb4ff9ca19bb249..be05ee57b904e18b28df96e2f9400886a76e017c 100644
--- a/nomad/app/api/upload.py
+++ b/nomad/app/api/upload.py
@@ -27,7 +27,7 @@ import os
 import io
 from functools import wraps
 
-from nomad import config, utils, files, datamodel
+from nomad import config, utils, files, search
 from nomad.processing import Upload, FAILURE
 from nomad.processing import ProcessAlreadyRunning
 from nomad.app import common
@@ -42,13 +42,6 @@ ns = api.namespace(
     'uploads',
     description='Uploading data and tracing uploaded data and its processing.')
 
-
-class CalcMetadata(fields.Raw):
-    def format(self, value):
-        entry_metadata = datamodel.EntryMetadata.m_from_dict(value)
-        return entry_metadata.a_elastic.create_index_entry().to_dict()
-
-
 proc_model = api.model('Processing', {
     'tasks': fields.List(fields.String),
     'current_task': fields.String,
@@ -96,7 +89,9 @@ calc_model = api.inherit('UploadCalculationProcessing', proc_model, {
     'mainfile': fields.String,
     'upload_id': fields.String,
     'parser': fields.String,
-    'metadata': CalcMetadata(description='The repository metadata for this entry.')
+    'metadata': fields.Raw(
+        attribute='_entry_metadata',
+        description='The repository metadata for this entry.')
 })
 
 upload_with_calcs_model = api.inherit('UploadWithPaginatedCalculations', upload_model, {
@@ -381,13 +376,24 @@ class UploadResource(Resource):
 
             order_by = ('-%s' if order == -1 else '+%s') % order_by
 
-        calcs = upload.all_calcs((page - 1) * per_page, page * per_page, order_by=order_by)
+        # load upload's calcs
+        calcs = list(upload.all_calcs(
+            (page - 1) * per_page, page * per_page, order_by=order_by))
+
+        calc_ids = [calc.calc_id for calc in calcs]
+        search_results = {
+            hit['calc_id']: hit
+            for hit in search.SearchRequest().search_parameter('calc_id', calc_ids).execute_scan()}
+
+        for calc in calcs:
+            calc._entry_metadata = search_results.get(calc.calc_id)
+
         failed_calcs = upload.failed_calcs
         result = ProxyUpload(upload, {
             'pagination': dict(
                 total=upload.total_calcs, page=page, per_page=per_page,
                 successes=upload.processed_calcs - failed_calcs, failures=failed_calcs),
-            'results': [calc for calc in calcs]
+            'results': calcs
         })
 
         return result, 200
diff --git a/nomad/app/optimade/endpoints.py b/nomad/app/optimade/endpoints.py
index accc47f0d2b66260b580d76cf14f3bd42c3ff234..8b48fa4da007bf080261fde012d2740a2c79e791 100644
--- a/nomad/app/optimade/endpoints.py
+++ b/nomad/app/optimade/endpoints.py
@@ -12,11 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List, Dict, Any
 from flask_restplus import Resource, abort
 from flask import request
 from elasticsearch_dsl import Q
 
-from nomad import search
+from nomad import search, files, datamodel
 from nomad.datamodel import OptimadeEntry
 
 from .api import api, url
@@ -46,6 +47,31 @@ def base_search_request():
         Q('exists', field='dft.optimade.elements'))  # TODO use the elastic annotations when done
 
 
+def to_calc_with_metadata(results: List[Dict[str, Any]]):
+    ''' Translates search results into :class:`EntryMetadata` objects read from archive. '''
+
+    upload_files_cache: Dict[str, files.UploadFiles] = {}
+
+    def transform(result):
+        calc_id, upload_id = result['calc_id'], result['upload_id']
+        upload_files = upload_files_cache.get(upload_id)
+
+        if upload_files is None:
+            upload_files = files.UploadFiles.get(upload_id)
+            upload_files_cache[upload_id] = upload_files
+
+        archive = upload_files.read_archive(calc_id)  # , access='public')
+        metadata = archive[calc_id]['section_metadata'].to_dict()
+        return datamodel.EntryMetadata.m_from_dict(metadata)
+
+    result = [transform(result) for result in results]
+
+    for upload_files in upload_files_cache.values():
+        upload_files.close()
+
+    return result
+
+
 @ns.route('/calculations')
 class CalculationList(Resource):
     @api.doc('list_calculations')
@@ -65,7 +91,7 @@ class CalculationList(Resource):
         except Exception:
             abort(400, message='bad parameter types')  # TODO Specific json API error handling
 
-        search_request = base_search_request().include('calc_id')
+        search_request = base_search_request().include('calc_id', 'upload_id')
 
         if filter is not None:
             try:
@@ -79,7 +105,7 @@ class CalculationList(Resource):
         # order_by='optimade.%s' % sort)  # TODO map the Optimade property
 
         available = result['pagination']['total']
-        results = search.to_calc_with_metadata(result['results'])
+        results = to_calc_with_metadata(result['results'])
         assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
 
         return dict(
@@ -115,7 +141,7 @@ class Calculation(Resource):
             per_page=1)
 
         available = result['pagination']['total']
-        results = search.to_calc_with_metadata(result['results'])
+        results = to_calc_with_metadata(result['results'])
         assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
 
         if available == 0:
diff --git a/nomad/cli/admin/admin.py b/nomad/cli/admin/admin.py
index 88558c1104903720fd0d1c7b2a611e5267b9e8d8..54861ec2bb6f92f8dc4592434a10b0cd2641d983 100644
--- a/nomad/cli/admin/admin.py
+++ b/nomad/cli/admin/admin.py
@@ -158,7 +158,8 @@ def lift_embargo(dry, parallel):
                 uploads_to_repack.append(upload)
                 upload.save()
 
-                search.index_all(upload.entries_metadata())
+                with upload.entries_metadata() as entries:
+                    search.index_all(entries)
 
     if not dry:
         __run_processing(uploads_to_repack, parallel, lambda upload: upload.re_pack(), 're-packing')
diff --git a/nomad/cli/admin/uploads.py b/nomad/cli/admin/uploads.py
index f2fa2abfbe1f4058eb77b1b0c0b7795a4a77e4fb..56434bff8c223891d5666c9545fc6aad0cadce20 100644
--- a/nomad/cli/admin/uploads.py
+++ b/nomad/cli/admin/uploads.py
@@ -146,16 +146,17 @@ def chown(ctx, username, uploads):
         upload.user_id = user.user_id
         calcs = upload.entries_metadata()
 
-        def create_update(calc):
+        def create_update(calc_id):
             return UpdateOne(
-                {'_id': calc.calc_id},
+                {'_id': calc_id},
                 {'$set': {'metadata.uploader': user.user_id}})
 
-        proc.Calc._get_collection().bulk_write([create_update(calc) for calc in calcs])
+        proc.Calc._get_collection().bulk_write(
+            [create_update(calc_id) for calc_id in upload.entry_ids()])
         upload.save()
 
-        calcs = upload.entries_metadata()
-        search.index_all(calcs, do_refresh=False)
+        with upload.entries_metadata() as calcs:
+            search.index_all(calcs, do_refresh=False)
         search.refresh()
 
 
@@ -192,9 +193,9 @@ def index(ctx, uploads):
 
     i, failed = 0, 0
     for upload in uploads:
-        calcs = upload.entries_metadata()
-        failed += search.index_all(calcs)
-        i += 1
+        with upload.entries_metadata() as calcs:
+            failed += search.index_all(calcs)
+            i += 1
 
         print('   indexed %d of %d uploads, failed to index %d entries' % (i, uploads_count, failed))
 
diff --git a/nomad/cli/client/mirror.py b/nomad/cli/client/mirror.py
index 8bc6ded4a4f99fad29498ac278c30bd93ac1422d..89787f0c161d27aaf7ee220b01b395da53332c02 100644
--- a/nomad/cli/client/mirror.py
+++ b/nomad/cli/client/mirror.py
@@ -318,7 +318,8 @@ def mirror(
             proc.Calc._get_collection().insert(upload_data.calcs)
 
             # index es
-            search.index_all(upload.entries_metadata())
+            with upload.entries_metadata() as entries:
+                search.index_all(entries)
 
         print(
             'Mirrored %s with %d calcs at %s' %
diff --git a/nomad/cli/parse.py b/nomad/cli/parse.py
index 55e39848efca1eed8605c2882dec6ae653b68bdc..5f311861e738043d1d7abf1081cd62cc67b904f8 100644
--- a/nomad/cli/parse.py
+++ b/nomad/cli/parse.py
@@ -4,7 +4,7 @@ import json
 import click
 import sys
 
-from nomad import config, utils
+from nomad import utils
 from nomad.parsing import Backend, parser_dict, match_parser, MatchingParser
 from nomad.normalizing import normalizers
 from nomad.datamodel import EntryMetadata
@@ -48,14 +48,6 @@ def parse(
     if not parser_backend.status[0] == 'ParseSuccess':
         logger.error('parsing was not successful', status=parser_backend.status)
 
-    parser_backend.openNonOverlappingSection('section_entry_info')
-    parser_backend.addValue('upload_id', config.services.unavailable_value)
-    parser_backend.addValue('calc_id', config.services.unavailable_value)
-    parser_backend.addValue('calc_hash', "no hash")
-    parser_backend.addValue('mainfile', mainfile)
-    parser_backend.addValue('parser_name', parser_name)
-    parser_backend.closeNonOverlappingSection('section_entry_info')
-
     logger.info('ran parser')
     return parser_backend
 
diff --git a/nomad/client.py b/nomad/client.py
index 7567f8d12fb5be5e57c04b8690cbdc47f7c1da88..2265da0396798f1452f7ea55a6859abab12da24f 100644
--- a/nomad/client.py
+++ b/nomad/client.py
@@ -40,8 +40,12 @@ from collections import Sequence
 import requests
 from urllib.parse import urlparse
 
-from nomad import config, metainfo, parsing
+from nomad import config
 from nomad.cli.client.client import KeycloakAuthenticator
+from nomad.datamodel import EntryArchive
+
+# TODO this import is necessary to load all metainfo defintions that the parsers are using
+from nomad import parsing  # pylint: disable=unused-import
 
 
 class ArchiveQuery(Sequence):
@@ -123,13 +127,7 @@ class ArchiveQuery(Sequence):
         results = data.get('results', [])
 
         for result in results:
-            parser_name = result['parser_name']
-            parser = parsing.parser_dict[parser_name]
-            metainfo_env = parser.metainfo_env
-
-            root_section_key = next(iter(result['archive']))
-            section_def = metainfo_env.resolve_definition(root_section_key, metainfo.Section)
-            archive = section_def.section_cls.m_from_dict(result['archive'][root_section_key])
+            archive = EntryArchive.m_from_dict(result['archive'])
 
             self._results.append(archive)
 
diff --git a/nomad/datamodel/__init__.py b/nomad/datamodel/__init__.py
index c3fea7df964a01f7104665b0290ac3c1d6328048..e0a1db9997d9d11ee8dcd4106037a3fdff7620ba 100644
--- a/nomad/datamodel/__init__.py
+++ b/nomad/datamodel/__init__.py
@@ -32,9 +32,9 @@ The class :class:`Dataset` is used to represent datasets and their attributes.
 .. autoclass:: nomad.datamodel.Dataset
     :members:
 
-The class :class:`UserMetadata` is used to represent user determined entry metadata.
+The class :class:`MongoMetadata` is used to tag metadata stored in mongodb.
 
-.. autoclass:: nomad.datamodel.UserMetadata
+.. autoclass:: nomad.datamodel.MongoMetadata
     :members:
 
 The class :class:`EntryMetadata` is used to represent all metadata about an entry.
@@ -56,7 +56,7 @@ In addition there are domain specific metadata classes:
 
 from .dft import DFTMetadata
 from .ems import EMSMetadata
-from .datamodel import Dataset, User, EditableUserMetadata, UserMetadata, EntryMetadata
+from .datamodel import Dataset, User, EditableUserMetadata, MongoMetadata, EntryMetadata, EntryArchive
 from .optimade import OptimadeEntry, Species
 
 domains = {
diff --git a/nomad/datamodel/datamodel.py b/nomad/datamodel/datamodel.py
index 8288ad688bcbe3e9274fd2ecaf2f4b59d6fa0ffa..69d2695423685f2872a58b919acf3ebf8be90c77 100644
--- a/nomad/datamodel/datamodel.py
+++ b/nomad/datamodel/datamodel.py
@@ -26,6 +26,8 @@ from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument
 
 from .dft import DFTMetadata
 from .ems import EMSMetadata
+from .metainfo.public import section_run
+from .metainfo.general_experimental import section_experiment
 
 
 def _only_atoms(atoms):
@@ -189,8 +191,8 @@ class EditableUserMetadata(metainfo.MCategory):
     ''' NOMAD entry quantities that can be edited by the user after publish. '''
 
 
-class UserMetadata(metainfo.MCategory):
-    ''' NOMAD entry quantities that are given by the user or determined by user actions. '''
+class MongoMetadata(metainfo.MCategory):
+    ''' NOMAD entry quantities that are stored in mongodb and not necessarely in the archive. '''
     pass
 
 
@@ -249,6 +251,7 @@ class EntryMetadata(metainfo.MSection):
     calc_hash = metainfo.Quantity(
         type=str,
         description='A raw file content based checksum/hash.',
+        categories=[MongoMetadata],
         a_search=Search(
             many_or='append', metric_name='unique_entries', metric='cardinality'))
 
@@ -280,39 +283,48 @@ class EntryMetadata(metainfo.MSection):
     pid = metainfo.Quantity(
         type=int,
         description='The unique, sequentially enumerated, integer persistent identifier',
+        categories=[MongoMetadata],
         a_search=Search(many_or='append'))
 
     raw_id = metainfo.Quantity(
         type=str,
         description='A raw format specific id that was acquired from the files of this entry',
+        categories=[MongoMetadata],
         a_search=Search(many_or='append'))
 
     domain = metainfo.Quantity(
         type=metainfo.MEnum('dft', 'ems'),
         description='The material science domain',
+        categories=[MongoMetadata],
         a_search=Search())
 
     published = metainfo.Quantity(
         type=bool, default=False,
         description='Indicates if the entry is published',
+        categories=[MongoMetadata],
         a_search=Search())
 
     processed = metainfo.Quantity(
         type=bool, default=False,
         description='Indicates that the entry is successfully processed.',
+        categories=[MongoMetadata],
         a_search=Search())
 
     last_processing = metainfo.Quantity(
         type=metainfo.Datetime,
-        description='The datetime of the last attempted processing.')
+        description='The datetime of the last attempted processing.',
+        categories=[MongoMetadata],
+        a_search=Search())
 
     nomad_version = metainfo.Quantity(
         type=str,
         description='The NOMAD version used for the last processing attempt.',
+        categories=[MongoMetadata],
         a_search=Search(many_or='append'))
     nomad_commit = metainfo.Quantity(
         type=str,
         description='The NOMAD commit used for the last processing attempt.',
+        categories=[MongoMetadata],
         a_search=Search(many_or='append'))
     parser_name = metainfo.Quantity(
         type=str,
@@ -320,17 +332,17 @@ class EntryMetadata(metainfo.MSection):
         a_search=Search(many_or='append'))
 
     comment = metainfo.Quantity(
-        type=str, categories=[UserMetadata, EditableUserMetadata],
+        type=str, categories=[MongoMetadata, EditableUserMetadata],
         description='A user provided comment.',
         a_search=Search(mapping=Text()))
 
     references = metainfo.Quantity(
-        type=str, shape=['0..*'], categories=[UserMetadata, EditableUserMetadata],
+        type=str, shape=['0..*'], categories=[MongoMetadata, EditableUserMetadata],
         description='User provided references (URLs).',
         a_search=Search())
 
     uploader = metainfo.Quantity(
-        type=user_reference, categories=[UserMetadata],
+        type=user_reference, categories=[MongoMetadata],
         description='The uploader of the entry',
         a_flask=dict(admin_only=True, verify=User),
         a_search=[
@@ -343,7 +355,7 @@ class EntryMetadata(metainfo.MSection):
         ])
 
     coauthors = metainfo.Quantity(
-        type=user_reference, shape=['0..*'], default=[], categories=[UserMetadata, EditableUserMetadata],
+        type=user_reference, shape=['0..*'], default=[], categories=[MongoMetadata, EditableUserMetadata],
         description='A user provided list of co-authors.',
         a_flask=dict(verify=User))
 
@@ -357,7 +369,7 @@ class EntryMetadata(metainfo.MSection):
             many_or='append', search_field='authors.name.keyword', statistic_size=1000))
 
     shared_with = metainfo.Quantity(
-        type=user_reference, shape=['0..*'], default=[], categories=[UserMetadata, EditableUserMetadata],
+        type=user_reference, shape=['0..*'], default=[], categories=[MongoMetadata, EditableUserMetadata],
         description='A user provided list of userts to share the entry with.',
         a_flask=dict(verify=User))
 
@@ -370,24 +382,24 @@ class EntryMetadata(metainfo.MSection):
             many_or='append', search_field='owners.name.keyword'))
 
     with_embargo = metainfo.Quantity(
-        type=bool, default=False, categories=[UserMetadata, EditableUserMetadata],
+        type=bool, default=False, categories=[MongoMetadata, EditableUserMetadata],
         description='Indicated if this entry is under an embargo',
         a_search=Search())
 
     upload_time = metainfo.Quantity(
-        type=metainfo.Datetime, categories=[UserMetadata],
+        type=metainfo.Datetime, categories=[MongoMetadata],
         description='The datetime this entry was uploaded to nomad',
         a_flask=dict(admin_only=True),
         a_search=Search(order_default=True))
 
     upload_name = metainfo.Quantity(
-        type=str, categories=[UserMetadata],
+        type=str, categories=[MongoMetadata],
         description='The user provided upload name',
         a_search=Search(many_or='append'))
 
     datasets = metainfo.Quantity(
         type=dataset_reference, shape=['0..*'], default=[],
-        categories=[UserMetadata, EditableUserMetadata],
+        categories=[MongoMetadata, EditableUserMetadata],
         description='A list of user curated datasets this entry belongs to.',
         a_flask=dict(verify=Dataset),
         a_search=[
@@ -401,12 +413,12 @@ class EntryMetadata(metainfo.MSection):
                 description='Search for a particular dataset by its id.')])
 
     external_id = metainfo.Quantity(
-        type=str, categories=[UserMetadata],
+        type=str, categories=[MongoMetadata],
         description='A user provided external id.',
         a_search=Search(many_or='split'))
 
     last_edit = metainfo.Quantity(
-        type=metainfo.Datetime, categories=[UserMetadata],
+        type=metainfo.Datetime, categories=[MongoMetadata],
         description='The datetime the user metadata was edited last.',
         a_search=Search())
 
@@ -441,7 +453,24 @@ class EntryMetadata(metainfo.MSection):
 
     def apply_domain_metadata(self, backend):
         assert self.domain is not None, 'all entries must have a domain'
-        domain_section_def = self.m_def.all_sub_sections.get(self.domain).sub_section
+        domain_sub_section_def = self.m_def.all_sub_sections.get(self.domain)
+        domain_section_def = domain_sub_section_def.sub_section
         assert domain_section_def is not None, 'unknown domain %s' % self.domain
-        domain_section = self.m_create(domain_section_def.section_cls)
+
+        # add domain section if not already there
+        domain_section = self.m_get_sub_section(domain_sub_section_def, -1)
+        if domain_section is None:
+            domain_section = self.m_create(domain_section_def.section_cls)
+
         domain_section.apply_domain_metadata(backend)
+
+
+class EntryArchive(metainfo.MSection):
+
+    section_run = metainfo.SubSection(sub_section=section_run, repeats=True)
+    section_experiment = metainfo.SubSection(sub_section=section_experiment)
+    section_metadata = metainfo.SubSection(sub_section=EntryMetadata)
+
+    processing_logs = metainfo.Quantity(
+        type=Any, shape=['0..*'],
+        description='The processing logs for this entry as a list of structlog entries.')
diff --git a/nomad/datamodel/dft.py b/nomad/datamodel/dft.py
index 909ee11f7f7313ce1618503fc27947f0b9576396..1a557bc95af6ed98c9f4a8c52a595fa10aeacc77 100644
--- a/nomad/datamodel/dft.py
+++ b/nomad/datamodel/dft.py
@@ -24,7 +24,6 @@ from nomad.metainfo.search_extension import Search
 
 from .common import get_optional_backend_value
 from .optimade import OptimadeEntry
-from .metainfo.public import section_run
 
 
 xc_treatments = {
@@ -235,14 +234,11 @@ class DFTMetadata(MSection):
         n_total_energies = 0
         n_geometries = 0
 
-        for root_section in backend.resource.contents:
-            if not root_section.m_follows(section_run.m_def):
-                continue
-
-            quantities.add(root_section.m_def.name)
+        for section_run in backend.entry_archive.section_run:
+            quantities.add(section_run.m_def.name)
             n_quantities += 1
 
-            for section, property_def, _ in root_section.m_traverse():
+            for section, property_def, _ in section_run.m_traverse():
                 property_name = property_def.name
                 quantities.add(property_name)
                 n_quantities += 1
@@ -284,8 +280,3 @@ class DFTMetadata(MSection):
         if aflow_id is not None and aflow_label is not None:
             self.labels.append(Label(label=aflow_label, type='prototype', source='aflow_prototype_library'))
             self.labels.append(Label(label=aflow_id, type='prototype_id', source='aflow_prototype_library'))
-
-        # optimade
-        optimade = backend.get_mi2_section(OptimadeEntry.m_def)
-        if optimade is not None:
-            self.optimade = optimade.m_copy()
diff --git a/nomad/datamodel/ems.py b/nomad/datamodel/ems.py
index 3a85b46b65d63201ce2d65624d69030cecd72e91..f29816c0b9fd4d5541f6618f238fa069e5afc0ec 100644
--- a/nomad/datamodel/ems.py
+++ b/nomad/datamodel/ems.py
@@ -21,7 +21,6 @@ from nomad.metainfo import Quantity, MSection, Section, Datetime
 from nomad.metainfo.search_extension import Search
 
 from .common import get_optional_backend_value
-from .metainfo.general_experimental import section_experiment
 
 
 class EMSMetadata(MSection):
@@ -107,12 +106,9 @@ class EMSMetadata(MSection):
 
         quantities = set()
 
-        for root_section in backend.resource.contents:
-            if not root_section.m_follows(section_experiment.m_def):
-                continue
-
-            quantities.add(root_section.m_def.name)
-            for _, property_def, _ in root_section.m_traverse():
-                quantities.add(property_def.name)
+        root_section = backend.entry_archive.section_experiment
+        quantities.add(root_section.m_def.name)
+        for _, property_def, _ in root_section.m_traverse():
+            quantities.add(property_def.name)
 
         self.quantities = list(quantities)
diff --git a/nomad/files.py b/nomad/files.py
index c758ca26341b2264ea2217b6270e34b7897ae51d..672fda251e3712876e4c28d9836a35a468809cb9 100644
--- a/nomad/files.py
+++ b/nomad/files.py
@@ -230,7 +230,7 @@ class UploadFiles(DirectoryObject, metaclass=ABCMeta):
         with open(self._user_metadata_file.os_path, 'wb') as f:
             pickle.dump(data, f)
 
-    def to_staging_upload_files(self, create: bool = False) -> 'StagingUploadFiles':
+    def to_staging_upload_files(self, create: bool = False, **kwargs) -> 'StagingUploadFiles':
         ''' Casts to or creates corresponding staging upload files or returns None. '''
         raise NotImplementedError()
 
@@ -308,7 +308,7 @@ class StagingUploadFiles(UploadFiles):
         self._size = 0
         self._shared = DirectoryObject(config.fs.public, upload_id, create=create)
 
-    def to_staging_upload_files(self, create: bool = False) -> 'StagingUploadFiles':
+    def to_staging_upload_files(self, create: bool = False, **kwargs) -> 'StagingUploadFiles':
         return self
 
     @property
@@ -348,7 +348,11 @@ class StagingUploadFiles(UploadFiles):
         if not self._is_authorized():
             raise Restricted
 
-        return read_archive(self.archive_file_object(calc_id).os_path)
+        try:
+            return read_archive(self.archive_file_object(calc_id).os_path)
+
+        except FileNotFoundError:
+            raise KeyError(calc_id)
 
     def archive_file_object(self, calc_id: str) -> PathObject:
         return self._archive_dir.join_file('%s.%s' % (calc_id, 'msg'))
@@ -678,13 +682,14 @@ class PublicUploadFilesBasedStagingUploadFiles(StagingUploadFiles):
     def extract(self, include_archive: bool = False) -> None:
         assert next(self.raw_file_manifest(), None) is None, 'can only extract once'
         for access in ['public', 'restricted']:
-            super().add_rawfiles(
-                self.public_upload_files._zip_file_object('raw', access, 'plain').os_path,
-                force_archive=True)
+            raw_file_zip = self.public_upload_files._zip_file_object('raw', access, 'plain')
+            if raw_file_zip.exists():
+                super().add_rawfiles(raw_file_zip.os_path, force_archive=True)
 
             if include_archive:
                 with self.public_upload_files._open_msg_file('archive', access, 'msg') as archive:
                     for calc_id, data in archive.items():
+                        calc_id = calc_id.strip()
                         self.write_archive(calc_id, data.to_dict())
 
     def add_rawfiles(self, *args, **kwargs) -> None:
@@ -771,13 +776,15 @@ class PublicUploadFiles(UploadFiles):
     def to_staging_upload_files(self, create: bool = False, **kwargs) -> 'StagingUploadFiles':
         exists = False
         try:
-            staging_upload_files = PublicUploadFilesBasedStagingUploadFiles(self, is_authorized=lambda: True)
+            staging_upload_files = PublicUploadFilesBasedStagingUploadFiles(
+                self, is_authorized=lambda: True)
             exists = True
         except KeyError:
             if not create:
                 return None
 
-            staging_upload_files = PublicUploadFilesBasedStagingUploadFiles(self, create=True, is_authorized=lambda: True)
+            staging_upload_files = PublicUploadFilesBasedStagingUploadFiles(
+                self, create=True, is_authorized=lambda: True)
             staging_upload_files.extract(**kwargs)
 
         if exists and create:
@@ -854,10 +861,10 @@ class PublicUploadFiles(UploadFiles):
         for access in accesses:
             try:
                 archive = self._open_msg_file('archive', access, 'msg')
-                if access == 'restricted' and not self._is_authorized():
-                    raise Restricted
-
                 if calc_id in archive:
+                    if access == 'restricted' and not self._is_authorized():
+                        raise Restricted
+
                     return archive
             except FileNotFoundError:
                 pass
diff --git a/nomad/metainfo/example.py b/nomad/metainfo/example.py
index d5ec015378d0561d107bef3ff31d54d4e168cc67..f585a2c4a2aef8ddac7f6a012ff008f419535e2f 100644
--- a/nomad/metainfo/example.py
+++ b/nomad/metainfo/example.py
@@ -70,6 +70,8 @@ class System(MSection):
         type=bool, shape=[3], default=[False, False, False], categories=[SystemHash],
         description='A vector of booleans indicating in which dimensions the unit cell is repeated.')
 
+    system_type = Quantity(type=str)
+
 
 class SCC(MSection):
 
diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py
index d8554da8132309ee55111ca29ddf205f7e4d093c..5afc41201caa5e8e2f174932deaf4836bac37a63 100644
--- a/nomad/metainfo/metainfo.py
+++ b/nomad/metainfo/metainfo.py
@@ -520,8 +520,6 @@ class MResource():
             self.contents.append(section)
 
     def remove(self, section):
-        import traceback
-        traceback.print_stack()
         assert section.m_resource == self, 'Can only remove section from the resource that contains it.'
         section.m_resource = None
         self.__data.get(section.m_def).remove(section)
@@ -1029,7 +1027,9 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             raise TypeError('%s is of class Section' % definition)
         return self.m_def == definition or definition in self.m_def.all_base_sections
 
-    def m_to_dict(self, with_meta: bool = False, include_defaults: bool = False) -> Dict[str, Any]:
+    def m_to_dict(
+            self, with_meta: bool = False, include_defaults: bool = False,
+            categories: List[Union['Category', Type['MCategory']]] = None) -> Dict[str, Any]:
         '''
         Returns the data of this section as a json serializeable dictionary.
 
@@ -1037,7 +1037,19 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             with_meta: Include information about the section definition and the sections
                 position in its parent.
             include_defaults: Include default values of unset quantities.
+            categories: A list of category classes or category definitions that is used
+                to filter the included quantities and sub sections.
         '''
+        category_defs: List[Category] = None
+        if categories is not None:
+            category_defs = []
+            for category in categories:
+                if issubclass(category, MCategory):  # type: ignore
+                    category_defs.append(category.m_def)  # type: ignore
+                elif isinstance(category, Category):
+                    category_defs.append(category)
+                else:
+                    raise TypeError('%s is not a category' % category)
 
         def items() -> Iterable[Tuple[str, Any]]:
             # metadata
@@ -1050,6 +1062,12 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
 
             # quantities
             for name, quantity in self.m_def.all_quantities.items():
+                if categories is not None:
+                    if not any(
+                            quantity in category.get_all_definitions()
+                            for category in category_defs):
+                        continue
+
                 if quantity.virtual:
                     continue
 
@@ -1126,6 +1144,12 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
 
             # sub sections
             for name, sub_section_def in self.m_def.all_sub_sections.items():
+                if categories is not None:
+                    if not any(
+                            sub_section_def in category.get_all_definitions()
+                            for category in category_defs):
+                        continue
+
                 if sub_section_def.repeats:
                     if self.m_sub_section_count(sub_section_def) > 0:
                         yield name, [
@@ -1138,16 +1162,13 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
 
         return {key: value for key, value in items()}
 
-    @classmethod
-    def m_from_dict(cls: Type[MSectionBound], dct: Dict[str, Any]) -> MSectionBound:
-        ''' Creates a section from the given serializable data dictionary.
-
-        This is the 'opposite' of :func:`m_to_dict`. It takes a deserialised dict, e.g
-        loaded from JSON, and turns it into a proper section, i.e. instance of the given
-        section class.
+    def m_update_from_dict(self, dct: Dict[str, Any]) -> None:
         '''
-
-        section_def = cls.m_def
+        Updates this section with the serialized data from the given dict, e.g. data
+        produced by :func:`m_to_dict`.
+        '''
+        section_def = self.m_def
+        section = self
 
         # remove m_def, m_parent_index, m_parent_sub_section metadata,
         # they set themselves automatically
@@ -1155,8 +1176,6 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
         dct.pop('m_parent_index', None)
         dct.pop('m_parent_sub_section', None)
 
-        section = cls()
-
         for name, sub_section_def in section_def.all_sub_sections.items():
             if name in dct:
                 sub_section_value = dct.pop(name)
@@ -1191,6 +1210,16 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
 
                 section.__dict__[name] = quantity_value  # type: ignore
 
+    @classmethod
+    def m_from_dict(cls: Type[MSectionBound], dct: Dict[str, Any]) -> MSectionBound:
+        ''' Creates a section from the given serializable data dictionary.
+
+        This is the 'opposite' of :func:`m_to_dict`. It takes a deserialised dict, e.g
+        loaded from JSON, and turns it into a proper section, i.e. instance of the given
+        section class.
+        '''
+        section = cls()
+        section.m_update_from_dict(dct)
         return section
 
     def m_to_json(self, **kwargs):
diff --git a/nomad/normalizing/optimade.py b/nomad/normalizing/optimade.py
index 30bd8735b00389407d946d651e764b7a0c48357b..6e5dc4694f6510fa45aac74a640d3237878e091e 100644
--- a/nomad/normalizing/optimade.py
+++ b/nomad/normalizing/optimade.py
@@ -12,16 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Dict
+from typing import Any, Dict, cast
 import numpy as np
 import re
 import ase.data
 from string import ascii_uppercase
 import pint.quantity
 
+from nomad.parsing.legacy import Backend
 from nomad.normalizing.normalizer import SystemBasedNormalizer
 from nomad.metainfo import units
-from nomad.datamodel import OptimadeEntry, Species
+from nomad.datamodel import OptimadeEntry, Species, DFTMetadata, EntryMetadata
 
 species_re = re.compile(r'^([A-Z][a-z]?)(\d*)$')
 
@@ -35,13 +36,19 @@ class OptimadeNormalizer(SystemBasedNormalizer):
     def __init__(self, backend):
         super().__init__(backend, only_representatives=True)
 
-    def get_optimade_data(self, index) -> OptimadeEntry:
+    def add_optimade_data(self, index) -> OptimadeEntry:
         '''
         The 'main' method of this :class:`SystemBasedNormalizer`.
         Normalizes the section with the given `index`.
         Normalizes geometry, classifies, system_type, and runs symmetry analysis.
         '''
-        optimade = OptimadeEntry()
+
+        backend = cast(Backend, self._backend)
+        if backend.entry_archive.section_metadata is None:
+            backend.entry_archive.m_create(EntryMetadata)
+        if backend.entry_archive.section_metadata.dft is None:
+            backend.entry_archive.section_metadata.m_create(DFTMetadata)
+        optimade = backend.entry_archive.section_metadata.dft.m_create(OptimadeEntry)
 
         def get_value(key: str, default: Any = None, numpy: bool = False, unit=None) -> Any:
             try:
@@ -125,8 +132,7 @@ class OptimadeNormalizer(SystemBasedNormalizer):
             return False
 
         try:
-            optimade = self.get_optimade_data(index)
-            self._backend.add_mi2_section(optimade)
+            self.add_optimade_data(index)
             return True
 
         except Exception as e:
diff --git a/nomad/parsing/legacy.py b/nomad/parsing/legacy.py
index 7e1949ab6a0838e2d7def8b105957aee8846bc14..362d05c8ea48bf931ae147302070447c25c9ef49 100644
--- a/nomad/parsing/legacy.py
+++ b/nomad/parsing/legacy.py
@@ -168,14 +168,6 @@ class AbstractParserBackend(metaclass=ABCMeta):
         ''' Used to catch parser warnings. '''
         pass
 
-    # The following are extensions to the origin NOMAD-coe parser backend. And allow
-    # access to existing data
-
-    # @property
-    # @abstractmethod
-    # def data(self) -> Results:
-    #     pass
-
     @abstractmethod
     def get_sections(self, meta_name: str, g_index: int = -1) -> List[int]:
         ''' Return all gIndices for existing sections of the given meta_name and parent section index. '''
@@ -189,19 +181,6 @@ class AbstractParserBackend(metaclass=ABCMeta):
         '''
         pass
 
-    # def add_mi2_section(self, section: MSection):
-    #     ''' Allows to mix a metainfo2 style section into backend. '''
-    #     pass
-
-    # def get_mi2_section(self, section_def: MI2Section):
-    #     ''' Allows to mix a metainfo2 style section into backend. '''
-    #     pass
-
-    # def traverse(self, *args, **kwargs) -> Iterable[Tuple[str, str, Any]]:
-    #     ''' Traverses the backend data and yiels tuples with metainfo name, event type,
-    #     and value '''
-    #     pass
-
     @abstractmethod
     def __getitem__(self, key):
         pass
@@ -221,6 +200,7 @@ class Backend(AbstractParserBackend):
         domain: The domain that this backend contains data for.
         env: The metainfo environment (all available definitions).
         resource: The metainfo resource that contains all data.
+        entry_archive: The root section of the archive behind this backend.
         logger: A logger that can be used to log metainfo and backend operation related
             warnings and errors.
     '''
@@ -242,6 +222,8 @@ class Backend(AbstractParserBackend):
         self.env: LegacyMetainfoEnvironment = cast(LegacyMetainfoEnvironment, metainfo)
         self.__legacy_env = None
         self.resource = MResource()
+        self.entry_archive = datamodel.EntryArchive()
+        self.resource.add(self.entry_archive)
 
         self.__open_sections: Dict[Tuple[Section, int], MSection] = {}
         self.strict = False  # TODO
@@ -286,7 +268,7 @@ class Backend(AbstractParserBackend):
                 if section_def is None:
                     raise BadContextUri(context_uri)
                 i = 0
-                for content in self.resource.contents:
+                for content in self.entry_archive.m_contents():
                     if content.m_follows(section_def):
                         if i == index:
                             current = content
@@ -339,7 +321,7 @@ class Backend(AbstractParserBackend):
         section_def = self.resolve_definition(name, Section)
 
         if name in datamodel.root_sections:
-            section = self.resource.create(section_def.section_cls)
+            section = self.entry_archive.m_create(section_def.section_cls)
 
         else:
             sub_section_def = self.resolve_definition(name, SubSection)
@@ -483,14 +465,6 @@ class Backend(AbstractParserBackend):
 
         return value
 
-    def add_mi2_section(self, section: MSection):
-        self.resource.add(section)
-
-    def get_mi2_section(self, section_def: Section):
-        for content in self.resource.contents:
-            if content.m_def == section_def:
-                return content
-
     def startedParsingSession(
             self, mainFileUri, parserInfo, parserStatus=None, parserErrors=None):
         self.reset_status()
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index e25e9246323f9ac59aebe8d31fcfbc17c5b73ccd..1444db3aad4cdc655ffddda4b2b6078d43bf7992 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -24,7 +24,7 @@ calculations, and files
 
 '''
 
-from typing import cast, List, Any, Tuple, Generator, Dict, cast, Iterable
+from typing import cast, List, Any, Tuple, Iterator, Dict, cast, Iterable
 from mongoengine import StringField, DateTimeField, DictField, BooleanField, IntField
 import logging
 from structlog import wrap_logger
@@ -110,15 +110,7 @@ class Calc(Proc):
         self._upload_files: ArchiveBasedStagingUploadFiles = None
         self._calc_proc_logs: List[Any] = None
 
-    @classmethod
-    def from_entry_metadata(cls, entry_metadata):
-        calc = Calc.create(
-            calc_id=entry_metadata.calc_id,
-            upload_id=entry_metadata.upload_id,
-            mainfile=entry_metadata.mainfile,
-            metadata=entry_metadata.m_to_dict(include_defaults=True))
-
-        return calc
+        self._entry_metadata = None
 
     @classmethod
     def get(cls, id):
@@ -135,6 +127,61 @@ class Calc(Proc):
             self._upload.worker_hostname = self.worker_hostname
         return self._upload
 
+    def apply_entry_metadata(self, entry_metadata: datamodel.EntryMetadata):
+        self.metadata = entry_metadata.m_to_dict(
+            include_defaults=True,
+            categories=[datamodel.MongoMetadata])  # TODO use embedded doc?
+
+    def create_metadata(self) -> datamodel.EntryMetadata:
+        '''
+        Returns a :class:`nomad.datamodel.EntryMetadata` with values from this
+        processing object, not necessarely the user metadata nor the metadata from
+        the archive.
+        '''
+        entry_metadata = datamodel.EntryMetadata()
+        entry_metadata.domain = parser_dict[self.parser].domain
+        entry_metadata.upload_id = self.upload_id
+        entry_metadata.calc_id = self.calc_id
+        entry_metadata.mainfile = self.mainfile
+        entry_metadata.nomad_version = config.version
+        entry_metadata.nomad_commit = config.commit
+        entry_metadata.uploader = self.upload.user_id
+        entry_metadata.upload_time = self.upload.upload_time
+        entry_metadata.upload_name = self.upload.name
+
+        return entry_metadata
+
+    def entry_metadata(self, upload_files: UploadFiles) -> datamodel.EntryMetadata:
+        '''
+        Returns a complete set of :class:`nomad.datamodel.EntryMetadata` including
+        the user metadata and metadata from the archive.
+
+        Arguments:
+            upload_files:
+                The :class:`nomad.files.UploadFiles` instance to read the archive from.
+            cache:
+                A boolean that indicates if the archive file should be left unclosed,
+                e.g. if this method is called for many entries of the same upload.
+        '''
+        archive = upload_files.read_archive(self.calc_id)
+        entry_metadata = datamodel.EntryMetadata.m_from_dict(
+            archive[self.calc_id][datamodel.EntryArchive.section_metadata.name].to_dict())
+
+        entry_metadata.m_update_from_dict(self.metadata)
+
+        return entry_metadata
+
+    def user_metadata(self) -> datamodel.EntryMetadata:
+        '''
+        Returns a :class:`nomad.datamodel.EntryMetadata` with values from this
+        processing object and the user metadata, not necessarely the metadata from
+        the archive.
+        '''
+        entry_metadata = self.create_metadata()
+        entry_metadata.m_update_from_dict(self.metadata)
+
+        return entry_metadata
+
     @property
     def upload_files(self) -> ArchiveBasedStagingUploadFiles:
         if not self._upload_files:
@@ -176,10 +223,19 @@ class Calc(Proc):
         records.
         '''
         parser = match_parser(self.upload_files.raw_file_object(self.mainfile).os_path, strict=False)
+        logger = self.get_logger()
 
         if parser is None and not config.reprocess_unmatched:
             self.errors = ['no parser matches during re-process, will not re-process this calc']
 
+            try:
+                upload_files = PublicUploadFiles(self.upload_id, is_authorized=lambda: True)
+                with upload_files.read_archive(self.calc_id) as archive:
+                    self.upload_files.write_archive(self.calc_id, archive[self.calc_id].to_dict())
+
+            except Exception as e:
+                logger.error('could not copy archive for non matching, non reprocessed entry', exc_info=e)
+
             # mock the steps of actual processing
             self._continue_with('parsing')
             self._continue_with('normalizing')
@@ -187,7 +243,6 @@ class Calc(Proc):
             self._complete()
             return
 
-        logger = self.get_logger()
         if parser is None:
             self.get_logger().error('no parser matches during re-process, use the old parser')
             self.errors = ['no matching parser found during re-processing']
@@ -198,16 +253,12 @@ class Calc(Proc):
                 parser=parser.name)
 
         try:
-            entry_metadata = datamodel.EntryMetadata.m_from_dict(self.metadata)
-            entry_metadata.upload_id = self.upload_id
-            entry_metadata.calc_id = self.calc_id
-            entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
-            entry_metadata.mainfile = self.mainfile
-            entry_metadata.nomad_version = config.version
-            entry_metadata.nomad_commit = config.commit
-            entry_metadata.last_processing = datetime.utcnow()
-            entry_metadata.files = self.upload_files.calc_files(self.mainfile)
-            self.metadata = entry_metadata.m_to_dict(include_defaults=True)
+            self._entry_metadata = self.user_metadata()
+            self._entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
+            self._entry_metadata.nomad_version = config.version
+            self._entry_metadata.nomad_commit = config.commit
+            self._entry_metadata.last_processing = datetime.utcnow()
+            self._entry_metadata.files = self.upload_files.calc_files(self.mainfile)
 
             self.parsing()
             self.normalizing()
@@ -233,23 +284,12 @@ class Calc(Proc):
         try:
             # save preliminary minimum calc metadata in case processing fails
             # successful processing will replace it with the actual metadata
-            calc_metadata = datamodel.EntryMetadata()
-            calc_metadata.domain = parser_dict[self.parser].domain
-            calc_metadata.upload_id = self.upload_id
-            calc_metadata.calc_id = self.calc_id
-            calc_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
-            calc_metadata.mainfile = self.mainfile
-            calc_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
-            calc_metadata.nomad_version = config.version
-            calc_metadata.nomad_commit = config.commit
-            calc_metadata.last_processing = datetime.utcnow()
-            calc_metadata.files = self.upload_files.calc_files(self.mainfile)
-            calc_metadata.uploader = self.upload.user_id
-            calc_metadata.upload_time = self.upload.upload_time
-            calc_metadata.upload_name = self.upload.name
-            self.metadata = calc_metadata.m_to_dict(include_defaults=True)  # TODO use embedded doc?
-
-            if len(calc_metadata.files) >= config.auxfile_cutoff:
+            self._entry_metadata = self.create_metadata()
+            self._entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
+            self._entry_metadata.last_processing = datetime.utcnow()
+            self._entry_metadata.files = self.upload_files.calc_files(self.mainfile)
+
+            if len(self._entry_metadata.files) >= config.auxfile_cutoff:
                 self.warning(
                     'This calc has many aux files in its directory. '
                     'Have you placed many calculations in the same directory?')
@@ -269,22 +309,28 @@ class Calc(Proc):
         # in case of failure, index a minimum set of metadata and mark
         # processing failure
         try:
-            entry_metadata = datamodel.EntryMetadata.m_from_dict(self.metadata)
             if self.parser is not None:
                 parser = parser_dict[self.parser]
                 if hasattr(parser, 'code_name'):
-                    entry_metadata.code_name = parser.code_name
+                    self._entry_metadata.code_name = parser.code_name
 
-            entry_metadata.processed = False
-            self.metadata = entry_metadata.m_to_dict(include_defaults=True)
-            entry_metadata.a_elastic.index()
+            self._entry_metadata.processed = False
+            self.apply_entry_metadata(self._entry_metadata)
+            self._entry_metadata.a_elastic.index()
         except Exception as e:
-            self.get_logger().error('could not index after processing failure', exc_info=e)
+            self.get_logger().error(
+                'could not index after processing failure', exc_info=e)
 
         try:
-            self.upload_files.write_archive(self.calc_id, {'processing_logs': self._calc_proc_logs})
+            archive = datamodel.EntryArchive()
+            archive.m_add_sub_section(
+                datamodel.EntryArchive.section_metadata, self._entry_metadata)
+            archive.processing_logs = self._calc_proc_logs
+
+            self.upload_files.write_archive(self.calc_id, archive.m_to_dict())
         except Exception as e:
-            self.get_logger().error('could not write archive (logs) after processing failure', exc_info=e)
+            self.get_logger().error(
+                'could not write archive after processing failure', exc_info=e)
 
         super().fail(*errors, log_level=log_level, **kwargs)
 
@@ -302,7 +348,7 @@ class Calc(Proc):
         context = dict(parser=self.parser, step=self.parser)
         logger = self.get_logger(**context)
         parser = parser_dict[self.parser]
-        self.metadata['parser_name'] = self.parser
+        self._entry_metadata.parser_name = self.parser
 
         with utils.timer(logger, 'parser executed', input_size=self.mainfile_file.size):
             try:
@@ -315,26 +361,8 @@ class Calc(Proc):
                 self.fail('parser raised system exit', error='system exit', **context)
                 return
 
-        # add the non code specific calc metadata to the backend
-        # all other quantities have been determined by parsers/normalizers
-        self._parser_backend.openNonOverlappingSection('section_entry_info')
-        self._parser_backend.addValue('upload_id', self.upload_id)
-        self._parser_backend.addValue('calc_id', self.calc_id)
-        self._parser_backend.addValue('calc_hash', self.metadata['calc_hash'])
-        self._parser_backend.addValue('mainfile', self.mainfile)
-        self._parser_backend.addValue('parser_name', self.parser)
-        filepaths = self.metadata['files']
-        self._parser_backend.addValue('number_of_files', len(filepaths))
-        self._parser_backend.addValue('filepaths', filepaths)
-        uploader = self.upload.uploader
-        self._parser_backend.addValue(
-            'entry_uploader_name', '%s, %s' % (uploader.first_name, uploader.last_name))
-        self._parser_backend.addValue(
-            'entry_uploader_id', str(uploader.user_id))
-        self._parser_backend.addValue('entry_upload_time', int(self.upload.upload_time.timestamp()))
-        self._parser_backend.closeNonOverlappingSection('section_entry_info')
-
-        self.add_processor_info(self.parser)
+        self._parser_backend.entry_archive.m_add_sub_section(
+            datamodel.EntryArchive.section_metadata, self._entry_metadata)
 
         if self._parser_backend.status[0] != 'ParseSuccess':
             error = self._parser_backend.status[1]
@@ -344,27 +372,25 @@ class Calc(Proc):
     def use_parser_backend(self, processor_name):
         self._parser_backend.reset_status()
         yield self._parser_backend
-        self.add_processor_info(processor_name)
-
-    def add_processor_info(self, processor_name: str) -> None:
-        self._parser_backend.openContext('/section_entry_info/0')
-        self._parser_backend.openNonOverlappingSection('section_archive_processing_info')
-        self._parser_backend.addValue('archive_processor_name', processor_name)
 
         if self._parser_backend.status[0] == 'ParseSuccess':
             warnings = getattr(self._parser_backend, '_warnings', [])
+
             if len(warnings) > 0:
-                self._parser_backend.addValue('archive_processor_status', 'WithWarnings')
-                self._parser_backend.addValue('number_of_archive_processor_warnings', len(warnings))
-                self._parser_backend.addArrayValues('archive_processor_warnings', [str(warning) for warning in warnings])
+                self.get_logger().warn(
+                    'processor completed successful with warnings',
+                    processor=processor_name, warnings=[str(warning) for warning in warnings])
+
             else:
-                self._parser_backend.addValue('archive_processor_status', 'Success')
+                self.get_logger().info(
+                    'processor completed successful',
+                    processor=processor_name)
+
         else:
             errors = self._parser_backend.status[1]
-            self._parser_backend.addValue('archive_processor_error', str(errors))
-
-        self._parser_backend.closeNonOverlappingSection('section_archive_processing_info')
-        self._parser_backend.closeContext('/section_entry_info/0')
+            self.get_logger().error(
+                'processor completed with failure',
+                processor=processor_name, errors=str(errors))
 
     @task
     def normalizing(self):
@@ -401,29 +427,26 @@ class Calc(Proc):
         ''' The *task* that encapsulates all archival related actions. '''
         logger = self.get_logger()
 
-        entry_metadata = datamodel.EntryMetadata.m_from_dict(self.metadata)
-        entry_metadata.apply_domain_metadata(self._parser_backend)
-        entry_metadata.processed = True
+        self._entry_metadata.apply_domain_metadata(self._parser_backend)
+        self._entry_metadata.processed = True
 
         # persist the calc metadata
         with utils.timer(logger, 'saved calc metadata', step='metadata'):
-            self.metadata = entry_metadata.m_to_dict(include_defaults=True)
+            self.apply_entry_metadata(self._entry_metadata)
 
         # index in search
         with utils.timer(logger, 'indexed', step='index'):
-            entry_metadata.a_elastic.index()
+            self._entry_metadata.a_elastic.index()
 
         # persist the archive
         with utils.timer(
                 logger, 'archived', step='archive',
                 input_size=self.mainfile_file.size) as log_data:
 
-            archive_data = self._parser_backend.resource.m_to_dict(
-                lambda section: section.m_def.name in datamodel.root_sections)
-
-            archive_data['processing_logs'] = self._calc_proc_logs
+            self._parser_backend.entry_archive.processing_logs = self._calc_proc_logs
             self._calc_proc_logs = None
 
+            archive_data = self._parser_backend.entry_archive.m_to_dict()
             archive_size = self.upload_files.write_archive(self.calc_id, archive_data)
             log_data.update(archive_size=archive_size)
 
@@ -587,44 +610,45 @@ class Upload(Proc):
         logger.info('started to publish')
 
         with utils.lnr(logger, 'publish failed'):
-            calcs = self.entries_metadata(self.metadata)
+            with self.entries_metadata(self.metadata) as calcs:
 
-            with utils.timer(
-                    logger, 'upload metadata updated', step='metadata',
-                    upload_size=self.upload_files.size):
-
-                def create_update(calc):
-                    calc.published = True
-                    calc.with_embargo = calc.with_embargo if calc.with_embargo is not None else False
-                    return UpdateOne(
-                        {'_id': calc.calc_id},
-                        {'$set': {'metadata': calc.m_to_dict(include_defaults=True)}})
-
-                Calc._get_collection().bulk_write([create_update(calc) for calc in calcs])
-
-            if isinstance(self.upload_files, StagingUploadFiles):
                 with utils.timer(
-                        logger, 'staged upload files packed', step='pack',
+                        logger, 'upload metadata updated', step='metadata',
                         upload_size=self.upload_files.size):
-                    self.upload_files.pack(calcs)
 
-            with utils.timer(
-                    logger, 'index updated', step='index',
-                    upload_size=self.upload_files.size):
-                search.publish(calcs)
+                    def create_update(calc):
+                        calc.published = True
+                        calc.with_embargo = calc.with_embargo if calc.with_embargo is not None else False
+                        return UpdateOne(
+                            {'_id': calc.calc_id},
+                            {'$set': {'metadata': calc.m_to_dict(
+                                include_defaults=True, categories=[datamodel.MongoMetadata])}})
+
+                    Calc._get_collection().bulk_write([create_update(calc) for calc in calcs])
+
+                if isinstance(self.upload_files, StagingUploadFiles):
+                    with utils.timer(
+                            logger, 'staged upload files packed', step='pack',
+                            upload_size=self.upload_files.size):
+                        self.upload_files.pack(calcs)
 
-            if isinstance(self.upload_files, StagingUploadFiles):
                 with utils.timer(
-                        logger, 'staged upload deleted', step='delete staged',
+                        logger, 'index updated', step='index',
                         upload_size=self.upload_files.size):
-                    self.upload_files.delete()
-                    self.published = True
-                    self.publish_time = datetime.utcnow()
+                    search.publish(calcs)
+
+                if isinstance(self.upload_files, StagingUploadFiles):
+                    with utils.timer(
+                            logger, 'staged upload deleted', step='delete staged',
+                            upload_size=self.upload_files.size):
+                        self.upload_files.delete()
+                        self.published = True
+                        self.publish_time = datetime.utcnow()
+                        self.last_update = datetime.utcnow()
+                        self.save()
+                else:
                     self.last_update = datetime.utcnow()
                     self.save()
-            else:
-                self.last_update = datetime.utcnow()
-                self.save()
 
     @process
     def re_process_upload(self):
@@ -696,7 +720,7 @@ class Upload(Proc):
         self._continue_with('parse_all')
         self._continue_with('cleanup')
 
-        self.upload_files.re_pack(self.entries_metadata())
+        self.upload_files.re_pack(self.user_metadata())
         self.joined = True
         self._complete()
 
@@ -785,7 +809,7 @@ class Upload(Proc):
                     self.staging_upload_files.raw_file_object(path).os_path,
                     self.staging_upload_files.raw_file_object(stripped_path).os_path))
 
-    def match_mainfiles(self) -> Generator[Tuple[str, object], None, None]:
+    def match_mainfiles(self) -> Iterator[Tuple[str, object]]:
         '''
         Generator function that matches all files in the upload to all parsers to
         determine the upload's mainfiles.
@@ -908,7 +932,7 @@ class Upload(Proc):
                 logger, 'reprocessed staged upload packed', step='delete staged',
                 upload_size=self.upload_files.size):
 
-            staging_upload_files.pack(self.entries_metadata(), skip_raw=True)
+            staging_upload_files.pack(self.user_metadata(), skip_raw=True)
 
         with utils.timer(
                 logger, 'reprocessed staged upload deleted', step='delete staged',
@@ -982,16 +1006,19 @@ class Upload(Proc):
         ''' All successfully processed calculations. '''
         return Calc.objects(upload_id=self.upload_id, tasks_status=SUCCESS)
 
-    def entries_metadata(self, user_metadata: dict = None) -> Iterable[datamodel.EntryMetadata]:
+    @contextmanager
+    def entries_metadata(
+            self, user_metadata: dict = None) -> Iterator[Iterable[datamodel.EntryMetadata]]:
         '''
         This is the :py:mod:`nomad.datamodel` transformation method to transform
-        processing uploads into datamodel uploads. It will also implicitely transform
-        all calculations of this upload.
+        processing upload's entries into list of :class:`nomad.datamodel.EntryMetadata` objects.
 
         Arguments:
             user_metadata: A dict of user metadata that is applied to the resulting
                 datamodel data and the respective calculations.
         '''
+        upload_files = self.upload_files
+
         # prepare user metadata per upload and per calc
         if user_metadata is not None:
             entries_metadata_dict: Dict[str, Any] = dict()
@@ -1008,7 +1035,7 @@ class Upload(Proc):
             user_upload_name = upload_metadata.get('upload_name', None)
 
             def get_metadata(calc: Calc):
-                entry_metadata = datamodel.EntryMetadata.m_from_dict(calc.metadata)
+                entry_metadata = calc.entry_metadata(upload_files)
                 entry_user_metadata = dict(upload_metadata)
                 entry_user_metadata.pop('embargo_length', None)  # this is for uploads only
                 entry_user_metadata.update(entries_metadata_dict.get(calc.mainfile, {}))
@@ -1023,13 +1050,25 @@ class Upload(Proc):
             user_upload_time = None
 
             def get_metadata(calc: Calc):
-                entry_metadata = datamodel.EntryMetadata.m_from_dict(calc.metadata)
+                entry_metadata = calc.entry_metadata(upload_files)
                 entry_metadata.upload_time = self.upload_time
                 entry_metadata.upload_name = self.name
 
                 return entry_metadata
 
-        return [get_metadata(calc) for calc in Calc.objects(upload_id=self.upload_id)]
+        try:
+            yield [
+                get_metadata(calc)
+                for calc in Calc.objects(upload_id=self.upload_id)]
+
+        finally:
+            upload_files.close()
+
+    def entry_ids(self) -> Iterable[str]:
+        return [calc.calc_id for calc in Calc.objects(upload_id=self.upload_id)]
+
+    def user_metadata(self) -> Iterable[datamodel.EntryMetadata]:
+        return [calc.user_metadata() for calc in Calc.objects(upload_id=self.upload_id)]
 
     def compress_and_set_metadata(self, metadata: Dict[str, Any]) -> None:
         '''
diff --git a/nomad/search.py b/nomad/search.py
index 603d8dde3c50ae9b421c31a87f4176f29ca13e16..f9ac993acb6f6f06e706efaebb07eaa7e7baa740 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -23,7 +23,7 @@ from elasticsearch.exceptions import NotFoundError
 from datetime import datetime
 import json
 
-from nomad import config, datamodel, infrastructure, datamodel, utils, processing as proc
+from nomad import config, datamodel, infrastructure, datamodel, utils
 from nomad.metainfo.search_extension import search_quantities, metrics, order_default_quantities, default_statistics
 
 
@@ -638,14 +638,6 @@ class SearchRequest:
         return json.dumps(self._search.to_dict(), indent=2)
 
 
-def to_calc_with_metadata(results: List[Dict[str, Any]]):
-    ''' Translates search results into :class:`EntryMetadata` objects read from mongo. '''
-    ids = [result['calc_id'] for result in results]
-    return [
-        datamodel.EntryMetadata.m_from_dict(calc.metadata)
-        for calc in proc.Calc.objects(calc_id__in=ids)]
-
-
 def flat(obj, prefix=None):
     '''
     Helper that translates nested result objects into flattened dicts with
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index bf5a176f41fa1756433f54cec8243a24830b9668..5790b17ab6344d8df7e4fb6543f14f39d2d3249f 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -33,7 +33,7 @@ from nomad.files import UploadFiles, PublicUploadFiles
 from nomad.processing import Upload, Calc, SUCCESS
 from nomad.datamodel import EntryMetadata, User, Dataset
 
-from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
+from tests.conftest import create_auth_headers, clear_elastic, clear_raw_files
 from tests.test_files import example_file, example_file_mainfile, example_file_contents
 from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
 from tests.test_search import assert_search_upload
@@ -260,19 +260,19 @@ class TestUploads:
         assert upload_proc is not None
         assert upload_proc.published is True
         assert upload_proc.embargo_length == min(36, metadata.get('embargo_length', 36))
-        entries = upload_proc.entries_metadata()
-
-        for entry in entries:
-            for key, transform in {
-                    'comment': lambda e: e.comment,
-                    'with_embargo': lambda e: e.with_embargo,
-                    'references': lambda e: e.references,
-                    'coauthors': lambda e: [u.user_id for u in e.coauthors],
-                    '_uploader': lambda e: e.uploader.user_id,
-                    '_pid': lambda e: e.pid,
-                    'external_id': lambda e: e.external_id}.items():
-                if key in metadata:
-                    assert transform(entry) == metadata[key], key
+
+        with upload_proc.entries_metadata() as entries:
+            for entry in entries:
+                for key, transform in {
+                        'comment': lambda e: e.comment,
+                        'with_embargo': lambda e: e.with_embargo,
+                        'references': lambda e: e.references,
+                        'coauthors': lambda e: [u.user_id for u in e.coauthors],
+                        '_uploader': lambda e: e.uploader.user_id,
+                        '_pid': lambda e: e.pid,
+                        'external_id': lambda e: e.external_id}.items():
+                    if key in metadata:
+                        assert transform(entry) == metadata[key], key
 
         assert_upload_files(upload_id, entries, files.PublicUploadFiles, published=True)
         assert_search_upload(entries, additional_keys=additional_keys, published=True)
@@ -614,7 +614,10 @@ class TestArchive(UploadFilesBasedTests):
     def test_get(self, api, upload, auth_headers):
         rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
         assert rv.status_code == 200
-        assert json.loads(rv.data) is not None
+        data = json.loads(rv.data)
+        assert data is not None
+        assert 'section_metadata' in data
+        assert 'section_run' in data
 
     @UploadFilesBasedTests.ignore_authorization
     def test_get_signed(self, api, upload, _, test_user_signature_token):
@@ -707,7 +710,7 @@ class TestArchive(UploadFilesBasedTests):
 class TestRepo():
     @pytest.fixture(scope='class')
     def example_elastic_calcs(
-            self, elastic_infra, normalized: parsing.Backend,
+            self, elastic_infra, raw_files_infra, normalized: parsing.Backend,
             test_user: User, other_test_user: User):
         clear_elastic(elastic_infra)
 
@@ -748,6 +751,7 @@ class TestRepo():
         yield
 
         example_dataset.a_mongo.delete()
+        clear_raw_files()
 
     def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
         if rv.status_code != 200:
@@ -1183,21 +1187,30 @@ class TestEditRepo():
         Dataset.m_def.a_mongo.objects(name='new_ds').delete()
 
     @pytest.fixture(autouse=True)
-    def example_data(self, class_api, test_user, other_test_user):
-        def create_entry(id, user, **kwargs):
-            metadata = dict(uploader=user.user_id, **kwargs)
-            create_test_structure(id, 2, 1, [], 0, metadata=metadata)
+    def example_data(self, class_api, test_user, other_test_user, raw_files):
+        from tests.app.utils import Upload
+
+        uploads = {}
+        for i in range(1, 4):
+            upload_id = 'upload_%d' % i
+            upload = Upload()
+            upload.upload_id = upload_id
+            uploads[upload_id] = upload
 
         entries = [
-            dict(calc_id='1', upload_id='upload_1', user=test_user, published=True, with_embargo=False),
-            dict(calc_id='2', upload_id='upload_2', user=test_user, published=True, with_embargo=True),
-            dict(calc_id='3', upload_id='upload_2', user=test_user, published=False, with_embargo=False),
-            dict(calc_id='4', upload_id='upload_3', user=other_test_user, published=True, with_embargo=False)]
+            dict(upload_id='upload_1', user=test_user, published=True, with_embargo=False),
+            dict(upload_id='upload_2', user=test_user, published=True, with_embargo=True),
+            dict(upload_id='upload_2', user=test_user, published=False, with_embargo=False),
+            dict(upload_id='upload_3', user=other_test_user, published=True, with_embargo=False)]
+
+        for i, entry in enumerate(entries):
+            upload = uploads[entry.pop('upload_id')]
+            user = entry.pop('user')
+            metadata = dict(uploader=user.user_id, **entry)
+            upload.create_test_structure(i + 1, 2, 1, [], 0, metadata=metadata)
 
-        i = 0
-        for entry in entries:
-            create_entry(i, **entry)
-            i += 1
+        for upload in uploads.values():
+            upload.create_upload_files()
 
         search.refresh()
 
@@ -1242,7 +1255,7 @@ class TestEditRepo():
 
     def mongo(self, *args, edited: bool = True, **kwargs):
         for calc_id in args:
-            calc = Calc.objects(calc_id=str(calc_id)).first()
+            calc = Calc.objects(calc_id='test_calc_id_%d' % calc_id).first()
             assert calc is not None
             metadata = calc.metadata
             if edited:
@@ -1254,7 +1267,7 @@ class TestEditRepo():
 
     def elastic(self, *args, **kwargs):
         for calc_id in args:
-            for calc in search.SearchRequest().search_parameters(calc_id=str(calc_id)).execute_scan():
+            for calc in search.SearchRequest().search_parameters(calc_id='test_calc_id_%d' % calc_id).execute_scan():
                 for key, value in kwargs.items():
                     if key in ['authors', 'owners']:
                         ids = [user['user_id'] for user in calc.get(key)]
@@ -1800,13 +1813,16 @@ class TestDataset:
         assert rv.status_code == 404
 
     @pytest.fixture()
-    def example_dataset_with_entry(self, mongo, elastic, example_datasets):
+    def example_dataset_with_entry(self, mongo, elastic, raw_files, example_datasets):
         entry_metadata = EntryMetadata(
             domain='dft', calc_id='1', upload_id='1', published=True, with_embargo=False,
             datasets=['1'])
         Calc(
             calc_id='1', upload_id='1', create_time=datetime.datetime.now(),
             metadata=entry_metadata.m_to_dict()).save()
+        upload_files = files.StagingUploadFiles(upload_id='1', create=True)
+        upload_files.write_archive('1', dict(section_metadata=entry_metadata.m_to_dict()))
+        upload_files.close()
         entry_metadata.a_elastic.index(refresh=True)
 
     def test_delete_dataset(self, api, test_user_auth, example_dataset_with_entry):
diff --git a/tests/app/test_optimade.py b/tests/app/test_optimade.py
index 21e99ca7d9783b7045b8ab47b2adaae4ebcc8e1c..60cc09c7b9107e61aa5f5394ae2c13aec31aba6e 100644
--- a/tests/app/test_optimade.py
+++ b/tests/app/test_optimade.py
@@ -21,7 +21,7 @@ from nomad import search
 from nomad.app.optimade import parse_filter, url
 
 from tests.app.test_app import BlueprintClient
-from tests.conftest import clear_elastic, create_test_structure
+from tests.conftest import clear_elastic, clear_raw_files
 
 
 @pytest.fixture(scope='session')
@@ -33,14 +33,18 @@ def test_get_entry(published: Upload):
     calc_id = list(published.calcs)[0].calc_id
     with published.upload_files.read_archive(calc_id) as archive:
         data = archive[calc_id]
-    assert 'OptimadeEntry' in data
+        assert data['section_metadata']['dft']['optimade'] is not None
+
     search_result = search.SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
     assert 'dft.optimade.chemical_formula_hill' in search.flat(search_result)
 
 
-def test_no_optimade(mongo, elastic, api):
-    create_test_structure(1, 2, 1, [], 0)
-    create_test_structure(2, 2, 1, [], 0, optimade=False)
+def test_no_optimade(mongo, elastic, raw_files, api):
+    from tests.app.utils import Upload
+    upload = Upload()
+    upload.create_test_structure(1, 2, 1, [], 0)
+    upload.create_test_structure(2, 2, 1, [], 0, optimade=False)
+    upload.create_upload_files()
     search.refresh()
 
     rv = api.get('/calculations')
@@ -51,18 +55,22 @@ def test_no_optimade(mongo, elastic, api):
 
 
 @pytest.fixture(scope='module')
-def example_structures(elastic_infra, mongo_infra):
+def example_structures(elastic_infra, mongo_infra, raw_files_infra):
     clear_elastic(elastic_infra)
     mongo_infra.drop_database('test_db')
 
-    create_test_structure(1, 2, 1, [], 0)
-    create_test_structure(2, 2, 1, ['C'], 0)
-    create_test_structure(3, 2, 1, [], 1)
-    create_test_structure(4, 1, 1, [], 0)
+    from tests.app.utils import Upload
+    upload = Upload()
+    upload.create_test_structure(1, 2, 1, [], 0)
+    upload.create_test_structure(2, 2, 1, ['C'], 0)
+    upload.create_test_structure(3, 2, 1, [], 1)
+    upload.create_test_structure(4, 1, 1, [], 0)
+    upload.create_upload_files()
     search.refresh()
 
     yield
     clear_elastic(elastic_infra)
+    clear_raw_files()
 
 
 @pytest.mark.parametrize('query, results', [
@@ -155,6 +163,7 @@ def test_list_endpoint_request_fields(api, example_structures):
     assert rv.status_code == 200
     data = json.loads(rv.data)
     ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
+    data['data'] = sorted(data['data'], key=lambda x: x['id'])
     for i in range(len(data['data'])):
         rf = list(data['data'][i]['attributes'].keys())
         rf.sort()
diff --git a/tests/app/utils.py b/tests/app/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7730c7504e97d7452415f72edfc728f64fa9d67
--- /dev/null
+++ b/tests/app/utils.py
@@ -0,0 +1,110 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+import numpy as np
+
+from nomad import processing, files
+from nomad.datamodel import EntryMetadata, MongoMetadata, EntryArchive
+from nomad.parsing import Backend
+
+from tests.test_normalizing import run_normalize
+
+
+class Upload():
+
+    def __init__(self):
+        self.entries: List[EntryMetadata] = []
+        self.upload_id = 'test_upload_id'
+
+    def create_upload_files(self) -> None:
+        upload_files = files.StagingUploadFiles(self.upload_id, create=True)
+        for entry_metadata in self.entries:
+            archive = entry_metadata.m_parent
+            if archive is None:
+                archive = EntryArchive()
+                archive.m_add_sub_section(EntryArchive.section_metadata, entry_metadata)
+
+            upload_files.write_archive(entry_metadata.calc_id, archive.m_to_dict())
+
+        upload_files.pack(self.entries, skip_raw=True)
+        upload_files.delete()
+
+        assert files.UploadFiles.get(self.upload_id) is not None
+
+    def add_entry(self, entry_metadata: EntryMetadata):
+        self.entries.append(entry_metadata)
+
+        processing.Calc.create(
+            calc_id=entry_metadata.calc_id,
+            upload_id=entry_metadata.upload_id,
+            mainfile=entry_metadata.mainfile,
+            metadata=entry_metadata.m_to_dict(
+                include_defaults=True, categories=[MongoMetadata])).save()
+
+        entry_metadata.a_elastic.index()
+
+    def create_test_structure(
+            self, id: int, h: int, o: int, extra: List[str], periodicity: int,
+            optimade: bool = True, metadata: dict = None):
+        ''' Creates a calculation in Elastic and Mongodb with the given properties.
+
+        Does require initialized :func:`elastic_infra` and :func:`mongo_infra`.
+
+        Args:
+            meta_info: A legace metainfo env.
+            id: A number to create ``test_calc_id_<number>`` ids.
+            h: The amount of H atoms
+            o: The amount of O atoms
+            extra: A list of further atoms
+            periodicity: The number of dimensions to repeat the structure in
+            optimade: A boolean. Iff true the entry will have optimade metadata. Default is True.
+            metadata: Additional (user) metadata.
+        '''
+        atom_labels = ['H' for i in range(0, h)] + ['O' for i in range(0, o)] + extra
+        test_vector = np.array([0, 0, 0])
+
+        backend = Backend('public', False, True)  # type: ignore
+        backend.openSection('section_run')
+        backend.addValue('program_name', 'test_code')
+        backend.openSection('section_system')
+
+        backend.addArrayValues('atom_labels', np.array(atom_labels))
+        backend.addArrayValues(
+            'atom_positions', np.array([test_vector for i in range(0, len(atom_labels))]))
+        backend.addArrayValues(
+            'lattice_vectors', np.array([test_vector, test_vector, test_vector]))
+        backend.addArrayValues(
+            'configuration_periodic_dimensions',
+            np.array([True for _ in range(0, periodicity)] + [False for _ in range(periodicity, 3)]))
+
+        backend.closeSection('section_system', 0)
+        backend.closeSection('section_run', 0)
+
+        backend = run_normalize(backend)
+        entry_metadata = backend.entry_archive.section_metadata
+
+        entry_metadata.m_update(
+            domain='dft', upload_id=self.upload_id, calc_id='test_calc_id_%d' % id,
+            mainfile='test_mainfile', published=True, with_embargo=False)
+
+        entry_metadata.apply_domain_metadata(backend)
+
+        if metadata is not None:
+            entry_metadata.m_update(**metadata)
+
+        if not optimade:
+            entry_metadata.dft.optimade = None
+
+        self.add_entry(entry_metadata)
diff --git a/tests/conftest.py b/tests/conftest.py
index 4b7731e1503f99c052bc7f74dc0f93b89c343c4f..b6faf5b56252640abe45136d223287d42bb42555 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -27,19 +27,16 @@ from bravado.client import SwaggerClient
 from flask import request, g
 import elasticsearch.exceptions
 from typing import List
-import numpy as np
 import json
 import logging
 
 from nomad import config, infrastructure, parsing, processing, app, utils
-from nomad.datamodel import User, EntryMetadata
-from nomad.parsing import Backend
+from nomad.datamodel import User
 
 from tests import test_parsing, test_normalizing
 from tests.processing import test_data as test_processing
 from tests.test_files import example_file, empty_file
 from tests.bravado_flask import FlaskTestHttpClient
-from tests.test_normalizing import run_normalize
 
 test_log_level = logging.CRITICAL
 example_files = [empty_file, example_file]
@@ -82,11 +79,16 @@ def raw_files(raw_files_infra):
     try:
         yield
     finally:
-        for directory in directories:
-            try:
-                shutil.rmtree(directory)
-            except FileNotFoundError:
-                pass
+        clear_raw_files()
+
+
+def clear_raw_files():
+    directories = [config.fs.staging, config.fs.public, config.fs.tmp]
+    for directory in directories:
+        try:
+            shutil.rmtree(directory)
+        except FileNotFoundError:
+            pass
 
 
 @pytest.fixture(scope='session')
@@ -634,59 +636,3 @@ def reset_config():
 def reset_infra(mongo, elastic):
     ''' Fixture that resets infrastructure after deleting db or search index. '''
     yield None
-
-
-def create_test_structure(
-        id: int, h: int, o: int, extra: List[str], periodicity: int,
-        optimade: bool = True, metadata: dict = None):
-    ''' Creates a calculation in Elastic and Mongodb with the given properties.
-
-    Does require initialized :func:`elastic_infra` and :func:`mongo_infra`.
-
-    Args:
-        meta_info: A legace metainfo env.
-        id: A number to create ``test_calc_id_<number>`` ids.
-        h: The amount of H atoms
-        o: The amount of O atoms
-        extra: A list of further atoms
-        periodicity: The number of dimensions to repeat the structure in
-        optimade: A boolean. Iff true the entry will have optimade metadata. Default is True.
-        metadata: Additional (user) metadata.
-    '''
-
-    atom_labels = ['H' for i in range(0, h)] + ['O' for i in range(0, o)] + extra
-    test_vector = np.array([0, 0, 0])
-
-    backend = Backend('public', False, True)  # type: ignore
-    backend.openSection('section_run')
-    backend.addValue('program_name', 'test_code')
-    backend.openSection('section_system')
-
-    backend.addArrayValues('atom_labels', np.array(atom_labels))
-    backend.addArrayValues(
-        'atom_positions', np.array([test_vector for i in range(0, len(atom_labels))]))
-    backend.addArrayValues(
-        'lattice_vectors', np.array([test_vector, test_vector, test_vector]))
-    backend.addArrayValues(
-        'configuration_periodic_dimensions',
-        np.array([True for _ in range(0, periodicity)] + [False for _ in range(periodicity, 3)]))
-
-    backend.closeSection('section_system', 0)
-    backend.closeSection('section_run', 0)
-
-    backend = run_normalize(backend)
-    calc = EntryMetadata(
-        domain='dft', upload_id='test_uload_id', calc_id='test_calc_id_%d' % id,
-        mainfile='test_mainfile', published=True, with_embargo=False)
-    calc.apply_domain_metadata(backend)
-    if metadata is not None:
-        calc.m_update(**metadata)
-
-    if not optimade:
-        calc.dft.optimade = None
-
-    proc_calc = processing.Calc.from_entry_metadata(calc)
-    proc_calc.save()
-    calc.a_elastic.index()
-
-    assert processing.Calc.objects(calc_id__in=[calc.calc_id]).count() == 1
diff --git a/tests/metainfo/test_metainfo.py b/tests/metainfo/test_metainfo.py
index ea86b3cb73bba8e6b971ebc4f77293970f3733e5..f5fdcd0b546be733720115f749363f54f835aa0a 100644
--- a/tests/metainfo/test_metainfo.py
+++ b/tests/metainfo/test_metainfo.py
@@ -420,6 +420,14 @@ class TestM1:
 
         self.assert_example_data(new_example_data)
 
+    def test_to_dict_category_filter(self, example_data: Run):
+        system = example_data.systems[0]
+        system.system_type = 'bulk'
+        dct = system.m_to_dict(categories=[SystemHash])
+        assert 'atom_labels' in dct
+        assert 'n_atoms' not in dct  # derived
+        assert 'system_type' not in dct  # not system hash
+
     def test_to_dict_defaults(self, example_data):
         dct = example_data.m_to_dict()
         assert 'nomad_version' not in dct['parsing']
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index ef4182a41787647c8c36ea185c8b008f073cdb9f..e25ed455e646e2cf5c18f72cdd40909347da1a82 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -86,7 +86,7 @@ def assert_processing(upload: Upload, published: bool = False):
         with upload_files.read_archive(calc.calc_id) as archive:
             calc_archive = archive[calc.calc_id]
             assert 'section_run' in calc_archive
-            assert 'section_entry_info' in calc_archive
+            assert 'section_metadata' in calc_archive
             assert 'processing_logs' in calc_archive
 
             has_test_event = False
@@ -101,15 +101,19 @@ def assert_processing(upload: Upload, published: bool = False):
         with upload_files.raw_file(calc.mainfile) as f:
             f.read()
 
-        for path in calc.metadata['files']:
+        entry_metadata = calc.entry_metadata(upload_files)
+
+        for path in entry_metadata.files:
             with upload_files.raw_file(path) as f:
                 f.read()
 
         # check some domain metadata
-        assert calc.metadata['n_atoms'] > 0
-        assert len(calc.metadata['atoms']) > 0
+        assert entry_metadata.n_atoms > 0
+        assert len(entry_metadata.atoms) > 0
+
+        assert upload.get_calc(calc.calc_id) is not None
 
-        assert upload.get_calc(calc.calc_id).metadata is not None
+        upload_files.close()
 
 
 def test_processing(processed, no_warn, mails, monkeypatch):
@@ -139,10 +143,9 @@ def test_publish(non_empty_processed: Upload, no_warn, internal_example_user_met
     except Exception:
         pass
 
-    entries = processed.entries_metadata(internal_example_user_metadata)
-
-    assert_upload_files(processed.upload_id, entries, PublicUploadFiles, published=True)
-    assert_search_upload(entries, additional_keys, published=True)
+    with processed.entries_metadata(internal_example_user_metadata) as entries:
+        assert_upload_files(processed.upload_id, entries, PublicUploadFiles, published=True)
+        assert_search_upload(entries, additional_keys, published=True)
 
     assert_processing(Upload.get(processed.upload_id, include_published=True), published=True)
 
@@ -160,10 +163,9 @@ def test_republish(non_empty_processed: Upload, no_warn, internal_example_user_m
     processed.publish_upload()
     processed.block_until_complete(interval=.01)
 
-    entries = processed.entries_metadata(internal_example_user_metadata)
-
-    assert_upload_files(processed.upload_id, entries, PublicUploadFiles, published=True)
-    assert_search_upload(entries, additional_keys, published=True)
+    with processed.entries_metadata(internal_example_user_metadata) as entries:
+        assert_upload_files(processed.upload_id, entries, PublicUploadFiles, published=True)
+        assert_search_upload(entries, additional_keys, published=True)
 
 
 def test_publish_failed(
@@ -183,9 +185,8 @@ def test_publish_failed(
     except Exception:
         pass
 
-    entries = processed.entries_metadata(internal_example_user_metadata)
-
-    assert_search_upload(entries, additional_keys, published=True, processed=False)
+    with processed.entries_metadata(internal_example_user_metadata) as entries:
+        assert_search_upload(entries, additional_keys, published=True, processed=False)
 
 
 @pytest.mark.timeout(config.tests.default_timeout)
@@ -228,16 +229,20 @@ def test_re_processing(published: Upload, internal_example_user_metadata, monkey
     old_calc_time = first_calc.metadata['last_processing']
 
     with published.upload_files.read_archive(first_calc.calc_id) as archive:
-        old_logs = archive[first_calc.calc_id]['processing_logs']
+        archive[first_calc.calc_id]['processing_logs']
 
     old_archive_files = list(
         archive_file
         for archive_file in os.listdir(published.upload_files.os_path)
         if 'archive' in archive_file)
 
-    for archive_file in old_archive_files:
-        with open(published.upload_files.join_file(archive_file).os_path, 'wt') as f:
-            f.write('')
+    with published.entries_metadata(internal_example_user_metadata) as entries_generator:
+        entries = list(entries_generator)
+
+    if with_failure != 'not-matched':
+        for archive_file in old_archive_files:
+            with open(published.upload_files.join_file(archive_file).os_path, 'wt') as f:
+                f.write('')
 
     if with_failure == 'after':
         raw_files = 'tests/data/proc/examples_template_unparsable.zip'
@@ -249,8 +254,6 @@ def test_re_processing(published: Upload, internal_example_user_metadata, monkey
     shutil.copyfile(
         raw_files, published.upload_files.join_file('raw-restricted.plain.zip').os_path)
 
-    entries = published.entries_metadata(internal_example_user_metadata)
-
     # reprocess
     monkeypatch.setattr('nomad.config.version', 're_process_test_version')
     monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')
@@ -277,24 +280,11 @@ def test_re_processing(published: Upload, internal_example_user_metadata, monkey
     # assert changed archive files
     if with_failure == 'after':
         with published.upload_files.read_archive(first_calc.calc_id) as archive:
-            assert list(archive[first_calc.calc_id].keys()) == ['processing_logs']
-
-    elif with_failure == 'not-matched':
-        with published.upload_files.read_archive(first_calc.calc_id) as archive:
-            assert len(archive[first_calc.calc_id]) == 0
-
-    else:
-        with published.upload_files.read_archive(first_calc.calc_id) as archive:
-            assert len(archive[first_calc.calc_id]) > 1  # contains more then logs
-
-    # assert changed archive log files
-    if with_failure in ['not-matched']:
-        with published.upload_files.read_archive(first_calc.calc_id) as archive:
-            assert len(archive[first_calc.calc_id]) == 0
+            assert list(archive[first_calc.calc_id].keys()) == ['processing_logs', 'section_metadata']
 
     else:
         with published.upload_files.read_archive(first_calc.calc_id) as archive:
-            assert archive[first_calc.calc_id]['processing_logs'] != old_logs
+            assert len(archive[first_calc.calc_id]) > 2  # contains more then logs and metadata
 
     # assert maintained user metadata (mongo+es)
     assert_upload_files(published.upload_id, entries, PublicUploadFiles, published=True)
@@ -303,10 +293,13 @@ def test_re_processing(published: Upload, internal_example_user_metadata, monkey
         assert_processing(Upload.get(published.upload_id, include_published=True), published=True)
 
     # assert changed calc metadata (mongo)
+    entry_metadata = first_calc.entry_metadata(published.upload_files)
     if with_failure not in ['after', 'not-matched']:
-        assert first_calc.metadata['atoms'][0] == 'H'
+        assert entry_metadata.atoms[0] == 'H'
+    elif with_failure == 'not-matched':
+        assert entry_metadata.atoms[0] == 'Si'
     else:
-        assert first_calc.metadata['atoms'][0] == 'Si'
+        assert entry_metadata.atoms == []
 
 
 @pytest.mark.timeout(config.tests.default_timeout)
@@ -409,6 +402,6 @@ def test_ems_data(proc_infra, test_user):
     assert upload.total_calcs == 1
     assert len(upload.calcs) == 1
 
-    entries = upload.entries_metadata()
-    assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False)
-    assert_search_upload(entries, additional_keys, published=False)
+    with upload.entries_metadata() as entries:
+        assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False)
+        assert_search_upload(entries, additional_keys, published=False)
diff --git a/tests/test_archive.py b/tests/test_archive.py
index a380ecc444570044ce42ef987e1b518542691d12..35855761206a280b0ae25cde8c7cd3853fd42134 100644
--- a/tests/test_archive.py
+++ b/tests/test_archive.py
@@ -221,3 +221,11 @@ def test_query():
     assert query_archive(f, {'calc2': {'secA': {'subsecA1[0]': '*'}}}) == {'calc2': {'secA': {'subsecA1[0]': [{'propA1a': 2.0}]}}}
     # TODO
     # test [:][-1][0:1] ...
+
+
+if __name__ == '__main__':
+    import sys
+    import pprint
+    with open(sys.argv[1], 'rb') as f:
+        data = msgpack.unpack(f)
+        pprint.pprint(data)
diff --git a/tests/test_files.py b/tests/test_files.py
index 1269b98bba9f5e379569e1934f5c245b2ed5c1a3..d8ab592220a7a005b3bcc81453e97ff0b1dcaeba 100644
--- a/tests/test_files.py
+++ b/tests/test_files.py
@@ -44,7 +44,11 @@ example_file_contents = [
     'examples_template/4.aux']
 example_file_mainfile = 'examples_template/template.json'
 empty_file = 'tests/data/proc/empty.zip'
-example_archive_contents = {"archive": True, "processing_logs": [{"entry": "test"}]}
+example_archive_contents = {
+    "section_run": [],
+    "section_metadata": {},
+    "processing_logs": [{"entry": "test"}]
+}
 
 example_bucket = 'test_bucket'
 example_data = dict(test_key='test_value')
diff --git a/tests/test_normalizing.py b/tests/test_normalizing.py
index cf3d52fae737032120cadc4ea73244d3ac6f3b93..385c7c0ecec19e3b1143eea96bcac1fe7aac661a 100644
--- a/tests/test_normalizing.py
+++ b/tests/test_normalizing.py
@@ -226,7 +226,8 @@ def assert_normalized(backend: Backend):
         assert metadata.dft.crystal_system is not None
         assert metadata.dft.spacegroup is not None
 
-    exceptions = parser_exceptions.get(backend.get_value('parser_name'), [])
+    parser_name = backend.entry_archive.section_metadata.parser_name
+    exceptions = parser_exceptions.get(parser_name, [])
 
     if metadata.formula != config.services.unavailable_value:
         assert len(metadata.atoms) > 0
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index aa28ebff62c7cd168321e93b8ab7ea38e9bbd607..58a3743e1ae82a63b39e68266fbd284ced2cdcac 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -19,7 +19,7 @@ import pytest
 import os
 from shutil import copyfile
 
-from nomad import utils, files
+from nomad import utils, files, datamodel
 from nomad.parsing import parser_dict, match_parser, BrokenParser, BadContextUri, Backend
 
 
@@ -153,19 +153,17 @@ class TestBackend(object):
         backend.addValue('program_name', 't0')
         backend.closeSection('section_run', 0)
 
-        g_index = backend.openSection('section_entry_info')
-        assert g_index == 0
-        backend.addValue('parser_name', 'p0')
-        backend.closeSection('section_entry_info', 0)
+        g_index = backend.openSection('section_run')
+        assert g_index == 1
+        backend.addValue('program_name', 't1')
+        backend.closeSection('section_run', 1)
 
-        assert backend.get_sections('section_run') == [0]
-        assert backend.get_sections('section_entry_info') == [0]
+        assert backend.get_sections('section_run') == [0, 1]
 
         output = StringIO()
         json.dump(backend.resource.m_to_dict(), output)
         archive = json.loads(output.getvalue())
-        assert 'section_run' in archive
-        assert 'section_entry_info' in archive
+        assert 'section_run' in archive['EntryArchive']
 
     def test_subsection(self, backend: Backend, no_warn):
         backend.openSection('section_run')
@@ -341,14 +339,12 @@ def parsed_example(request) -> Backend:
 
 
 def add_calculation_info(backend: Backend, **kwargs) -> Backend:
-    backend.openNonOverlappingSection('section_entry_info')
-    backend.addValue('upload_id', 'test_upload_id')
-    backend.addValue('calc_id', 'test_calc_id')
-    backend.addValue('calc_hash', 'test_calc_hash')
-    backend.addValue('mainfile', 'test/mainfile.txt')
-    for key, value in kwargs.items():
-        backend.addValue(key, value)
-    backend.closeNonOverlappingSection('section_entry_info')
+    entry_metadata = backend.entry_archive.m_create(datamodel.EntryMetadata)
+    entry_metadata.upload_id = 'test_upload_id'
+    entry_metadata.calc_id = 'test_calc_id'
+    entry_metadata.calc_hash = 'test_calc_hash'
+    entry_metadata.mainfile = 'test/mainfile.txt'
+    entry_metadata.m_update(**kwargs)
     return backend
 
 
diff --git a/tests/test_search.py b/tests/test_search.py
index 6678017ccf41a685a1dd5948181cd8f56bf7698f..16324a259ccad14d447d4fd8028212f0b764430e 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -34,7 +34,8 @@ def test_index_skeleton_calc(elastic):
 
 
 def test_index_normalized_calc(elastic, normalized: parsing.Backend):
-    entry_metadata = datamodel.EntryMetadata(
+    entry_metadata = normalized.entry_archive.section_metadata
+    entry_metadata.m_update(
         domain='dft', upload_id='test upload id', calc_id='test id')
     entry_metadata.apply_domain_metadata(normalized)
 
@@ -49,7 +50,8 @@ def test_index_normalized_calc(elastic, normalized: parsing.Backend):
 
 def test_index_normalized_calc_with_metadata(
         elastic, normalized: parsing.Backend, internal_example_user_metadata: dict):
-    entry_metadata = datamodel.EntryMetadata(
+    entry_metadata = normalized.entry_archive.section_metadata
+    entry_metadata.m_update(
         domain='dft', upload_id='test upload id', calc_id='test id')
     entry_metadata.apply_domain_metadata(normalized)
     internal_example_user_metadata.pop('embargo_length')  # is for uploads only
@@ -67,7 +69,8 @@ def test_index_upload(elastic, processed: processing.Upload):
 
 @pytest.fixture()
 def example_search_data(elastic, normalized: parsing.Backend):
-    entry_metadata = datamodel.EntryMetadata(
+    entry_metadata = normalized.entry_archive.section_metadata
+    entry_metadata.m_update(
         domain='dft', upload_id='test upload id', calc_id='test id')
     entry_metadata.apply_domain_metadata(normalized)
     create_entry(entry_metadata)
@@ -78,7 +81,8 @@ def example_search_data(elastic, normalized: parsing.Backend):
 
 @pytest.fixture()
 def example_ems_search_data(elastic, parsed_ems: parsing.Backend):
-    entry_metadata = datamodel.EntryMetadata(
+    entry_metadata = parsed_ems.entry_archive.section_metadata
+    entry_metadata.m_update(
         domain='ems', upload_id='test upload id', calc_id='test id')
     entry_metadata.apply_domain_metadata(parsed_ems)
     create_entry(entry_metadata)