diff --git a/nomad/app/flask/api/upload.py b/nomad/app/flask/api/upload.py index e0bf42a02060d447329496259242939805de0659..57679044798cb6ba265b41883f20b3d914442be3 100644 --- a/nomad/app/flask/api/upload.py +++ b/nomad/app/flask/api/upload.py @@ -93,7 +93,7 @@ calc_model = api.inherit('UploadCalculationProcessing', proc_model, { 'calc_id': fields.String, 'mainfile': fields.String, 'upload_id': fields.String, - 'parser': fields.String, + 'parser_name': fields.String, 'metadata': fields.Raw( attribute='_entry_metadata', description='The repository metadata for this entry.') diff --git a/nomad/app/v1/routers/uploads.py b/nomad/app/v1/routers/uploads.py index d1e2b6b0ebd14d2899dbc664036c7fc73508464d..c6b6f4866745fbfbeeb868bf91756f2ce9f62d55 100644 --- a/nomad/app/v1/routers/uploads.py +++ b/nomad/app/v1/routers/uploads.py @@ -103,7 +103,7 @@ class EntryProcData(ProcData): entry_id: str = Field() mainfile: str = Field() upload_id: str = Field() - parser: str = Field() + parser_name: str = Field() entry_metadata: Optional[dict] = Field() @@ -130,7 +130,7 @@ class EntryProcDataPagination(Pagination): def validate_order_by(cls, order_by): # pylint: disable=no-self-argument if order_by is None: return 'mainfile' # Default value - assert order_by in ('mainfile', 'parser', 'process_status', 'current_process'), 'order_by must be a valid attribute' + assert order_by in ('mainfile', 'parser_name', 'process_status', 'current_process'), 'order_by must be a valid attribute' return order_by @validator('page_after_value') diff --git a/nomad/processing/data.py b/nomad/processing/data.py index ac6879d85c7c865c88c0b3620256864a4accb253..d314df10b98866b05f7ba484fafe489082ab702b 100644 --- a/nomad/processing/data.py +++ b/nomad/processing/data.py @@ -147,7 +147,7 @@ class Calc(Proc): Attributes: calc_id: the calc_id of this calc - parser: the name of the parser used to process this calc + parser_name: the name of the parser used to process this calc upload_id: the id of the upload used to create this calculation mainfile: the mainfile (including path in upload) that was used to create this calc @@ -156,7 +156,7 @@ class Calc(Proc): calc_id = StringField(primary_key=True) upload_id = StringField() mainfile = StringField() - parser = StringField() + parser_name = StringField() metadata = DictField() # Stores user provided metadata and system metadata (not archive metadata) @@ -164,9 +164,9 @@ class Calc(Proc): 'strict': False, 'indexes': [ 'upload_id', - 'parser', + 'parser_name', ('upload_id', 'mainfile'), - ('upload_id', 'parser'), + ('upload_id', 'parser_name'), ('upload_id', 'process_status'), ('upload_id', 'metadata.nomad_version'), 'metadata.processed', @@ -186,7 +186,7 @@ class Calc(Proc): self._entry_metadata: datamodel.EntryMetadata = None @classmethod - def get(cls, id): + def get(cls, id) -> 'Calc': return cls.get_by_id(id, 'calc_id') @property @@ -299,11 +299,11 @@ class Calc(Proc): entry_metadata.published = upload.published entry_metadata.with_embargo = (upload.embargo_length > 0) # Entry metadata - if self.parser is not None: - entry_metadata.parser_name = self.parser - parser = parser_dict[self.parser] + entry_metadata.parser_name = self.parser_name + if self.parser_name is not None: + parser = parser_dict[self.parser_name] if parser.domain: - entry_metadata.domain = parser_dict[self.parser].domain + entry_metadata.domain = parser.domain entry_metadata.calc_id = self.calc_id entry_metadata.mainfile = self.mainfile @@ -436,13 +436,13 @@ class Calc(Proc): logger.warn('no parser matches during re-process, not updating the entry') self.warnings = ['no matching parser found during processing'] else: - parser_changed = self.parser != parser.name and parser_dict[self.parser].name != parser.name + parser_changed = self.parser_name != parser.name and parser_dict[self.parser_name].name != parser.name if reparse_if_parser_unchanged and not parser_changed: should_parse = True elif reparse_if_parser_changed and parser_changed: should_parse = True - if should_parse and self.parser != parser.name: - if parser_dict[self.parser].name == parser.name: + if should_parse and self.parser_name != parser.name: + if parser_dict[self.parser_name].name == parser.name: logger.info( 'parser renamed, using new parser name', parser=parser.name) @@ -450,7 +450,7 @@ class Calc(Proc): logger.info( 'different parser matches during re-process, use new parser', parser=parser.name) - self.parser = parser.name # Parser changed or renamed + self.parser_name = parser.name # Parser changed or renamed # 2. Either parse the entry, or preserve it as it is. if should_parse: @@ -533,10 +533,10 @@ class Calc(Proc): def parsing(self): ''' The process step that encapsulates all parsing related actions. ''' self.set_process_step('parsing') - context = dict(parser=self.parser, step=self.parser) + context = dict(parser=self.parser_name, step=self.parser_name) logger = self.get_logger(**context) - parser = parser_dict[self.parser] - self._entry_metadata.parser_name = self.parser + parser = parser_dict[self.parser_name] + self._entry_metadata.parser_name = self.parser_name with utils.timer(logger, 'parser executed', input_size=self.mainfile_file.size): if not config.process_reuse_parser: @@ -567,7 +567,7 @@ class Calc(Proc): information in section_encyclopedia as well as the DFT domain metadata. """ try: - logger = self.get_logger(parser=self.parser, step=self.parser) + logger = self.get_logger(parser=self.parser_name, step=self.parser_name) # Open the archive of the phonon calculation. upload_files = StagingUploadFiles(self.upload_id) @@ -652,7 +652,7 @@ class Calc(Proc): datamodel.EntryArchive.metadata, self._entry_metadata) for normalizer in normalizers: - if normalizer.domain is not None and normalizer.domain != parser_dict[self.parser].domain: + if normalizer.domain is not None and normalizer.domain != parser_dict[self.parser_name].domain: continue normalizer_name = normalizer.__name__ @@ -1342,8 +1342,8 @@ class Upload(Proc): if entry.process_running: count_already_processing += 1 # Ensure that we update the parser if in staging - if not self.published and parser.name != entry.parser: - entry.parser = parser.name + if not self.published and parser.name != entry.parser_name: + entry.parser_name = parser.name entry.save() matched_entries.add(calc_id) except KeyError: @@ -1352,7 +1352,7 @@ class Upload(Proc): entry = Calc.create( calc_id=calc_id, mainfile=filename, - parser=parser.name, + parser_name=parser.name, worker_hostname=self.worker_hostname, upload_id=self.upload_id) entry.save() @@ -1428,7 +1428,7 @@ class Upload(Proc): # calculations. TODO: This should be replaced by a more # extensive mechanism that supports more complex dependencies # between calculations. - phonon_calculations = Calc.objects(upload_id=self.upload_id, parser="parsers/phonopy") + phonon_calculations = Calc.objects(upload_id=self.upload_id, parser_name="parsers/phonopy") for calc in phonon_calculations: calc.process_phonon() @@ -1586,7 +1586,7 @@ class Upload(Proc): metadata__nomad_version__ne=config.meta.version) @property - def calcs(self): + def calcs(self) -> Iterable[Calc]: ''' All successfully processed calculations. ''' return Calc.objects(upload_id=self.upload_id, process_status=ProcessStatus.SUCCESS) @@ -1810,7 +1810,7 @@ class Upload(Proc): 'upload.embargo_length', 'entries') required_keys_entry_level = ( - '_id', 'upload_id', 'mainfile', 'parser', 'process_status', 'create_time', 'metadata') + '_id', 'upload_id', 'mainfile', 'parser_name', 'process_status', 'create_time', 'metadata') required_keys_entry_metadata = ( 'upload_time', 'calc_hash') required_keys_datasets = ( @@ -1913,7 +1913,7 @@ class Upload(Proc): # Instantiate an entry object from the json, and validate it entry_keys_to_copy = ( - 'upload_id', 'mainfile', 'parser', 'metadata', 'errors', 'warnings', + 'upload_id', 'mainfile', 'parser_name', 'metadata', 'errors', 'warnings', 'last_status_message', 'current_process', 'current_process_step', 'create_time', 'complete_time', 'worker_hostname', 'celery_task_id') try: diff --git a/tests/app/v1/routers/test_uploads.py b/tests/app/v1/routers/test_uploads.py index 8a62bdff3fc0ddc9b9864f6bf989a0ee53c0a59d..5e5cd165d7e28bd6984093ade13ae85d1e66f266 100644 --- a/tests/app/v1/routers/test_uploads.py +++ b/tests/app/v1/routers/test_uploads.py @@ -557,13 +557,13 @@ def test_get_upload( id='pag-out-of-rage-page_after_value'), pytest.param( dict( - query_args={'page_size': 1, 'order_by': 'parser'}, + query_args={'page_size': 1, 'order_by': 'parser_name'}, expected_data_len=1, expected_response={'processing_successful': 2, 'processing_failed': 0}, expected_pagination={ - 'total': 2, 'page': 1, 'page_after_value': None, 'next_page_after_value': '0', 'order_by': 'parser', + 'total': 2, 'page': 1, 'page_after_value': None, 'next_page_after_value': '0', 'order_by': 'parser_name', 'page_url': Any, 'next_page_url': Any, 'prev_page_url': None, 'first_page_url': Any}), - id='pag-order_by-parser'), + id='pag-order_by-parser_name'), pytest.param( dict( query_args={'page_size': 1, 'order_by': 'calc_id'}, diff --git a/tests/normalizing/test_encyclopedia.py b/tests/normalizing/test_encyclopedia.py index 6977ddb1d2241b49123db6d5c7fe6255a2383095..fd573c0fc5dd21a322f981d5864097ca20c9b3ee 100644 --- a/tests/normalizing/test_encyclopedia.py +++ b/tests/normalizing/test_encyclopedia.py @@ -509,10 +509,10 @@ def test_phonon(test_user, proc_infra): # Read the resulting archive upload_id = upload.upload_id - calcs = upload.calcs() + calcs = upload.calcs phonon_id = None for calc in calcs: - if calc.parser == "parsers/phonopy": + if calc.parser_name == "parsers/phonopy": phonon_id = calc.calc_id break upload_file = UploadFiles.get(upload_id) diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py index 8b1cb97afc827519a7be76b2c237d3bfdfd52c6a..0138402aafbfa2b220882c43b898838611c70b8f 100644 --- a/tests/processing/test_data.py +++ b/tests/processing/test_data.py @@ -92,7 +92,7 @@ def assert_processing(upload: Upload, published: bool = False, process='process_ assert isinstance(upload_files, StagingUploadFiles) for calc in Calc.objects(upload_id=upload.upload_id): - assert calc.parser is not None + assert calc.parser_name is not None assert calc.mainfile is not None assert calc.process_status == ProcessStatus.SUCCESS diff --git a/tests/utils.py b/tests/utils.py index ba293947a7efa4e1715eb72657ad1c1024462c36..8013d6f395478c31cff1c01d5088c68f486590b0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -189,7 +189,7 @@ class ExampleData: calc_id=entry_metadata.calc_id, upload_id=entry_metadata.upload_id, mainfile=entry_metadata.mainfile, - parser='parsers/vasp', + parser_name='parsers/vasp', process_status=proc.ProcessStatus.SUCCESS) upload_dict = self.uploads.get(entry_metadata.upload_id) if upload_dict: