Commit 8a93b7f6 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixed linting and tests after improving matching regexps.

parent 61f4c44e
Pipeline #48180 canceled with stages
in 21 minutes and 56 seconds
......@@ -23,7 +23,7 @@ import multiprocessing
import queue
import json
from nomad import config, infrastructure, utils
from nomad import config, infrastructure
from nomad.migration import NomadCOEMigration, SourceCalc, Package, missing_calcs_data
from .main import cli
......
......@@ -213,12 +213,12 @@ def missing_calcs_data():
'ftp_upload_for_uid_502_2012-11-26-14-56-17-339064',
'ftp_upload_for_uid_502_2012-12-03-09-52-02-714224',
'ftp_upload_for_uid_502_2012-12-10-20-09-30-463926',
'ftp_upload_for_uid_502_2011-08-17-14-29-25-505869']
'ftp_upload_for_uid_502_2011-08-17-14-29-25-505869']
# aggregate missing calcs based on uploads
source_uploads = SourceCalc._get_collection().aggregate([
{'$match': {'migration_version': -1, 'upload': {'$nin':not_check_uploads}}},
{'$group': {'_id': '$upload', 'calcs': {'$push': { 'mainfile': '$mainfile', 'pid': '$metadata.pid'}}}}])
{'$match': {'migration_version': -1, 'upload': {'$nin': not_check_uploads}}},
{'$group': {'_id': '$upload', 'calcs': {'$push': {'mainfile': '$mainfile', 'pid': '$metadata.pid'}}}}])
source_uploads = list(source_uploads)
for source_upload in source_uploads:
......@@ -297,7 +297,7 @@ def missing_calcs_data():
if mainfile in all_files:
exist += 1
example_exists_mainfile = mainfile
else:
else:
not_exist += 1
example_mainfile = mainfile
example_pid = calc['pid']
......@@ -347,7 +347,7 @@ def missing_calcs_data():
summary[key] = 0
for value in values:
summary[key] += value['calcs']
summary.overall_missing += value['calcs']
summary['overall_missing'] += value['calcs']
results.summary = summary
......
......@@ -371,7 +371,7 @@ parsers = [
MissingParser(
name='parsers/qbox', code_name='qbox', domain='DFT',
mainfile_contents_re=(r'http://qboxcode.org')
),
),
MissingParser(
name='parsers/dmol', code_name='DMol3', domain='DFT',
mainfile_name_re=r'.*\.outmol'
......@@ -397,19 +397,19 @@ parsers = [
LegacyParser(
name='parsers/crystal', code_name='Crystal',
parser_class_name='crystalparser.CrystalParser',
mainfile_name_re='.*\.cryst\.out'
mainfile_name_re=r'.*\.cryst\.out'
),
# We already have wien2k with mainfile_contents_re, but this one does not always properly match
LegacyParser(
name='parsers/wien2k', code_name='WIEN2k',
parser_class_name='wien2kparser.Wien2kParser',
mainfile_name_re='.*\.scf'
mainfile_name_re=r'.*\.scf'
),
# We already have fhi-aims with mainfile_contents_re, but this one does not always properly match
LegacyParser(
name='parsers/fhi-aims', code_name='FHI-aims',
parser_class_name='fhiaimsparser.FHIaimsParser',
mainfile_name_re='.*\.fhiaims'
mainfile_name_re=r'.*\.fhiaims'
),
BrokenParser()
]
......
......@@ -118,8 +118,8 @@ class MatchingParser(Parser):
supported_compressions: List[str] = []) -> None:
super().__init__()
self.name = name
self.code_name = code_name
self.domain = domain
self._mainfile_mime_re = re.compile(mainfile_mime_re)
self._mainfile_name_re = re.compile(mainfile_name_re)
......@@ -161,7 +161,7 @@ class MissingParser(MatchingParser):
class LegacyParser(MatchingParser):
"""
A parser implementation for legacy NOMAD-coe parsers. It assumes that parsers
are installed to the python environment.
are installed to the python environment.
Arguments:
parser_class_name: the main parser class that implements NOMAD-coe's
......
......@@ -22,7 +22,7 @@ from nomadcore.local_meta_info import loadJsonFile
import nomad_meta_info
from nomad import utils, files
from nomad.parsing import JSONStreamWriter, parser_dict, match_parser
from nomad.parsing import JSONStreamWriter, parser_dict, match_parser, BrokenParser
from nomad.parsing import LocalBackend, BadContextURI
parser_examples = [
......@@ -70,7 +70,7 @@ for parser, mainfile in parser_examples:
parser_examples = fixed_parser_examples
correct_num_output_files = 34
correct_num_output_files = 38
class TestLocalBackend(object):
......@@ -333,7 +333,7 @@ def test_match(raw_files, no_warn):
count = 0
for mainfile in upload_files.raw_file_manifest():
parser = match_parser(mainfile, upload_files)
if parser is not None:
if parser is not None and not isinstance(parser, BrokenParser):
count += 1
assert count == correct_num_output_files
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment