From fd783d7f8c7dbdf6ce2f3eeffd08ada9b18e4539 Mon Sep 17 00:00:00 2001 From: Jungho Shin <jungho@physik.hu-berlin.de> Date: Wed, 13 Sep 2017 16:56:36 +0200 Subject: [PATCH] enhance interface to the Repository codebase --- normalizer/normalizer-repo-tags/0.GIT_ADD.txt | 1 - .../calculate_repo_tags.py | 38 +++++++++---------- .../normalizer-repo-tags/convert_2to3.sh | 6 --- .../normalizer-repo-tags/extract_from_repo.py | 14 +++++++ .../normalizer-repo-tags/extract_from_repo.sh | 4 ++ .../normalizer-repo-tags/requirements.txt | 3 -- .../normalizer-repo-tags/setup_paths.py | 10 ++--- .../normalizer-repo-tags/setup_paths_repo.py | 6 +++ 8 files changed, 45 insertions(+), 37 deletions(-) delete mode 100644 normalizer/normalizer-repo-tags/0.GIT_ADD.txt delete mode 100755 normalizer/normalizer-repo-tags/convert_2to3.sh create mode 100644 normalizer/normalizer-repo-tags/extract_from_repo.py create mode 100755 normalizer/normalizer-repo-tags/extract_from_repo.sh delete mode 100755 normalizer/normalizer-repo-tags/requirements.txt create mode 100644 normalizer/normalizer-repo-tags/setup_paths_repo.py diff --git a/normalizer/normalizer-repo-tags/0.GIT_ADD.txt b/normalizer/normalizer-repo-tags/0.GIT_ADD.txt deleted file mode 100644 index 2285620..0000000 --- a/normalizer/normalizer-repo-tags/0.GIT_ADD.txt +++ /dev/null @@ -1 +0,0 @@ -model_base.py? diff --git a/normalizer/normalizer-repo-tags/calculate_repo_tags.py b/normalizer/normalizer-repo-tags/calculate_repo_tags.py index f1d057d..58bca7e 100644 --- a/normalizer/normalizer-repo-tags/calculate_repo_tags.py +++ b/normalizer/normalizer-repo-tags/calculate_repo_tags.py @@ -1,13 +1,11 @@ import setup_paths import json -import os.path, sys +import os.path, sys, subprocess from nomadcore.local_meta_info import loadJsonFile, InfoKindEl from nomadcore.parser_backend import JsonParseEventsWriterBackend from nomadcore.parse_streamed_dicts import * import logging -from nomadrepo.core.api import API - base_path = os.path.abspath(os.path.dirname(__file__)) def calculateTags(inputDict, backend, calcUri): @@ -18,21 +16,21 @@ def calculateTags(inputDict, backend, calcUri): backend.openContext(calcUri) repoSect = backend.openSection("section_repository_info") - work = API() - for calc, error in work._parse("/nomad/jshin/work/normalizer/scalaOut1.txt", "normalizerRepo"): - calc, error = work.classify(calc) + repo_dic_sub = subprocess.Popen([os.path.join(base_path, 'extract_from_repo.sh'), '/scratch/u/jungho/NomadRepositoryParser/', base_path, '/nomad/jshin/work/normalizer/scalaOut1.txt'], stdout = subprocess.PIPE) + + repo_dic = json.load(repo_dic_sub.stdout) - backend.addValue("repository_checksum", calc.get_checksum()) - backend.addValue("repository_chemical_formula", calc.info['formula'] - #backend.addValue("repository_parser_id", value) - backend.addValue("repository_atomic_elements", calc.info['elements']) - backend.addValue("repository_basis_set_type", calc.info['ansatz']) - #backend.addValue("repository_code_version", value) - backend.addValue("repository_crystal_system", calc.info['symmetry']) - #backend.addValue("repository_program_name", calc.info['']) - backend.addValue("repository_spacegroup_nr", calc.info['ng']) - backend.addValue("repository_system_type", calc.info['periodicity']) - backend.addValue("repository_xc_treatment", calc.info['H_types']) + backend.addValue("repository_checksum", repo_dic['checksum']) + backend.addValue("repository_chemical_formula", repo_dic['formula']) + backend.addValue("repository_parser_id", repo_dic['prog_name'] + ' v1.0') + backend.addValue("repository_atomic_elements", repo_dic['elements']) + backend.addValue("repository_basis_set_type", repo_dic['ansatz']) + backend.addValue("repository_code_version", repo_dic['prog'].split(repo_dic['prog_name'])[0]) + backend.addValue("repository_crystal_system", repo_dic['symmetry']) + backend.addValue("repository_program_name", repo_dic['prog_name']) + backend.addValue("repository_spacegroup_nr", repo_dic['ng']) + backend.addValue("repository_system_type", repo_dic['periodicity']) + backend.addValue("repository_xc_treatment", repo_dic['H_types']) if repoSectUri: backend.closeContext(repoSectUri) @@ -57,14 +55,14 @@ def main(): calcUri, parserInfo = {'name':'RepoTagsNormalizer', 'version': '1.0'}) - #dictReader=ParseStreamedDicts(sys.stdin) - dictReader=ParseStreamedDicts(open("/u/jungho/myscratch/nomad-lab-base/normalizers/repo-tags/test/examples/scalaOut1.txt", 'r')) + dictReader=ParseStreamedDicts(sys.stdin) + #dictReader=ParseStreamedDicts(open("/u/jungho/myscratch/nomad-lab-base/normalizers/repo-tags/test/examples/scalaOut1.txt", 'r')) while True: inputDict=dictReader.readNextDict() if inputDict is None: break - calculateTags(inputDict,backend, calcUri) + calculateTags(inputDict, backend, calcUri) backend.finishedParsingSession("ParseSuccess", None) sys.stdout.flush() diff --git a/normalizer/normalizer-repo-tags/convert_2to3.sh b/normalizer/normalizer-repo-tags/convert_2to3.sh deleted file mode 100755 index 3f78ed0..0000000 --- a/normalizer/normalizer-repo-tags/convert_2to3.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -for python_file in NomadRepositoryParser/python/nomadrepo/core/settings.py NomadRepositoryParser/python/nomadrepo/core/symmetry.py -do - 2to3 -w $python_file -done diff --git a/normalizer/normalizer-repo-tags/extract_from_repo.py b/normalizer/normalizer-repo-tags/extract_from_repo.py new file mode 100644 index 0000000..dfaf745 --- /dev/null +++ b/normalizer/normalizer-repo-tags/extract_from_repo.py @@ -0,0 +1,14 @@ +import setup_paths_repo +import os.path, sys, json +from nomadrepo.core.api import API + +base_path = os.path.abspath(os.path.dirname(__file__)) + +inp_str = "".join(sys.argv[1:]) + +work = API() +for calc, error in work._parse(inp_str, "normalizerRepo"): + calc, error = work.classify(calc) + calc.info['oadate'] = None + calc.info['checksum'] = calc.get_checksum() + json.dump(calc.info, sys.__stdout__) diff --git a/normalizer/normalizer-repo-tags/extract_from_repo.sh b/normalizer/normalizer-repo-tags/extract_from_repo.sh new file mode 100755 index 0000000..8cb3f33 --- /dev/null +++ b/normalizer/normalizer-repo-tags/extract_from_repo.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +source $1/bin/activate +python $2/extract_from_repo.py $3 diff --git a/normalizer/normalizer-repo-tags/requirements.txt b/normalizer/normalizer-repo-tags/requirements.txt deleted file mode 100755 index 477ad42..0000000 --- a/normalizer/normalizer-repo-tags/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sqlalchemy == 1.0.6 -httplib2 -bcrypt diff --git a/normalizer/normalizer-repo-tags/setup_paths.py b/normalizer/normalizer-repo-tags/setup_paths.py index b3214e6..820841d 100644 --- a/normalizer/normalizer-repo-tags/setup_paths.py +++ b/normalizer/normalizer-repo-tags/setup_paths.py @@ -1,14 +1,10 @@ import sys, os, os.path baseDir = os.path.dirname(os.path.abspath(__file__)) commonDir = os.path.normpath(os.path.join(baseDir,"../../../../python-common/common/python")) -repoParserDir = os.path.normpath(os.path.join(baseDir,"/u/jungho/myscratch/NomadRepositoryParser/python/")) -#repoParserCoreDir = os.path.normpath(os.path.join(baseDir,"NomadRepositoryParser/python/nomadrepo/core/")) +#repoParserDir = os.path.normpath(os.path.join(baseDir,"/u/jungho/myscratch/NomadRepositoryParser/python/")) if not commonDir in sys.path: sys.path.insert(0, commonDir) -if not repoParserDir in sys.path: - sys.path.insert(0, repoParserDir) - -#if not repoParserCoreDir in sys.path: -# sys.path.insert(0, repoParserCoreDir) +#if not repoParserDir in sys.path: +# sys.path.insert(0, repoParserDir) diff --git a/normalizer/normalizer-repo-tags/setup_paths_repo.py b/normalizer/normalizer-repo-tags/setup_paths_repo.py new file mode 100644 index 0000000..392d70c --- /dev/null +++ b/normalizer/normalizer-repo-tags/setup_paths_repo.py @@ -0,0 +1,6 @@ +import sys, os, os.path +baseDir = os.path.dirname(os.path.abspath(__file__)) +repoParserDir = os.path.normpath(os.path.join(baseDir,"/u/jungho/myscratch/NomadRepositoryParser/python/")) + +if not repoParserDir in sys.path: + sys.path.insert(0, repoParserDir) -- GitLab