From fd783d7f8c7dbdf6ce2f3eeffd08ada9b18e4539 Mon Sep 17 00:00:00 2001
From: Jungho Shin <jungho@physik.hu-berlin.de>
Date: Wed, 13 Sep 2017 16:56:36 +0200
Subject: [PATCH] enhance interface to the Repository codebase

---
 normalizer/normalizer-repo-tags/0.GIT_ADD.txt |  1 -
 .../calculate_repo_tags.py                    | 38 +++++++++----------
 .../normalizer-repo-tags/convert_2to3.sh      |  6 ---
 .../normalizer-repo-tags/extract_from_repo.py | 14 +++++++
 .../normalizer-repo-tags/extract_from_repo.sh |  4 ++
 .../normalizer-repo-tags/requirements.txt     |  3 --
 .../normalizer-repo-tags/setup_paths.py       | 10 ++---
 .../normalizer-repo-tags/setup_paths_repo.py  |  6 +++
 8 files changed, 45 insertions(+), 37 deletions(-)
 delete mode 100644 normalizer/normalizer-repo-tags/0.GIT_ADD.txt
 delete mode 100755 normalizer/normalizer-repo-tags/convert_2to3.sh
 create mode 100644 normalizer/normalizer-repo-tags/extract_from_repo.py
 create mode 100755 normalizer/normalizer-repo-tags/extract_from_repo.sh
 delete mode 100755 normalizer/normalizer-repo-tags/requirements.txt
 create mode 100644 normalizer/normalizer-repo-tags/setup_paths_repo.py

diff --git a/normalizer/normalizer-repo-tags/0.GIT_ADD.txt b/normalizer/normalizer-repo-tags/0.GIT_ADD.txt
deleted file mode 100644
index 2285620..0000000
--- a/normalizer/normalizer-repo-tags/0.GIT_ADD.txt
+++ /dev/null
@@ -1 +0,0 @@
-model_base.py?
diff --git a/normalizer/normalizer-repo-tags/calculate_repo_tags.py b/normalizer/normalizer-repo-tags/calculate_repo_tags.py
index f1d057d..58bca7e 100644
--- a/normalizer/normalizer-repo-tags/calculate_repo_tags.py
+++ b/normalizer/normalizer-repo-tags/calculate_repo_tags.py
@@ -1,13 +1,11 @@
 import setup_paths
 import json
-import os.path, sys
+import os.path, sys, subprocess
 from nomadcore.local_meta_info import loadJsonFile, InfoKindEl
 from nomadcore.parser_backend import JsonParseEventsWriterBackend
 from nomadcore.parse_streamed_dicts import *
 import logging
 
-from nomadrepo.core.api import API
-
 base_path = os.path.abspath(os.path.dirname(__file__))
 
 def calculateTags(inputDict, backend, calcUri):
@@ -18,21 +16,21 @@ def calculateTags(inputDict, backend, calcUri):
         backend.openContext(calcUri)
         repoSect = backend.openSection("section_repository_info")
 
-    work = API()
-    for calc, error in work._parse("/nomad/jshin/work/normalizer/scalaOut1.txt", "normalizerRepo"):
-        calc, error = work.classify(calc)
+    repo_dic_sub = subprocess.Popen([os.path.join(base_path, 'extract_from_repo.sh'), '/scratch/u/jungho/NomadRepositoryParser/', base_path, '/nomad/jshin/work/normalizer/scalaOut1.txt'], stdout = subprocess.PIPE)
+
+    repo_dic = json.load(repo_dic_sub.stdout)
 
-    backend.addValue("repository_checksum", calc.get_checksum())
-    backend.addValue("repository_chemical_formula", calc.info['formula']
-    #backend.addValue("repository_parser_id", value)
-    backend.addValue("repository_atomic_elements", calc.info['elements'])
-    backend.addValue("repository_basis_set_type", calc.info['ansatz'])
-    #backend.addValue("repository_code_version", value)
-    backend.addValue("repository_crystal_system", calc.info['symmetry'])
-    #backend.addValue("repository_program_name", calc.info[''])
-    backend.addValue("repository_spacegroup_nr", calc.info['ng'])
-    backend.addValue("repository_system_type", calc.info['periodicity'])
-    backend.addValue("repository_xc_treatment", calc.info['H_types'])
+    backend.addValue("repository_checksum", repo_dic['checksum'])
+    backend.addValue("repository_chemical_formula", repo_dic['formula'])
+    backend.addValue("repository_parser_id", repo_dic['prog_name'] + ' v1.0')
+    backend.addValue("repository_atomic_elements", repo_dic['elements'])
+    backend.addValue("repository_basis_set_type", repo_dic['ansatz'])
+    backend.addValue("repository_code_version", repo_dic['prog'].split(repo_dic['prog_name'])[0])
+    backend.addValue("repository_crystal_system", repo_dic['symmetry'])
+    backend.addValue("repository_program_name", repo_dic['prog_name'])
+    backend.addValue("repository_spacegroup_nr", repo_dic['ng'])
+    backend.addValue("repository_system_type", repo_dic['periodicity'])
+    backend.addValue("repository_xc_treatment", repo_dic['H_types'])
 
     if repoSectUri:
         backend.closeContext(repoSectUri)
@@ -57,14 +55,14 @@ def main():
         calcUri,
         parserInfo = {'name':'RepoTagsNormalizer', 'version': '1.0'})
 
-    #dictReader=ParseStreamedDicts(sys.stdin)
-    dictReader=ParseStreamedDicts(open("/u/jungho/myscratch/nomad-lab-base/normalizers/repo-tags/test/examples/scalaOut1.txt", 'r'))
+    dictReader=ParseStreamedDicts(sys.stdin)
+    #dictReader=ParseStreamedDicts(open("/u/jungho/myscratch/nomad-lab-base/normalizers/repo-tags/test/examples/scalaOut1.txt", 'r'))
 
     while True:
         inputDict=dictReader.readNextDict()
         if inputDict is None:
             break
-        calculateTags(inputDict,backend, calcUri)
+        calculateTags(inputDict, backend, calcUri)
     backend.finishedParsingSession("ParseSuccess", None)
     sys.stdout.flush()
 
diff --git a/normalizer/normalizer-repo-tags/convert_2to3.sh b/normalizer/normalizer-repo-tags/convert_2to3.sh
deleted file mode 100755
index 3f78ed0..0000000
--- a/normalizer/normalizer-repo-tags/convert_2to3.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-for python_file in NomadRepositoryParser/python/nomadrepo/core/settings.py NomadRepositoryParser/python/nomadrepo/core/symmetry.py
-do
-  2to3 -w $python_file
-done
diff --git a/normalizer/normalizer-repo-tags/extract_from_repo.py b/normalizer/normalizer-repo-tags/extract_from_repo.py
new file mode 100644
index 0000000..dfaf745
--- /dev/null
+++ b/normalizer/normalizer-repo-tags/extract_from_repo.py
@@ -0,0 +1,14 @@
+import setup_paths_repo
+import os.path, sys, json
+from nomadrepo.core.api import API
+
+base_path = os.path.abspath(os.path.dirname(__file__))
+
+inp_str = "".join(sys.argv[1:])
+
+work = API()
+for calc, error in work._parse(inp_str, "normalizerRepo"):
+    calc, error = work.classify(calc)
+    calc.info['oadate'] = None
+    calc.info['checksum'] = calc.get_checksum()
+    json.dump(calc.info, sys.__stdout__)
diff --git a/normalizer/normalizer-repo-tags/extract_from_repo.sh b/normalizer/normalizer-repo-tags/extract_from_repo.sh
new file mode 100755
index 0000000..8cb3f33
--- /dev/null
+++ b/normalizer/normalizer-repo-tags/extract_from_repo.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+source $1/bin/activate
+python $2/extract_from_repo.py $3
diff --git a/normalizer/normalizer-repo-tags/requirements.txt b/normalizer/normalizer-repo-tags/requirements.txt
deleted file mode 100755
index 477ad42..0000000
--- a/normalizer/normalizer-repo-tags/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-sqlalchemy == 1.0.6
-httplib2
-bcrypt
diff --git a/normalizer/normalizer-repo-tags/setup_paths.py b/normalizer/normalizer-repo-tags/setup_paths.py
index b3214e6..820841d 100644
--- a/normalizer/normalizer-repo-tags/setup_paths.py
+++ b/normalizer/normalizer-repo-tags/setup_paths.py
@@ -1,14 +1,10 @@
 import sys, os, os.path
 baseDir = os.path.dirname(os.path.abspath(__file__))
 commonDir = os.path.normpath(os.path.join(baseDir,"../../../../python-common/common/python"))
-repoParserDir = os.path.normpath(os.path.join(baseDir,"/u/jungho/myscratch/NomadRepositoryParser/python/"))
-#repoParserCoreDir = os.path.normpath(os.path.join(baseDir,"NomadRepositoryParser/python/nomadrepo/core/"))
+#repoParserDir = os.path.normpath(os.path.join(baseDir,"/u/jungho/myscratch/NomadRepositoryParser/python/"))
 
 if not commonDir in sys.path:
     sys.path.insert(0, commonDir)
 
-if not repoParserDir in sys.path:
-    sys.path.insert(0, repoParserDir)
-
-#if not repoParserCoreDir in sys.path:
-#    sys.path.insert(0, repoParserCoreDir)
+#if not repoParserDir in sys.path:
+#    sys.path.insert(0, repoParserDir)
diff --git a/normalizer/normalizer-repo-tags/setup_paths_repo.py b/normalizer/normalizer-repo-tags/setup_paths_repo.py
new file mode 100644
index 0000000..392d70c
--- /dev/null
+++ b/normalizer/normalizer-repo-tags/setup_paths_repo.py
@@ -0,0 +1,6 @@
+import sys, os, os.path
+baseDir = os.path.dirname(os.path.abspath(__file__))
+repoParserDir = os.path.normpath(os.path.join(baseDir,"/u/jungho/myscratch/NomadRepositoryParser/python/"))
+
+if not repoParserDir in sys.path:
+    sys.path.insert(0, repoParserDir)
-- 
GitLab