dependencies.py 6.35 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
16
17
This module allows to configure and install all necessary legecy nomad GIT repositories
to process (parser, normalizer, etc.) uploaded calculations.
18
19
20
21
22

Parsers are developed as independed, individual python programs in their own GIT repositories.
They are build on a common modules called *python-common*, also in a separate GIT.
All parsers depend on the *meta-info*, which is also maintained in its own GIT.

23
24
Preparing dependencies
----------------------
25

26
27
To make GIT maintained python modules available, we use:

28
.. autoclass:: PythonGit
29
    :members:
30
31


32
Dependencies are configured in
33
34
35
36
37
38
39

.. autodata:: dependencies


To install all dependencies use

.. autofunction:: prepare
40
"""
41
import sys
42
43
import os
import os.path
44
45
import logging
import subprocess
46
47

_meta_info_path = './submodules/nomad-meta-info/meta_info/nomad_meta_info/'
48
_logger = logging.getLogger(__name__)
49
50
51
base_dir = './.dependencies'


52
class PythonGitError(Exception):
53
54
55
56
    def __init__(self, msg, repo):
        msg = '%s [%s]' % (msg, repo)
        super().__init__(msg)

57

58
class PythonGit():
59
60
61
62
    """
    Represents a python module in a git repository. It allows to fetch a specific commit,
    install all requirements to the current python environment, and check the installation
    via module import.
63
64
65
66

    This is only useful before you want to use the respective module in a different
    python process, because it will not try to reload any already loaded modules into
    the current python process.
67

68
    Arguments:
69
70
71
72
73
        name: A name that determines the download path, can contain '/' for sub dirs.
                Names are important, because modules might use relatives paths between
                them.
        git_url: A publically available and fetchable url to the GIT repository.
        git_commit: The full commit SHA of the desired commit.
74
    """
75
    def __init__(self, name: str, git_url: str, git_commit: str) -> None:
76
77
78
79
        self.name = name
        self.git_url = git_url
        self.git_commit = git_commit

80
81
82
83
    def _run_pip_install(self, *args):
        pipcode = 0

        # some weird interaction of pip and virtualenv causes a bug that does
84
        # not allow to install in docker due to a wrong PIP_REQ_TRACKER path. This
85
86
87
88
89
90
        # is a workarround.
        pip_req_tracker_key = 'PIP_REQ_TRACKER'
        env = dict(os.environ)
        if pip_req_tracker_key in env:
            del(env['PIP_REQ_TRACKER'])
        pipcode = subprocess.call(
91
            [sys.executable, '-m', 'pip', 'install'] + list(args),
92
93
94
95
96
97
            env=env)

        if pipcode != 0:
            raise PythonGitError(
                'Could not install (pip return code=%s)' % pipcode, repo=self)

98
99
100
    def prepare(self) -> None:
        """
        Makes sure that the repository is fetched, at the right commit, and installed.
101
102

        Raises:
103
            PythonGitError: if something went wrong.
104
        """
105
106
107
        # import late because git will not be available in production
        from git import Repo, Git

108
109
110
111
112
113
114
115
        # check/change working directory
        old_cwd = os.getcwd()
        try:
            cwd = os.path.join(base_dir, self.name)
            if not os.path.exists(cwd):
                os.makedirs(cwd)
            os.chdir(cwd)

116
            _logger.info('check git/do init with origin %s for %s' % (self.git_url, self.name))
117
118
119
120
121
122
123
124
            if os.path.exists('.git'):
                git = Repo('./')
            else:
                git_cmd = Git('./')
                git_cmd.init()
                git = Repo('./')
                origin = git.create_remote('origin', self.git_url)

125
126
            _logger.info('pull %s for %s' % (self.git_commit, self.name))
            origin = git.remote('origin')
Markus Scheidgen's avatar
Markus Scheidgen committed
127
            origin.pull(self.git_commit, depth=1)
128

129
            if os.path.exists('requirements.txt'):
130
131
132
                _logger.info('install requirements.txt for %s' % self.name)
                self._run_pip_install('-r', 'requirements.txt')

133
            if os.path.exists('setup.py'):
134
135
136
137
                _logger.info('install setup.py for %s' % self.name)
                self._run_pip_install('-e', '.')

        except PythonGitError as e:
138
139
            raise e
        except Exception as e:
140
            raise PythonGitError(
141
142
143
144
145
                'Unexpected exception during preparation: %s' % e, repo=self)
        finally:
            os.chdir(old_cwd)
        pass

146
147
148
    def __repr__(self):
        return self.name

149

150
dependencies = [
151
152
153
    PythonGit(
        name='nomad-meta-info',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-info.git',
154
        git_commit='nomad-xt'),
155
156
157
158
    PythonGit(
        name='python_common',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/python-common.git',
        git_commit='master'),  # COMMIT b6f64de2149f95da6a79c4f86fd909b1dcfc23e8
159
160
161
162
    PythonGit(
        name='parsers/vasp',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/parser-vasp.git',
        git_commit='nomad-xt'),
Markus Scheidgen's avatar
Markus Scheidgen committed
163
164
165
166
    PythonGit(
        name='parsers/exciting',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/parser-exciting.git',
        git_commit='nomad-xt'),
167
168
169
    PythonGit(
        name='normalizers/stats',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/normalizer-stats.git',
170
171
172
173
        git_commit='nomad-xt'),
    PythonGit(
        name='normalizers/symmetry',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/normalizer-symmetry',
174
175
176
177
178
        git_commit='nomad-xt'),
    PythonGit(
        name='normalizers/system-type',
        git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/normalizer-system-type',
        git_commit='nomad-xt')
179
]
180

181
182
dependencies_dict = {dependency.name: dependency for dependency in dependencies}

183

184
def prepare() -> None:
185
186
187
188
    """
    Installs all dependencies from :data:`dependencies` and :data:`parsers`.
    """
    for python_git in dependencies:
189
190
        python_git.prepare()

191
if __name__ == '__main__':
192
193
    _logger.setLevel(logging.DEBUG)
    prepare()