diff --git a/README.md b/README.md
index 65fb412cb4d0d02804932789a6604d2ce6105989..232098f30beefca87dd503f8da10ab8971c816dd 100644
--- a/README.md
+++ b/README.md
@@ -23,27 +23,48 @@ $ git pull --rebae
And then the update to a new version of the core needs to be commited.
-## Deploying
-To deploy to devel ( For deploy of production, use according inventory):
+## Deploying using the edd_tool (Recommended)
+The edd_tool helps with clean build and deployments of the EDD.
-```
-$ ansible-playbook -i effelsberg_devel effelsberg_config.yml --tags=buildbase,build --ask-vault-password
-$ ansible-playbook -i effelsberg_devel effelsberg_config.yml --ask-vault-password
-```
+Requirements
+ ansible (pip install ansible)
+ coloredlogs (pip install coloredlogs)
-It is important, that these commands are executed in a clean environment, i.e.
-with the locally checked out provision-core submodule put manually to the
-correct state as defined in the `requirements.txt`. Use the
-```
-$./clean_deploy effelsberg_production TAG
-```
+Example Usage:
+
+1) List all available versions
+ `$ ./edd_tool -i effelsberg_devel list-versions`
+
+2) Deploy a specific version
+ `$ ./edd_tool -i ./effelsberg_devel deploy --version=221219.0`
+
+3) Build all containers of a version [necessary only on new releases]
+ `$ ./edd_tool buildall -i effelsberg_production --version=VERSION buildall`
-for a clean deployment of the system with a given TAG.
+4) Limit Re-deploy to a specific host, e.g. a single packetizer
+ `./edd_tool -i ./effelsberg_devel deploy --version=221219.0`
+For more information check the help
+`$ ./edd_tool --help`
+## Manual deployment (not recommended)
+System can stil be deployed manyually. However, care must be taken to checkout
+/ install the correct versions of the base system
+
+Assuming a checkout with submodules
+```
+$ cd ansible_collections/EDD/core
+$ git co CORRECT_TAG
+$ cd ../../../
+$ git co CORRECT_TAG
+$ ansible-playbook -i effelsberg_devel effelsberg_config.yml --tags=buildbase,build --ask-vault-password
+$ ansible-playbook -i effelsberg_devel effelsberg_config.yml --ask-vault-password
+```
+
+
# Retagging only
To retag the current 'latest' containers as production verions, you can use
diff --git a/clean_deploy b/clean_deploy
deleted file mode 100755
index f1d1aa032c2be9bd0937a2d9c18738491ea1b85a..0000000000000000000000000000000000000000
--- a/clean_deploy
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/python3
-__doc__ = "Deploy the EDD using a clean environment"
-
-import argparse
-import os
-import sys
-import tempfile
-import shlex
-import subprocess
-import getpass
-import pathlib
-
-CFG_FILE = 'effelsberg_config.yml'
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('inventory', help='Inventory to deploy to')
- parser.add_argument('tag', help='tags')
-
- args = parser.parse_args()
-
- vault_password = getpass.getpass('Please enter vault password: ')
-
- cwd = os.getcwd()
- with tempfile.TemporaryDirectory() as build_directory:
- print(f"--> Created temporary build directory: {build_directory}")
- os.chdir(build_directory)
-
- print(f"--> Cloning repository and switching to {args.tag}")
- subprocess.run(shlex.split(f'git clone {cwd} .'), check=True)
- subprocess.run(shlex.split(f'git checkout {args.tag}'), check=True)
-
- pwdfile = os.path.join(build_directory, 'vault_pwd')
- P = pathlib.Path(pwdfile).touch(mode=0o600)
- with open(pwdfile, 'w', encoding='utf8') as f:
- f.write(vault_password)
-
- env = {}
- env.update(os.environ)
- env['ANSIBLE_COLLECTIONS_PATH'] = build_directory
- requirements_file = os.path.join(args.inventory, 'requirements.yml')
-
- print(f'--> Install collections from {requirements_file}')
- print()
- with open(requirements_file) as f:
- print(f.read())
-
- try:
- subprocess.run(shlex.split(f'ansible-galaxy collection install -r {requirements_file} -p {build_directory}'), check=True)
- except subprocess.CalledProcessError:
- print('!!! ERROR during installation of collection. Deployment was NOT started!')
- sys.exit(-1)
-
- print('--> Building base in clean environment')
- try:
- subprocess.run(shlex.split(f'ansible-playbook -i {args.inventory} {CFG_FILE} --tags=buildbase --vault-password-file={pwdfile}'), env=env, check=True)
- except subprocess.CalledProcessError:
- print('!!! Error during buildbase. Deployment unsuccessfull !')
- sys.exit(-1)
-
- print('--> Building in clean environment')
- try:
- subprocess.run(shlex.split(f'ansible-playbook -i {args.inventory} {CFG_FILE} --tags=build --vault-password-file={pwdfile}'), env=env, check=True)
- except subprocess.CalledProcessError:
- print('!!! Error during build. Deployment unsuccessfull !')
- sys.exit(-1)
-
- print('--> Deploying from clean build')
- try:
- subprocess.run(shlex.split(f'ansible-playbook -i {args.inventory} {CFG_FILE} --vault-password-file={pwdfile}'), env=env, check=True)
- except subprocess.CalledProcessError:
- print('!!! Error during deployment. deployment unsuccessfull !')
- sys.exit(-1)
-
- os.chdir(cwd)
- print(f'--> Successfully deployed the EDD on {args.inventory}!')
diff --git a/deploy_tool/app.py b/deploy_tool/app.py
new file mode 100755
index 0000000000000000000000000000000000000000..6b2aefcb589d3e91c47c4782503d2827589a678b
--- /dev/null
+++ b/deploy_tool/app.py
@@ -0,0 +1,287 @@
+#!/usr/bin/python3
+__doc__ = "Deploy the EDD using a clean environment"
+
+import argparse
+import os
+import sys
+import tempfile
+import shlex
+import shutil
+import subprocess
+import getpass
+import pathlib
+import logging
+import datetime
+import dataclasses
+import git
+import coloredlogs
+
+
+_log = logging.getLogger("EDD")
+
+coloredlogs.install(
+ fmt=("[ %(levelname)s - %(asctime)s - %(name)s "
+ "- %(filename)s:%(lineno)s] %(message)s"),
+ level=1, # We manage the log level via the logger, not the handler
+ logger=_log)
+
+env_level = os.getenv("LOG_LEVEL")
+if env_level:
+ _log.setLevel(env_level.upper())
+else:
+ _log.setLevel("INFO")
+
+CFG_FILE = 'effelsberg_config.yml'
+
+
+class TempRepository:
+ """
+ Checkout a repository into a temporary folder for version management with EDD versions.
+ """
+ def __init__(self, origin):
+ self.origin = origin
+ self._tmpdir = tempfile.TemporaryDirectory()
+ self.repo = None
+
+ def __del__(self):
+ self._tmpdir.cleanup()
+
+ def init(self):
+ """Initialize repository"""
+ _log.debug('Initializing tmp repo')
+ self.repo = git.Repo.clone_from(self.origin, self._tmpdir.name)
+
+
+ def get_next_version(self, dto=None):
+ """Return next verson in EDD versioning scheme"""
+ _log.debug('getting next version')
+ if dto is None:
+ _log.debug('No date provided, using today')
+ dto = datetime.datetime.now()
+
+ base = dto.strftime("%y%m%d")
+ today_tags = [t.name for t in self.repo.tags if t.name.startswith(base)]
+ _log.debug('Found %i tags for given date', len(today_tags))
+
+ return f"{base}.{len(today_tags)}"
+
+
+ def release(self, dto=None):
+ """
+ Create a new tag in the repository and push it to the origin
+ """
+ nt = self.get_next_version(dto)
+ _log.debug('Creating release %s', nt)
+ self.repo.create_tag(nt)
+ self.repo.remote().push(tags=True)
+
+
+@dataclasses.dataclass
+class BuildContext:
+ """Any informatino needed by the build command"""
+ build_directory: str
+ env: dict
+ inventory: str
+ no_act: bool
+ limit: str
+ config_file: str
+
+
+
+class CleanEnvironment:
+ """Contextmanager for checkouts in a clean build environment"""
+ def __init__(self, inventory, config_file, version, no_act, limit, keep_tempdir):
+
+ self.inventory = inventory
+ self.config_file = config_file
+ self.version = version
+ self.no_act = no_act
+ self.keep_tempdir = keep_tempdir
+
+ if limit:
+ self.limit = f"--limit={limit}"
+ else:
+ self.limit = ""
+
+ self.cwd = None
+ self.build_directory = None
+
+
+ def __enter__(self):
+ self.cwd = os.getcwd()
+ # Resolve origin before changing directory
+ origin = os.path.abspath(os.path.dirname(self.inventory))
+
+ _log.debug('Creating temp directory')
+ self.build_directory = tempfile.mkdtemp()
+ os.chdir(self.build_directory)
+
+ _log.debug('Cloning %s -> %s', origin, self.build_directory)
+ repo = git.Git()
+
+ repo.clone(origin, self.build_directory)
+
+ if self.version:
+ _log.debug('Checking out version %s', self.version)
+ try:
+ repo.checkout(self.version)
+ except git.exc.GitCommandError as E:
+ _log.exception(E)
+ _log.error("Version '%s' not known to git", self.version)
+ sys.exit(-1)
+
+ vault_password = getpass.getpass('Please enter vault password: ')
+
+ pwdfile = os.path.join(self.build_directory, 'vault_pwd')
+ pathlib.Path(pwdfile).touch(mode=0o600)
+ with open(pwdfile, 'w', encoding='utf8') as f:
+ f.write(vault_password)
+
+ env = {}
+ env.update(os.environ)
+ env['ANSIBLE_COLLECTIONS_PATH'] = self.build_directory
+
+ requirements_file = os.path.join(self.build_directory, os.path.basename(self.inventory), 'requirements.yml')
+ print('Install collections:')
+ with open(requirements_file) as f:
+ print(f.read())
+
+ try:
+ subprocess.run(shlex.split(f'ansible-galaxy collection install -r {requirements_file} -p {self.build_directory}'), check=True)
+ except subprocess.CalledProcessError as E:
+ _log.error('ERROR during installation of collection')
+ raise RuntimeError from E
+
+ return BuildContext(build_directory=self.build_directory, env=env, inventory=self.inventory, no_act=self.no_act, limit=self.limit, config_file=self.config_file)
+
+ def __exit__(self, *args, **kwargs):
+ _log.debug("Change back to initial working dir")
+ os.chdir(self.cwd)
+ if self.keep_tempdir:
+ _log.info('Temporary directory %s was not deleted as requested', self.build_directory)
+ else:
+ _log.debug("Cleaning up temporary directory")
+ shutil.rmtree(self.build_directory)
+
+
+def get_version_list(remote):
+ """Get list of tags without cloning repository"""
+ R = subprocess.run(shlex.split(f"git -c 'versionsort.suffix=-' ls-remote --tags --sort='v:refname' {remote}"), capture_output=True, check=True)
+ _log.debug('Got reply from remote:\n%s', R.stdout.decode())
+ result = [s.split('/')[-1] for s in R.stdout.decode().split('\n') if s]
+ return result
+
+
+def list_versions(inventory, **kwargs):
+ """List all versions in the provision_repository"""
+ origin = os.path.abspath(os.path.dirname(inventory))
+ R = get_version_list(origin)
+# R.sort()
+ print(f"Versions in {origin}:\n")
+ print("\n".join(R))
+
+
+def buildbase(build_context):
+ """Execute ansible buildbase command"""
+ print('--> Building base in clean environment')
+ if build_context.no_act:
+ no_act = '-C'
+ else:
+ no_act = ''
+ try:
+ subprocess.run(shlex.split(f'ansible-playbook {no_act} -i {build_context.inventory} {build_context.config_file} --tags=buildbase --vault-password-file=vault_pwd {build_context.limit}'), env=build_context.env, check=True)
+ except subprocess.CalledProcessError as E:
+ print('!!! Error during buildbase. Deployment unsuccessfull !')
+ raise RuntimeError from E
+
+
+def build(build_context):
+ """Execute ansible build command"""
+ print('--> Building in clean environment')
+ if build_context.no_act:
+ no_act = '-C'
+ else:
+ no_act = ''
+ try:
+ subprocess.run(shlex.split(f'ansible-playbook {no_act} -i {build_context.inventory} {build_context.config_file} --tags=build --vault-password-file=vault_pwd {build_context.limit}'), env=build_context.env, check=True)
+ except subprocess.CalledProcessError as E:
+ print('!!! Error during build. Deployment unsuccessfull !')
+ raise RuntimeError from E
+
+
+def deploy(build_context):
+ """Execute ansible deploy command"""
+ print('--> Deploying from clean build')
+ if build_context.no_act:
+ no_act = '-C'
+ else:
+ no_act = ''
+ try:
+ subprocess.run(shlex.split(f'ansible-playbook {no_act} -i {build_context.inventory} {build_context.config_file} --vault-password-file=vault_pwd {build_context.limit}'), env=build_context.env, check=True)
+ except subprocess.CalledProcessError as E:
+ print('!!! Error during deployment. deployment unsuccessfull !')
+ raise RuntimeError from E
+
+
+def buildall_action(**kwargs):
+ """Build base images and images"""
+ with CleanEnvironment(**kwargs) as ctx:
+ buildbase(ctx)
+ build(ctx)
+
+
+def buildbase_action(**kwargs):
+ """Build base images only"""
+ with CleanEnvironment(**kwargs) as ctx:
+ buildbase(ctx)
+
+def build_action(**kwargs):
+ """Build all imagese xcept base images"""
+ with CleanEnvironment(**kwargs) as ctx:
+ build(ctx)
+
+def deploy_action(**kwargs):
+ """Deploy system"""
+ with CleanEnvironment(**kwargs) as ctx:
+ deploy(ctx)
+
+
+
+
+actions = {}
+actions['list-versions'] = list_versions
+actions['build'] = build_action
+actions['buildbase'] = buildbase_action
+actions['buildall'] = buildall_action
+actions['deploy'] = deploy_action
+
+
+def action_help():
+ """
+ Compile help string from all actions
+ """
+ res = []
+ for k, v in actions.items():
+ res.append(f"{k:16} - {v.__doc__}")
+ return "\n".join(res)
+
+
+def main():
+ """edd_tool main"""
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
+ parser.add_argument('-i', '--inventory', help='Inventory to deploy to', required=True)
+ parser.add_argument('--version', help='Version to use. Defaults to head of branch', default=None)
+ parser.add_argument('-n', '--no-act', help='Do not execute ansible but add check parameter', action='store_true', default=False)
+ parser.add_argument('--limit', help='Limit action to selected host group', metavar='HOSTGROUP')
+ parser.add_argument('--keep-tempdir', help='Do not delete temporary build directory', action='store_true')
+ parser.add_argument('action', help=f'Action to perform. Valid actions are:\n{action_help()}', default=None)
+
+ args = parser.parse_args()
+
+ #with open(os.path.join(args.inventory, 'group_vars', 'all.yml')) as f:
+ # inventory_settings = yaml.load(f, Loader=yaml.SafeLoader)
+ actions[args.action](
+ version=args.version, config_file=CFG_FILE, inventory=args.inventory, no_act=args.no_act, limit=args.limit, keep_tempdir=args.keep_tempdir)
+
+if __name__ == "__main__":
+ main()
diff --git a/edd_tool b/edd_tool
new file mode 120000
index 0000000000000000000000000000000000000000..8dbb8c09764474cc2b6710dfe2fdc2d65d8a1189
--- /dev/null
+++ b/edd_tool
@@ -0,0 +1 @@
+deploy_tool/app.py
\ No newline at end of file
diff --git a/tests/test_deploy_tool.py b/tests/test_deploy_tool.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d467382f0409cca1bc6d54bf93951e96590b84c
--- /dev/null
+++ b/tests/test_deploy_tool.py
@@ -0,0 +1,93 @@
+import unittest
+import unittest.mock
+import git
+import tempfile
+import os
+import io
+import datetime
+
+import deploy_tool.app as deploy_tool
+
+
+class TestTempRepository(unittest.TestCase):
+ def setUp(self):
+ # Set up a origin repository
+ self.tmpdir = tempfile.TemporaryDirectory()
+
+ self.origin_repo = git.Repo.init(self.tmpdir.name)
+ foo_file = os.path.join(self.tmpdir.name, 'requirements.yml')
+ with open(foo_file, 'w') as f:
+ f.write('---')
+ self.origin_repo.index.add(foo_file)
+ self.origin_repo.index.commit('Added foo file')
+
+ def tearDown(self):
+ self.tmpdir.cleanup()
+
+ def test_init(self):
+ # After init, the repository should be cloned
+ repo = deploy_tool.TempRepository(self.tmpdir.name)
+ repo.init()
+
+ with open(os.path.join(repo._tmpdir.name, 'requirements.yml')) as f:
+ data = f.read()
+ self.assertEqual(data, '---')
+
+ def test_next_version(self):
+ # If no version exists, a well defined tag should be created
+ repo = deploy_tool.TempRepository(self.tmpdir.name)
+ repo.init()
+
+ v = repo.get_next_version(datetime.datetime(2022, 3, 24))
+ self.assertEqual(v, "220324.0")
+ repo.repo.create_tag(v)
+
+ v = repo.get_next_version(datetime.datetime(2022, 3, 24))
+ self.assertEqual(v, "220324.1")
+
+ # There should be no tag for today, so there should be one ending with
+ # 0 next
+ vt = repo.get_next_version()
+ self.assertTrue(vt.endswith('.0'))
+ repo.repo.create_tag(vt)
+ vt = repo.get_next_version()
+ self.assertTrue(vt.endswith('.1'))
+
+ def test_release(self):
+ # After release, the new tag should exist in the origin repository
+ repo = deploy_tool.TempRepository(self.tmpdir.name)
+ repo.init()
+
+ nt = repo.get_next_version()
+
+ repo.release()
+
+ self.assertIn(nt, [t.name for t in self.origin_repo.tags])
+
+
+ def test_list_version(self):
+ VL = deploy_tool.get_version_list(self.tmpdir.name)
+ self.assertListEqual(VL, [])
+
+ self.origin_repo.create_tag('foo')
+ VL = deploy_tool.get_version_list(self.tmpdir.name)
+ self.assertListEqual(VL, ['foo'])
+
+
+ def test_cleanenvironment_passphrase(self):
+ pwd = 'scientiavinceretenebras'
+ # password should be queried and saved to a file in the tem directory
+
+ inventory = self.tmpdir.name
+ inventory_settings = {}
+ version = '230201.0'
+
+ with unittest.mock.patch('getpass.getpass', side_effect=[pwd]) as stdin, unittest.mock.patch('subprocess.run') as sub, deploy_tool.CleanEnvironment(inventory, inventory_settings, version) as ctx:
+ with open(os.path.join(ctx.build_directory, 'vault_pwd')) as f:
+ d = f.read()
+ self.assertEqual(pwd, d)
+
+
+
+if __name__ == "__main__":
+ unittest.main()