From 2e8054c3e26c56b7039fa9e27648c81126a8aa3e Mon Sep 17 00:00:00 2001 From: Stefan Possanner <spossann@ipp.mpg.de> Date: Thu, 6 Jul 2023 11:04:13 +0200 Subject: [PATCH 1/2] add stages to ci --- .gitlab-ci.yml | 74 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 859594b..aa8bc67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ default: # Stupid GitLab fails to load the cache for second jobs. cache: # Global `cache`. paths: - - g2p-env-$CI_PIPELINE_ID + - g2p-env-$CI_PIPELINE_ID/ artifacts: reports: @@ -28,31 +28,67 @@ variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" stages: + - startup + - build - install_and_test - release -# Changes (e.g. apt install, pip install) do not persist across different GitLab CI jobs. -# => Consolidate everything in GitLab CI into a single job, because the design is dumb. -# => Since we only have one job, and GitLab Pages requires the job to be called `pages` for it to work, we now call this job `pages`. +# python virtual environment into cache +default_venv: + stage: startup + rules: + - if: $CI_COMMIT_TAG + when: never + - if: $CI_PIPELINE_SOURCE == "push" + before_script: + - uname -s # get system info + - uname -v + - uname -m + - uname -o + - echo "Hello ${GITLAB_USER_NAME} !" + - echo "This is job ${CI_JOB_ID}" + - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev + script: + - echo "Creating virtual environment..." + - pip install -U virtualenv + - python3 -m venv g2p-env-$CI_PIPELINE_ID + - ls + - pwd + - source g2p-env-$CI_PIPELINE_ID/bin/activate + - python3 -m pip install --upgrade pip + - pip install -U pip build pytest pytest-cov coverage twine + - pip list + +# build on mpcdf server +default_build: + stage: build + rules: + - if: $CI_COMMIT_TAG + when: never + - if: $CI_PIPELINE_SOURCE == "push" + needs: ['default_venv'] + script: + - pip list + - python3 -m build # build gvec_to_python (.whl) + artifacts: + paths: + - dist/ + install_and_test: stage: install_and_test rules: - if: $CI_COMMIT_TAG when: never - if: $CI_COMMIT_REF_PROTECTED == "false" && $CI_PIPELINE_SOURCE == "push" + needs: ['default_build'] before_script: # Overrides global `before_script`. - echo "Hello, $GITLAB_USER_LOGIN!" - python -V # Print out python version for debugging. - pwd - # Install everything. - - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev libopenmpi-dev openmpi-bin - - python -m venv g2p-env-$CI_PIPELINE_ID + - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev - source g2p-env-$CI_PIPELINE_ID/bin/activate - - pip install -U pip build pytest pytest-cov coverage twine script: - # Build: build python package, install package. - - echo "Run python build:" - - python -m build + # install package - ls -lah dist - echo "Install dist:" - pip install -U dist/gvec_to_python-*.whl --force-reinstall @@ -77,7 +113,7 @@ deploy: - if: $CI_COMMIT_TAG when: never - if: $CI_COMMIT_REF_NAME == "master" - needs: ["install_and_test"] + needs: ["default_build"] script: - twine upload dist/* @@ -119,18 +155,4 @@ release_job: url: 'https://pypi.org/project/gvec_to_python/' -# OLD, TO BE DELETED -# deploy: -# stage: release -# rules: -# - if: $CI_COMMIT_TAG -# when: never -# - if: $CI_COMMIT_REF_NAME == "master" -# before_script: -# - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev libopenmpi-dev openmpi-bin -# - pip install -U pip build twine -# script: -# - python -m build -# - ls -lah dist -# - twine upload dist/* -- GitLab From 324ab029baf598e4673edbe14c4bc336516eddf1 Mon Sep 17 00:00:00 2001 From: Stefan Possanner <spossann@ipp.mpg.de> Date: Thu, 6 Jul 2023 11:17:02 +0200 Subject: [PATCH 2/2] use before_script in ci --- .gitlab-ci.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa8bc67..3836494 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,9 +8,21 @@ default: # Try to use only one job per stage. # Stupid GitLab fails to load the cache for second jobs. cache: # Global `cache`. + key: 'python-env' paths: - g2p-env-$CI_PIPELINE_ID/ + before_script: + - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev + - source g2p-env-$CI_PIPELINE_ID/bin/activate + - pip list + - ENV_PATH=$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))") + #- ENV_PATH=$(python3 -m site --user-site) + - LIBDIR=$(python3 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") + - echo $ENV_PATH + - echo $LIBDIR + - ls + artifacts: reports: coverage_report: @@ -18,7 +30,7 @@ default: path: coverage.xml paths: - logs/execution.log - - dist/* + - dist/ after_script: # Global `after_script`. - ls g2p-env-$CI_PIPELINE_ID @@ -47,7 +59,6 @@ default_venv: - uname -o - echo "Hello ${GITLAB_USER_NAME} !" - echo "This is job ${CI_JOB_ID}" - - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev script: - echo "Creating virtual environment..." - pip install -U virtualenv @@ -68,11 +79,7 @@ default_build: - if: $CI_PIPELINE_SOURCE == "push" needs: ['default_venv'] script: - - pip list - python3 -m build # build gvec_to_python (.whl) - artifacts: - paths: - - dist/ install_and_test: stage: install_and_test @@ -81,12 +88,6 @@ install_and_test: when: never - if: $CI_COMMIT_REF_PROTECTED == "false" && $CI_PIPELINE_SOURCE == "push" needs: ['default_build'] - before_script: # Overrides global `before_script`. - - echo "Hello, $GITLAB_USER_LOGIN!" - - python -V # Print out python version for debugging. - - pwd - - apt-get -qq update && apt-get -qq install -y gcc gfortran libblas-dev liblapack-dev - - source g2p-env-$CI_PIPELINE_ID/bin/activate script: # install package - ls -lah dist -- GitLab