From 5271286fb987b2b43661e08736ac594ec894be77 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Tue, 22 Mar 2022 22:07:35 +0000 Subject: [PATCH 1/7] Remove defined stages from pipeline for now --- .gitlab-ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8d5af2..f3a8fd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,6 @@ # SAFETY_API_KEY: # Set to your API key for accessing up-to-date package security information -stages: -- prepare -- test .python: image: python:3.9 @@ -22,7 +19,6 @@ stages: Pin: # Pin dependencies in requirements.txt for reproducing pipeline results - stage: prepare extends: [.python] script: - pip install --prefer-binary -e . @@ -32,7 +28,6 @@ Pin: Dependency Check: - stage: test image: pyupio/safety:latest needs: [Pin] allow_failure: true @@ -41,7 +36,7 @@ Dependency Check: Quality Gate: - stage: prepare + stage: .pre image: docker.kodo.org.uk/ci-images/pre-commit:latest variables: PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit @@ -57,7 +52,6 @@ Quality Gate: Unit Tests: - stage: test extends: [.python] needs: [Pin] script: -- GitLab From 87980da0dda5cbe6234087e249ab011063ad082e Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Tue, 22 Mar 2022 22:08:24 +0000 Subject: [PATCH 2/7] Add workflow rules to pipeline The rules should prevent duplicate pipelines for branches awaiting merging. --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3a8fd9..9c38665 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,14 @@ # Set to your API key for accessing up-to-date package security information +workflow: + rules: + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + + .python: image: python:3.9 variables: -- GitLab From 2ef449429145648703b35d54d0457e68ed7ac563 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Tue, 22 Mar 2022 22:13:14 +0000 Subject: [PATCH 3/7] Move QA pipeline job above other jobs This makes it's relationship with other jobs a little clearer, but otherwise has no effect. --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c38665..6ec6cdd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,22 @@ workflow: - pip install "pip>=21.3" +Quality Gate: + stage: .pre + image: docker.kodo.org.uk/ci-images/pre-commit:latest + variables: + PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit + cache: + key: $CI_JOB_IMAGE + paths: [cache] + rules: + - if: $CI_PIPELINE_SOURCE == "push" + script: + - source .gitlab-ci.pre-commit-run.bash + - pre_commit_run --hook-stage=commit + - pre_commit_run --hook-stage=push + + Pin: # Pin dependencies in requirements.txt for reproducing pipeline results extends: [.python] @@ -43,22 +59,6 @@ Dependency Check: - safety check -r requirements.txt -Quality Gate: - stage: .pre - image: docker.kodo.org.uk/ci-images/pre-commit:latest - variables: - PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit - cache: - key: $CI_JOB_IMAGE - paths: [cache] - rules: - - if: $CI_PIPELINE_SOURCE == "push" - script: - - source .gitlab-ci.pre-commit-run.bash - - pre_commit_run --hook-stage=commit - - pre_commit_run --hook-stage=push - - Unit Tests: extends: [.python] needs: [Pin] -- GitLab From d73af69f4319193344bbbd15fd4066f31b276a04 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Fri, 1 Apr 2022 14:32:32 +0100 Subject: [PATCH 4/7] Add build, check version, and upload jobs to pipeline --- .gitlab-ci.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ec6cdd..f4246f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,8 +41,19 @@ Quality Gate: - pre_commit_run --hook-stage=push +Build Package: + stage: build + extends: [.python] + script: + - pip install build + - python -m build + artifacts: + paths: [dist] + + Pin: # Pin dependencies in requirements.txt for reproducing pipeline results + stage: test extends: [.python] script: - pip install --prefer-binary -e . @@ -52,6 +63,7 @@ Pin: Dependency Check: + stage: test image: pyupio/safety:latest needs: [Pin] allow_failure: true @@ -60,6 +72,7 @@ Dependency Check: Unit Tests: + stage: test extends: [.python] needs: [Pin] script: @@ -78,3 +91,40 @@ Unit Tests: reports: cobertura: results/coverage.xml junit: results/xunit.xml + + +Check Tag: + stage: test + extends: [.python] + needs: ["Build Package"] + rules: + - if: $CI_COMMIT_TAG =~ /^v[0-9]/ + script: + - pip install packaging pkginfo + - | + python <<-END + from glob import glob + from packaging.version import Version + from pkginfo import Wheel + + wheel_path = glob("dist/*.whl")[0] + wheel = Wheel(wheel_path) + assert Version("$CI_COMMIT_TAG") == Version(wheel.version) + END + + +Upload Package: + stage: deploy + extends: [.python] + needs: ["Build Package"] + rules: + - if: $CI_COMMIT_TAG =~ /^v[0-9]/ + script: + - pip install twine + - TWINE_USERNAME=gitlab-ci-token + TWINE_PASSWORD=$CI_JOB_TOKEN + twine upload + --verbose + --non-interactive + --repository-url $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/pypi + dist/* -- GitLab From 6227d8051f57027682c93316d2d7cf930c52a6a5 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Thu, 7 Apr 2022 17:59:18 +0100 Subject: [PATCH 5/7] Run QA job on pushes to MR (& other MR activity) --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f4246f7..fb14c6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ Quality Gate: paths: [cache] rules: - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" script: - source .gitlab-ci.pre-commit-run.bash - pre_commit_run --hook-stage=commit -- GitLab From 342ba6a4e00452692df4dec9b6fac06634fb82f3 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Tue, 29 Mar 2022 14:27:19 +0100 Subject: [PATCH 6/7] Refactor pre_commit_run test code with correction Refactors the code to set `CI_*` variables in the Bash wrapper script. At the same time corrects the assumption that `CI_COMMIT_BRANCH` is always set; it's now never set for simulated `merge_request_event` pipelines. --- .../unit/gitlab_ci/pre_commit_run/runner.bash | 74 +++++++++++++++- tests/unit/gitlab_ci/pre_commit_run/tests.py | 88 ++++++++----------- 2 files changed, 111 insertions(+), 51 deletions(-) diff --git a/tests/unit/gitlab_ci/pre_commit_run/runner.bash b/tests/unit/gitlab_ci/pre_commit_run/runner.bash index bef850f..e0e85a3 100644 --- a/tests/unit/gitlab_ci/pre_commit_run/runner.bash +++ b/tests/unit/gitlab_ci/pre_commit_run/runner.bash @@ -15,6 +15,21 @@ # # A runner for pre_commit_run which mocks the "pre-commit" command # +# Usage: runner.bash REPO-PATH TARGET-SCRIPT COMMIT-BRANCH SOURCE-REPO [ARG [...]] +# +# REPO-PATH: +# Path to the test worktree/repo +# TARGET-SCRIPT: +# The absolute path to the script under test (containing the pre_commit_run +# function) +# COMMIT-BRANCH: +# A head reference name on the repo at CI_REPOSITORY_URL to be pulled as the +# pipeline target +# SOURCE-REPO: +# The path to a repository fixture to use as an upstream source +# ARG: +# Additional arguments to pass to pre_commit_run +# set -eu shopt -s expand_aliases @@ -24,7 +39,64 @@ pre_commit() { printf "%s\0" "$@" } +assert() { + eval "$1" && return + echo "Assert: $1" + if [[ $# -gt 1 ]]; then + shift + echo "Fatal: $*" + fi + exit 3 +} >&2 + +get_sha() { + GIT_DIR=$1 git rev-parse --revs-only --verify $2 +} + +get_head() { + GIT_DIR=$1 git symbolic-ref HEAD | sed 's@refs/heads/@@' +} + +# Handle arguments cd "${1? The directory to run from, this should be the top of the git repository}" source "${2? The full path to the Bash file containing pre_commit_run}" -shift 2 +declare COMMIT_BRANCH=${3? Need the head reference of a target commmit} +declare SOURCE_REPO=${4? Need the path to the source repository fixture} +shift 4 + +# Constants +declare -r NULL_SHA=0000000000000000000000000000000000000000 + +# Declare exported CI_* variables used +declare -x CI_COMMIT_SHA CI_COMMIT_BEFORE_SHA CI_COMMIT_BRANCH +declare -x CI_DEFAULT_BRANCH CI_PIPELINE_SOURCE CI_REPOSITORY_URL +declare -x CI_MERGE_REQUEST_SOURCE_BRANCH_NAME +declare -x CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +# Set appropriate values for CI_* variables +CI_REPOSITORY_URL=file://$SOURCE_REPO +CI_DEFAULT_BRANCH=$(get_head $SOURCE_REPO) +CI_COMMIT_SHA=$(get_sha $SOURCE_REPO $COMMIT_BRANCH) + +case ${CI_PIPELINE_SOURCE:=push} in + merge_request_event) + assert '[[ ! -v CI_COMMIT_BEFORE_SHA ]]' \ + "CI_COMMIT_BEFORE_SHA cannot be set for merge request pipelines" + : ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:=$CI_DEFAULT_BRANCH} + CI_MERGE_REQUEST_SOURCE_BRANCH_NAME=$COMMIT_BRANCH + CI_COMMIT_BEFORE_SHA=$NULL_SHA + ;; + *) + CI_COMMIT_BRANCH=$COMMIT_BRANCH + ;; +esac + +if [[ -v CI_COMMIT_BRANCH ]]; then + git fetch $CI_REPOSITORY_URL -q --depth=2 $CI_COMMIT_BRANCH:$CI_COMMIT_BRANCH + git symbolic-ref HEAD refs/heads/$CI_COMMIT_BRANCH +else + git fetch $CI_REPOSITORY_URL -q --depth=2 $CI_COMMIT_SHA + echo "$CI_COMMIT_SHA" >HEAD +fi + pre_commit_run "$@" diff --git a/tests/unit/gitlab_ci/pre_commit_run/tests.py b/tests/unit/gitlab_ci/pre_commit_run/tests.py index 7d5b9f3..dc087bd 100644 --- a/tests/unit/gitlab_ci/pre_commit_run/tests.py +++ b/tests/unit/gitlab_ci/pre_commit_run/tests.py @@ -81,88 +81,80 @@ class PreCommitRunTests(unittest.TestCase): ref: str, args: Iterable[str|Path] = tuple(), **env: str, - ) -> tuple[list[str], dict[str, str]]: + ) -> list[str]: """ - Execute the test runner script, return the pre-commit arguments and the environment + Execute the test runner script, return the pre-commit arguments """ - self.test_repo.command("fetch", self.repo.url, "-q", "--depth=2", f"{ref}:{ref}") - self.test_repo.command("symbolic-ref", "HEAD", f"refs/heads/{ref}") - env.update( - CI_REPOSITORY_URL=self.repo.url, - CI_DEFAULT_BRANCH=MAIN, - CI_COMMIT_BRANCH=ref, - CI_COMMIT_SHA=self.repo.get_sha(ref), - ) proc = subprocess.run( - ["bash", TEST_RUNNER, self.test_repo.directory, TEST_SCRIPT, *args], + [ + "bash", TEST_RUNNER, + self.test_repo.directory, TEST_SCRIPT, ref, self.repo.directory, + *args, + ], check=True, text=True, stdout=subprocess.PIPE, env=env, ) - return proc.stdout.split("\0")[:-1], env + return proc.stdout.split("\0")[:-1] def test_with_before_sha(self) -> None: """ Check that calling with CI_COMMIT_BEFORE_SHA set ensures the commit is available - - CI_COMMIT_BEFORE_SHA ~! /0{40}/ """ - args, env = self.run_script( + from_sha = self.repo.get_sha(f"{MAIN}~4") + to_sha = self.repo.get_sha(MAIN) + + args = self.run_script( MAIN, - CI_COMMIT_BEFORE_SHA=self.repo.get_sha(f"{MAIN}~4"), + CI_COMMIT_BEFORE_SHA=from_sha, ) expect = [ "run", - f"--from-ref={env['CI_COMMIT_BEFORE_SHA']}", - f"--to-ref={env['CI_COMMIT_SHA']}", + f"--from-ref={from_sha}", + f"--to-ref={to_sha}", ] self.assertListEqual(expect, args) def test_with_target_branch(self) -> None: """ Check that calling in merge request pipelines finds the LCA with the target branch - - CI_MERGE_REQUEST_TARGET_BRANCH_NAME == CI_DEFAULT_BRANCH - CI_COMMIT_BRANCH == FORK """ - args, env = self.run_script( + args = self.run_script( FORK, - CI_MERGE_REQUEST_TARGET_BRANCH_NAME=MAIN, + CI_PIPELINE_SOURCE="merge_request_event", ) - lca = self.branches[LCA] - expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"] + expect = [ + "run", + f"--from-ref={self.branches[LCA].head}", + f"--to-ref={self.repo.get_sha(FORK)}", + ] self.assertListEqual(expect, args) def test_new_branch(self) -> None: """ Check that pushing to a new fork gets the LCA with the default branch - CI_COMMIT_BEFORE_SHA ~= /0{40}/ CI_COMMIT_BRANCH != CI_DEFAULT_BRANCH """ - args, env = self.run_script( - FORK, - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(FORK) - lca = self.branches[LCA] - expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"] + expect = [ + "run", + f"--from-ref={self.branches[LCA].head}", + f"--to-ref={self.repo.get_sha(FORK)}", + ] self.assertListEqual(expect, args) def test_new_default_branch(self) -> None: """ Check that pushing to a brand new repo checks all files - CI_COMMIT_BEFORE_SHA ~= /0{40}/ CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH """ - args, env = self.run_script( - MAIN, - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(MAIN) expect = ["run", "--all-files"] self.assertListEqual(expect, args) @@ -170,13 +162,8 @@ class PreCommitRunTests(unittest.TestCase): def test_orphan_branch(self) -> None: """ Check that pushing to a new orphan branch check all files - - CI_COMMIT_BEFORE_SHA ~= /0{40}/ """ - args, env = self.run_script( - ORPHAN, - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(ORPHAN) expect = ["run", "--all-files"] self.assertListEqual(expect, args) @@ -185,10 +172,7 @@ class PreCommitRunTests(unittest.TestCase): """ Check additional arguments passed to pre_commit_run are passed to pre-commit """ - args, env = self.run_script( - MAIN, ["more", "arguments"], - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(MAIN, ["more", "arguments"]) expect = ["run", "more", "arguments", "--all-files"] self.assertListEqual(expect, args) @@ -204,11 +188,15 @@ class PreCommitRunTests(unittest.TestCase): ssh or file: "fatal: no commits selected for shallow requests" http: "fatal: error processing shallow info: 4" """ - args, env = self.run_script( + args = self.run_script( MAIN, + CI_PIPELINE_SOURCE="merge_request_event", CI_MERGE_REQUEST_TARGET_BRANCH_NAME=LCA, ) - lca = self.branches[LCA] - expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"] + expect = [ + "run", + f"--from-ref={self.branches[LCA].head}", + f"--to-ref={self.repo.get_sha(MAIN)}", + ] self.assertListEqual(expect, args) -- GitLab From 263463a7a632826f6c62e0678d77f8f410b568f2 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Tue, 29 Mar 2022 14:31:15 +0100 Subject: [PATCH 7/7] Correct pre_commit_run assumption that CI_COMMIT_BRANCH always available --- .gitlab-ci.pre-commit-run.bash | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.pre-commit-run.bash b/.gitlab-ci.pre-commit-run.bash index d8b97a5..704e716 100644 --- a/.gitlab-ci.pre-commit-run.bash +++ b/.gitlab-ci.pre-commit-run.bash @@ -21,8 +21,8 @@ pre_commit_run() ( declare -a PRE_COMMIT_ARGS find_lca() { - local repo=$CI_REPOSITORY_URL current_branch=$CI_COMMIT_BRANCH - local other_branch=$1 + local repo=$CI_REPOSITORY_URL + local current_branch=$1 other_branch=$2 # See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1 # and https://stackoverflow.com/questions/4698759/converting-git-repository-to-shallow/53245223#53245223 @@ -43,9 +43,9 @@ pre_commit_run() ( if [[ -v CI_COMMIT_BEFORE_SHA ]] && [[ ! $CI_COMMIT_BEFORE_SHA =~ ^0{40}$ ]]; then fetch_ref $CI_COMMIT_BEFORE_SHA elif [[ -v CI_MERGE_REQUEST_TARGET_BRANCH_NAME ]]; then - find_lca $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + find_lca $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME elif [[ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]]; then - find_lca $CI_DEFAULT_BRANCH + find_lca $CI_COMMIT_BRANCH $CI_DEFAULT_BRANCH fi if [[ -v FROM_REF ]]; then -- GitLab