diff --git a/.gitlab-ci.pre-commit-run.bash b/.gitlab-ci.pre-commit-run.bash index d8b97a536713a869627068e863ab38355062eb85..704e716956ec58c44775b2b11d696e71560a6650 100644 --- a/.gitlab-ci.pre-commit-run.bash +++ b/.gitlab-ci.pre-commit-run.bash @@ -21,8 +21,8 @@ pre_commit_run() ( declare -a PRE_COMMIT_ARGS find_lca() { - local repo=$CI_REPOSITORY_URL current_branch=$CI_COMMIT_BRANCH - local other_branch=$1 + local repo=$CI_REPOSITORY_URL + local current_branch=$1 other_branch=$2 # See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1 # and https://stackoverflow.com/questions/4698759/converting-git-repository-to-shallow/53245223#53245223 @@ -43,9 +43,9 @@ pre_commit_run() ( if [[ -v CI_COMMIT_BEFORE_SHA ]] && [[ ! $CI_COMMIT_BEFORE_SHA =~ ^0{40}$ ]]; then fetch_ref $CI_COMMIT_BEFORE_SHA elif [[ -v CI_MERGE_REQUEST_TARGET_BRANCH_NAME ]]; then - find_lca $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + find_lca $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME elif [[ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]]; then - find_lca $CI_DEFAULT_BRANCH + find_lca $CI_COMMIT_BRANCH $CI_DEFAULT_BRANCH fi if [[ -v FROM_REF ]]; then diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8d5af22666b7ab8c56eedc94749f0afcd849ff6..fb14c6dbd13b507a8f45610ebd3e79ab4a6a27a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,14 @@ # SAFETY_API_KEY: # Set to your API key for accessing up-to-date package security information -stages: -- prepare -- test + +workflow: + rules: + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + .python: image: python:3.9 @@ -20,9 +25,36 @@ stages: - pip install "pip>=21.3" +Quality Gate: + stage: .pre + image: docker.kodo.org.uk/ci-images/pre-commit:latest + variables: + PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit + cache: + key: $CI_JOB_IMAGE + paths: [cache] + rules: + - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + script: + - source .gitlab-ci.pre-commit-run.bash + - pre_commit_run --hook-stage=commit + - pre_commit_run --hook-stage=push + + +Build Package: + stage: build + extends: [.python] + script: + - pip install build + - python -m build + artifacts: + paths: [dist] + + Pin: # Pin dependencies in requirements.txt for reproducing pipeline results - stage: prepare + stage: test extends: [.python] script: - pip install --prefer-binary -e . @@ -40,22 +72,6 @@ Dependency Check: - safety check -r requirements.txt -Quality Gate: - stage: prepare - image: docker.kodo.org.uk/ci-images/pre-commit:latest - variables: - PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit - cache: - key: $CI_JOB_IMAGE - paths: [cache] - rules: - - if: $CI_PIPELINE_SOURCE == "push" - script: - - source .gitlab-ci.pre-commit-run.bash - - pre_commit_run --hook-stage=commit - - pre_commit_run --hook-stage=push - - Unit Tests: stage: test extends: [.python] @@ -76,3 +92,40 @@ Unit Tests: reports: cobertura: results/coverage.xml junit: results/xunit.xml + + +Check Tag: + stage: test + extends: [.python] + needs: ["Build Package"] + rules: + - if: $CI_COMMIT_TAG =~ /^v[0-9]/ + script: + - pip install packaging pkginfo + - | + python <<-END + from glob import glob + from packaging.version import Version + from pkginfo import Wheel + + wheel_path = glob("dist/*.whl")[0] + wheel = Wheel(wheel_path) + assert Version("$CI_COMMIT_TAG") == Version(wheel.version) + END + + +Upload Package: + stage: deploy + extends: [.python] + needs: ["Build Package"] + rules: + - if: $CI_COMMIT_TAG =~ /^v[0-9]/ + script: + - pip install twine + - TWINE_USERNAME=gitlab-ci-token + TWINE_PASSWORD=$CI_JOB_TOKEN + twine upload + --verbose + --non-interactive + --repository-url $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/pypi + dist/* diff --git a/tests/unit/gitlab_ci/pre_commit_run/runner.bash b/tests/unit/gitlab_ci/pre_commit_run/runner.bash index bef850f577392cd84148357fde565127ad3bcfa8..e0e85a357b11061e0d593841d3fe46e2a4c65fd5 100644 --- a/tests/unit/gitlab_ci/pre_commit_run/runner.bash +++ b/tests/unit/gitlab_ci/pre_commit_run/runner.bash @@ -15,6 +15,21 @@ # # A runner for pre_commit_run which mocks the "pre-commit" command # +# Usage: runner.bash REPO-PATH TARGET-SCRIPT COMMIT-BRANCH SOURCE-REPO [ARG [...]] +# +# REPO-PATH: +# Path to the test worktree/repo +# TARGET-SCRIPT: +# The absolute path to the script under test (containing the pre_commit_run +# function) +# COMMIT-BRANCH: +# A head reference name on the repo at CI_REPOSITORY_URL to be pulled as the +# pipeline target +# SOURCE-REPO: +# The path to a repository fixture to use as an upstream source +# ARG: +# Additional arguments to pass to pre_commit_run +# set -eu shopt -s expand_aliases @@ -24,7 +39,64 @@ pre_commit() { printf "%s\0" "$@" } +assert() { + eval "$1" && return + echo "Assert: $1" + if [[ $# -gt 1 ]]; then + shift + echo "Fatal: $*" + fi + exit 3 +} >&2 + +get_sha() { + GIT_DIR=$1 git rev-parse --revs-only --verify $2 +} + +get_head() { + GIT_DIR=$1 git symbolic-ref HEAD | sed 's@refs/heads/@@' +} + +# Handle arguments cd "${1? The directory to run from, this should be the top of the git repository}" source "${2? The full path to the Bash file containing pre_commit_run}" -shift 2 +declare COMMIT_BRANCH=${3? Need the head reference of a target commmit} +declare SOURCE_REPO=${4? Need the path to the source repository fixture} +shift 4 + +# Constants +declare -r NULL_SHA=0000000000000000000000000000000000000000 + +# Declare exported CI_* variables used +declare -x CI_COMMIT_SHA CI_COMMIT_BEFORE_SHA CI_COMMIT_BRANCH +declare -x CI_DEFAULT_BRANCH CI_PIPELINE_SOURCE CI_REPOSITORY_URL +declare -x CI_MERGE_REQUEST_SOURCE_BRANCH_NAME +declare -x CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +# Set appropriate values for CI_* variables +CI_REPOSITORY_URL=file://$SOURCE_REPO +CI_DEFAULT_BRANCH=$(get_head $SOURCE_REPO) +CI_COMMIT_SHA=$(get_sha $SOURCE_REPO $COMMIT_BRANCH) + +case ${CI_PIPELINE_SOURCE:=push} in + merge_request_event) + assert '[[ ! -v CI_COMMIT_BEFORE_SHA ]]' \ + "CI_COMMIT_BEFORE_SHA cannot be set for merge request pipelines" + : ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:=$CI_DEFAULT_BRANCH} + CI_MERGE_REQUEST_SOURCE_BRANCH_NAME=$COMMIT_BRANCH + CI_COMMIT_BEFORE_SHA=$NULL_SHA + ;; + *) + CI_COMMIT_BRANCH=$COMMIT_BRANCH + ;; +esac + +if [[ -v CI_COMMIT_BRANCH ]]; then + git fetch $CI_REPOSITORY_URL -q --depth=2 $CI_COMMIT_BRANCH:$CI_COMMIT_BRANCH + git symbolic-ref HEAD refs/heads/$CI_COMMIT_BRANCH +else + git fetch $CI_REPOSITORY_URL -q --depth=2 $CI_COMMIT_SHA + echo "$CI_COMMIT_SHA" >HEAD +fi + pre_commit_run "$@" diff --git a/tests/unit/gitlab_ci/pre_commit_run/tests.py b/tests/unit/gitlab_ci/pre_commit_run/tests.py index 7d5b9f3c5f453b612e9dcc171ab6a71595945fce..dc087bd9231578e5bd808e1cb74d94d2b39cb288 100644 --- a/tests/unit/gitlab_ci/pre_commit_run/tests.py +++ b/tests/unit/gitlab_ci/pre_commit_run/tests.py @@ -81,88 +81,80 @@ class PreCommitRunTests(unittest.TestCase): ref: str, args: Iterable[str|Path] = tuple(), **env: str, - ) -> tuple[list[str], dict[str, str]]: + ) -> list[str]: """ - Execute the test runner script, return the pre-commit arguments and the environment + Execute the test runner script, return the pre-commit arguments """ - self.test_repo.command("fetch", self.repo.url, "-q", "--depth=2", f"{ref}:{ref}") - self.test_repo.command("symbolic-ref", "HEAD", f"refs/heads/{ref}") - env.update( - CI_REPOSITORY_URL=self.repo.url, - CI_DEFAULT_BRANCH=MAIN, - CI_COMMIT_BRANCH=ref, - CI_COMMIT_SHA=self.repo.get_sha(ref), - ) proc = subprocess.run( - ["bash", TEST_RUNNER, self.test_repo.directory, TEST_SCRIPT, *args], + [ + "bash", TEST_RUNNER, + self.test_repo.directory, TEST_SCRIPT, ref, self.repo.directory, + *args, + ], check=True, text=True, stdout=subprocess.PIPE, env=env, ) - return proc.stdout.split("\0")[:-1], env + return proc.stdout.split("\0")[:-1] def test_with_before_sha(self) -> None: """ Check that calling with CI_COMMIT_BEFORE_SHA set ensures the commit is available - - CI_COMMIT_BEFORE_SHA ~! /0{40}/ """ - args, env = self.run_script( + from_sha = self.repo.get_sha(f"{MAIN}~4") + to_sha = self.repo.get_sha(MAIN) + + args = self.run_script( MAIN, - CI_COMMIT_BEFORE_SHA=self.repo.get_sha(f"{MAIN}~4"), + CI_COMMIT_BEFORE_SHA=from_sha, ) expect = [ "run", - f"--from-ref={env['CI_COMMIT_BEFORE_SHA']}", - f"--to-ref={env['CI_COMMIT_SHA']}", + f"--from-ref={from_sha}", + f"--to-ref={to_sha}", ] self.assertListEqual(expect, args) def test_with_target_branch(self) -> None: """ Check that calling in merge request pipelines finds the LCA with the target branch - - CI_MERGE_REQUEST_TARGET_BRANCH_NAME == CI_DEFAULT_BRANCH - CI_COMMIT_BRANCH == FORK """ - args, env = self.run_script( + args = self.run_script( FORK, - CI_MERGE_REQUEST_TARGET_BRANCH_NAME=MAIN, + CI_PIPELINE_SOURCE="merge_request_event", ) - lca = self.branches[LCA] - expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"] + expect = [ + "run", + f"--from-ref={self.branches[LCA].head}", + f"--to-ref={self.repo.get_sha(FORK)}", + ] self.assertListEqual(expect, args) def test_new_branch(self) -> None: """ Check that pushing to a new fork gets the LCA with the default branch - CI_COMMIT_BEFORE_SHA ~= /0{40}/ CI_COMMIT_BRANCH != CI_DEFAULT_BRANCH """ - args, env = self.run_script( - FORK, - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(FORK) - lca = self.branches[LCA] - expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"] + expect = [ + "run", + f"--from-ref={self.branches[LCA].head}", + f"--to-ref={self.repo.get_sha(FORK)}", + ] self.assertListEqual(expect, args) def test_new_default_branch(self) -> None: """ Check that pushing to a brand new repo checks all files - CI_COMMIT_BEFORE_SHA ~= /0{40}/ CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH """ - args, env = self.run_script( - MAIN, - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(MAIN) expect = ["run", "--all-files"] self.assertListEqual(expect, args) @@ -170,13 +162,8 @@ class PreCommitRunTests(unittest.TestCase): def test_orphan_branch(self) -> None: """ Check that pushing to a new orphan branch check all files - - CI_COMMIT_BEFORE_SHA ~= /0{40}/ """ - args, env = self.run_script( - ORPHAN, - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(ORPHAN) expect = ["run", "--all-files"] self.assertListEqual(expect, args) @@ -185,10 +172,7 @@ class PreCommitRunTests(unittest.TestCase): """ Check additional arguments passed to pre_commit_run are passed to pre-commit """ - args, env = self.run_script( - MAIN, ["more", "arguments"], - CI_COMMIT_BEFORE_SHA="0" * 40, - ) + args = self.run_script(MAIN, ["more", "arguments"]) expect = ["run", "more", "arguments", "--all-files"] self.assertListEqual(expect, args) @@ -204,11 +188,15 @@ class PreCommitRunTests(unittest.TestCase): ssh or file: "fatal: no commits selected for shallow requests" http: "fatal: error processing shallow info: 4" """ - args, env = self.run_script( + args = self.run_script( MAIN, + CI_PIPELINE_SOURCE="merge_request_event", CI_MERGE_REQUEST_TARGET_BRANCH_NAME=LCA, ) - lca = self.branches[LCA] - expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"] + expect = [ + "run", + f"--from-ref={self.branches[LCA].head}", + f"--to-ref={self.repo.get_sha(MAIN)}", + ] self.assertListEqual(expect, args)