Commit 224ecef1 authored by Dom Sekotill's avatar Dom Sekotill
Browse files

Merge branch 'pipeline-improvement' into 'main'

Improvements to the CI pipeline

See merge request !1
parents 880a6187 263463a7
Loading
Loading
Loading
Loading
Loading
+4 −4
Original line number Diff line number Diff line
@@ -21,8 +21,8 @@ pre_commit_run() (
	declare -a PRE_COMMIT_ARGS

	find_lca() {
		local repo=$CI_REPOSITORY_URL current_branch=$CI_COMMIT_BRANCH
		local other_branch=$1
		local repo=$CI_REPOSITORY_URL
		local current_branch=$1 other_branch=$2

		# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1
		# and https://stackoverflow.com/questions/4698759/converting-git-repository-to-shallow/53245223#53245223
@@ -43,9 +43,9 @@ pre_commit_run() (
	if [[ -v CI_COMMIT_BEFORE_SHA ]] && [[ ! $CI_COMMIT_BEFORE_SHA =~ ^0{40}$ ]]; then
		fetch_ref $CI_COMMIT_BEFORE_SHA
	elif [[ -v CI_MERGE_REQUEST_TARGET_BRANCH_NAME ]]; then
		find_lca $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
		find_lca $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
	elif [[ $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH ]]; then
		find_lca $CI_DEFAULT_BRANCH
		find_lca $CI_COMMIT_BRANCH $CI_DEFAULT_BRANCH
	fi

	if [[ -v FROM_REF ]]; then
+73 −20
Original line number Diff line number Diff line
@@ -3,9 +3,14 @@
# SAFETY_API_KEY:
#   Set to your API key for accessing up-to-date package security information

stages:
- prepare
- test

workflow:
  rules:
  - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
    when: never
  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  - if: $CI_COMMIT_BRANCH


.python:
  image: python:3.9
@@ -20,9 +25,36 @@ stages:
  - pip install "pip>=21.3"


Quality Gate:
  stage: .pre
  image: docker.kodo.org.uk/ci-images/pre-commit:latest
  variables:
    PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit
  cache:
    key: $CI_JOB_IMAGE
    paths: [cache]
  rules:
  - if: $CI_PIPELINE_SOURCE == "push"
  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  script:
  - source .gitlab-ci.pre-commit-run.bash
  - pre_commit_run --hook-stage=commit
  - pre_commit_run --hook-stage=push


Build Package:
  stage: build
  extends: [.python]
  script:
  - pip install build
  - python -m build
  artifacts:
    paths: [dist]


Pin:
  # Pin dependencies in requirements.txt for reproducing pipeline results
  stage: prepare
  stage: test
  extends: [.python]
  script:
  - pip install --prefer-binary -e .
@@ -40,22 +72,6 @@ Dependency Check:
  - safety check -r requirements.txt


Quality Gate:
  stage: prepare
  image: docker.kodo.org.uk/ci-images/pre-commit:latest
  variables:
    PRE_COMMIT_HOME: $CI_PROJECT_DIR/cache/pre-commit
  cache:
    key: $CI_JOB_IMAGE
    paths: [cache]
  rules:
  - if: $CI_PIPELINE_SOURCE == "push"
  script:
  - source .gitlab-ci.pre-commit-run.bash
  - pre_commit_run --hook-stage=commit
  - pre_commit_run --hook-stage=push


Unit Tests:
  stage: test
  extends: [.python]
@@ -76,3 +92,40 @@ Unit Tests:
    reports:
      cobertura: results/coverage.xml
      junit: results/xunit.xml


Check Tag:
  stage: test
  extends: [.python]
  needs: ["Build Package"]
  rules:
  - if: $CI_COMMIT_TAG =~ /^v[0-9]/
  script:
  - pip install packaging pkginfo
  - |
    python <<-END
    from glob import glob
    from packaging.version import Version
    from pkginfo import Wheel

    wheel_path = glob("dist/*.whl")[0]
    wheel = Wheel(wheel_path)
    assert Version("$CI_COMMIT_TAG") == Version(wheel.version)
    END


Upload Package:
  stage: deploy
  extends: [.python]
  needs: ["Build Package"]
  rules:
  - if: $CI_COMMIT_TAG =~ /^v[0-9]/
  script:
  - pip install twine
  - TWINE_USERNAME=gitlab-ci-token
    TWINE_PASSWORD=$CI_JOB_TOKEN
    twine upload
    --verbose
    --non-interactive
    --repository-url $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/pypi
    dist/*
+73 −1
Original line number Diff line number Diff line
@@ -15,6 +15,21 @@
#
# A runner for pre_commit_run which mocks the "pre-commit" command
#
# Usage: runner.bash REPO-PATH TARGET-SCRIPT COMMIT-BRANCH SOURCE-REPO [ARG [...]]
#
# REPO-PATH:
#   Path to the test worktree/repo
# TARGET-SCRIPT:
#   The absolute path to the script under test (containing the pre_commit_run
#   function)
# COMMIT-BRANCH:
#   A head reference name on the repo at CI_REPOSITORY_URL to be pulled as the
#   pipeline target
# SOURCE-REPO:
#   The path to a repository fixture to use as an upstream source
# ARG:
#   Additional arguments to pass to pre_commit_run
#

set -eu
shopt -s expand_aliases
@@ -24,7 +39,64 @@ pre_commit() {
	printf "%s\0" "$@"
}

assert() {
	eval "$1" && return
	echo "Assert: $1"
	if [[ $# -gt 1 ]]; then
		shift
		echo "Fatal: $*"
	fi
	exit 3
} >&2

get_sha() {
	GIT_DIR=$1 git rev-parse --revs-only --verify $2
}

get_head() {
	GIT_DIR=$1 git symbolic-ref HEAD | sed 's@refs/heads/@@'
}

# Handle arguments
cd "${1? The directory to run from, this should be the top of the git repository}"
source "${2? The full path to the Bash file containing pre_commit_run}"
shift 2
declare COMMIT_BRANCH=${3? Need the head reference of a target commmit}
declare SOURCE_REPO=${4? Need the path to the source repository fixture}
shift 4

# Constants
declare -r NULL_SHA=0000000000000000000000000000000000000000

# Declare exported CI_* variables used
declare -x CI_COMMIT_SHA CI_COMMIT_BEFORE_SHA CI_COMMIT_BRANCH
declare -x CI_DEFAULT_BRANCH CI_PIPELINE_SOURCE CI_REPOSITORY_URL
declare -x CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
declare -x CI_MERGE_REQUEST_TARGET_BRANCH_NAME

# Set appropriate values for CI_* variables
CI_REPOSITORY_URL=file://$SOURCE_REPO
CI_DEFAULT_BRANCH=$(get_head $SOURCE_REPO)
CI_COMMIT_SHA=$(get_sha $SOURCE_REPO $COMMIT_BRANCH)

case ${CI_PIPELINE_SOURCE:=push} in
	merge_request_event)
		assert '[[ ! -v CI_COMMIT_BEFORE_SHA ]]' \
			"CI_COMMIT_BEFORE_SHA cannot be set for merge request pipelines"
		: ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:=$CI_DEFAULT_BRANCH}
		CI_MERGE_REQUEST_SOURCE_BRANCH_NAME=$COMMIT_BRANCH
		CI_COMMIT_BEFORE_SHA=$NULL_SHA
		;;
	*)
		CI_COMMIT_BRANCH=$COMMIT_BRANCH
		;;
esac

if [[ -v CI_COMMIT_BRANCH ]]; then
	git fetch $CI_REPOSITORY_URL -q --depth=2 $CI_COMMIT_BRANCH:$CI_COMMIT_BRANCH
	git symbolic-ref HEAD refs/heads/$CI_COMMIT_BRANCH
else
	git fetch $CI_REPOSITORY_URL -q --depth=2 $CI_COMMIT_SHA
	echo "$CI_COMMIT_SHA" >HEAD
fi

pre_commit_run "$@"
+38 −50
Original line number Diff line number Diff line
@@ -81,88 +81,80 @@ class PreCommitRunTests(unittest.TestCase):
		ref: str,
		args: Iterable[str|Path] = tuple(),
		**env: str,
	) -> tuple[list[str], dict[str, str]]:
	) -> list[str]:
		"""
		Execute the test runner script, return the pre-commit arguments and the environment
		Execute the test runner script, return the pre-commit arguments
		"""
		self.test_repo.command("fetch", self.repo.url, "-q", "--depth=2", f"{ref}:{ref}")
		self.test_repo.command("symbolic-ref", "HEAD", f"refs/heads/{ref}")
		env.update(
			CI_REPOSITORY_URL=self.repo.url,
			CI_DEFAULT_BRANCH=MAIN,
			CI_COMMIT_BRANCH=ref,
			CI_COMMIT_SHA=self.repo.get_sha(ref),
		)
		proc = subprocess.run(
			["bash", TEST_RUNNER, self.test_repo.directory, TEST_SCRIPT, *args],
			[
				"bash", TEST_RUNNER,
				self.test_repo.directory, TEST_SCRIPT, ref, self.repo.directory,
				*args,
			],
			check=True,
			text=True,
			stdout=subprocess.PIPE,
			env=env,
		)
		return proc.stdout.split("\0")[:-1], env
		return proc.stdout.split("\0")[:-1]

	def test_with_before_sha(self) -> None:
		"""
		Check that calling with CI_COMMIT_BEFORE_SHA set ensures the commit is available

		CI_COMMIT_BEFORE_SHA ~! /0{40}/
		"""
		args, env = self.run_script(
		from_sha = self.repo.get_sha(f"{MAIN}~4")
		to_sha = self.repo.get_sha(MAIN)

		args = self.run_script(
			MAIN,
			CI_COMMIT_BEFORE_SHA=self.repo.get_sha(f"{MAIN}~4"),
			CI_COMMIT_BEFORE_SHA=from_sha,
		)

		expect = [
			"run",
			f"--from-ref={env['CI_COMMIT_BEFORE_SHA']}",
			f"--to-ref={env['CI_COMMIT_SHA']}",
			f"--from-ref={from_sha}",
			f"--to-ref={to_sha}",
		]
		self.assertListEqual(expect, args)

	def test_with_target_branch(self) -> None:
		"""
		Check that calling in merge request pipelines finds the LCA with the target branch

		CI_MERGE_REQUEST_TARGET_BRANCH_NAME == CI_DEFAULT_BRANCH
		CI_COMMIT_BRANCH == FORK
		"""
		args, env = self.run_script(
		args = self.run_script(
			FORK,
			CI_MERGE_REQUEST_TARGET_BRANCH_NAME=MAIN,
			CI_PIPELINE_SOURCE="merge_request_event",
		)

		lca = self.branches[LCA]
		expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"]
		expect = [
			"run",
			f"--from-ref={self.branches[LCA].head}",
			f"--to-ref={self.repo.get_sha(FORK)}",
		]
		self.assertListEqual(expect, args)

	def test_new_branch(self) -> None:
		"""
		Check that pushing to a new fork gets the LCA with the default branch

		CI_COMMIT_BEFORE_SHA ~= /0{40}/
		CI_COMMIT_BRANCH != CI_DEFAULT_BRANCH
		"""
		args, env = self.run_script(
			FORK,
			CI_COMMIT_BEFORE_SHA="0" * 40,
		)
		args = self.run_script(FORK)

		lca = self.branches[LCA]
		expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"]
		expect = [
			"run",
			f"--from-ref={self.branches[LCA].head}",
			f"--to-ref={self.repo.get_sha(FORK)}",
		]
		self.assertListEqual(expect, args)

	def test_new_default_branch(self) -> None:
		"""
		Check that pushing to a brand new repo checks all files

		CI_COMMIT_BEFORE_SHA ~= /0{40}/
		CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH
		"""
		args, env = self.run_script(
			MAIN,
			CI_COMMIT_BEFORE_SHA="0" * 40,
		)
		args = self.run_script(MAIN)

		expect = ["run", "--all-files"]
		self.assertListEqual(expect, args)
@@ -170,13 +162,8 @@ class PreCommitRunTests(unittest.TestCase):
	def test_orphan_branch(self) -> None:
		"""
		Check that pushing to a new orphan branch check all files

		CI_COMMIT_BEFORE_SHA ~= /0{40}/
		"""
		args, env = self.run_script(
			ORPHAN,
			CI_COMMIT_BEFORE_SHA="0" * 40,
		)
		args = self.run_script(ORPHAN)

		expect = ["run", "--all-files"]
		self.assertListEqual(expect, args)
@@ -185,10 +172,7 @@ class PreCommitRunTests(unittest.TestCase):
		"""
		Check additional arguments passed to pre_commit_run are passed to pre-commit
		"""
		args, env = self.run_script(
			MAIN, ["more", "arguments"],
			CI_COMMIT_BEFORE_SHA="0" * 40,
		)
		args = self.run_script(MAIN, ["more", "arguments"])

		expect = ["run", "more", "arguments", "--all-files"]
		self.assertListEqual(expect, args)
@@ -204,11 +188,15 @@ class PreCommitRunTests(unittest.TestCase):
			ssh or file: "fatal: no commits selected for shallow requests"
			http: "fatal: error processing shallow info: 4"
		"""
		args, env = self.run_script(
		args = self.run_script(
			MAIN,
			CI_PIPELINE_SOURCE="merge_request_event",
			CI_MERGE_REQUEST_TARGET_BRANCH_NAME=LCA,
		)

		lca = self.branches[LCA]
		expect = ["run", f"--from-ref={lca.head}", f"--to-ref={env['CI_COMMIT_SHA']}"]
		expect = [
			"run",
			f"--from-ref={self.branches[LCA].head}",
			f"--to-ref={self.repo.get_sha(MAIN)}",
		]
		self.assertListEqual(expect, args)