From cf9f928ef4ee3a6e1d8658ad8eecf9ab7db79bc1 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Thu, 10 Oct 2019 22:51:48 +0100 Subject: [PATCH 1/7] Rework Docker tagging in CI build --- .gitlab-ci.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a30b5de..1c92ece 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,14 +4,9 @@ stages: - publish - deploy -variables: &global - BUILD_TAG: ${CI_REGISTRY_IMAGE}/build/${CI_COMMIT_REF_SLUG}:${CI_PIPELINE_IID} - LATEST_TAG: ${CI_REGISTRY_IMAGE}/build/${CI_COMMIT_REF_SLUG}:latest - .python: image: python:slim variables: - <<: *global PYTHONPATH: . PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip cache: @@ -56,7 +51,12 @@ lint:publish: .docker: image: docker:stable variables: - <<: *global + BUILD_TAG: ${CI_REGISTRY_IMAGE}/pipeline:${CI_PIPELINE_IID} + BUILD_TAG_COMMIT: ${CI_REGISTRY_IMAGE}/commit:${CI_COMMIT_SHORT_SHA} + BUILD_TAG_REF: ${CI_REGISTRY_IMAGE}/ref/${CI_COMMIT_REF_SLUG}:${CI_PIPELINE_IID} + BUILD_TAG_LATEST: ${CI_REGISTRY_IMAGE}/ref/${CI_COMMIT_REF_SLUG}:latest + RELEASE_TAG: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG#v} + RELEASE_TAG_LATEST: ${CI_REGISTRY_IMAGE}:latest DOCKER_HOST: "tcp://docker:2375/" DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" @@ -73,7 +73,11 @@ build: except: [ master ] script: - docker build . --pull=true --tag=${BUILD_TAG} + - docker tag ${BUILD_TAG} ${BUILD_TAG_COMMIT} - docker push ${BUILD_TAG} + - docker push ${BUILD_TAG_COMMIT} + - test -z "${CI_COMMIT_TAG}" && docker tag ${BUILD_TAG} ${BUILD_TAG_REF} + - test -z "${CI_COMMIT_TAG}" && docker push ${BUILD_TAG_REF} deploy:branch: stage: deploy @@ -81,8 +85,8 @@ deploy:branch: except: [ tags, master ] script: - docker pull ${BUILD_TAG} - - docker tag ${BUILD_TAG} ${LATEST_TAG} - - docker push ${LATEST_TAG} + - docker tag ${BUILD_TAG} ${BUILD_TAG_LATEST} + - docker push ${BUILD_TAG_LATEST} deploy:tag: stage: deploy @@ -93,7 +97,7 @@ deploy:tag: - /^v[^0-9]/ script: - docker pull ${BUILD_TAG} - - docker tag ${BUILD_TAG} ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG#v} - - docker tag ${BUILD_TAG} ${CI_REGISTRY_IMAGE}:latest - - docker push ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG#v} - - docker push ${CI_REGISTRY_IMAGE}:latest + - docker tag ${BUILD_TAG} ${RELEASE_TAG} + - docker tag ${BUILD_TAG} ${RELEASE_TAG_LATEST} + - docker push ${RELEASE_TAG} + - docker push ${RELEASE_TAG_LATEST} -- GitLab From b742c79651f6c94403ba4ed9d902e67c694b3632 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Fri, 11 Oct 2019 00:25:47 +0100 Subject: [PATCH 2/7] Make JSON '.messages' match builtin JSON reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … this is what pylint-json2html expects. --- pylint_reporter/reporter.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pylint_reporter/reporter.py b/pylint_reporter/reporter.py index 7b4b998..8c53c28 100644 --- a/pylint_reporter/reporter.py +++ b/pylint_reporter/reporter.py @@ -29,31 +29,31 @@ class JSONStatsReporter: """ def __init__(self, options, parent=None): + self.parent_reporter = parent or pylint.reporters.BaseReporter() + self.json_reporter = pylint.reporters.JSONReporter(parent.out if parent else None) self.options = options - self.parent = parent or pylint.reporters.BaseReporter() - self.messages = [] def __getattr__(self, name): - return getattr(self.parent, name) + return getattr(self.parent_reporter, name) @property def linter(self): """Proxy of the parent's linter""" - return self.parent.linter + return self.parent_reporter.linter @linter.setter def linter(self, linter): - self.parent.linter = linter + self.parent_reporter.linter = linter def handle_message(self, msg): """Lint message handling callback""" - self.messages.append(msg) - self.parent.handle_message(msg) + self.json_reporter.handle_message(msg) + self.parent_reporter.handle_message(msg) def on_close(self, stats, previous_stats): """Post-linting callback; save the lint data as a JSON file""" report = dict( - messages=self.messages, + messages=self.json_reporter.messages, stats=stats, previous=previous_stats, ) -- GitLab From 55cd31ffab2208c99fee322ea316266619b2b34d Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Fri, 7 Aug 2020 00:08:00 +0100 Subject: [PATCH 3/7] Switch argparser help text to docstrings --- pylint_reporter/main.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pylint_reporter/main.py b/pylint_reporter/main.py index 73af639..9c2c8cc 100644 --- a/pylint_reporter/main.py +++ b/pylint_reporter/main.py @@ -29,7 +29,7 @@ def main(argv=None): parser = argparse.ArgumentParser(__package__) subparsers = parser.add_subparsers(required=True) - score_parser = subparsers.add_parser('score') + score_parser = subparsers.add_parser('score', help=cmd_score.__doc__) score_parser.set_defaults(cmd=cmd_score) score_parser.add_argument( '--minimum', '--min', @@ -49,12 +49,7 @@ def main(argv=None): """ ) - badge_parser = subparsers.add_parser( - 'badge', - help=""" - Generate a shield.io badge to display the score - """ - ) + badge_parser = subparsers.add_parser('badge', help=cmd_badge.__doc__) badge_parser.set_defaults(cmd=cmd_badge) badge_parser.add_argument( 'input', @@ -101,7 +96,9 @@ def cmd_score(args) -> int: def cmd_badge(args) -> int: - """Make a lint score badge from lint stats""" + """ + Generate a shield.io badge to display the score + """ with args.input as fh: results = json.load(fh) score = reporter.calculate_score(results['stats']) -- GitLab From 01b12904efa8c5d83c54395c23f0cc804d5c7399 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Fri, 7 Aug 2020 00:20:20 +0100 Subject: [PATCH 4/7] Add a run command to the cli --- pylint_reporter/main.py | 56 ++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/pylint_reporter/main.py b/pylint_reporter/main.py index 9c2c8cc..fe29cf5 100644 --- a/pylint_reporter/main.py +++ b/pylint_reporter/main.py @@ -1,5 +1,5 @@ # -# Copyright 2019 Dominik Sekotill +# Copyright 2019,2020 Dominik Sekotill # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,18 +19,36 @@ CLI command and subcommands import argparse import json +import subprocess import sys +from pylint import lint + from . import badge, errors, reporter +def set_command(parser, cmd): + parser.set_defaults(cmd=cmd, parser=parser) + + def main(argv=None): """Script and runpy entrypoint""" parser = argparse.ArgumentParser(__package__) subparsers = parser.add_subparsers(required=True) + run_parser = subparsers.add_parser( + 'run', + help=cmd_run.__doc__, + usage=f'{parser.prog} run [options] PYLINT-ARGUMENT [PYLINT-ARGUMENT ...]', + ) + run_parser.add_argument( + '--extras', '-e', + action='append', + help="The names of optional requirements sets to install", + ) + set_command(run_parser, cmd_run) + score_parser = subparsers.add_parser('score', help=cmd_score.__doc__) - score_parser.set_defaults(cmd=cmd_score) score_parser.add_argument( '--minimum', '--min', type=float, @@ -48,9 +66,9 @@ def main(argv=None): A Pylint report serialised as a JSON file """ ) + set_command(score_parser, cmd_score) badge_parser = subparsers.add_parser('badge', help=cmd_badge.__doc__) - badge_parser.set_defaults(cmd=cmd_badge) badge_parser.add_argument( 'input', nargs='?', @@ -71,8 +89,10 @@ def main(argv=None): Save the badge to this file name. """ ) + set_command(badge_parser, cmd_badge) - args = parser.parse_args(argv or sys.argv[1:]) + args, additional = parser.parse_known_args(argv or sys.argv[1:]) + args.additional = additional try: args.cmd(args) @@ -80,7 +100,27 @@ def main(argv=None): parser.exit(3, str(exc) + '\n') -def cmd_score(args) -> int: +def cmd_run(args): + """ + Install dependencies in the current environment and run pylint on the given files + """ + if not args.additional: + args.parser.error("at least one file to lint is required") + + cmd = ['pip', 'install', '-e', '.'] + if args.extras: + cmd.append(".[{0}]".format(','.join(args.extras))) + + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.returncode: + args.parser.exit(3, proc.stdout) + + cmd = ['--load-plugins=pylint_reporter.reporter'] + cmd.extend(args.additional) + lint.Run(cmd) + + +def cmd_score(args): """ Display the linter score, and optionally compare it with a minimum score """ @@ -90,12 +130,10 @@ def cmd_score(args) -> int: print("Lint Score: {0:.2f}".format(score)) if args.minimum: - return int(score < args.minimum) - - return 0 + args.parser.exit(int(score < args.minimum)) -def cmd_badge(args) -> int: +def cmd_badge(args): """ Generate a shield.io badge to display the score """ -- GitLab From 163b31c51b7b91e60b3dbdb75044d963005ec20a Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Fri, 7 Aug 2020 03:08:08 +0100 Subject: [PATCH 5/7] Add a pre-commit hooks configuration --- .pre-commit-hooks.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .pre-commit-hooks.yaml diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 0000000..8c94ee0 --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,7 @@ +- id: pylint + name: PyLint + entry: pylint-reporter run + language: python + require_serial: true + types: [python] + verbose: true -- GitLab From cd43282541c4b39653e3ac966bc57487a18e612a Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Sun, 9 Aug 2020 15:41:17 +0100 Subject: [PATCH 6/7] Stop virtualenvs prefixed with .venv from copying to docker --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index ff01e40..9b4802e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ .eggs/ +.venv*/ -- GitLab From c10a319761c7d8ac36545cd02f7780e7745f2002 Mon Sep 17 00:00:00 2001 From: Dom Sekotill Date: Mon, 17 Aug 2020 01:32:12 +0100 Subject: [PATCH 7/7] Add packages to ci-image for building extension modules --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4c58f21..1a65c14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,3 +4,6 @@ ARG PY_VERSION=3.7 FROM python:$PY_VERSION RUN --mount=type=bind,rw,target=/src \ pip install file:///src/#egg=pylint_reporter[badges] +RUN apt-get update && apt-get install -y \ + build-essential \ + python3-dev -- GitLab