diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..5847d929b444082a79f5b073ae37d5dfac0e80ec --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# Python (for tests) +*.py[co] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a30af8098d3f865f849dc2eeecd952fc812d4d00..4c71b48d3447ba4bf3186928b5a6fdc6b96a4c42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,84 +1,90 @@ -image: docker:stable - variables: - DOCKER_HOST: "tcp://docker:2375/" - DOCKER_DRIVER: "overlay2" - DOCKER_TLS_CERTDIR: "" - DOCKER_BUILDKIT: "1" - -services: -- docker:dind + WORDPRESS_VERSION: + value: 5.8.1 + description: WordPress release + PHP_VERSION: + value: 8.0.0 + description: PHP release to build into the backend image + NGINX_VERSION: + value: 1.21.3 + description: Nginx release for the frontend image -before_script: -- docker info -- docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" +.changes: &change-files + changes: + - .gitlab-ci.yml + - Dockerfile + - data/* + - plugins/* + - scripts/* -.build: &build +.build: stage: build + image: docker.kodo.org.uk/ci-images/buildkit/buildctl:latest + tags: [buildkit] + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - << : *change-files script: - - TARGET=${CI_JOB_NAME##*:} - - BUILD_TAG=${CI_REGISTRY_IMAGE}/${TARGET}/build:${CI_PIPELINE_ID} - - docker build . - --pull=true - --tag=${BUILD_TAG} - --target=${TARGET} - ${NGINX_VERSION:+--build-arg=nginx_version=$NGINX_VERSION} - ${PHP_VERSION:+--build-arg=php_version=$PHP_VERSION} - ${WORDPRESS_VERSION:+--build-arg=wp_version=$WORDPRESS_VERSION} - - docker push ${BUILD_TAG} + - BUILD_TAG=${CI_REGISTRY_IMAGE}/${TARGET}:build-${CI_PIPELINE_IID} + - buildctl build + --frontend=dockerfile.v0 + --local context=. + --local dockerfile=. + --opt target=${TARGET} + --opt build-arg:nginx_version=${NGINX_VERSION} + --opt build-arg:php_version=${PHP_VERSION} + --opt build-arg:wp_version=${WORDPRESS_VERSION} + --opt label:nginx.version=${NGINX_VERSION} + --opt label:php.version=${PHP_VERSION} + --opt label:wordpress.version=${WORDPRESS_VERSION} + --output type=image,name=${BUILD_TAG},push=true -.changes: &only-changes - only: &change-files - changes: - - .gitlab-ci.yml - - Dockerfile - - data/* - - plugins/* - - scripts/* -.merge-requests: &only-merge-requests - only: +.tag: + stage: deploy + image: docker.kodo.org.uk/ci-images/docker-reg:latest + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" + variables: + TAG: latest + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH << : *change-files - refs: - - merge_requests - -build-master:fastcgi: - << : [ *build ] - only: [ master, schedules ] -build-master:nginx: - << : [ *build ] - only: [ master, schedules ] + variables: + TAG: latest + - if: $CI_COMMIT_BRANCH == "develop" + << : *change-files + variables: + TAG: unstable + script: | + BUILD_TAG=${CI_REGISTRY_IMAGE}/${TARGET}:build-${CI_PIPELINE_IID} + docker-reg $BUILD_TAG retag $TAG + if [ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]; then + docker-reg $BUILD_TAG retag $VERSION + fi -build-mr:fastcgi: - << : [ *build, *only-merge-requests ] -build-mr:nginx: - << : [ *build, *only-merge-requests ] -build:fastcgi: - << : [ *build, *only-changes ] - except: [ merge_requests, master, schedules ] -build:nginx: - << : [ *build, *only-changes ] - except: [ merge_requests, master, schedules ] +Build Wordpress: + extends: [.build] + variables: + TARGET: fastcgi +Build Nginx: + extends: [.build] + variables: + TARGET: nginx -.push-tags: &push-tags - stage: deploy - only: - << : *change-files - refs: [ master, develop, schedules ] - script: | - BUILD_REPO=${CI_REGISTRY_IMAGE}/${CI_JOB_NAME##*:}/build:${CI_PIPELINE_ID} - DEPLOY_REPO=${CI_REGISTRY_IMAGE}/${CI_JOB_NAME##*:} - VERSION=`eval "docker run --rm ${BUILD_REPO} ${GET_VERSION}"` - . scripts/deploy.sh -push:fastcgi: - <<: *push-tags +Tag Wordpress Image: + extends: [.tag] variables: - GET_VERSION: wp core version -push:nginx: - <<: *push-tags + TARGET: fastcgi + VERSION: $WORDPRESS_VERSION + +Tag Nginx Image: + extends: [.tag] variables: - GET_VERSION: nginx -V 2>&1 | sed -n '/nginx version:/s/.*nginx\///p' + TARGET: nginx + VERSION: $NGINX_VERSION diff --git a/.lint.cfg b/.lint.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c56b8d61f60b8ee811c8b58ffeed7bd3d4de702e --- /dev/null +++ b/.lint.cfg @@ -0,0 +1,103 @@ +[isort] +force_single_line = true + +[mypy] +strict = true +warn_unused_configs = true +warn_unreachable = true +mypy_path = tests, tests/stubs +plugins = + trio_typing.plugin + +[flake8] +max-line-length = 92 +max-doc-length = 92 +use-flake8-tabs = true +blank-lines-indent = never +indent-tabs-def = 1 +format = pylint +select = C,D,E,ET,F,SFS,T,W,WT + +per-file-ignores = + **/__init__.py: D104 + **/__main__.py: D100, E702 + +ignore = + ;[ '%s' imported but unused ] + ; Handled by pylint, which does it better + F401 + + ;[ Missing docstring in public method ] + ; Handled by pylint, which does it better + D102 + + ;[ Missing docstring in magic method ] + ; Magic/dunder methods are well-known + D105 + + ;[ Misisng docstring in __init__ ] + ; Document basic construction in the class docstring + D107 + + ;[ One-line docstring should fit on one line with quotes ] + ; Prefer top-and-bottom style always + D200 + + ;[ Docstring should be indented with spaces, not tabs ] + ; Tabs, absolutely always + D206 + + ;[ Use r""" if any backslashes in a docstring ] + ; If I want to put escape chars in a docstring, I will + D301 + + ;[ Use u""" for Unicode docstrings ] + ; This must be for Python 2? + D302 + + ;[ First line should end with a period ] + ; First line should *NEVER* end with a period + D400 + + ;[ First line should be in the imperative mood ] + ; I like this for functions and methods, not for properties. This stands until + ; pydocstyle splits a new code for properties or flake8 adds some way of + ; filtering codes with line regexes like golangci-lint. + D401 + + ;[ No blank lines allowed between a section header and its content ] + D412 + + ;[ missing whitespace around bitwise or shift operator ] + E227 + + ;[ Line too long ] + ; Prefer B950 implementation + E501 + + ;[ multiple statements on one line (def) ] + ; Dosen't work well with short @overload definitions + E704 + + ;[ unexpected number of tabs and spaces at start of statement ] + ET128 + + ;[ Line break before binary operator ] + ; Not considered current + W503 + + ;[ Format-method string formatting ] + ; Allow this style + SFS201 + + ;[ f-string string formatting ] + ; Allow this style + SFS301 + +include = + ;[ First word of the docstring should not be This ] + D404 + + ; flake8-bugbear plugin + ; B950 is a replacement for E501 + B0 B903 B950 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52f815c5d232fc68369157548190e09506e42c42..e7292c36580f9c2ad6e27cc686095f670e95711c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,47 +1,100 @@ +default_stages: [commit] repos: - repo: meta hooks: - id: check-hooks-apply + - id: check-useless-excludes - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.4.0 hooks: - id: check-added-large-files - stages: [commit] - id: check-case-conflict - stages: [commit] + - id: check-docstring-first - id: check-merge-conflict - stages: [commit] - id: check-yaml args: [--allow-multiple-documents] - stages: [commit] + - id: debug-statements - id: destroyed-symlinks - stages: [commit] - id: end-of-file-fixer - stages: [commit] + stages: [commit, manual] - id: fix-byte-order-marker - stages: [commit] + - id: fix-encoding-pragma + args: [--remove] - id: mixed-line-ending args: [--fix=lf] - stages: [commit] + stages: [commit, manual] - id: trailing-whitespace exclude_types: [markdown, plain-text] - stages: [commit] + stages: [commit, manual] - repo: https://github.com/jorisroovers/gitlint rev: v0.15.0 hooks: - id: gitlint -- repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 2.1.5 - hooks: - - id: protect-first-parent - - repo: https://code.kodo.org.uk/dom/pre-commit-hooks - rev: v0.5.1 + rev: v0.6 hooks: + - id: check-executable-modes - id: check-for-squash - id: copyright-notice exclude: ^data/ + - id: protect-first-parent + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.8.0 + hooks: + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + +- repo: https://github.com/hakancelik96/unimport + rev: 0.9.2 + hooks: + - id: unimport + args: ["--remove", "--include=\\.pyi?$"] + types: [] + types_or: [python, pyi] + stages: [commit, manual] + +- repo: https://github.com/pycqa/isort + rev: 5.9.3 + hooks: + - id: isort + args: ["--settings=.lint.cfg"] + stages: [commit, manual] + +- repo: https://github.com/asottile/add-trailing-comma + rev: v2.1.0 + hooks: + - id: add-trailing-comma + args: [--py36-plus] + types: [] + types_or: [python, pyi] + stages: [commit, manual] + +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + args: ["--config=.lint.cfg"] + additional_dependencies: + - flake8-bugbear + - flake8-docstrings + - flake8-print + - flake8-requirements + - flake8-return + - flake8-sfs + - flake8-tabs + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + args: ["--config-file=.lint.cfg"] + additional_dependencies: + - trio-typing + - types-requests + - git+https://code.kodo.org.uk/dom/type-stubs.git#type-stubs[jsonpath,parse] diff --git a/Dockerfile b/Dockerfile index 1e5131c23ea160b10c0fe999edc892d032ac7226..3bc22e57df86b1eef31526b081501770b678cd7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,10 @@ RUN --mount=type=bind,source=scripts/install-deps.sh,target=/stage /stage FROM deps as compile RUN --mount=type=bind,source=scripts/install-build-deps.sh,target=/stage /stage RUN --mount=type=bind,source=scripts/compile-dist-ext.sh,target=/stage /stage -RUN --mount=type=bind,source=scripts/compile-imagick.sh,target=/stage /stage + +ARG imagick_version +RUN --mount=type=bind,source=scripts/compile-imagick.sh,target=/stage \ + /stage ${imagick_version} FROM deps as fastcgi diff --git a/behave.ini b/behave.ini new file mode 100644 index 0000000000000000000000000000000000000000..a7da884dd886a00783d67737c55c93f7e030f46a --- /dev/null +++ b/behave.ini @@ -0,0 +1,2 @@ +[behave] +paths = tests diff --git a/data/composer.json b/data/composer.json index c028f78b051f0f31074739ee76689ab6e3514fbd..51f5bb2ee44a45dde0c1bcc81c94eb14dff1d9cb 100644 --- a/data/composer.json +++ b/data/composer.json @@ -8,7 +8,8 @@ } ], "require": { - "humanmade/s3-uploads": "dev-develop" + "humanmade/s3-uploads": "dev-develop", + "ayesh/wordpress-password-hash": "2.*" }, "config": { "gitlab-domains": [ "code.kodo.org.uk" ] @@ -17,5 +18,10 @@ "installer-paths": { "wp-content/mu-plugins/{$name}/": [ "type:wordpress-plugin" ] } + }, + "scripts": { + "post-update-cmd": [ + "cd */mu-plugins;for f in */*.php;do [ `basename $f` != index.php ] && ln -s $f `dirname $f`.php; done" + ] } } diff --git a/data/nginx/server.conf b/data/nginx/server.conf index 566e5603ebfd7986e0d15f9d25819b8c0983defd..afc77233eb1802a70c98f7c4d092acb30b4a6cda 100644 --- a/data/nginx/server.conf +++ b/data/nginx/server.conf @@ -23,18 +23,19 @@ server { # Add Cache-Control headers for static files, removed in *.php location add_header Cache-Control "public, max-age=7776000, stale-while-revalidate=86400, stale-if-error=604800"; - error_page 404 @not-found; - error_page 502 /errors/502.html; + error_page 404 /errors/generated/404.html; + error_page 502 /errors/static/502.html; - location /errors { + location /errors/ { internal; - alias /etc/nginx/html; - } - location @not-found { - include fastcgi.conf; - fastcgi_cache ERR; - fastcgi_cache_valid 404 1d; + location /errors/static/ { + alias /etc/nginx/html/; + } + + location /errors/generated/ { + alias /app/static/errors/; + } } location @index { @@ -77,6 +78,20 @@ server { include cache-bust.conf; } + location = /wp-comments-post.php { + error_page 403 = @post-only; + limit_except POST { + deny all; + } + include fastcgi-script.conf; + include cache-bust.conf; + } + + location @post-only { + add_header Allow "POST" always; + return 405; + } + location /wp-admin/ { try_files $uri $uri/index.php; diff --git a/plugins/media-url.php b/plugins/docker-integration.php similarity index 50% rename from plugins/media-url.php rename to plugins/docker-integration.php index 955cc97efdfeb5932785cb6b042e549ef08c74b3..7d90fdae1b1a1ab9ee2c9b4879971856b899e5db 100644 --- a/plugins/media-url.php +++ b/plugins/docker-integration.php @@ -1,14 +1,19 @@ + * + * Plugin Name: Docker Image Integration * Plugin URI: https://code.kodo.org.uk/singing-chimes.co.uk/wordpress/tree/master/plugins - * Description: Adjusts the media URL path base to /media, where the Nginx instance is hosting it. + * Description: Hooks in behaviour for operating cleanly in a Docker environment * Licence: MPL-2.0 * Licence URI: https://www.mozilla.org/en-US/MPL/2.0/ * Author: Dominik Sekotill * Author URI: https://code.kodo.org.uk/dom */ + +// Media URL Fix + add_action( 'plugins_loaded', function() { add_filter( 'upload_dir', function( $paths ) { $baseurl = parse_url( $paths['baseurl'] ); @@ -25,6 +30,70 @@ add_action( 'plugins_loaded', function() { }); }); + +// Block File Modification + +add_filter( + 'file_mod_allowed', + + function( $setting, $context ) { + if ( $context == 'automatic_updater' ) { + return true; + } + return $setting; + }, + + 10, 2 +); + + +// Disable Plugins & Themes + +add_filter( + 'user_has_cap', + + function( $allcaps, $caps, $args ) { + switch ($args[0]) { + case 'delete_plugins': + case 'delete_themes': + case 'edit_plugins': + case 'edit_themes': + case 'install_plugins': + case 'install_themes': + case 'update_plugins': + case 'update_themes': + $allcaps[$caps[0]] = false; + } + return $allcaps; + }, + + 10, 3 +); + + +// S3-Uploads Integration + +if ( defined( 'S3_UPLOADS_ENDPOINT_URL' ) || defined( 'WP_CLI' ) ): + +add_filter( + 's3_uploads_s3_client_params', + + function ( $params ) { + $params['endpoint'] = S3_UPLOADS_ENDPOINT_URL; + $params['bucket_endpoint'] = true; + $params['disable_host_prefix_injection'] = true; + $params['use_path_style_endpoint'] = true; + $params['debug'] = WP_DEBUG && WP_DEBUG_DISPLAY; + $params['region'] = ''; + return $params; + } +); + +endif; + + +// Functions + function unparse_url( array $parts ) { return ( (isset($parts['scheme']) ? "{$parts['scheme']}://" : '') . diff --git a/plugins/override_file_mod.php b/plugins/override_file_mod.php deleted file mode 100644 index d34dfa85e90db4f4f4a4338edbb611892b2334cd..0000000000000000000000000000000000000000 --- a/plugins/override_file_mod.php +++ /dev/null @@ -1,19 +0,0 @@ - +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + set -eux +shopt -s lastpipe + +VERSION=${1-} + + +int_version() +{ + local IFS=. major minor patch + read major minor patch <<<"$1" + printf "%d%d%d" $((major*10000)) $((minor*100)) $patch +} + +get_latest() +{ + declare -n array=$1 + local IFS=$'\n' + sort -nr <<<"${!array[*]}" | + head -n1 +} + +get_tarball() +{ + local release url version + declare -A URLS=() RELEASES=() + + curl -sS https://api.github.com/repos/imagick/imagick/tags | + jq -r '.[] | [.name, .tarball_url] | @tsv' | + while read release url; do + [[ $release =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && + RELEASES[$(int_version $release)]=$release + URLS[$release]=$url + done + + if [[ -n $VERSION ]]; then + url=${URLS[$VERSION]} + else + release=${RELEASES[`get_latest RELEASES`]} + url=${URLS[$release]} + fi + + curl -sSL $url | gunzip -c +} + cd $(mktemp -d) -git clone --depth 1 https://github.com/imagick/imagick.git . +get_tarball | tar -xf- --strip-components=1 phpize ./configure make install diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100755 index 8555c151b8d2a7b378e9f096d8d21a30445ac557..0000000000000000000000000000000000000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -set -eu - -: ${BUILD_REPO?} -: ${DEPLOY_REPO?} - -case ${CI_COMMIT_REF_NAME-develop} in - master) tags="latest ${VERSION-}" ;; - develop) tags="unstable" ;; - *) exit 3 ;; -esac - -set -x - -docker pull ${BUILD_REPO} - -for tag in $tags; do - docker tag ${BUILD_REPO} ${DEPLOY_REPO}:${tag} - docker push ${DEPLOY_REPO}:${tag} -done diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 6f32da8bbab5f2a8778a3dc0c2650b27906d8253..b552687f819da5b83fd6dbdfbf664d09145e7464 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -223,6 +223,12 @@ collect_static() . static/ } +generate_static() +{ + mkdir -p static/errors + wp eval 'get_template_part("404");' >static/errors/404.html +} + deactivate_missing_plugins() { # Output active plugin entrypoints as a JSON array @@ -299,6 +305,7 @@ case "$1" in setup_components setup_media collect_static + generate_static timestamp "Completed Wordpress preparation" run_background_cron exec "$@" "${extra_args[@]}" diff --git a/scripts/install-build-deps.sh b/scripts/install-build-deps.sh index 0d963e779adf6e84dfd056d2691ef8a6c90942ac..5b7e7e1bad58f08c6e8e603b2e2025f48518fc57 100755 --- a/scripts/install-build-deps.sh +++ b/scripts/install-build-deps.sh @@ -1,4 +1,10 @@ #!/bin/sh +# Copyright 2019-2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + set -eux # Install packaged dependencies @@ -6,7 +12,6 @@ apk update apk add \ autoconf \ build-base \ - git \ gmp-dev \ imagemagick-dev \ jpeg-dev \ diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh index 51d0f812f8a75dd8c3cdcfb7e1506ee6a537768e..5dd7e75fb774ece0370e004f64c91c0a53abb120 100755 --- a/scripts/install-deps.sh +++ b/scripts/install-deps.sh @@ -14,6 +14,7 @@ apk add \ imagemagick-libs \ jq \ libgmpxx \ + libgomp \ libjpeg \ libpng \ libwebp \ diff --git a/scripts/install-wp.sh b/scripts/install-wp.sh index 55d1fb70ba91aae2680df75e8666d5708d081310..32f27dfd5a79779037ee4da76c8f4cf77cd7e58d 100755 --- a/scripts/install-wp.sh +++ b/scripts/install-wp.sh @@ -1,9 +1,14 @@ #!/bin/bash +# Copyright 2019-2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + set -eux COMPOSER_INSTALLER_URL=https://getcomposer.org/installer WP_CLI_URL=https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar -WP_PASSWORD_HASH=https://raw.githubusercontent.com/Ayesh/WordPress-Password-Hash/1.5.1 # Install Composer curl -sSL ${COMPOSER_INSTALLER_URL} | @@ -24,10 +29,6 @@ rm wp-config-sample.php mkdir --mode=go+w media mkdir -p wp-content/mu-plugins -# Install non-optional plugins -curl ${WP_PASSWORD_HASH}/wp-php-password-hash.php \ - >wp-content/mu-plugins/password-hash.php - # Install composer managed dependencies export COMPOSER_ALLOW_SUPERUSER=1 composer install --prefer-dist diff --git a/scripts/wp.sh b/scripts/wp.sh index 0021371d78b9eb98dd68a633f926e056f34733a8..05a3bf1fac457398b1614c08385c30c8e14d9463 100755 --- a/scripts/wp.sh +++ b/scripts/wp.sh @@ -9,4 +9,4 @@ # installing it as root is idiocy. WP needs to be installed owned by a user # seperate from the server's user. 'root' is available for such, besides which # root in a container is not really root. -exec php -d memory_limit=512M /usr/local/lib/wp-cli.phar --allow-root "$@" +exec php -d memory_limit=512M -d display_errors=stderr /usr/local/lib/wp-cli.phar --allow-root "$@" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..989f7b0cc56781e5f782a6651170a0a1f2c8155b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,74 @@ +Behaviour Testing +================= + +These behaviour tests use "Behave", a Python framework. + + +Requirements +------------ + +### Docker (>=18.09) + +Docker is required for running the project; a minimum version of 18.09 is required to build +the images. + +### Python (>=3.9) + +The tests are coordinated and run by "behave", a Python testing framework. In order to +check the correctness of the test code it has been written with the latest typing features +of Python. + + +Installing +---------- + +There are a small number of Python package dependencies listed in [requirements.txt]() which +must be installed; it is recommended that they are installed in a [virtual +environment][venv]: + +```bash +env=venv # You may choose any directory name here +python -m venv $env +$env/bin/python -m pip install -r tests/requirements.txt +``` + +### OPTIONAL: Make `behave` runnable without a full path + +The virtual environment's *bin/* directory (*Scripts/* for Windows builds of Python) can be +added to the executable search variable "PATH". This will make `behave` and other installed +tools runnable without having to supply a full path to the executable. The 'venv' tool +supplies handy scripts for this purpose. + +All the following example in this document assume this has been done; if not simply replace +`behave` with `$env/bin/behave` where `$env` expands to the virtual environment created +above. + +For Bash and Zsh use the following, for other shells see the [venv][] documentation: + +```bash +source $env/bin/activate +``` + +[venv]: + https://docs.python.org/3/library/venv.html + "Documentation for 'venv'" + + +Usage +----- + +From the top directory of the project or the *tests* subdirectory, Behave can be called with +no arguments to run all scenarios: + +```bash +behave +``` + +Behave can be run with path arguments in which case it can be run from any directory. +Feature files (matching `*.feature`) may be specified to run their scenarios, or if the path +is a directory the tree will be searched for feature files. + +```bash +behave tests # Run scenarios for all features +behave tests/regression-*.feature # Run regression scenarios +``` diff --git a/tests/environment.py b/tests/environment.py new file mode 100644 index 0000000000000000000000000000000000000000..ee5727fe1121d5f3558e437608675d371a65e7c6 --- /dev/null +++ b/tests/environment.py @@ -0,0 +1,193 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Setup module for Behave tests + +This module prepares test fixtures and global context items. + +https://behave.readthedocs.io/en/stable/tutorial.html#environmental-controls +""" + +from __future__ import annotations + +import sys +from contextlib import contextmanager +from os import environ +from pathlib import Path +from typing import TYPE_CHECKING +from typing import Any +from typing import Iterator +from typing import NamedTuple + +from behave import fixture +from behave import use_fixture +from behave.model import Feature +from behave.model import Scenario +from behave.runner import Context +from requests.sessions import Session +from utils import URL +from utils import make_secret +from utils import redirect +from wp import Mysql +from wp import Wordpress +from wp.docker import Container +from wp.docker import Image +from wp.docker import IPv4Address +from wp.docker import Network + +if TYPE_CHECKING: + from behave.runner import FeatureContext + from behave.runner import ScenarioContext + +SITE_URL = URL("http://test.example.com") +BUILD_CONTEXT = Path(__file__).parent.parent + + +def before_all(context: Context) -> None: + """ + Setup fixtures for all tests + """ + context.site = use_fixture(setup_test_cluster, context, SITE_URL) + + +def before_feature(context: FeatureContext, feature: Feature) -> None: + """ + Setup/revert fixtures before each feature + """ + use_fixture(db_snapshot_rollback, context) + + +def before_scenario(context: ScenarioContext, scenario: Scenario) -> None: + """ + Setup tools for each scenario + """ + context.session = use_fixture(requests_session, context) + + +class Site(NamedTuple): + """ + A named-tuple of information about the containers for a site fixture + """ + + url: str + address: IPv4Address + frontend: Container + backend: Wordpress + database: Mysql + + +# Todo(dom.sekotill): When PEP-612 is properly implemented in mypy the [*a, **k] and default +# values nonsense can be removed from fixtures + +@fixture +def setup_test_cluster(context: Context, /, site_url: str|None = None, *a: Any, **k: Any) -> Iterator[Site]: + """ + Prepare and return the details of a site fixture + """ + assert site_url is not None, \ + "site_url is required, but default supplied until PEP-612 supported" + with test_cluster(site_url) as site: + yield site + + +@fixture +def requests_session(context: ScenarioContext, /, *a: Any, **k: Any) -> Iterator[Session]: + """ + Create and configure a `requests` session for accessing site fixtures + """ + site = context.site + with Session() as session: + redirect(session, site.url, site.address) + yield session + + +@fixture +def db_snapshot_rollback(context: FeatureContext, /, *a: Any, **k: Any) -> Iterator[None]: + """ + Manage the state of a site's database as a revertible fixture + """ + db = context.site.database + snapshot = db.mysqldump("--all-databases", deserialiser=bytes) + yield + db.mysql(input=snapshot) + + +@contextmanager +def test_cluster(site_url: str) -> Iterator[Site]: + """ + Configure and start all the necessary containers for use as test fixtures + """ + test_dir = Path(__file__).parent + + db_secret = make_secret(20) + db_init = test_dir / "mysql-init.sql" + + with Network() as network: + database = Mysql( + Image.pull("mysql/mysql-server"), + network=network, + volumes={ + Path("/var/lib/mysql"), + (db_init, Path("/docker-entrypoint-initdb.d") / db_init.name), + }, + env=dict( + MYSQL_DATABASE="test-db", + MYSQL_USER="test-db-user", + MYSQL_PASSWORD=db_secret, + ), + ) + frontend = Container( + Image.build( + BUILD_CONTEXT, + target='nginx', + nginx_version=environ.get("NGINX_VERSION"), + ), + network=network, + volumes=[ + ("static", Path("/app/static")), + ("media", Path("/app/media")), + ], + ) + backend = Wordpress( + Image.build( + BUILD_CONTEXT, + php_version=environ.get("PHP_VERSION"), + wp_version=environ.get("WP_VERSION"), + ), + network=network, + volumes=frontend.volumes, + env=dict( + SITE_URL=site_url, + SITE_ADMIN_EMAIL="test@kodo.org.uk", + DB_NAME="test-db", + DB_USER="test-db-user", + DB_PASS=db_secret, + DB_HOST="database:3306", + ), + ) + + backend.connect(network, "upstream") + database.connect(network, "database") + + with database.started(), backend.started(), frontend.started(): + addr = frontend.inspect( + f"$.NetworkSettings.Networks.{network}.IPAddress", + str, IPv4Address, + ) + yield Site(site_url, addr, frontend, backend, database) + + +if __name__ == "__main__": + from subprocess import run + + with test_cluster(SITE_URL) as site: + run([environ.get("SHELL", "/bin/sh")]) + +elif not sys.stderr.isatty(): + import logging + + logging.basicConfig(level=logging.DEBUG, stream=sys.stderr) diff --git a/tests/mysql-init.sql b/tests/mysql-init.sql new file mode 100644 index 0000000000000000000000000000000000000000..c7b9c584a6729fcee0dfcd6769e76987e950235b --- /dev/null +++ b/tests/mysql-init.sql @@ -0,0 +1,3 @@ +INSTALL PLUGIN auth_socket SONAME 'auth_socket.so'; + +ALTER USER 'root'@'localhost' IDENTIFIED WITH auth_socket; diff --git a/tests/pages.feature b/tests/pages.feature new file mode 100644 index 0000000000000000000000000000000000000000..49fa56540da2e5e6d10d93a75b432fcde37aa00f --- /dev/null +++ b/tests/pages.feature @@ -0,0 +1,19 @@ +Feature: Pages + Pages should be returned when their URL is requested. + + Background: A page exists + Given a page exists containing + """ + This is some page content + """ + + Scenario: A page type post + When the page is requested + Then OK is returned + And we will see the page text + + Scenario: Static homepage + Given the page is configured as the homepage + When the homepage is requested + Then OK is returned + And we will see the page text diff --git a/tests/posts.feature b/tests/posts.feature new file mode 100644 index 0000000000000000000000000000000000000000..cf92e5a140228bfd42d8d1793d892cc03afb91f7 --- /dev/null +++ b/tests/posts.feature @@ -0,0 +1,25 @@ +Feature: Posts + Posts should be returned when their URL is requested. The post index page + should also be accessible. + + Background: A post exists + Given a post exists containing + """ + This is some page content + """ + + Scenario: Individual posts + When the post is requested + Then OK is returned + And we will see the post text + + Scenario: Homepage post index + When the homepage is requested + Then OK is returned + And we will see the post text + + Scenario: Non-homepage post index + Given a blank page exists + And is configured as the post index + When the page is requested + Then we will see the post text diff --git a/tests/regression-14.feature b/tests/regression-14.feature new file mode 100644 index 0000000000000000000000000000000000000000..0a96f03d2a691f70fe34d62eed212ca3aa6e1aa1 --- /dev/null +++ b/tests/regression-14.feature @@ -0,0 +1,27 @@ +Feature: Return 404 for unknown path + Regression check for "#14": don't redirect to the homepage when a 404 would + be expected. + + Scenario Outline: Not found + Given does not exist + When is requested + Then "Not Found" is returned + + Examples: + | path | + | /this/is/missing | + | /this/is/missing/ | + + Scenario Outline: Bad pagination paths + Given a page exists containing + """ + Some content + """ + When the page suffixed with is requested + Then is returned + + Examples: + | suffix | result | + | / | 301 | + | /0 | OK | + | /foo | Not Found | diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..77b00734a9b6bf9c5eb9e68a03386ee4569b7352 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +Python ~=3.9; python_version < '3.9' + +behave +jsonpath-python ~=1.0 +requests ~=2.26 +trio ~=0.19.0 diff --git a/tests/script-access.feature b/tests/script-access.feature new file mode 100644 index 0000000000000000000000000000000000000000..f2aad15e0dc2b0d7a5864b564762a67a7ca10cf3 --- /dev/null +++ b/tests/script-access.feature @@ -0,0 +1,69 @@ +Feature: Script Access and Restrictions + The user-facing parts of a WordPress application should all be either static + resources or channeled through the root *index.php* entrypoint. However PHP + is architectured in such a way that, if left unrestricted, any PHP file + could be accessed as a script. + + In many cases protections have been put in place in WordPress' PHP files to + prevent circumvention of access restriction, as well as some plugins. + However this should never be relied on as it introduces additional + complexity and may not have been thoroughly tested. It could also be + considered a UI bug if a non-404 code is returned. + + To confuse matters the administration interface *is* accessed in a + one-script-per-endpoint manner. + + Scenario Outline: Direct file access + When is requested + Then is returned + + Examples: Static files + | path | result | + | /wp-includes/images/w-logo-blue.png | OK | + | /wp-admin/images/w-logo-blue.png | OK | + | /readme.html | Not Found | + | /composer.json | Not Found | + | /composer.lock | Not Found | + + Examples: Non-entrypoint PHP files + | path | result | + | /wp-activate.php | Not Found | + | /wp-blog-header.php | Not Found | + | /wp-config.php | Not Found | + | /wp-cron.php | Not Found | + | /wp-load.php | Not Found | + | /wp-mail.php | Not Found | + | /wp-settings.php | Not Found | + | /wp-signup.php | Not Found | + | /wp-trackback.php | Not Found | + | /xmlrpc.php | Not Found | + | /wp-includes/user.php | Not Found | + + Examples: Entrypoint PHP files + | path | result | + | / | OK | + | /index.php | 301 | + | /wp-login.php | OK | + | /wp-admin/ | 302 | + | /wp-admin/index.php | 302 | + | /wp-comments-post.php | 405 | + + Scenario: Check the JSON API is accessible + When /wp-json/wp/v2/ is requested + Then OK is returned + And the response body is JSON + + Scenario: "GET /wp-comments-post.php" is not allowed + When /wp-comments-post.php is requested + Then 405 is returned + And the "Allow" header's value is "POST" + + Scenario: "POST /wp-contents-post.php" accepts content + Given a blank post exists + When data is sent with POST to /wp-comments-post.php + """ + comment_post_id={context.post[ID]}&author=John+Smith&email=j.smith@example.com&comment=First+%F0%9F%8D%86 + """ + Then OK is returned + # (Why 200 instead of 201? Probably the same reason 200 is returned when + # there are missing values?! It's WordPress.) diff --git a/tests/steps/commands.py b/tests/steps/commands.py new file mode 100644 index 0000000000000000000000000000000000000000..897f21e9f6de4657aa95538ee7db6302f1eca920 --- /dev/null +++ b/tests/steps/commands.py @@ -0,0 +1,92 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Step implementations involving running commands in fixture containers +""" + +from __future__ import annotations + +import json +import shlex +from typing import TYPE_CHECKING + +from behave import then +from behave import when +from utils.behave import PatternEnum +from utils.behave import register_pattern +from wp import Container + +if TYPE_CHECKING: + from behave.runner import Context + + +@register_pattern(r".+") +class Arguments(list[str]): + """ + Step pattern for command lines + """ + + def __init__(self, cmdline: str): + self.extend(shlex.split(cmdline)) + + +class Stream(PatternEnum): + """ + Pattern matching enum for stdio stream names + """ + + STDOUT = "stdout" + STDERR = "stderr" + + stdout = STDOUT + stderr = STDERR + + +@when(""""{args:Arguments}" is run""") +@when("""'{args:Arguments}' is run""") +def run_command(context: Context, args: Arguments) -> None: + """ + Run a command in the appropriate site container + """ + if len(args) == 0: + raise ValueError("No arguments in argument list") + if args[0] in ('wp', 'php'): + container: Container = context.site.backend + else: + raise ValueError(f"Unknown command: {args[0]}") + context.process = container.run(args, capture_output=True) + + +@then("nothing is seen from {stream:Stream}") +def check_empty_stream(context: Context, stream: Stream) -> None: + """ + Check there is no output on the given stream of a previous command + """ + output = getattr(context.process, stream.value) + assert not output, f"Unexpected output seen from {stream.name}: {output}" + + +@then("JSON is seen from {stream:Stream}") +def check_json_stream(context: Context, stream: Stream) -> None: + """ + Check there is no output on the given stream of a previous command + """ + output = getattr(context.process, stream.value) + try: + json.loads(output) + except json.JSONDecodeError: + raise AssertionError(f"Expecting JSON from {stream.name}; got {output}") + + +@then('"{response}" is seen from {stream:Stream}') +def check_stream(context: Context, response: str, stream: Stream) -> None: + """ + Check the output streams of a previous command for the given response + """ + output = getattr(context.process, stream.value) + assert output.strip() == response.encode(), \ + f"Expected output from {stream.name}: {response.encode()!r}\ngot: {output!r}" diff --git a/tests/steps/pages.py b/tests/steps/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..151e78cbf254868f3e8451f4a2359cc9bab7bca0 --- /dev/null +++ b/tests/steps/pages.py @@ -0,0 +1,193 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Step implementations involving creating and requesting WP posts (and pages) +""" + +from __future__ import annotations + +from codecs import decode as utf8_decode +from typing import Any +from typing import Iterator + +from behave import fixture +from behave import given +from behave import then +from behave import use_fixture +from behave import when +from behave.runner import Context +from request_steps import get_request +from utils import URL +from utils import JSONArray +from utils import JSONObject +from utils import PatternEnum + +DEFAULT_CONTENT = """ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut +labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco +laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in +voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat +cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +""" + + +class PostType(PatternEnum): + """ + Enumeration for matching WP post types in step texts + """ + + post = "post" + page = "page" + + +@given("{path} does not exist") +def assert_not_exist(context: Context, path: str) -> None: + """ + Assert that the path does not route to any resource + """ + cmd = [ + "post", "list", "--field=url", "--format=json", + "--post_type=post,page", "--post_status=publish", + ] + urls = {*context.site.backend.cli(*cmd, deserialiser=JSONArray.from_string)} + assert context.site.url / path not in urls, \ + f"{context.site.url / path} exists" + + +@given("a blank {post_type:PostType} exists") +@given("a {post_type:PostType} exists containing") +def create_post(context: Context, post_type: PostType, text: str|None = None) -> None: + """ + Create a WP post of the given type and store it in the context with the type as the name + """ + post = use_fixture(wp_post, context, post_type, text or getattr(context, "text", "")) + setattr(context, post_type.value, post) + + +@given("the page is configured as the homepage") +def set_homepage(context: Context) -> None: + """ + Set the WP page from the context as the configured front page + """ + use_fixture(set_specials, context, homepage=context.page) + + +@given("is configured as the post index") +def set_post_index(context: Context) -> None: + """ + Set the WP page from the context as the post index page + """ + use_fixture(set_specials, context, posts=context.page) + + +@when("the {post_type:PostType} is requested") +def request_page(context: Context, post_type: PostType) -> None: + """ + Request the specified WP post of the given type in the context + """ + post = getattr(context, post_type.value) + get_request(context, post.path("$.url", URL)) + + +@when("the {post_type:PostType} suffixed with {suffix} is requested") +def request_page_with_suffix(context: Context, post_type: PostType, suffix: str) -> None: + """ + Like `request_page`, with additional URL components appended to the post's URL + """ + post = getattr(context, post_type.value) + get_request(context, post.path("$.url", URL) + suffix) + + +@then("we will see the {post_type:PostType} text") +def assert_contains( + context: Context, + post_type: PostType = PostType.post, + text: str|None = None, +) -> None: + """ + Assert that the text is in the response from a previous step + + The text can be supplied directly or taken from a WP post of the type specified, taken + from the context. + """ + if not text: + post = getattr(context, post_type.value) + text = post.path("$.post_content", str) + assert text in context.response.text + + +@fixture +def wp_post( + context: Context, /, + post_type: PostType|None = None, + content: str = DEFAULT_CONTENT, + *a: Any, + **k: Any, +) -> Iterator[JSONObject]: + """ + Create a WP post fixture of the given type with the given content + """ + assert post_type is not None, \ + "post_type MUST be supplied to use_fixture when calling with wp_post" + + wp = context.site.backend + postid = wp.cli( + "post", "create", + f"--post_type={post_type.value}", "--post_status=publish", + f"--post_name=test-{post_type.value}", + f"--post_title=Test {post_type.name.capitalize()}", + "-", "--porcelain", + input=content, deserialiser=utf8_decode, + ).strip() + post = wp.cli("post", "get", postid, "--format=json", deserialiser=JSONObject.from_string) + post.update( + url=URL( + wp.cli( + "post", "list", "--field=url", + f"--post__in={postid}", f"--post_type={post_type.value}", + deserialiser=utf8_decode, + ).strip(), + ), + ) + yield post + wp.cli("post", "delete", postid) + + +@fixture +def set_specials( + context: Context, /, + homepage: JSONObject|None = None, + posts: JSONObject|None = None, + *a: Any, + **k: Any, +) -> Iterator[None]: + """ + Set the homepage and post index to new pages, creating default pages if needed + + Pages are reset at the end of a scenario + """ + wp = context.site.backend + + options = { + opt["option_name"]: opt["option_value"] + for opt in wp.cli("option", "list", "--format=json", deserialiser=JSONArray.from_string) + } + + homepage = homepage or use_fixture(wp_post, context, PostType.page) + wp.cli("option", "update", "page_on_front", homepage.path("$.ID", int, str)) + wp.cli("option", "update", "show_on_front", "page") + + posts = posts or use_fixture(wp_post, context, PostType.page) + wp.cli("option", "update", "page_for_posts", posts.path("$.ID", int, str)) + + yield + + for name in ["page_on_front", "show_on_front", "page_for_posts"]: + try: + wp.cli("option", "update", name, options[name]) + except KeyError: + wp.cli("option", "delete", name) diff --git a/tests/steps/request_steps.py b/tests/steps/request_steps.py new file mode 100644 index 0000000000000000000000000000000000000000..6af8d985e50370ff340cc3bbd092a61777709be5 --- /dev/null +++ b/tests/steps/request_steps.py @@ -0,0 +1,138 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Step implementations dealing with HTTP requests +""" + +from __future__ import annotations + +import json +from typing import Any + +from behave import then +from behave import when +from behave.runner import Context +from utils import URL +from utils import PatternEnum + + +class Method(PatternEnum): + """ + HTTP methods + """ + + GET = "GET" + POST = "POST" + PUT = "PUT" + # add more methods as needed… + + +class ResponseCode(int, PatternEnum): + """ + HTTP response codes + """ + + ok = 200 + moved_permanently = 301 + found = 302 + not_modified = 304 + temporary_redirect = 307 + permanent_redirect = 308 + not_found = 404 + method_not_allowed = 405 + + # Aliases for the above codes, for mapping natural language in feature files to enums + ALIASES = { + "OK": 200, + "Not Found": 404, + "Method Not Allowed": 405, + } + + @staticmethod + def member_filter(attr: dict[str, Any], member_names: list[str]) -> None: + """ + Add natural language aliases and stringified code values to members + + Most will be accessible only though a class call, which is acceptable as that is how + step implementations look up the values. + """ + additional = { + str(value): value + for name in member_names + for value in [attr[name]] + if isinstance(value, int) + } + additional.update(attr["ALIASES"]) + member_names.remove("ALIASES") + member_names.extend(additional) + attr.update(additional) + + +@when("{url:URL} is requested") +def get_request(context: Context, url: URL) -> None: + """ + Assign the response from making a GET request to "url" to the context + """ + context.response = context.session.get(context.site.url / url, allow_redirects=False) + + +@when("data is sent with {method:Method} to {url:URL}") +def post_request(context: Context, method: Method, url: URL) -> None: + """ + Send context text to a URL endpoint and assign the response to the context + """ + if context.text is None: + raise ValueError("Missing data, please add as text to step definition") + context.response = context.session.request( + method.value, + context.site.url / url, + data=context.text.strip().format(context=context).encode("utf-8"), + allow_redirects=False, + ) + + +@when("the homepage is requested") +def get_homepage(context: Context) -> None: + """ + Assign the response from making a GET request to the base URL to the context + """ + get_request(context, '/') + + +@then('"{response:ResponseCode}" is returned') +@then('{response:ResponseCode} is returned') +def assert_response(context: Context, response: ResponseCode) -> None: + """ + Assert that the expected response was received during a previous step + + "response" can be a numeric or phrasal response in ResponseCode + """ + assert context.response.status_code == response, \ + f"Expected response {response}: got {context.response.status_code}" + + +@then('''the "{header_name}" header's value is "{header_value}"''') +def assert_header(context: Context, header_name: str, header_value: str) -> None: + """ + Assert that an expected header was received during a previous step + """ + headers = context.response.headers + assert header_name in headers, \ + f"Expected header not found in response: {header_name!r}" + assert headers[header_name] == header_value, \ + f"Expected header value not found: got {headers[header_name]!r}" + + +@then("the response body is JSON") +def assert_is_json(context: Context) -> None: + """ + Assert the response body of a previous step contains a JSON document + """ + try: + context.response.json() + except json.JSONDecodeError: + raise AssertionError("Response is not a JSON document") diff --git a/tests/stubs/behave/__init__.pyi b/tests/stubs/behave/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..289273d53bdb0b40e7b62731c4e267acd99c0ce9 --- /dev/null +++ b/tests/stubs/behave/__init__.pyi @@ -0,0 +1,30 @@ +from .fixture import fixture +from .fixture import use_fixture +from .matchers import register_type +from .matchers import use_step_matcher +from .step_registry import Given +from .step_registry import Step +from .step_registry import Then +from .step_registry import When +from .step_registry import given +from .step_registry import step +from .step_registry import then +from .step_registry import when + +# .matchers.step_matcher is deprecated, not including + +__version__: str +__all__ = [ + "Given", + "Step", + "Then", + "When", + "fixture", + "given", + "register_type", + "step", + "then", + "use_fixture", + "use_step_matcher", + "when", +] diff --git a/tests/stubs/behave/capture.pyi b/tests/stubs/behave/capture.pyi new file mode 100644 index 0000000000000000000000000000000000000000..8bf42bfc66c6ec3992d4c50261897b8c43e40a27 --- /dev/null +++ b/tests/stubs/behave/capture.pyi @@ -0,0 +1,43 @@ +from io import StringIO +from typing import IO + +from .log_capture import LoggingCapture +from .model import Configuration +from .runner import Context + +class Captured: + + stdout: str + stderr: str + log_output: str + + @property + def output(self) -> str: ... + + def __init__(self, stdout: str = ..., stderr: str = ..., log_output: str = ...): ... + def __add__(self, other: Captured) -> Captured: ... + + def reset(self) -> None: ... + def add(self, captured: Captured) -> Captured: ... + def make_report(self) -> str: ... + + +class CaptureController: + + config: Configuration + stdout_capture: StringIO + stderr_capture: StringIO + log_capture: LoggingCapture + old_stdout: IO[str] + old_stderr: IO[str] + + @property + def capture(self) -> Captured: ... + + def __init__(self, config: Configuration): ... + + def setup_capture(self, context: Context) -> None: ... + def start_capture(self) -> None: ... + def stop_capture(self) -> None: ... + def teardown_capture(self) -> None: ... + def make_capture_report(self) -> str: ... diff --git a/tests/stubs/behave/fixture.pyi b/tests/stubs/behave/fixture.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4af0fab1073053d19ca42c53ba699b253b513515 --- /dev/null +++ b/tests/stubs/behave/fixture.pyi @@ -0,0 +1,118 @@ +import sys +from typing import Any +from typing import Iterator +from typing import Protocol +from typing import TypeVar +from typing import overload + +from .runner import Context + +C = TypeVar("C", bound=Context) +C_con = TypeVar("C_con", bound=Context, contravariant=True) + +R = TypeVar("R") +R_co = TypeVar("R_co", covariant=True) + + +# There's a lot of @overload-ed functions here as fixtures come in two varieties: +# 1) A @contextlib.contextmanager-like generator that yields an arbitrary object once. +# 2) A simple function that returns an arbitrary object +# +# "use_fixture" allows both types of fixture callables to be used in the same way + +if sys.version_info >= (3, 10) and False: + # This depends on complete support of ParamSpec in mypy so is disabled for now. + + from typing import ParamSpec + + P = ParamSpec("P") + + + class FixtureCoroutine(Protocol[C_con, P, R_co]): + def __call__(self, _: C_con, /, *__a: P.args, **__k: P.kwargs) -> Iterator[R_co]: ... + + class FixtureFunction(Protocol[C_con, P, R_co]): + def __call__(self, _: C_con, /, *__a: P.args, **__k: P.kwargs) -> R_co: ... + + + @overload + def use_fixture( + fixture_func: FixtureCoroutine[C_con, P, R], + context: C_con, + *a: P.args, + **k: P.kwargs, + ) -> R: ... + + @overload + def use_fixture( + fixture_func: FixtureFunction[C_con, P, R], + context: C_con, + *a: P.args, + **k: P.kwargs, + ) -> R: ... + +else: + # Without ParamSpec no checking is done to ensure the arguments passed to use_fixture + # match the fixture's arguments; fixtures must be able to handle arguments not being + # supplied (except the context); and fixtures must accept ANY arbitrary keyword + # arguments. + + P = TypeVar("P", bound=None) + P_co = TypeVar("P_co", covariant=True) # unused + + + class FixtureCoroutine(Protocol[C_con, P_co, R_co]): + def __call__(self, _: C_con, /, *__a: Any, **__k: Any) -> Iterator[R_co]: ... + + class FixtureFunction(Protocol[C_con, P_co, R_co]): + def __call__(self, _: C_con, /, *__a: Any, **__k: Any) -> R_co: ... + + + @overload + def use_fixture( + fixture_func: FixtureCoroutine[C_con, P_co, R_co], + context: C_con, + *a: Any, + **k: Any, + ) -> R_co: ... + + @overload + def use_fixture( + fixture_func: FixtureFunction[C_con, P_co, R_co], + context: C_con, + *a: Any, + **k: Any, + ) -> R_co: ... + + +# "fixture" is a decorator used to mark both types of fixture callables. It can also return +# a decorator, when called without the "func" argument. + +@overload +def fixture( + func: FixtureCoroutine[C, P, R], + name: str = ..., + pattern: str = ..., +) -> FixtureCoroutine[C, P, R]: ... + +@overload +def fixture( + func: FixtureFunction[C, P, R], + name: str = ..., + pattern: str = ..., +) -> FixtureFunction[C, P, R]: ... + +@overload +def fixture( + name: str = ..., + pattern: str = ..., +) -> FixtureDecorator: ... + + +class FixtureDecorator(Protocol): + + @overload + def __call__(self, _: FixtureCoroutine[C, P, R], /) -> FixtureCoroutine[C, P, R]: ... + + @overload + def __call__(self, _: FixtureFunction[C, P, R], /) -> FixtureFunction[C, P, R]: ... diff --git a/tests/stubs/behave/formatter/__init__.pyi b/tests/stubs/behave/formatter/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/stubs/behave/formatter/base.pyi b/tests/stubs/behave/formatter/base.pyi new file mode 100644 index 0000000000000000000000000000000000000000..df454b57618c2f4e204512c2f9259b2b675d4500 --- /dev/null +++ b/tests/stubs/behave/formatter/base.pyi @@ -0,0 +1,58 @@ +from typing import IO +from typing import ClassVar +from typing import Protocol + +from ..matchers import Match +from ..model import Background +from ..model import Configuration +from ..model import Feature +from ..model import Scenario +from ..model import Step + +class StreamOpener: + + default_encoding: ClassVar[str] + + name: str + stream: IO[str] + encoding: str + should_close_stream = bool + + def __init__( + self, + filename: str = ..., + stream: IO[str] = ..., + encoding: str = ..., + ): ... + + @staticmethod + def ensure_dir_exists(directory: str) -> None: ... + + @classmethod + def ensure_stream_with_encoder(cls, stream: IO[str], encoding: str = ...) -> IO[str]: ... + + def open(self) -> IO[str]: ... + def close(self) -> bool: ... + + +class Formatter(Protocol): + + name: str + description: str + + def __init__(self, stream_opener: StreamOpener, config: Configuration): ... + + @property + def stdout_mode(self) -> bool: ... + + def open(self) -> IO[str]: ... + def uri(self, uri: str) -> None: ... + def feature(self, feature: Feature) -> None: ... + def background(self, background: Background) -> None: ... + def scenario(self, scenario: Scenario) -> None: ... + def step(self, step: Step) -> None: ... + def match(self, match: Match) -> None: ... + def result(self, step: Step) -> None: ... + def eof(self) -> None: ... + def close(self) -> None: ... + def close_stream(self) -> None: ... diff --git a/tests/stubs/behave/log_capture.pyi b/tests/stubs/behave/log_capture.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4692f7ef4d29470281d0d589a2a419b06967036d --- /dev/null +++ b/tests/stubs/behave/log_capture.pyi @@ -0,0 +1,55 @@ +from logging import Handler +from logging import LogRecord +from logging.handlers import BufferingHandler +from typing import Any +from typing import Callable +from typing import Protocol +from typing import TypeVar +from typing import overload + +from .model import Configuration +from .runner import ScenarioContext + +class RecordFilter: + + include: set[str] + exclude: set[str] + + def __init__(self, names: str): ... + + def filter(self, record: LogRecord) -> bool: ... + + +class LoggingCapture(BufferingHandler): + + config: Configuration + old_handlers: list[Handler] + old_level: int|None + + def __init__(self, config: Configuration, level: int = ...): ... + + def flush(self) -> None: ... + def truncate(self) -> None: ... + def getvalue(self) -> str: ... + def find_event(self, pattern: str) -> bool: ... + def any_errors(self) -> bool: ... + def inveigle(self) -> None: ... + def abandon(self) -> None: ... + + +MemoryHandler = LoggingCapture + + +class Hook(Protocol): + def __call__(self, _: ScenarioContext, /, *a: Any, **k: Any) -> None: ... + + +H = TypeVar("H", bound=Hook) + + +@overload +def capture(level: int = ...) -> Callable[[H], H]: ... + + +@overload +def capture(func: H, level: int = ...) -> H: ... diff --git a/tests/stubs/behave/matchers.pyi b/tests/stubs/behave/matchers.pyi new file mode 100644 index 0000000000000000000000000000000000000000..8ff68f0644ca0951fece02da991afdfef58db3b0 --- /dev/null +++ b/tests/stubs/behave/matchers.pyi @@ -0,0 +1,86 @@ +from typing import Any +from typing import Callable +from typing import ClassVar +from typing import Optional +from typing import Sequence +from typing import TypeVar + +from parse import Parser +from parse import TypeConverter + +from .model_core import Argument +from .model_core import FileLocation +from .runner import Context +from .step_registry import StepFunction + +Arguments = Optional[list[Argument]] + +T = TypeVar("T") + + +matcher_mapping: dict[str, Matcher] +current_matcher: Matcher + + +def register_type(**kw: Callable[[str], Any]) -> None: ... +def use_step_matcher(name: str) -> None: ... +def get_matcher(func: StepFunction, pattern: str) -> Matcher: ... + + +class StepParserError(ValueError): ... + + +class Match: + + func: StepFunction + arguments: Sequence[Argument]|None + location: int + + def __init__(self, func: StepFunction, arguments: Sequence[Argument] = ...): ... + def with_arguments(self, arguments: Sequence[Argument]) -> Match: ... + def run(self, context: Context) -> None: ... + + @staticmethod + def make_location(step_function: StepFunction) -> FileLocation: ... + + +class NoMatch(Match): + + def __init__(self) -> None: ... + + +class MatchWithError(Match): + + def __init__(self, func: StepFunction, error: BaseException): ... + + +class Matcher: + + schema: ClassVar[str] + + pattern: str + func: StepFunction + + def __init__(self, func: StepFunction, pattern: str, step_type: str = ...): ... + + @property + def location(self) -> FileLocation: ... + + @property + def regex_pattern(self) -> str: ... + + def describe(self, schema: str = ...) -> str: ... + def check_match(self, step: str) -> Arguments: ... + def match(self, step: StepFunction) -> Match: ... + + +class ParseMatcher(Matcher): + + custom_types: ClassVar[dict[str, TypeConverter[Any]]] + parser_class: ClassVar[type[Parser]] + + +class CFParseMatcher(ParseMatcher): ... +class RegexMatcher(Matcher): ... +class SimplifiedRegexMatcher(RegexMatcher): ... +class CucumberRegexMatcher(RegexMatcher): ... diff --git a/tests/stubs/behave/model.pyi b/tests/stubs/behave/model.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9372d29b7090d351af6604d6fe1706f70fed73d2 --- /dev/null +++ b/tests/stubs/behave/model.pyi @@ -0,0 +1,250 @@ +from typing import Any +from typing import ClassVar +from typing import Iterable +from typing import Iterator +from typing import Literal +from typing import Protocol +from typing import Sequence + +from .model_core import BasicStatement +from .model_core import Replayable +from .model_core import Status +from .model_core import TagAndStatusStatement +from .runner import ModelRunner +from .tag_expression import TagExpression + +Configuration = dict[str, Any] + + +def reset_model(model_elements: Iterable[ModelElement]) -> None: ... + + +class ModelElement(Protocol): + + def reset(self) -> None: ... + + +class Feature(TagAndStatusStatement, Replayable): + + type: ClassVar[Literal["feature"]] + + keyword: str + name: str + description: list[str] + background: Background + scenarios: list[Scenario] + tags: list[Tag] + hook_failed: bool + filename: str + line: int + language: str + + @property + def status(self) -> Status: ... + + @property + def duration(self) -> float: ... + + def __init__( + self, + filename: str, + line: int, + keyword: str, + name: str, + tags: Sequence[Tag] = ..., + description: str = ..., + scenarios: Sequence[Scenario] = ..., + background: Background = ..., + language: str = ..., + ): ... + def __iter__(self) -> Iterator[Scenario]: ... + + def reset(self) -> None: ... + def add_scenario(self, scenario: Scenario) -> None: ... + def compute_status(self) -> Status: ... + def walk_scenarios(self, with_outlines: bool = ...) -> list[Scenario]: ... + def should_run(self, config: Configuration = ...) -> bool: ... + def should_run_with_tags(self, tag_expression: TagExpression) -> bool: ... + def mark_skipped(self) -> None: ... + def skip(self, reason: str = ..., require_not_executed: bool = ...) -> None: ... + def run(self, runner: ModelRunner) -> bool: ... + + +class Background(BasicStatement, Replayable): + + type: ClassVar[Literal["background"]] + + keyword: str + name: str + steps: list[Step] + filename: str + line: int + + @property + def duration(self) -> float: ... + + def __init__( + self, + filename: str, + line: int, + keyword: str, + name: str, + steps: Sequence[Step] = ..., + ): ... + def __iter__(self) -> Iterator[Step]: ... + + +class Scenario(TagAndStatusStatement, Replayable): + + type: ClassVar[Literal["scenario"]] + continue_after_failed_step: ClassVar[bool] + + keyword: str + name: str + description: str + feature: Feature + steps: Sequence[Step] + tags: Sequence[Tag] + hook_failed: bool + filename: str + line: int + + @property + def background_steps(self) -> list[Step]: ... + + @property + def all_steps(self) -> Iterator[Step]: ... + + @property + def duration(self) -> float: ... + + @property + def effective_tags(self) -> list[Tag]: ... + + def __init__( + self, + filename: str, + line: int, + keyword: str, + name: str, + tags: Sequence[Tag] = ..., + steps: Sequence[Step] = ..., + description: str = ..., + ): ... + def __iter__(self) -> Iterator[Step]: ... + + def reset(self) -> None: ... + def compute_status(self) -> Status: ... + def should_run(self, config: Configuration = ...) -> bool: ... + def should_run_with_tags(self, tag_expression: TagExpression) -> bool: ... + def should_run_with_name_select(self, config: Configuration) -> bool: ... + def mark_skipped(self) -> None: ... + def skip(self, reason: str = ..., require_not_executed: bool = ...) -> None: ... + def run(self, runner: ModelRunner) -> bool: ... + + +class Step(BasicStatement, Replayable): + + type: ClassVar[Literal["step"]] + + keyword: str + name: str + step_type: str + text: Text|None + table: Table|None + status: Status + hook_failed: bool + duration: float + error_message: str|None + filename: str + line: int + + def __init__( + self, + filename: str, + line: int, + keyword: str, + step_type: str, + name: str, + text: Text = ..., + table: Table = ..., + ): ... + def __eq__(self, other: Any) -> bool: ... + def __hash__(self) -> int: ... + + def reset(self) -> None: ... + def run(self, runner: ModelRunner, quiet: bool = ..., capture: bool = ...) -> bool: ... + + +class Table(Replayable): + + type: ClassVar[Literal["table"]] + + headings: Sequence[str] + line: int|None + rows: list[Row] + + def __init__(self, headings: Sequence[str], line: int = ..., rows: Sequence[Row] = ...): ... + def __eq__(self, other: Any) -> bool: ... + def __iter__(self) -> Iterator[Row]: ... + def __getitem__(self, index: int) -> Row: ... + + def add_row(self, row: Sequence[str], line: int) -> None: ... + def add_column(self, column_name: str, values: Iterable[str], default_value: str = ...) -> int: ... + def remove_column(self, column_name: str) -> None: ... + def remove_columns(self, column_names: Iterable[str]) -> None: ... + def has_column(self, column_name: str) -> bool: ... + def get_column_index(self, column_name: str) -> int: ... + def require_column(self, column_name: str) -> int: ... + def require_columns(self, column_names: Iterable[str]) -> None: ... + def ensure_column_exists(self, column_name: str) -> int: ... + def assert_equals(self, data: Table|Iterable[Row]) -> None: ... + + +class Row: + + headings: Sequence[str] + cells: Sequence[str] + line: int|None + comments: Sequence[str]|None + + def __init__( + self, + headings: Sequence[str], + cells: Sequence[str], + line: int = ..., + comments: Sequence[str] = ..., + ): ... + def __getitem__(self, index: int) -> str: ... + def __eq__(self, other: Any) -> bool: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[str]: ... + + def items(self) -> Iterator[tuple[str, str]]: ... + def get(self, key: int, default: str = ...) -> str: ... + def as_dict(self) -> dict[str, str]: ... + + +class Tag(str): + + allowed_chars: ClassVar[str] + quoting_chars: ClassVar[str] + + line: int + + def __init__(self, name: str, line: int): ... + + @classmethod + def make_name(cls, text: str, unexcape: bool = ..., allowed_chars: str = ...) -> str: ... + + +class Text(str): + + content_type: Literal["text/plain"] + line: int + + def __init__(self, value: str, content_type: Literal["text/plain"] = ..., line: int = ...): ... + + def line_range(self) -> tuple[int, int]: ... + def replace(self, old: str, new: str, count: int = ...) -> Text: ... + def assert_equals(self, expected: str) -> bool: ... diff --git a/tests/stubs/behave/model_core.pyi b/tests/stubs/behave/model_core.pyi new file mode 100644 index 0000000000000000000000000000000000000000..44949165ce78a5bd04944542023cbfe1220f4e64 --- /dev/null +++ b/tests/stubs/behave/model_core.pyi @@ -0,0 +1,123 @@ +from enum import Enum +from types import TracebackType +from typing import Any +from typing import Callable +from typing import ClassVar +from typing import Sequence +from typing import TypeVar + +from .capture import Captured +from .formatter.base import Formatter +from .model import Tag +from .tag_expression import TagExpression + +class Status(Enum): + + untested = 0 + skipped = 1 + passed = 2 + failed = 3 + undefined = 4 + executing = 5 + + @classmethod + def from_name(cls, name: str) -> Status: ... + + +class Argument: + + original: str + value: Any + name: str|None + start: int + end: int + + def __init__(self, start: int, end: int, original: str, value: Any, name: str = ...): ... + + +class FileLocation: + + T = TypeVar("T", bound="FileLocation") + + filename: str + line: int + + def __init__(self, filename: str, line: int): ... + def __eq__(self, other: Any) -> bool: ... + def __ne__(self, other: Any) -> bool: ... + def __le__(self, other: str|FileLocation) -> bool: ... + def __gt__(self, other: str|FileLocation) -> bool: ... + def __ge__(self, other: str|FileLocation) -> bool: ... + def __str__(self) -> str: ... + + def get(self) -> str: ... + def abspath(self) -> str: ... + def basename(self) -> str: ... + def dirname(self) -> str: ... + def relpath(self, start: str = ...) -> str: ... + def exists(self) -> str: ... + + @classmethod + def for_function(cls: type[T], func: Callable[..., Any], curdir: str = ...) -> T: ... + + +class BasicStatement: + + location: FileLocation + keyword: str + name: str + captured: Captured + exception: Exception|None + exc_traceback: TracebackType|None + error_message: str|None + + @property + def filename(self) -> str: ... + + @property + def line(self) -> int: ... + + def __init__(self, filename: str, line: int, keyword: str, name: str): ... + def __hash__(self) -> int: ... + def __eq__(self, other: Any) -> bool: ... + def __ne__(self, other: Any) -> bool: ... + def __lt__(self, other: BasicStatement) -> bool: ... + def __le__(self, other: BasicStatement) -> bool: ... + def __gt__(self, other: BasicStatement) -> bool: ... + def __ge__(self, other: BasicStatement) -> bool: ... + + def reset(self) -> None: ... + def store_exception_context(self, exception: Exception) -> None: ... + + +class TagStatement(BasicStatement): + + tags: Sequence[Tag] + + def __init__(self, filename: str, line: int, keyword: str, name: str, tags: Sequence[Tag]): ... + def should_run_with_tags(self, tag_expression: TagExpression) -> bool: ... + + +class TagAndStatusStatement(BasicStatement): + + final_status: ClassVar[tuple[Status]] + + tags: Sequence[Tag] + should_skip: bool + skip_reason: str|None + + @property + def status(self) -> Status: ... + + def __init__(self, filename: str, line: int, keyword: str, name: str, tags: Sequence[Tag]): ... + + def should_run_with_tags(self, tag_expression: TagExpression) -> bool: ... + def set_status(self, value: Status) -> None: ... + def clear_status(self) -> None: ... + def reset(self) -> None: ... + def compute_status(self) -> Status: ... + + +class Replayable: + + def replay(self, formatter: Formatter) -> None: ... diff --git a/tests/stubs/behave/runner.pyi b/tests/stubs/behave/runner.pyi new file mode 100644 index 0000000000000000000000000000000000000000..48deb102760d3929c1f1587c0e4146cd7b96d184 --- /dev/null +++ b/tests/stubs/behave/runner.pyi @@ -0,0 +1,148 @@ +import contextlib +from io import StringIO +from typing import Any +from typing import Callable +from typing import Iterator +from typing import Literal +from typing import Protocol +from typing import Sequence +from typing import Union + +from .capture import CaptureController +from .formatter.base import Formatter +from .log_capture import LoggingCapture +from .model import Configuration +from .model import Feature +from .model import Row +from .model import Scenario +from .model import Step +from .model import Table +from .model import Tag +from .model_core import FileLocation +from .step_registry import StepRegistry + +Mode = Union[Literal["behave"], Literal["user"]] + + +@contextlib.contextmanager +def use_context_with_mode(context: Context, mode: Mode) -> Iterator[None]: ... + +@contextlib.contextmanager +def scoped_context_layer(context: Context, layer_name: str|None = None) -> Iterator[Context]: ... + +def path_getrootdir(path: str)-> str: ... + + +class CleanupError(RuntimeError): ... +class ContextMaskWarning(UserWarning): ... + + +class Context(Protocol): + + def __getattr__(self, name: str) -> Any: ... + def __setattr__(self, name: str, value: Any) -> None: ... + def __contains__(self, name: str) -> bool: ... + + def add_cleanup(self, cleanup_func: Callable[..., None], *a: Any, **k: Any) -> None: ... + + @property + def config(self) -> Configuration: ... + + @property + def aborted(self) -> bool: ... + + @property + def failed(self) -> bool: ... + + @property + def log_capture(self) -> LoggingCapture|None: ... + + @property + def stdout_capture(self) -> StringIO|None: ... + + @property + def stderr_capture(self) -> StringIO|None: ... + + @property + def cleanup_errors(self) -> int: ... + + # Feature always present, None outside of feature namespace + + @property + def feature(self) -> Feature|None: ... + + # Step values always present, may be None even in step namespace + + @property + def active_outline(self) -> Row|None: ... + + @property + def table(self) -> Table|None: ... + + @property + def text(self) -> str|None: ... + + +class FeatureContext(Protocol, Context): + + def execute_steps(self, steps_text: str) -> bool: ... + + @property + def feature(self) -> Feature: ... + + @property + def tags(self) -> set[Tag]: ... + + +class ScenarioContext(Protocol, FeatureContext): + + @property + def scenario(self) -> Scenario: ... + + +class Hook(Protocol): + + def __call__(self, context: Context, *args: Any) -> None: ... + + +class ModelRunner: + + config: Configuration + features: Sequence[Feature] + step_registry: StepRegistry + hooks: dict[str, Hook] + formatters: list[Formatter] + undefined_steps: list[Step] + capture_controller: CaptureController + context: Context|None + feature: Feature|None + hook_failures: int + + # is a property in concrete class + aborted: bool + + def __init__( + self, + config: Configuration, + features: Sequence[Feature]|None, + step_registry: StepRegistry|None, + ): ... + + def run_hook(self, name: str, context: Context, *args: Any) -> None: ... + def setup_capture(self) -> None: ... + def start_capture(self) -> None: ... + def stop_capture(self) -> None: ... + def teardown_capture(self) -> None: ... + def run_model(self, features: Sequence[Feature]|None) -> bool: ... + def run(self) -> bool: ... + + +class Runner(ModelRunner): + + def __init__(self, config: Configuration): ... + def setup_paths(self) -> None: ... + def before_all_default_hook(self, context: Context) -> None: ... + def load_hooks(self, filename: str = ...) -> None: ... + def load_step_definitions(self, extra_step_paths: Sequence[str] = ...) -> None: ... + def feature_locations(self) -> list[FileLocation]: ... + def run_with_paths(self) -> bool: ... diff --git a/tests/stubs/behave/step_registry.pyi b/tests/stubs/behave/step_registry.pyi new file mode 100644 index 0000000000000000000000000000000000000000..8609d1ceef0647905a22cd58e8ffa7ff7e2bf7ff --- /dev/null +++ b/tests/stubs/behave/step_registry.pyi @@ -0,0 +1,27 @@ +from typing import Callable +from typing import TypeVar + +C = TypeVar("C") +StepDecorator = Callable[[str], Callable[[C], C]] +StepFunction = Callable[..., None] + + +Given: StepDecorator[StepFunction] +given: StepDecorator[StepFunction] + +When: StepDecorator[StepFunction] +when: StepDecorator[StepFunction] + +Then: StepDecorator[StepFunction] +then: StepDecorator[StepFunction] + +Step: StepDecorator[StepFunction] +step: StepDecorator[StepFunction] + + +class AmbiguousStep(ValueError): ... + + +class StepRegistry: + + steps: dict[str, list[StepFunction]] diff --git a/tests/stubs/behave/tag_expression.pyi b/tests/stubs/behave/tag_expression.pyi new file mode 100644 index 0000000000000000000000000000000000000000..cdb025b366fcc2bd780376f604bf42aa33be1c50 --- /dev/null +++ b/tests/stubs/behave/tag_expression.pyi @@ -0,0 +1,22 @@ +from typing import Iterable +from typing import Sequence + +from .model import Tag + +class TagExpression: + + ands: list[tuple[str]] + limits: dict[tuple[str], int] + + def __init__(self, tag_expressions: Iterable[str]): ... + def __len__(self) -> int: ... + + def check(self, tags: Sequence[Tag]) -> bool: ... + + @staticmethod + def normalize_tag(tag: str) -> str: ... + + @classmethod + def normalized_tags_from_or(cls, expr: str) -> Iterable[str]: ... + + def store_and_extract_limits(self, tags: Iterable[str]) -> None: ... diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8c8fde56fa08ba7d68b44acf0c17d4fa131d9b6e --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +A toolkit of helpful functions and classes for step implementations +""" + +from __future__ import annotations + +from .behave import PatternEnum +from .behave import register_pattern +from .http import redirect +from .json import JSONArray +from .json import JSONObject +from .secret import make_secret +from .url import URL + +__all__ = ( + "JSONArray", + "JSONObject", + "PatternEnum", + "URL", + "make_secret", + "redirect", + "register_pattern", +) diff --git a/tests/utils/behave.py b/tests/utils/behave.py new file mode 100644 index 0000000000000000000000000000000000000000..765b62abb92b5945559d6a2b5000224af599d28a --- /dev/null +++ b/tests/utils/behave.py @@ -0,0 +1,115 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Utilities for "behave" interactions +""" + +from __future__ import annotations + +import enum +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol +from typing import TypeVar +from typing import overload + +import behave +import parse + +T = TypeVar("T") + +__all__ = [ + "PatternEnum", + "register_pattern", +] + + +class PatternConverter(Protocol): + + __name__: str + + def __call__(self, match: str) -> Any: ... + + +class Decorator(Protocol[T]): + + def __call__(self, converter: T) -> T: ... + + +@overload +def register_pattern(pattern: str) -> Decorator[PatternConverter]: ... + + +@overload +def register_pattern( + pattern: str, + converter: PatternConverter, + name: str = ..., +) -> PatternConverter: ... + + +def register_pattern( + pattern: str, + converter: PatternConverter|None = None, + name: str = "", +) -> PatternConverter|Decorator[PatternConverter]: + """ + Register a pattern and converter for a step parser type + + The type is named after the converter. + """ + pattern_decorator = parse.with_pattern(pattern) + + def decorator(converter: PatternConverter) -> PatternConverter: + nonlocal name + name = name or converter.__name__ + behave.register_type(**{name: pattern_decorator(converter)}) + return converter + + if converter: + return decorator(converter) + return decorator + + +class EnumMeta(enum.EnumMeta): + + MEMBER_FILTER = 'member_filter' + + T = TypeVar("T", bound="EnumMeta") + + def __new__(mtc: type[T], name: str, bases: tuple[type, ...], attr: dict[str, Any], **kwds: Any) -> T: + member_names: list[str] = attr._member_names # type: ignore + member_filter = attr.pop(mtc.MEMBER_FILTER, None) + if member_filter: + assert isinstance(member_filter, staticmethod) + member_filter.__func__(attr, member_names) + cls = enum.EnumMeta.__new__(mtc, name, bases, attr, **kwds) + decorator = parse.with_pattern('|'.join(member for member in cls.__members__)) + behave.register_type(**{name: decorator(cls)}) + return cls + + +class PatternEnum(enum.Enum, metaclass=EnumMeta): + """ + An enum class that self registers as a pattern type for step implementations + + Enum names are used to match values in step texts, so a value can be aliased multiple + times to provide alternates for matching, including alternative languages. + To supply names that are not valid identifiers the functional Enum API must be used, + supplying mapped values: + https://docs.python.org/3/library/enum.html#functional-api + + Enum values may be anything meaningful; for instance a command keyword that identifies a + type. + """ + + if TYPE_CHECKING: + C = TypeVar("C", bound="PatternEnum") + + @classmethod + def _missing_(cls: type[C], key: Any) -> C: + return cls[key] diff --git a/tests/utils/http.py b/tests/utils/http.py new file mode 100644 index 0000000000000000000000000000000000000000..b88484c777c6e4329bfcb25213a8c22d0f044569 --- /dev/null +++ b/tests/utils/http.py @@ -0,0 +1,70 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Extensions for "requests" +""" + +from __future__ import annotations + +import ipaddress +from typing import Any +from typing import Mapping +from urllib.parse import urlparse + +import requests.adapters +from requests.packages.urllib3 import connection +from requests.packages.urllib3 import connectionpool + + +def redirect(session: requests.Session, prefix: str, address: ipaddress.IPv4Address) -> None: + """ + Redirect all requests for "prefix" to a given address + + This function allows a user to completely override DNS and local name lookups, allowing + fixtures to be contacted via any configured URL without having to mess with the system's + name resolution services. + + "prefix" is formated as either "{hostname}[:{port}]" or "{schema}://{hostname}[:{port}]" + where "schema" defaults to (and currently only supports) "http". + """ + if prefix.startswith("https://"): + raise ValueError("https:// prefixes not currently supported") + if not prefix.startswith("http://"): + prefix = f"http://{prefix}" + session.mount(prefix, LocalHTTPAdapter(address)) + + +class LocalHTTPAdapter(requests.adapters.HTTPAdapter): + """ + An alternative HTTP adapter that directs all connections to a configured address + + Instances of this class are mounted on a `requests.Session` as adapters for specific URL + prefixes. + + Rather than using this class directly the easiest way to use it is with the `redirect` + function. + """ + + def __init__(self, destination: ipaddress.IPv4Address): + super().__init__() + self.destination = destination + + def get_connection(self, url: str, proxies: Mapping[str, str]|None = None) -> _HTTPConnectionPool: + parts = urlparse(url) + return _HTTPConnectionPool(parts.hostname, parts.port, address=self.destination) + + +class _HTTPConnectionPool(connectionpool.HTTPConnectionPool): + + class ConnectionCls(connection.HTTPConnection): + + # Undo the damage done by parent class which makes 'host' a property with magic + host = "" + + def __init__(self, /, address: ipaddress.IPv4Address, **kwargs: Any): + connection.HTTPConnection.__init__(self, **kwargs) + self._dns_host = str(address) diff --git a/tests/utils/json.py b/tests/utils/json.py new file mode 100644 index 0000000000000000000000000000000000000000..fd9d43bba0fc33bdbaab5930e1511c8c8743aa15 --- /dev/null +++ b/tests/utils/json.py @@ -0,0 +1,68 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +JSON classes for container types (objects and arrays) +""" + +from __future__ import annotations + +import json +from typing import Any +from typing import Callable +from typing import TypeVar +from typing import overload + +from jsonpath import JSONPath + +__all__ = [ + "JSONObject", + "JSONArray", +] + + +class JSONPathMixin: + + T = TypeVar('T', bound=object) + C = TypeVar('C', bound=object) + + @overload + def path(self, path: str, kind: type[T], convert: None = None) -> T: ... + + @overload + def path(self, path: str, kind: type[T], convert: Callable[[T], C]) -> C: ... + + def path(self, path: str, kind: type[T], convert: Callable[[T], C]|None = None) -> T|C: + result = JSONPath(path).parse(self)[0] + if convert is not None: + return convert(result) + elif isinstance(result, kind): + return result + raise ValueError(f"{path} is wrong type; expected {kind}; got {type(result)}") + + +class JSONObject(JSONPathMixin, dict[str, Any]): + """ + A dict for JSON objects that implements `.path` for getting child items by a JSON path + """ + + T = TypeVar("T", bound="JSONObject") + + @classmethod + def from_string(cls: type[T], string: bytes) -> T: + return cls(json.loads(string)) + + +class JSONArray(JSONPathMixin, list[Any]): + """ + A list for JSON arrays that implements `.path` for getting child items by a JSON path + """ + + T = TypeVar("T", bound="JSONArray") + + @classmethod + def from_string(cls: type[T], string: bytes) -> T: + return cls(json.loads(string)) diff --git a/tests/utils/secret.py b/tests/utils/secret.py new file mode 100644 index 0000000000000000000000000000000000000000..b5caa58bb2c1c34bf83ce360dccdd3120556b166 --- /dev/null +++ b/tests/utils/secret.py @@ -0,0 +1,23 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Utility for generating secrets +""" + +from __future__ import annotations + +import string +from secrets import choice + +CHARS = string.ascii_letters + string.digits + + +def make_secret(size: int) -> str: + """ + Generate a string of alphanumeric characters for use as a password + """ + return ''.join(choice(CHARS) for _ in range(size)) diff --git a/tests/utils/url.py b/tests/utils/url.py new file mode 100644 index 0000000000000000000000000000000000000000..2fc6f9bd301297aff4a9a2588ae59c18c33bbd8e --- /dev/null +++ b/tests/utils/url.py @@ -0,0 +1,28 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +URL types and pattern matcher +""" + +from __future__ import annotations + +from urllib.parse import urljoin + +from .behave import register_pattern + + +@register_pattern(r"(?:https?://\S+|/\S*)") +class URL(str): + """ + A subclass for URL strings which also acts as a pattern match type + """ + + def __truediv__(self, other: str) -> URL: + return URL(urljoin(self, other)) + + def __add__(self, other: str) -> URL: + return URL(str(self) + other) diff --git a/tests/wp-cli.feature b/tests/wp-cli.feature new file mode 100644 index 0000000000000000000000000000000000000000..24a514e17a32f54131b4bb1ccef8b59cad24b420 --- /dev/null +++ b/tests/wp-cli.feature @@ -0,0 +1,30 @@ +Feature: WP-CLI management tool + + Scenario Outline: Setting test commands + When "" is run + Then nothing is seen from stderr + + Examples: + | cmd | + | wp option update timezone_string Europe/London | + + + Scenario Outline: Getting test commands + When "" is run + Then "" is seen from stdout + And nothing is seen from stderr + + Examples: + | cmd | response | + | wp option get timezone_string | Europe/London | + + + Scenario Outline: Getting JSON test commands + When "" is run + Then JSON is seen from stdout + And nothing is seen from stderr + + Examples: + | cmd | + | wp option get timezone_string --format=json | + | wp theme list --format=json | diff --git a/tests/wp/__init__.py b/tests/wp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a6f487b957b9006e1f39cf79baa0e8ecbfa6141 --- /dev/null +++ b/tests/wp/__init__.py @@ -0,0 +1,177 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Management and control for WordPress fixtures +""" + +from __future__ import annotations + +from contextlib import contextmanager +from subprocess import CalledProcessError +from time import sleep +from time import time +from typing import Any +from typing import Callable +from typing import Iterator +from typing import Literal +from typing import SupportsBytes +from typing import TypeVar +from typing import overload + +from .docker import Container as Container +from .proc import PathArg +from .proc import coerce_args +from .proc import exec_io + + +def wait(predicate: Callable[[], bool], timeout: float = 120.0) -> None: + """ + Block and periodically call "predictate" until it returns True, or the time limit passes + """ + end = time() + timeout + left = timeout + while left > 0.0: + sleep( + 10 if left > 60.0 else + 5 if left > 10.0 else + 1, + ) + left = end - time() + if predicate(): + return + raise TimeoutError + + +class Cli: + """ + Manage calling executables in a container + + Any arguments passed to the constructor will prefix the arguments passed when the object + is called. + """ + + T = TypeVar("T") + + def __init__(self, container: Container, *cmd: PathArg): + self.container = container + self.cmd = cmd + + @overload + def __call__( + self, + *args: PathArg, + input: str|SupportsBytes|None = ..., + deserialiser: Callable[[bytes], T], + query: Literal[False], + **kwargs: Any, + ) -> T: ... + + @overload + def __call__( + self, + *args: PathArg, + input: str|SupportsBytes|None = ..., + deserialiser: None = None, + query: Literal[True], + **kwargs: Any, + ) -> int: ... + + @overload + def __call__( + self, + *args: PathArg, + input: str|SupportsBytes|None = ..., + deserialiser: None = None, + query: Literal[False], + **kwargs: Any, + ) -> None: ... + + def __call__( + self, + *args: PathArg, + input: str|SupportsBytes|None = None, + deserialiser: Callable[[bytes], T]|None = None, + query: bool = False, + **kwargs: Any, + ) -> Any: + # deserialiser = kwargs.pop('deserialiser', None) + assert not deserialiser or not query + + data = ( + b"" if input is None else + input.encode() if isinstance(input, str) else + bytes(input) + ) + cmd = self.container.get_exec_args([*self.cmd, *args], interactive=bool(data)) + + if deserialiser: + return exec_io(cmd, data, deserialiser=deserialiser, **kwargs) + + rcode = exec_io(cmd, data, **kwargs) + if query: + return rcode + if not isinstance(rcode, int): + raise TypeError(f"got rcode {rcode!r}") + if 0 != rcode: + raise CalledProcessError(rcode, ' '.join(coerce_args(cmd))) + return None + + +class Wordpress(Container): + """ + Container subclass for a WordPress PHP-FPM container + """ + + @property + def cli(self) -> Cli: + """ + Run WP-CLI commands + """ + return Cli(self, "wp") + + @contextmanager + def started(self) -> Iterator[Container]: + with self: + self.start() + cmd = ["bash", "-c", "[[ /proc/1/exe -ef `which php-fpm` ]]"] + wait(lambda: self.is_running() and self.run(cmd).returncode == 0, timeout=600) + yield self + + +class Mysql(Container): + """ + Container subclass for a database container + """ + + @property + def mysql(self) -> Cli: + """ + Run "mysql" commands + """ + return Cli(self, "mysql") + + @property + def mysqladmin(self) -> Cli: + """ + Run "mysqladmin" commands + """ + return Cli(self, "mysqladmin") + + @property + def mysqldump(self) -> Cli: + """ + Run "mysqldump" commands + """ + return Cli(self, "mysqldump") + + @contextmanager + def started(self) -> Iterator[Container]: + with self: + self.start() + sleep(20) + wait(lambda: self.run(['/healthcheck.sh']).returncode == 0) + yield self diff --git a/tests/wp/docker.py b/tests/wp/docker.py new file mode 100644 index 0000000000000000000000000000000000000000..9a28d3ceccc258affcf67a6dab964a97bd804f38 --- /dev/null +++ b/tests/wp/docker.py @@ -0,0 +1,414 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Commands for managing Docker for fixtures +""" + +from __future__ import annotations + +import ipaddress +import json +from contextlib import contextmanager +from pathlib import Path +from secrets import token_hex +from subprocess import DEVNULL +from subprocess import PIPE +from subprocess import CompletedProcess +from subprocess import Popen +from subprocess import run +from types import TracebackType +from typing import IO +from typing import Any +from typing import Callable +from typing import Iterable +from typing import Iterator +from typing import Optional +from typing import Tuple +from typing import TypeVar +from typing import Union +from typing import overload + +from jsonpath import JSONPath + +from .proc import Arguments +from .proc import Environ +from .proc import MutableArguments +from .proc import PathArg +from .proc import PathLike +from .proc import coerce_args + +T_co = TypeVar('T_co', covariant=True) + +HostMount = tuple[PathLike, PathLike] +NamedMount = tuple[str, PathLike] +AnonMount = PathLike +Mount = Union[HostMount, NamedMount, AnonMount] +Volumes = Iterable[Mount] + +DOCKER = 'docker' + + +def docker(*args: PathArg, **env: str) -> None: + """ + Run a Docker command, with output going to stdout + """ + run([DOCKER, *coerce_args(args)], env=env, check=True) + + +def docker_output(*args: PathArg, **env: str) -> str: + """ + Run a Docker command, capturing and returning its stdout + """ + proc = run([DOCKER, *coerce_args(args)], env=env, check=True, stdout=PIPE, text=True) + return proc.stdout.strip() + + +def docker_quiet(*args: PathArg, **env: str) -> None: + """ + Run a Docker command, directing its stdout to /dev/null + """ + run([DOCKER, *coerce_args(args)], env=env, check=True, stdout=DEVNULL) + + +class IPv4Address(ipaddress.IPv4Address): + """ + Subclass of IPv4Address that handle's docker idiosyncratic tendency to add a mask suffix + """ + + T = TypeVar("T", bound="IPv4Address") + + @classmethod + def with_suffix(cls: type[T], address: str) -> T: + """ + Construct an instance with a suffixed bitmask size + """ + address, *_ = address.partition("/") + return cls(address) + + +class Item: + """ + A mix-in for Docker items that can be inspected + """ + + T = TypeVar('T', bound=object) + C = TypeVar('C', bound=object) + + _data: Optional[dict[str, Any]] = None + + def __init__(self, name: str): + self.name = name + self._data: Any = None + + def get_id(self) -> str: + """ + Return an identifier for the Docker item + """ + return self.name + + @overload + def inspect(self, path: str, kind: type[T], convert: None = None) -> T: ... + + @overload + def inspect(self, path: str, kind: type[T], convert: Callable[[T], C]) -> C: ... + + def inspect(self, path: str, kind: type[T], convert: Callable[[T], C]|None = None) -> T|C: + """ + Extract a value from an item's information by JSON path + + "kind" is the type of the extracted value, while "convert" is an optional callable + that can turn that type to another type. The return type will be the return type of + "convert" if provided, "kind" otherwise. + """ + if self._data is None: + with Popen([DOCKER, 'inspect', self.get_id()], stdout=PIPE) as proc: + assert proc.stdout is not None + results = json.load(proc.stdout) + assert isinstance(results, list) + assert len(results) == 1 and isinstance(results[0], dict) + self._data = results[0] + result = JSONPath(path).parse(self._data) + if "*" not in path: + try: + result = result[0] + except IndexError: + raise KeyError(path) from None + if not isinstance(result, kind): + raise TypeError(f"{path} is wrong type; expected {kind}; got {type(result)}") + if convert is None: + return result + return convert(result) + + +class Image(Item): + """ + Docker image items + """ + + T = TypeVar('T', bound='Image') + + def __init__(self, iid: str): + self.iid = iid + + @classmethod + def build(cls: type[T], context: Path, target: str = "", **build_args: str|None) -> T: + """ + Build an image from the given context + + Build arguments are ignored if they are None to make it easier to supply (or not) + arguments from external lookups without complex argument composing. + """ + cmd: Arguments = [ + 'build', context, f"--target={target}", + *(f"--build-arg={arg}={val}" for arg, val in build_args.items() if val is not None), + ] + docker(*cmd, DOCKER_BUILDKIT='1') + iid = docker_output(*cmd, '-q', DOCKER_BUILDKIT='1') + return cls(iid) + + @classmethod + def pull(cls: type[T], repository: str) -> T: + """ + Pull an image from a registry + """ + docker('pull', repository) + iid = Item(repository).inspect('$.Id', str) + return cls(iid) + + def get_id(self) -> str: + return self.iid + + +class Container(Item): + """ + Docker container items + + Instances can be used as context managers that ensure the container is stopped on + exiting the context. + """ + + DEFAULT_ALIASES = tuple[str]() + + def __init__( + self, + image: Image, + cmd: Arguments = [], + volumes: Volumes = [], + env: Environ = {}, + network: Network|None = None, + entrypoint: HostMount|PathArg|None = None, + ): + if isinstance(entrypoint, tuple): + volumes = [*volumes, entrypoint] + entrypoint = entrypoint[1] + + self.image = image + self.cmd = cmd + self.volumes = volumes + self.env = env + self.entrypoint = entrypoint + self.networks = dict[Network, Tuple[str, ...]]() + self._id: str|None = None + + if network: + self.networks[network] = Container.DEFAULT_ALIASES + + def __enter__(self) -> Container: + return self + + def __exit__(self, etype: type[BaseException], exc: BaseException, tb: TracebackType) -> None: + if self._id and exc: + self.show_logs() + self.stop(rm=True) + + @contextmanager + def started(self) -> Iterator[Container]: + """ + A context manager that ensures the container is started when the context is entered + """ + with self: + self.start() + yield self + + def is_running(self) -> bool: + """ + Return whether the container is running + """ + if self._id is None: + return False + item = Item(self._id) + if item.inspect('$.State.Status', str) == 'exited': + code = item.inspect('$.State.ExitCode', int) + raise ProcessLookupError(f"container {self._id} exited ({code})") + return ( + self._id is not None + and item.inspect('$.State.Running', bool) + ) + + def get_id(self) -> str: + if self._id is not None: + return self._id + + networks = set[Network]() + opts: MutableArguments = [ + *( + (f"--volume={vol[0]}:{vol[1]}" if isinstance(vol, tuple) else f"--volume={vol}") + for vol in self.volumes + ), + *(f"--env={name}={val}" for name, val in self.env.items()), + ] + + if self.entrypoint: + opts.append(f"--entrypoint={self.entrypoint}") + if self.networks: + networks.update(self.networks) + net = networks.pop() + opts.append(f"--network={net}") + opts.extend(f"--network-alias={alias}" for alias in self.networks[net]) + + self._id = docker_output('container', 'create', *opts, self.image.iid, *self.cmd) + assert self._id + return self._id + + def start(self) -> None: + """ + Start the container + """ + docker_quiet('container', 'start', self.get_id()) + + def stop(self, rm: bool = False) -> None: + """ + Stop the container + """ + if self._id is None: + return + docker_quiet('container', 'stop', self._id) + if rm: + docker_quiet('container', 'rm', self._id) + self._id = None + + def connect(self, network: Network, *aliases: str) -> None: + """ + Connect the container to a Docker network + + Any aliases supplied will be resolvable to the container by other containers on the + network. + """ + is_running = self.is_running() + if network in self.networks: + if self.networks[network] == aliases: + return + if is_running: + docker('network', 'disconnect', str(network), self.get_id()) + if is_running: + docker( + 'network', 'connect', + *(f'--alias={a}' for a in aliases), + str(network), self.get_id(), + ) + self.networks[network] = aliases + + def show_logs(self) -> None: + """ + Print the container logs to stdout + """ + if self._id: + docker('logs', self._id) + + def get_exec_args(self, cmd: Arguments, interactive: bool = False) -> MutableArguments: + """ + Return a full argument list for running "cmd" inside the container + """ + return [DOCKER, "exec", *(("-i",) if interactive else ""), self.get_id(), *coerce_args(cmd)] + + def run( + self, + cmd: Arguments, + *, + stdin: IO[Any]|int|None = None, + stdout: IO[Any]|int|None = None, + stderr: IO[Any]|int|None = None, + capture_output: bool = False, + check: bool = False, + input: bytes|None = None, + timeout: float|None = None, + ) -> CompletedProcess[bytes]: + """ + Run "cmd" to completion inside the container and return the result + """ + return run( + self.get_exec_args(cmd), + stdin=stdin, stdout=stdout, stderr=stderr, + capture_output=capture_output, + check=check, timeout=timeout, input=input, + ) + + def exec( + self, + cmd: Arguments, + *, + stdin: IO[Any]|int|None = None, + stdout: IO[Any]|int|None = None, + stderr: IO[Any]|int|None = None, + ) -> Popen[bytes]: + """ + Execute "cmd" inside the container and return a process object once started + """ + return Popen( + self.get_exec_args(cmd), + stdin=stdin, stdout=stdout, stderr=stderr, + ) + + +class Network: + """ + A Docker network + """ + + def __init__(self, name: str|None = None) -> None: + self._name = name or f"br{token_hex(6)}" + + def __str__(self) -> str: + return self._name + + def __repr__(self) -> str: + cls = type(self) + return f"<{cls.__module__}.{cls.__name__} {self._name}>" + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Network): + return self._name == str(other) + return self._name == other._name + + def __hash__(self) -> int: + return self._name.__hash__() + + def __enter__(self) -> Network: + self.create() + return self + + def __exit__(self, etype: type[BaseException], exc: BaseException, tb: TracebackType) -> None: + self.destroy() + + @property + def name(self) -> str: + return self._name + + def get_id(self) -> str: + return self._name + + def create(self) -> None: + """ + Create the network + """ + docker_quiet("network", "create", self._name) + + def destroy(self) -> None: + """ + Remove the network + """ + docker_quiet("network", "rm", self._name) diff --git a/tests/wp/proc.py b/tests/wp/proc.py new file mode 100644 index 0000000000000000000000000000000000000000..e368af68a13cf1428a3e62a5afca37b9229dc8c5 --- /dev/null +++ b/tests/wp/proc.py @@ -0,0 +1,158 @@ +# Copyright 2021 Dominik Sekotill +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Manage processes asynchronously +""" + +from __future__ import annotations + +import io +import logging +import os +import sys +from subprocess import DEVNULL +from subprocess import PIPE +from typing import IO +from typing import Any +from typing import Callable +from typing import Iterator +from typing import Mapping +from typing import MutableSequence +from typing import Sequence +from typing import TypeVar +from typing import Union +from typing import overload + +import trio.abc + +T = TypeVar('T') +Deserialiser = Callable[[bytes], T] + +PathLike = os.PathLike[str] +PathArg = Union[PathLike, str] +Arguments = Sequence[PathArg] +MutableArguments = MutableSequence[PathArg] +Environ = Mapping[str, str] + +_logger = logging.getLogger(__name__) + + +def coerce_args(args: Arguments) -> Iterator[str]: + """ + Ensure path-like arguments are converted to strings + """ + return (os.fspath(a) for a in args) + + +@overload +def exec_io( + cmd: Arguments, + data: bytes = b'', + deserialiser: Deserialiser[T] = ..., + **kwargs: Any, +) -> T: ... + + +@overload +def exec_io( + cmd: Arguments, + data: bytes = b'', + deserialiser: None = None, + **kwargs: Any, +) -> int: ... + + +def exec_io( + cmd: Arguments, + data: bytes = b'', + deserialiser: Deserialiser[Any]|None = None, + **kwargs: Any, +) -> Any: + """ + Execute a command, handling output asynchronously + + If data is provided it will be fed to the process' stdin. + If a deserialiser is provided it will be used to parse stdout data from the process. + + Stderr and stdout (if no deserialiser is provided) will be written to `sys.stderr` and + `sys.stdout` respectively. + + Note that the data is written, not redirected. If either `sys.stdout` or `sys.stderr` + is changed to an IO-like object with no file descriptor, this will still work. + """ + if deserialiser and 'stdout' in kwargs: + raise TypeError("Cannot provide 'deserialiser' with 'stdout' argument") + if data and 'stdin' in kwargs: + raise TypeError("Cannot provide 'data' with 'stdin' argument") + stdout: IO[str]|IO[bytes]|int = io.BytesIO() if deserialiser else kwargs.pop('stdout', sys.stdout) + stderr: IO[str]|IO[bytes]|int = kwargs.pop('stderr', sys.stderr) + _logger.debug("executing: %s", cmd) + proc = trio.run(_exec_io, cmd, data, stdout, stderr, kwargs) + if deserialiser: + assert isinstance(stdout, io.BytesIO) + return deserialiser(stdout.getvalue()) + return proc.returncode + + +async def _exec_io( + cmd: Arguments, + data: bytes, + stdout: IO[str]|IO[bytes]|int, + stderr: IO[str]|IO[bytes]|int, + kwargs: dict[str, Any], +) -> trio.Process: + proc = await trio.open_process( + [*coerce_args(cmd)], + stdin=PIPE if data else DEVNULL, + stdout=PIPE, + stderr=PIPE, + **kwargs, + ) + async with proc, trio.open_nursery() as nursery: + assert proc.stdout is not None and proc.stderr is not None + nursery.start_soon(_passthru, proc.stderr, stderr) + nursery.start_soon(_passthru, proc.stdout, stdout) + if data: + assert proc.stdin is not None + async with proc.stdin as stdin: + await stdin.send_all(data) + return proc + + +async def _passthru(in_stream: trio.abc.ReceiveStream, out_stream: IO[str]|IO[bytes]|int) -> None: + try: + if not isinstance(out_stream, int): + out_stream = out_stream.fileno() + except (OSError, AttributeError): + # cannot get file descriptor, probably a memory buffer + if isinstance(out_stream, io.BytesIO): + async def write(data: bytes) -> None: + assert isinstance(out_stream, io.BytesIO) + out_stream.write(data) + elif isinstance(out_stream, io.StringIO): + async def write(data: bytes) -> None: + assert isinstance(out_stream, io.StringIO) + out_stream.write(data.decode()) + else: + raise TypeError(f"Unknown IO type: {type(out_stream)}") + else: + # is/has a file descriptor, out_stream is now that file descriptor + async def write(data: bytes) -> None: + assert isinstance(out_stream, int) + data = memoryview(data) + remaining = len(data) + while remaining: + await trio.lowlevel.wait_writable(out_stream) + written = os.write(out_stream, data) + data = data[written:] + remaining -= written + + while True: + data = await in_stream.receive_some() + if not data: + return + await write(data)