Alpha (#1)

* add test cases for invalid arrange
* add invalid tests for act
* add tests for missing arrange and act
* add cases for assert
* add initial checks for arrange
* write checks for arrange
* add checks for act start
* add test for filename
* add integration tests
* add integration tests for docs pattern argument
* add support for custom indent size
* add tests against own repository
* add hypothesis checks
* update changelog
* add documentation
* add workflows

Author: jdkandersson

.devcontainer/devcontainer.json

@@ -0,0 +1,10 @@
+{
+    "name": "Python 3",
+    "image": "mcr.microsoft.com/devcontainers/python:0-3.11",
+    "features": {
+        "ghcr.io/devcontainers/features/python:1": {},
+        "ghcr.io/devcontainers-contrib/features/black:1": {},
+        "ghcr.io/devcontainers-contrib/features/tox:1": {},
+        "ghcr.io/devcontainers-contrib/features/isort:1": {}
+    }
+}

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+max-line-length = 99
+max-doc-length = 99
+extend-ignore = E203,W503
+per-file-ignores =
+    tests/*:D205,D400
+    flake8_test_docs.py:N802
+test-docs-pattern = given/when/then

.github/workflows/ci-cd.yaml

@@ -0,0 +1,141 @@
+name: CI-CD
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+"
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Run linting
+        run: tox -e lint
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Run testing
+        run: tox -e test
+  build:
+    runs-on: ubuntu-latest
+    needs:
+      - test
+      - lint
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install poetry
+        run: pip install poetry
+      - uses: actions/cache@v3
+        id: cache-poetry
+        with:
+          path: ~/.virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock', 'poetry.toml') }}
+      - name: Configure poetry for ci
+        run: |
+          poetry config virtualenvs.in-project false --local
+          poetry config virtualenvs.path ~/.virtualenvs --local
+      - name: Install dependencies
+        if: steps.cache-poetry.outputs.cache-hit != 'true'
+        run: |
+          poetry install
+      - name: Build packages
+        run: poetry build
+      - name: Upload artifacts for release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheel
+          path: dist/
+  release-test-pypi:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - build
+    steps:
+      - name: Retrieve packages
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: dist/
+      - name: Publish distribution 📦 to Test PyPI
+        uses: pypa/gh-action-pypi-publish@v1.6.4
+        with:
+          password: ${{ secrets.test_pypi_password }}
+          repository_url: https://test.pypi.org/legacy/
+  release-pypi:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - release-test-pypi
+    steps:
+      - name: Retrieve packages
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: dist/
+      - name: Publish distribution 📦 to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@v1.6.4
+        with:
+          password: ${{ secrets.pypi_password }}
+          repository_url: https://upload.pypi.org/legacy/
+  release-github:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - release-pypi
+    steps:
+      - name: Get version from tag
+        id: tag_name
+        run: |
+          echo ::set-output name=current_version::${GITHUB_REF#refs/tags/v}
+        shell: bash
+      - uses: actions/checkout@v3
+      - name: Get latest Changelog Entry
+        id: changelog_reader
+        uses: mindsers/changelog-reader-action@v2
+        with:
+          version: v${{ steps.tag_name.outputs.current_version }}
+          path: ./CHANGELOG.md
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.changelog_reader.outputs.version }}
+          release_name: Release ${{ steps.changelog_reader.outputs.version }}
+          body: ${{ steps.changelog_reader.outputs.changes }}
+          prerelease: ${{ steps.changelog_reader.outputs.status == 'prereleased' }}
+          draft: ${{ steps.changelog_reader.outputs.status == 'unreleased' }}

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.analysis.typeCheckingMode": "basic",
+    "python.linting.pylintEnabled": true,
+    "python.linting.enabled": true
+}

CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+## [Unreleased]
+
+## [v1.0.0] - 2022-12-23
+
+### Added
+
+- Lint checks for test docs using the arrange/act/assert pattern
+- Lint checks for longer descriptions of each stage
+- `--test-docs-pattern` argument to customise the docstring pattern
+- `--test-docs-filename-pattern` argument to customise the test file discovery
+- `--test-docs-function-pattern` argument to customise the test function
+  discovery
+- support for flake8 `--indent-size` argument
+
+[//]: # "Release links"
+[v1.0.0]: https://github.com/jdkandersson/flake8-test-docs/releases/v1.0.0

README.md

@@ -1,2 +1,181 @@
 # flake8-test-docs
-Linter that checks test docstrings for the arrange/act/assert or given/when/then structure
+
+Have you ever needed to understand a new project and started reading the tests
+only to find that you have no idea what the tests are doing? Good test
+documentation is critical during test definition and when reviewing tests
+written in the past or by someone else. This linter checks that the test
+function docstring includes a description of the test setup, execution and
+checks.
+
+## Getting Started
+
+```shell
+python -m venv venv
+source ./venv/bin/activate
+pip install flake8 flake8-test-docs
+flake8 test_source.py
+```
+
+On the following code:
+
+```Python
+# test_source.py
+def test_foo():
+    value = foo()
+    assert value == "bar"
+```
+
+This will produce warnings such as:
+
+```shell
+flake8 test_source.py
+test_source.py:2:1: TDO001 Docstring not defined on test function, more information: https://github.com/jdkandersson/flake8-test-docs#fix-tdo001
+```
+
+This can be resolved by changing the code to:
+
+```Python
+# test_source.py
+def test_foo():
+    """
+    arrange: given foo that returns bar
+    act: when foo is called
+    assert: then bar is returned
+    """
+    value = foo()
+    assert value == "bar"
+```
+
+## Configuration
+
+The plugin adds the following configurations to `flake8`:
+
+* `--test-docs-patter`: The pattern the test documentation should follow,
+  e.g., `given/when/then`. Defaults to `arrange/act/assert`.
+* `--test-docs-filename-pattern`: The filename pattern for test files. Defaults
+  to `test_.*.py`.
+* `--test-docs-function-pattern`: The function pattern for test functions.
+  Defaults to `test_.*`.
+
+
+## Rules
+
+A few rules have been defined to allow for selective suppression:
+
+* `TDO001`: checks that test functions have a docstring.
+* `TDO002`: checks that test function docstrings follow the documentation
+  pattern.
+
+### Fix TDO001
+
+This linting rule is triggered by a test function in a test file without a
+docstring. For example:
+
+```Python
+# test_source.py
+def test_foo():
+    result = foo()
+    assert result == "bar"
+```
+
+This example can be fixed by:
+
+```Python
+# test_source.py
+def test_foo():
+    """
+    arrange: given foo that returns bar
+    act: when foo is called
+    assert: then bar is returned
+    """
+    result = foo()
+    assert result == "bar"
+```
+
+### Fix TDO002
+
+This linting rule is triggered by a test function in a test file with a
+docstring that doesn't follow the documentation pattern. For example:
+
+```Python
+# test_source.py
+def test_foo():
+    """Test foo."""
+    result = foo()
+    assert result == "bar"
+```
+
+This example can be fixed by:
+
+```Python
+# test_source.py
+def test_foo():
+    """
+    arrange: given foo that returns bar
+    act: when foo is called
+    assert: then bar is returned
+    """
+    result = foo()
+    assert result == "bar"
+```
+
+The message of the linting rule should give you the specific problem with the
+documentation. In general, the pattern is:
+
+* start on the second line with the same indentation is the start of the
+  docstring
+* the starting line should begin with `arrange:` (or whatever was set using
+  `--test-docs-patter`) followed by at least some words describing the test
+  setup
+* long test setup descriptions can be broken over multiple lines by indenting
+  the lines after the first by one level (e.g., 4 spaces by default)
+* this is followed by similar sections starting with `act:` describing the test
+  execution and `assert:` describing the checks
+* the last line should be indented the same as the start of the docstring
+
+Below are some valid examples. Starting with a vanilla example:
+
+```Python
+# test_source.py
+def test_foo():
+    """
+    arrange: given foo that returns bar
+    act: when foo is called
+    assert: then bar is returned
+    """
+    result = foo()
+    assert result == "bar"
+```
+
+Here is an example where the test function is in a nested scope:
+
+```Python
+# test_source.py
+class TestSuite:
+
+    def test_foo():
+        """
+        arrange: given foo that returns bar
+        act: when foo is called
+        assert: then bar is returned
+        """
+        result = foo()
+        assert result == "bar"
+```
+
+Here is an example where each of the descriptions go over multiple lines:
+
+```Python
+# test_source.py
+def test_foo():
+    """
+    arrange: given foo
+        that returns bar
+    act: when foo
+        is called
+    assert: then bar
+        is returned
+    """
+    result = foo()
+    assert result == "bar"
+```