Initial commit

Author: PPB InfoSec Engineering

.dockerignore

@@ -0,0 +1,11 @@
+# file set with Dockerfile.mysql (for run_tests script) in mind - review if used for anything else
+.jenkins
+**/.coverage
+**/__pycache__
+**/*.pyc
+dist
+*.egg-info
+.git
+.reports
+.dockerignore
+**/.pytest_cache

.gitignore

@@ -0,0 +1,8 @@
+*.pyc
+__pycache__
+/dist
+*.egg-info
+.coverage
+.reports
+.pytest_cache
+.vscode

.jenkins/Dockerfile.tests

@@ -0,0 +1,8 @@
+FROM docker.app.betfair/appsec/base/python:3.9-slim-buster
+
+RUN apt update
+RUN apt install -y libmariadb-dev build-essential
+COPY . /app
+
+WORKDIR /app/testapp
+RUN pip install -r requirements.txt

.jenkins/ci_build

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+cd ..
+
+if [ ! -e /.dockerenv ]; then
+    # not in container
+    .jenkins/run_tests
+    docker run -v $(pwd):/work \
+               -w /work \
+               -e ARTIFACTORY_USER \
+               -e ARTIFACTORY_PASSWORD \
+               docker.app.betfair/appsec/base/python:3.9-slim-buster \
+               /work/.jenkins/ci_build
+    exit 0
+fi
+
+export TWINE_USERNAME=${ARTIFACTORY_USER}
+export TWINE_PASSWORD=${ARTIFACTORY_PASSWORD}
+export TWINE_NON_INTERACTIVE=1
+
+# check if package-version already uploaded (pypi.org blocks but ppb artifactory does not)
+PN=$(python setup.py --name)
+PV=$(python setup.py --version)
+# requests required by twine anyway, not a waste :)
+pip install 'requests<3'
+python <<EOF
+import requests, sys
+c=requests.get('https://artifactory-prd.prd.betfair/artifactory/api/pypi/pypi/$PN/$PV/').status_code
+if c != 404:
+  print(f'ERROR: version already exists? [{c}]', file=sys.stderr)
+  exit(2)
+EOF
+
+pip install twine
+
+
+python setup.py sdist
+python -m twine upload --repository-url https://artifactory-prd.prd.betfair/artifactory/api/pypi/pypi dist/*

.jenkins/mr_test

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+./run_tests

.jenkins/run_tests

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+cd ..
+
+create_mysql() {
+    # to make sure the latest is in JKS cache... can comment out sometime in the future :P
+    MYSQLDOCKER=$(docker run \
+		           -e MYSQL_ROOT_PASSWORD=root \
+		           --rm -d \
+				   --health-cmd "mysqladmin ping" \
+				   --health-interval 10s \
+				   --health-timeout 5s \
+				   --health-retries=5 \
+				   mysql:5)
+    trap "rm_mysql" EXIT
+}
+
+wait_mysql() {
+    echo "Waiting for mysql container (${MYSQLDOCKER})..."
+    while [ 1 ]; do
+        docker inspect --format='{{.State.Health}}' ${MYSQLDOCKER} | grep -q '^{healthy' && break
+        sleep 0.5
+    done
+}
+
+rm_mysql() {
+    echo "Removing mysql container"
+    docker rm -f ${MYSQLDOCKER} > /dev/null
+}
+
+if [ ! -e /.dockerenv ]; then
+    # not in container
+    create_mysql
+    PACKAGE_NAME=$(cat setup.cfg | grep ^"name = " | cut -d ' ' -f3-)
+    docker build -t ppb_${PACKAGE_NAME}_tests_image -f .jenkins/Dockerfile.tests .
+    wait_mysql
+    docker run -v $(pwd):/work \
+               -w /work \
+               --link ${MYSQLDOCKER}:mysql \
+               -e DBCLEANUP_TEST_MYSQL_HOST=mysql \
+               -e DBCLEANUP_TEST_MYSQL_PORT=3306 \
+               -e COVERAGE_REPORT_XML=/work/.reports/coverage.xml \
+               -e JUNIT_REPORT_XML=/work/.reports/junit.xml \
+               ppb_${PACKAGE_NAME}_tests_image \
+               /work/.jenkins/run_tests
+    exit 0
+fi
+
+python -m pytest testapp \
+       --cov \
+       --no-cov-on-fail \
+       --junit-xml ${JUNIT_REPORT_XML:-/dev/null} \
+       --cov-report xml:${COVERAGE_REPORT_XML:-/dev/null} \
+       --cov-report term-missing \
+       --ds testapp.settings_mysql

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2021 PaddyPowerBetfair
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include LICENSE

Makefile

@@ -0,0 +1,56 @@
+# test setup based on https://github.com/fopina/django-bulk-update-or-create/blob/master/Makefile
+TEST_CONTAINER := django-dbcleanup-test
+
+.PHONY: style
+style:
+	black --target-version=py36 \
+	      --line-length=120 \
+		  --skip-string-normalization \
+		  dbcleanup testapp setup.py
+
+.PHONY: style_check
+style_check:
+	black --target-version=py36 \
+	      --line-length=120 \
+		  --skip-string-normalization \
+		  --check \
+		  dbcleanup testapp setup.py
+
+.PHONY: startmysql
+startmysql:
+	@docker inspect ${TEST_CONTAINER}-mysql | grep -q '"Running": true' || \
+		docker run --name ${TEST_CONTAINER}-mysql \
+		           -e MYSQL_ROOT_PASSWORD=root \
+		           --rm -p 8877:3306 -d \
+				   --health-cmd "mysqladmin ping" \
+				   --health-interval 10s \
+				   --health-timeout 5s \
+				   --health-retries=5 \
+				   mysql:8
+	until [ "`docker inspect -f {{.State.Health.Status}} ${TEST_CONTAINER}-mysql`" == "healthy" ]; do sleep 0.1; done;
+
+startpg:
+	@docker inspect ${TEST_CONTAINER}-pg | grep -q '"Running": true' || \
+		docker run --name ${TEST_CONTAINER}-pg \
+		           -e POSTGRES_USER=postgres \
+          		   -e POSTGRES_PASSWORD=postgres \
+				   -e POSTGRES_DB=postgres \
+		           --rm -p 8878:5432 -d \
+				   --health-cmd pg_isready \
+				   --health-interval 10s \
+				   --health-timeout 5s \
+				   --health-retries 5 \
+				   postgres:10
+	until [ "`docker inspect -f {{.State.Health.Status}} ${TEST_CONTAINER}-pg`" == "healthy" ]; do sleep 0.1; done;
+
+testpg: startpg
+	DJANGO_SETTINGS_MODULE="testapp.settings_postgresql" \
+		testapp/manage.py test $${TEST_ARGS:-tests}
+
+test: startmysql
+	testapp/manage.py test $${TEST_ARGS:-tests}
+
+coverage: startmysql
+	PYTHONPATH="testapp" \
+		python -b -W always -m coverage run testapp/manage.py test $${TEST_ARGS:-tests}
+	coverage report

README.md

@@ -0,0 +1,7 @@
+# django-dbcleanup
+
+Easily monitor database usage - and clean it up (based on your django models)
+
+## ToDo
+
+Gitlab Issues used to track it