From 0eabc3bd9509c964b6d6b8e19ebd346d166bc37a Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 25 Jun 2025 15:41:31 -0400 Subject: [PATCH] Refactor --- django_mongodb_cli/repo.py | 601 +++++++++--------- django_mongodb_cli/{config.py => settings.py} | 42 +- django_mongodb_cli/utils.py | 502 +++++++++++---- qe.py | 2 +- .../allauth_apps.py => test/apps/allauth.py | 0 .../apps/debug_toolbar.py | 0 .../apps/django_filter.py | 0 .../apps/django_mongodb_extensions.py | 0 .../apps/rest_framework.py | 0 .../wagtail_apps.py => test/apps/wagtail.py | 0 .../settings/allauth.py | 0 .../settings/debug_toolbar.py | 0 .../settings/django.py | 13 +- .../settings/django_filter.py | 0 .../settings/django_mongodb_extensions.py | 0 .../settings/rest_framework.py | 0 .../settings/rest_framework_migrations.py | 0 .../settings/wagtail.py | 0 18 files changed, 690 insertions(+), 470 deletions(-) rename django_mongodb_cli/{config.py => settings.py} (86%) rename config/allauth/allauth_apps.py => test/apps/allauth.py (100%) rename config/debug_toolbar/debug_toolbar_apps.py => test/apps/debug_toolbar.py (100%) rename config/extensions/debug_toolbar_apps.py => test/apps/django_filter.py (100%) rename config/filter/filter_apps.py => test/apps/django_mongodb_extensions.py (100%) rename config/rest_framework/rest_framework_apps.py => test/apps/rest_framework.py (100%) rename config/wagtail/wagtail_apps.py => test/apps/wagtail.py (100%) rename config/allauth/allauth_settings.py => test/settings/allauth.py (100%) rename config/debug_toolbar/debug_toolbar_settings.py => test/settings/debug_toolbar.py (100%) rename config/django/django_settings.py => test/settings/django.py (67%) rename config/filter/filter_settings.py => test/settings/django_filter.py (100%) rename config/extensions/debug_toolbar_settings.py => test/settings/django_mongodb_extensions.py (100%) rename config/rest_framework/rest_framework_settings.py => test/settings/rest_framework.py (100%) rename config/rest_framework/rest_framework_migrate.py => test/settings/rest_framework_migrations.py (100%) rename config/wagtail/wagtail_settings.py => test/settings/wagtail.py (100%) diff --git a/django_mongodb_cli/repo.py b/django_mongodb_cli/repo.py index aa58804..50c0be3 100644 --- a/django_mongodb_cli/repo.py +++ b/django_mongodb_cli/repo.py @@ -2,16 +2,19 @@ import subprocess import click -from .config import test_settings_map +from rich import print as rprint +from black import format_str, Mode + +from .settings import test_settings_map from .utils import ( + clone_repo, copy_mongo_apps, copy_mongo_migrations, copy_mongo_settings, get_management_command, get_repos, - repo_clone, - repo_install, - repo_status, + get_status, + install_package, ) @@ -30,6 +33,15 @@ def __repr__(self): pass_repo = click.make_pass_decorator(Repo) +def get_repo_name_map(repos, url_pattern): + """Return a dict mapping repo_name to repo_url from a list of repo URLs.""" + return { + os.path.basename(url_pattern.search(url).group(0)): url + for url in repos + if url_pattern.search(url) + } + + @click.group(invoke_without_command=True) @click.option( "-l", @@ -38,18 +50,18 @@ def __repr__(self): help="List all repositories in `pyproject.toml`.", ) @click.pass_context -def repo(ctx, list_repos): +def repo(context, list_repos): """ Run Django fork and third-party library tests. """ - ctx.obj = Repo() + context.obj = Repo() repos, url_pattern, branch_pattern = get_repos("pyproject.toml") if list_repos: for repo_entry in repos: click.echo(repo_entry) return - if ctx.invoked_subcommand is None: - click.echo(ctx.get_help()) + if context.invoked_subcommand is None: + click.echo(context.get_help()) @repo.command() @@ -58,378 +70,333 @@ def repo(ctx, list_repos): "-a", "--all-repos", is_flag=True, + help="Clone all repositories listed in pyproject.toml.", ) @click.option( "-i", "--install", is_flag=True, + help="Install repository after cloning.", ) @click.pass_context @pass_repo -def clone(repo, ctx, repo_names, all_repos, install): - """Clone and install repositories from `pyproject.toml`.""" +def clone(repo, context, repo_names, all_repos, install): + """Clone and optionally install repositories from pyproject.toml.""" repos, url_pattern, branch_pattern = get_repos("pyproject.toml") + repo_name_map = get_repo_name_map(repos, url_pattern) + # If specific repo names are given if repo_names: - for repo_name in repo_names: - not_found = set() - for repo_entry in repos: - if ( - os.path.basename(url_pattern.search(repo_entry).group(0)) - == repo_name - ): - repo_clone(repo_entry, url_pattern, branch_pattern, repo) - if install: - clone_path = os.path.join(ctx.obj.home, repo_name) - if os.path.exists(clone_path): - repo_install(clone_path) - return - else: - not_found.add(repo_name) - click.echo(f"Repository '{not_found.pop()}' not found.") + not_found = [] + for name in repo_names: + repo_url = repo_name_map.get(name) + if repo_url: + clone_repo(repo_url, url_pattern, branch_pattern, repo) + if install: + clone_path = os.path.join(context.obj.home, name) + if os.path.exists(clone_path): + install_package(clone_path) + else: + not_found.append(name) + if not_found: + for name in not_found: + click.echo(f"Repository '{name}' not found.") return + # If -a/--all-repos is given if all_repos: click.echo(f"Cloning {len(repos)} repositories...") - for repo_entry in repos: - repo_clone(repo_entry, url_pattern, branch_pattern, repo) + for name, repo_url in repo_name_map.items(): + clone_repo(repo_url, url_pattern, branch_pattern, repo) + if install: + clone_path = os.path.join(context.obj.home, name) + if os.path.exists(clone_path): + install_package(clone_path) return - if ctx.args == []: - click.echo(ctx.get_help()) - - -# @repo.command() -# @click.argument("repo_names", nargs=-1) -# @click.option( -# "-a", -# "--all-repos", -# is_flag=True, -# ) -# @click.pass_context -# @pass_repo -# def update(repo, ctx, repo_names, all_repos): -# """Update cloned repositories with `git pull`.""" -# repos, url_pattern, _ = get_repos("pyproject.toml") -# if repo_names: -# for repo_name in repo_names: -# for repo_entry in repos: -# if ( -# os.path.basename(url_pattern.search(repo_entry).group(0)) -# == repo_name -# ): -# repo_update(repo_entry, url_pattern, repo) -# return -# click.echo(f"Repository '{repo_name}' not found.") -# return -# -# if all_repos: -# click.echo(f"Updating {len(repos)} repositories...") -# for repo_entry in repos: -# repo_update(repo_entry, url_pattern, repo) -# return -# -# if ctx.args == []: -# click.echo(ctx.get_help()) -# + # If no args/options, show help + click.echo(context.get_help()) @repo.command(context_settings={"ignore_unknown_options": True}) @click.argument("repo_name", required=False) @click.argument("args", nargs=-1) @click.pass_context -def makemigrations( - ctx, - repo_name, - args, -): - """Run `makemigrations` for cloned repositories.""" +def makemigrations(context, repo_name, args): + """Run `makemigrations` for a cloned repository.""" + repos, url_pattern, _ = get_repos("pyproject.toml") - repos, url_pattern, branch_pattern = get_repos("pyproject.toml") if repo_name: - for repo_entry in repos: - url_match = url_pattern.search(repo_entry) - if url_match: - repo_url = url_match.group(0) - if ( - repo_name in test_settings_map.keys() - and repo_name == os.path.basename(repo_url) - ): - try: - copy_mongo_apps(repo_name) - copy_mongo_settings( - test_settings_map[repo_name]["settings"]["migrations"][ - "source" - ], - test_settings_map[repo_name]["settings"]["migrations"][ - "target" - ], - ) - except FileNotFoundError: - click.echo( - click.style( - f"Settings for '{repo_name}' not found.", fg="red" - ) - ) - return - command = get_management_command("makemigrations") - command.extend( - [ - "--settings", - test_settings_map[repo_name]["settings"]["module"][ - "migrations" - ], - ] - ) - if not repo_name == "django-filter": - command.extend( - [ - "--pythonpath", - os.path.join( - os.getcwd(), - test_settings_map[repo_name]["test_dir"], - ), - ] - ) - click.echo(f"Running command {' '.join(command)} {' '.join(args)}") - subprocess.run(command + [*args]) + repo_name_map = get_repo_name_map(repos, url_pattern) + # Check if repo exists + if repo_name not in repo_name_map or repo_name not in test_settings_map: + click.echo(click.style(f"Repository '{repo_name}' not found.", fg="red")) + return + + # Try settings copy + try: + copy_mongo_apps(repo_name) + copy_mongo_settings( + test_settings_map[repo_name]["settings"]["migrations"]["source"], + test_settings_map[repo_name]["settings"]["migrations"]["target"], + ) + except FileNotFoundError: + click.echo(click.style(f"Settings for '{repo_name}' not found.", fg="red")) + return + + command = get_management_command("makemigrations") + command.extend( + [ + "--settings", + test_settings_map[repo_name]["settings"]["module"]["migrations"], + ] + ) + if repo_name != "django-filter": + command.extend( + [ + "--pythonpath", + os.path.join(os.getcwd(), test_settings_map[repo_name]["test_dir"]), + ] + ) + if args: + command.extend(args) + click.echo(f"Running command: {' '.join(command)}") + subprocess.run(command) return - if ctx.args == []: - click.echo(ctx.get_help()) + + # No repo_name provided, show help + click.echo(context.get_help()) @repo.command() @click.argument("repo_name", required=False) @click.argument("modules", nargs=-1) @click.option("-k", "--keyword", help="Filter tests by keyword") -@click.option("-l", "--list-tests", help="List tests", is_flag=True) -@click.option("-s", "--show", help="Show settings", is_flag=True) +@click.option("-l", "--list-tests", is_flag=True, help="List tests") +@click.option("-s", "--show", is_flag=True, help="Show settings") +@click.option("--keepdb", is_flag=True, help="Keep db") @click.pass_context -def test( - ctx, - repo_name, - modules, - keyword, - list_tests, - show, -): - """ - Run tests for Django fork and third-party libraries. - """ - repos, url_pattern, branch_pattern = get_repos("pyproject.toml") +def test(context, repo_name, modules, keyword, list_tests, show, keepdb): + """Run tests for Django fork and third-party libraries.""" + repos, url_pattern, _ = get_repos("pyproject.toml") + repo_name_map = get_repo_name_map(repos, url_pattern) + if repo_name: - # Show test settings + if repo_name not in repo_name_map or repo_name not in test_settings_map: + click.echo( + click.style( + f"Repository/settings for '{repo_name}' not found.", fg="red" + ) + ) + return + if show: click.echo(f"āš™ļø Test settings for šŸ“¦ {repo_name}:") - if repo_name in test_settings_map.keys(): - from rich import print - from black import format_str as format - from black import Mode + settings_dict = dict(sorted(test_settings_map[repo_name].items())) + formatted = format_str(str(settings_dict), mode=Mode()) + rprint(formatted) + return + + settings = test_settings_map[repo_name] + test_dirs = settings.get("test_dirs", []) + if list_tests: + for test_dir in test_dirs: + click.echo(f"šŸ“‚ {test_dir}") + try: + test_modules = sorted(os.listdir(test_dir)) + for module in test_modules: + if module not in ("__pycache__", "__init__.py"): + click.echo(click.style(f" ā””ā”€ā”€ {module}", fg="green")) + click.echo() + except FileNotFoundError: + click.echo( + click.style(f"Directory '{test_dir}' not found.", fg="red") + ) + return + + # Prepare and copy settings, etc. + if "settings" in settings: + repo_dir = os.path.join(context.obj.home, repo_name) + if not os.path.exists(repo_dir): click.echo( - print( - format( - str(dict(sorted(test_settings_map[repo_name].items()))), - mode=Mode(), - ) + click.style( + f"Repository '{repo_name}' not found on disk.", fg="red" ) ) return - else: - click.echo( - click.style(f"Settings for '{repo_name}' not found.", fg="red") + copy_mongo_settings( + settings["settings"]["test"]["source"], + settings["settings"]["test"]["target"], + ) + else: + click.echo(click.style(f"Settings for '{repo_name}' not found.", fg="red")) + return + + command = [settings["test_command"]] + copy_mongo_migrations(repo_name) + copy_mongo_apps(repo_name) + + if ( + settings["test_command"] == "./runtests.py" + and repo_name != "django-rest-framework" + ): + command.extend( + [ + "--settings", + settings["settings"]["module"]["test"], + "--parallel", + "1", + "--verbosity", + "3", + "--debug-sql", + "--noinput", + ] + ) + if keyword: + command.extend(["-k", keyword]) + if keepdb: + command.append("--keepdb") + + if repo_name in { + "django-debug-toolbar", + "django-allauth", + "django-mongodb-extensions", + }: + os.environ["DJANGO_SETTINGS_MODULE"] = settings["settings"]["module"][ + "test" + ] + command.extend( + [ + "--continue-on-collection-errors", + "--html=report.html", + "--self-contained-html", + ] + ) + elif repo_name == "mongo-python-driver": + command.extend(["test", "-s"]) + + command.extend(modules) + if os.environ.get("DJANGO_SETTINGS_MODULE"): + click.echo( + click.style( + f"DJANGO_SETTINGS_MODULE={os.environ['DJANGO_SETTINGS_MODULE']}", + fg="blue", ) - return - for repo_entry in repos: - url_match = url_pattern.search(repo_entry) - repo_url = url_match.group(0) - if repo_name == os.path.basename(repo_url): - if repo_name in test_settings_map.keys(): - test_dirs = test_settings_map[repo_name]["test_dirs"] - if list_tests: - for test_dir in test_dirs: - click.echo(f"šŸ“‚ {test_dir}") - try: - modules = sorted(os.listdir(test_dir)) - count = 0 - for module in modules: - count += 1 - if ( - module != "__pycache__" - and module != "__init__.py" - ): - if count == len(modules): - click.echo( - click.style( - f" ā””ā”€ā”€ {module}", fg="green" - ) - ) - else: - click.echo( - click.style( - f" ā””ā”€ā”€ {module}", fg="green" - ) - ) - click.echo() - except FileNotFoundError: - click.echo( - click.style( - f"Directory '{test_dir}' not found.", fg="red" - ) - ) - return - # Copy settings for test run - if "settings" in test_settings_map[repo_name]: - if os.path.exists(os.path.join(ctx.obj.home, repo_name)): - copy_mongo_settings( - test_settings_map[repo_name]["settings"]["test"][ - "source" - ], - test_settings_map[repo_name]["settings"]["test"][ - "target" - ], - ) - else: - click.echo( - click.style( - f"Repository '{repo_name}' not found.", fg="red" - ) - ) - return - command = [test_settings_map[repo_name]["test_command"]] - copy_mongo_migrations(repo_name) - copy_mongo_apps(repo_name) - - # Configure test command - if ( - test_settings_map[repo_name]["test_command"] == "./runtests.py" - and repo_name != "django-rest-framework" - ): - command.extend( - [ - "--settings", - test_settings_map[repo_name]["settings"]["module"][ - "test" - ], - "--parallel", - "1", - "--verbosity", - "3", - "--debug-sql", - "--noinput", - "--keepdb", - ] - ) - if keyword: - command.extend(["-k", keyword]) - if ( - repo_name == "django-debug-toolbar" - or repo_name == "django-allauth" - or repo_name == "django-mongodb-extensions" - ): - os.environ["DJANGO_SETTINGS_MODULE"] = test_settings_map[ - repo_name - ]["settings"]["module"]["test"] - - command.extend( - [ - "--continue-on-collection-errors", - "--html=report.html", - "--self-contained-html", - ] - ) - elif repo_name == "mongo-python-driver": - command.extend(["test", "-s"]) - - command.extend(modules) - if os.environ.get("DJANGO_SETTINGS_MODULE"): - click.echo( - click.style( - f"DJANGO_SETTINGS_MODULE={os.environ['DJANGO_SETTINGS_MODULE']}", - fg="blue", - ) - ) - click.echo(click.style(f"Running {' '.join(command)}", fg="blue")) - # Run test command - subprocess.run( - command, cwd=test_settings_map[repo_name]["test_dir"] - ) - else: - click.echo(f"Settings for '{repo_name}' not found.") + ) + click.echo(click.style(f"Running {' '.join(command)}", fg="blue")) + subprocess.run(command, cwd=settings["test_dir"]) return - if ctx.args == []: - click.echo(ctx.get_help()) + + # No repo_name, show help + click.echo(context.get_help()) @repo.command() @click.argument("repo_names", nargs=-1) -@click.option( - "-a", - "--all-repos", - is_flag=True, -) -@click.option( - "-r", - "--reset", - is_flag=True, -) -@click.option( - "-d", - "--diff", - is_flag=True, -) -@click.option( - "-b", - "--branch", - is_flag=True, -) -@click.option( - "-u", - "--update", - is_flag=True, -) +@click.option("-a", "--all-repos", is_flag=True, help="Status for all repositories") +@click.option("-r", "--reset", is_flag=True, help="Reset") +@click.option("-d", "--diff", is_flag=True, help="Show diff") +@click.option("-b", "--branch", is_flag=True, help="Show branch") +@click.option("-u", "--update", is_flag=True, help="Update repos") +@click.option("-l", "--log", is_flag=True, help="Show log") @click.pass_context @pass_repo -def status(repo, ctx, repo_names, all_repos, reset, diff, branch, update): +def status(repo, context, repo_names, all_repos, reset, diff, branch, update, log): """Repository status.""" repos, url_pattern, _ = get_repos("pyproject.toml") + repo_name_map = get_repo_name_map(repos, url_pattern) + + # Status for specified repo names if repo_names: + not_found = [] for repo_name in repo_names: - not_found = set() - for repo_entry in repos: - if ( - os.path.basename(url_pattern.search(repo_entry).group(0)) - == repo_name - ): - repo_status( - repo_entry, - url_pattern, - repo, - reset=reset, - diff=diff, - branch=branch, - update=update, - ) - return - else: - not_found.add(repo_name) - if not_found: - click.echo(f"Repository '{not_found.pop()}' not found.") + repo_url = repo_name_map.get(repo_name) + if repo_url: + get_status( + repo_url, + url_pattern, + repo, + reset=reset, + diff=diff, + branch=branch, + update=update, + log=log, + ) + else: + not_found.append(repo_name) + for name in not_found: + click.echo(f"Repository '{name}' not found.") return + # Status for all repos if all_repos: click.echo(f"Status of {len(repos)} repositories...") - for repo_entry in repos: - repo_status( - repo_entry, + for repo_name, repo_url in repo_name_map.items(): + get_status( + repo_url, url_pattern, repo, reset=reset, diff=diff, branch=branch, update=update, + log=log, + ) + return + + # Show help if nothing selected + click.echo(context.get_help()) + + +@repo.command() +@click.argument("repo_names", nargs=-1) +@click.option("-a", "--all-repos", is_flag=True, help="Update all repositories") +@click.pass_context +@pass_repo +def update(repo, context, repo_names, all_repos): + """Update repositories (like 'status -u').""" + repos, url_pattern, _ = get_repos("pyproject.toml") + repo_name_map = get_repo_name_map(repos, url_pattern) + + if all_repos and repo_names: + click.echo("Cannot specify both repo names and --all-repos") + return + + if all_repos: + click.echo(f"Updating {len(repo_name_map)} repositories...") + for repo_name, repo_url in repo_name_map.items(): + get_status( + repo_url, + url_pattern, + repo, + update=True, # Just like '-u' in status + reset=False, + diff=False, + branch=False, + log=False, ) return - if ctx.args == []: - click.echo(ctx.get_help()) + if repo_names: + not_found = [] + for repo_name in repo_names: + repo_url = repo_name_map.get(repo_name) + if repo_url: + get_status( + repo_url, + url_pattern, + repo, + update=True, + reset=False, + diff=False, + branch=False, + log=False, + ) + else: + not_found.append(repo_name) + for name in not_found: + click.echo(f"Repository '{name}' not found.") + return + + click.echo(context.get_help()) diff --git a/django_mongodb_cli/config.py b/django_mongodb_cli/settings.py similarity index 86% rename from django_mongodb_cli/config.py rename to django_mongodb_cli/settings.py index b276c6a..6fd9e2b 100644 --- a/django_mongodb_cli/config.py +++ b/django_mongodb_cli/settings.py @@ -25,11 +25,11 @@ }, "settings": { "test": { - "source": join("config", "django", "django_settings.py"), + "source": join("test", "settings", "django.py"), "target": join("src", "django", "tests", "mongo_settings.py"), }, "migrations": { - "source": join("config", "django", "django_settings.py"), + "source": join("test", "settings", "django.py"), "target": join("src", "django", "tests", "mongo_settings.py"), }, "module": { @@ -44,7 +44,7 @@ }, "django-filter": { "apps_file": { - "source": join("config", "filter", "filter_apps.py"), + "source": join("test", "apps", "django_filter.py"), "target": join("src", "django-filter", "tests", "mongo_apps.py"), }, "test_command": "./runtests.py", @@ -61,11 +61,11 @@ "clone_dir": join("src", "django-filter"), "settings": { "test": { - "source": join("config", "filter", "filter_settings.py"), + "source": join("test", "settings", "django_filter.py"), "target": join("src", "django-filter", "tests", "settings.py"), }, "migrations": { - "source": join("config", "filter", "filter_settings.py"), + "source": join("test", "settings", "django_filter.py"), "target": join("src", "django-filter", "tests", "settings.py"), }, "module": { @@ -77,7 +77,7 @@ }, "django-rest-framework": { "apps_file": { - "source": join("config", "rest_framework", "rest_framework_apps.py"), + "source": join("test", "apps", "rest_framework.py"), "target": join("src", "django-rest-framework", "tests", "mongo_apps.py"), }, "migrations_dir": { @@ -89,13 +89,11 @@ "clone_dir": join("src", "django-rest-framework"), "settings": { "test": { - "source": join( - "config", "rest_framework", "rest_framework_settings.py" - ), + "source": join("test", "settings", "rest_framework.py"), "target": join("src", "django-rest-framework", "tests", "conftest.py"), }, "migrations": { - "source": join("config", "rest_framework", "rest_framework_migrate.py"), + "source": join("test", "settings", "rest_framework_migrations.py"), "target": join("src", "django-rest-framework", "tests", "conftest.py"), }, "module": { @@ -107,7 +105,7 @@ }, "wagtail": { "apps_file": { - "source": join("config", "wagtail", "wagtail_apps.py"), + "source": join("test", "apps", "wagtail.py"), "target": join("src", "wagtail", "wagtail", "test", "mongo_apps.py"), }, "migrations_dir": { @@ -124,13 +122,13 @@ "clone_dir": join("src", "wagtail"), "settings": { "test": { - "source": join("config", "wagtail", "wagtail_settings.py"), + "source": join("test", "settings", "wagtail.py"), "target": join( "src", "wagtail", "wagtail", "test", "mongo_settings.py" ), }, "migrations": { - "source": join("config", "wagtail", "settings_wagtail.py"), + "source": join("test", "settings", "wagtail.py"), "target": join( "src", "wagtail", "wagtail", "test", "mongo_settings.py" ), @@ -147,7 +145,7 @@ }, "django-debug-toolbar": { "apps_file": { - "source": join("config", "debug_toolbar", "debug_toolbar_apps.py"), + "source": join("test", "apps", "debug_toolbar.py"), "target": join( "src", "django-debug-toolbar", "debug_toolbar", "mongo_apps.py" ), @@ -157,13 +155,13 @@ "clone_dir": join("src", "django-debug-toolbar"), "settings": { "test": { - "source": join("config", "debug_toolbar", "debug_toolbar_settings.py"), + "source": join("test", "settings", "debug_toolbar.py"), "target": join( "src", "django-debug-toolbar", "debug_toolbar", "mongo_settings.py" ), }, "migrations": { - "source": join("config", "debug_toolbar", "debug_toolbar_settings.py"), + "source": join("test", "settings", "debug_toolbar.py"), "target": join( "src", "django-debug-toolbar", "debug_toolbar", "mongo_settings.py" ), @@ -179,7 +177,7 @@ }, "django-mongodb-extensions": { "apps_file": { - "source": join("config", "extensions", "debug_toolbar_apps.py"), + "source": join("test", "apps", "django_mongodb_extensions.py"), "target": join( "src", "django-mongodb-extensions", @@ -192,7 +190,7 @@ "clone_dir": join("src", "django-mongodb-extensions"), "settings": { "test": { - "source": join("config", "extensions", "debug_toolbar_settings.py"), + "source": join("test", "settings", "django_mongodb_extensions.py"), "target": join( "src", "django-mongodb-extensions", @@ -201,7 +199,7 @@ ), }, "migrations": { - "source": join("config", "extensions", "debug_toolbar_settings.py"), + "source": join("test", "extensions", "debug_toolbar_settings.py"), "target": join( "src", "django-mongodb-extensions", @@ -225,16 +223,16 @@ "test_dir": join("src", "django-allauth"), "clone_dir": join("src", "django-allauth"), "apps_file": { - "source": join("config", "allauth", "allauth_apps.py"), + "source": join("test", "apps", "allauth.py"), "target": join("src", "django-allauth", "allauth", "mongo_apps.py"), }, "settings": { "test": { - "source": join("config", "allauth", "allauth_settings.py"), + "source": join("test", "settings", "allauth.py"), "target": join("src", "django-allauth", "allauth", "mongo_settings.py"), }, "migrations": { - "source": join("config", "allauth", "allauth_settings.py"), + "source": join("test", "settings", "allauth.py"), "target": join("src", "django-allauth", "allauth", "mongo_settings.py"), }, "module": { diff --git a/django_mongodb_cli/utils.py b/django_mongodb_cli/utils.py index 92c84af..e203e31 100644 --- a/django_mongodb_cli/utils.py +++ b/django_mongodb_cli/utils.py @@ -8,49 +8,132 @@ import subprocess -from .config import test_settings_map +from .settings import test_settings_map def copy_mongo_apps(repo_name): - """Copy the appropriate mongo_apps file based on the repo name.""" - if "apps_file" in test_settings_map[repo_name] and repo_name != "django": + """ + Copy the appropriate mongo_apps file based on the repo name. + + Requires test_settings_map[repo_name]['apps_file']['source'] and + test_settings_map[repo_name]['apps_file']['target'] to be defined. + """ + settings = test_settings_map.get(repo_name) + if not settings or "apps_file" not in settings or repo_name == "django": + return # Nothing to copy for this repo + + apps_file = settings["apps_file"] + source = apps_file.get("source") + target = apps_file.get("target") + + if not source or not target: click.echo( click.style( - f"Copying {os.path.join(test_settings_map[repo_name]['apps_file']['source'])} to {os.path.join(test_settings_map[repo_name]['apps_file']['target'])}", - fg="blue", + f"[copy_mongo_apps] Source or target path missing for '{repo_name}'.", + fg="yellow", ) ) - shutil.copyfile( - os.path.join(test_settings_map[repo_name]["apps_file"]["source"]), - os.path.join(test_settings_map[repo_name]["apps_file"]["target"]), + return + + try: + click.echo(click.style(f"Copying {source} ā†’ {target}", fg="blue")) + shutil.copyfile(source, target) + click.echo( + click.style(f"Copied {source} to {target} successfully.", fg="green") ) + except FileNotFoundError as e: + click.echo(click.style(f"File not found: {e.filename}", fg="red")) + except Exception as e: + click.echo(click.style(f"Failed to copy: {e}", fg="red")) def copy_mongo_migrations(repo_name): - """Copy mongo_migrations to the specified test directory.""" - if "migrations_dir" in test_settings_map[repo_name]: - if not os.path.exists(test_settings_map[repo_name]["migrations_dir"]["target"]): - click.echo( - click.style( - f"Copying migrations from {test_settings_map[repo_name]['migrations_dir']['source']} to {test_settings_map[repo_name]['migrations_dir']['target']}", - fg="blue", - ) + """ + Copy mongo_migrations to the specified test directory for this repo. + + test_settings_map[repo_name]['migrations_dir']['source'] and + test_settings_map[repo_name]['migrations_dir']['target'] must be defined. + Does nothing if 'migrations_dir' not present. + """ + settings = test_settings_map.get(repo_name) + if not settings or "migrations_dir" not in settings: + click.echo(click.style("No migrations to copy.", fg="yellow")) + return + + migrations = settings["migrations_dir"] + source = migrations.get("source") + target = migrations.get("target") + + if not source or not target: + click.echo( + click.style( + f"[copy_mongo_migrations] Source/target path missing for '{repo_name}'.", + fg="yellow", ) - shutil.copytree( - test_settings_map[repo_name]["migrations_dir"]["source"], - test_settings_map[repo_name]["migrations_dir"]["target"], + ) + return + + if not os.path.exists(source): + click.echo( + click.style( + f"Source migrations directory does not exist: {source}", fg="red" ) - else: - click.echo(click.style("No migrations to copy", fg="yellow")) + ) + return + + if os.path.exists(target): + click.echo( + click.style( + f"Target migrations already exist: {target} (skipping copy)", fg="cyan" + ) + ) + return + + try: + click.echo( + click.style(f"Copying migrations from {source} to {target}", fg="blue") + ) + shutil.copytree(source, target) + click.echo( + click.style(f"Copied migrations to {target} successfully.", fg="green") + ) + except Exception as e: + click.echo(click.style(f"Failed to copy migrations: {e}", fg="red")) def copy_mongo_settings(source, target): - """Copy mongo_settings to the specified test directory.""" - click.echo(click.style(f"Copying {source} to {target}", fg="blue")) - shutil.copyfile(source, target) + """ + Copy mongo_settings to the specified test directory. + Args: + source (str): Path to the source settings file. + target (str): Path to the target location. -def get_management_command(command=None): + Shows a message and handles errors gracefully. + """ + if not source or not target: + click.echo(click.style("Source or target not specified.", fg="yellow")) + return + if not os.path.exists(source): + click.echo(click.style(f"Source file does not exist: {source}", fg="red")) + exit(1) + + try: + click.echo(click.style(f"Copying {source} ā†’ {target}", fg="blue")) + shutil.copyfile(source, target) + click.echo(click.style(f"Copied settings to {target}.", fg="green")) + except Exception as e: + click.echo(click.style(f"Failed to copy settings: {e}", fg="red")) + + +def get_management_command(command=None, *args): + """ + Construct a Django management command suitable for subprocess execution. + + If 'manage.py' exists in the current directory, use it; otherwise, fall back to 'django-admin'. + Commands in REQUIRES_MANAGE_PY require 'manage.py' to be present. + Pass additional arguments via *args for command options. + """ REQUIRES_MANAGE_PY = { "createsuperuser", "migrate", @@ -58,86 +141,171 @@ def get_management_command(command=None): "shell", "startapp", } - manage_py_exists = os.path.exists("manage.py") + manage_py = "manage.py" + manage_py_exists = os.path.isfile(manage_py) if not manage_py_exists and (command is None or command in REQUIRES_MANAGE_PY): - exit( - click.style( - "manage.py is required to run this command. Please run this command in the project directory.", - fg="red", - ) + raise click.ClickException( + "manage.py is required to run this command. " + "Please run this command in the project directory." ) - base_command = ( - [sys.executable, "manage.py"] if manage_py_exists else ["django-admin"] - ) + base_command = [sys.executable, manage_py] if manage_py_exists else ["django-admin"] if command: full_command = base_command + [command] - return full_command - - return base_command - - -def get_repos(pyproject_path): - with open(pyproject_path, "r") as f: - pyproject_data = toml.load(f) - repos = pyproject_data.get("tool", {}).get("django_mongodb_cli", {}).get("dev", []) - - url_pattern = re.compile(r"git\+ssh://(?:[^@]+@)?([^/]+)/([^@]+)") + else: + full_command = base_command + + if args: + # *args allows for further options/args (e.g. get_management_command("makemigrations", "--verbosity", "2")) + full_command.extend(args) + + return full_command + + +URL_PATTERN = re.compile(r"git\+ssh://(?:[^@]+@)?([^/]+)/([^@]+)") +BRANCH_PATTERN = re.compile( + r"git\+ssh://git@github\.com/[^/]+/[^@]+@([a-zA-Z0-9_\-\.]+)\b" +) + + +def get_repos(pyproject_path, section="dev"): + """ + Load repository info from a pyproject.toml file. + + Args: + pyproject_path (str): Path to the pyproject.toml file. + section (str): Which section ('dev', etc.) of [tool.django_mongodb_cli] to use. + + Returns: + repos (list): List of repository spec strings. + url_pattern (Pattern): Compiled regex to extract repo basename. + branch_pattern (Pattern): Compiled regex to extract branch from url. + """ + try: + with open(pyproject_path, "r") as f: + pyproject_data = toml.load(f) + except FileNotFoundError: + raise click.ClickException(f"Could not find {pyproject_path}") + except toml.TomlDecodeError as e: + raise click.ClickException(f"Failed to parse TOML: {e}") + + tool_section = pyproject_data.get("tool", {}).get("django_mongodb_cli", {}) + repos = tool_section.get(section, []) + if not isinstance(repos, list): + raise click.ClickException( + f"Expected a list of repos in [tool.django_mongodb_cli.{section}]" + ) - branch_pattern = re.compile( - r"git\+ssh://git@github\.com/[^/]+/[^@]+@([a-zA-Z0-9_\-\.]+)\b" - ) - return repos, url_pattern, branch_pattern + return repos, URL_PATTERN, BRANCH_PATTERN -def repo_clone(repo_entry, url_pattern, branch_pattern, repo): - """Helper function to clone a single repo entry.""" +def clone_repo(repo_entry, url_pattern, branch_pattern, repo): + """ + Clone a single repository into repo.home given a repo spec entry. + If the repo already exists at the destination, skips. + Tries to check out a branch if one is found in the entry, defaults to 'main'. + """ url_match = url_pattern.search(repo_entry) - branch_match = branch_pattern.search(repo_entry) if not url_match: - click.echo(f"Invalid repository entry: {repo_entry}") + click.echo(click.style(f"Invalid repository entry: {repo_entry}", fg="red")) return repo_url = url_match.group(0) repo_name = os.path.basename(repo_url) - branch = branch_match.group(1) if branch_match else "main" + branch = ( + branch_pattern.search(repo_entry).group(1) + if branch_pattern.search(repo_entry) + else "main" + ) clone_path = os.path.join(repo.home, repo_name) if os.path.exists(clone_path): - click.echo(f"Skipping {repo_name}: already exists at {clone_path}") - else: click.echo( - f"Cloning {repo_name} from {repo_url} into {clone_path} (branch: {branch})" - ) - if not os.path.exists(clone_path): - click.echo(f"Cloning {repo_url} into {clone_path} (branch: {branch})") - try: - git.Repo.clone_from(repo_url, clone_path, branch=branch) - except git.exc.GitCommandError: - try: - git.Repo.clone_from(repo_url, clone_path) - except git.exc.GitCommandError as e: - click.echo(f"Failed to clone repository: {e}") - subprocess.run(["pre-commit", "install"], cwd=clone_path) + click.style( + f"Skipping {repo_name}: already exists at {clone_path}", fg="yellow" + ) + ) + return + + click.echo( + click.style( + f"Cloning {repo_name} from {repo_url} into {clone_path} (branch: {branch})", + fg="blue", + ) + ) + try: + git.Repo.clone_from(repo_url, clone_path, branch=branch) + except git.exc.GitCommandError as e: + click.echo( + click.style( + f"Warning: Failed to clone branch '{branch}'. Trying default branch instead... ({e})", + fg="yellow", + ) + ) + try: + git.Repo.clone_from(repo_url, clone_path) + except git.exc.GitCommandError as e2: + click.echo(click.style(f"Failed to clone repository: {e2}", fg="red")) + return + + # Optionally install pre-commit hooks if available + pre_commit_cfg = os.path.join(clone_path, ".pre-commit-config.yaml") + if os.path.exists(pre_commit_cfg): + click.echo( + click.style(f"Installing pre-commit hooks for {repo_name}...", fg="green") + ) + result = subprocess.run(["pre-commit", "install"], cwd=clone_path) + if result.returncode == 0: + click.echo(click.style("pre-commit installed successfully.", fg="green")) else: - click.echo(f"Skipping {repo_url}: already exists at {clone_path}") + click.echo(click.style("pre-commit installation failed.", fg="red")) + else: + click.echo( + click.style(f"No pre-commit config found in {repo_name}.", fg="yellow") + ) -def repo_install(clone_path): - install_path = clone_path +def install_package(clone_path): + """ + Install a package from the given clone path. - if clone_path.endswith("mongo-arrow") or clone_path.endswith("libmongocrypt"): - install_path = os.path.join(clone_path, "bindings", "python") + - If clone_path ends with 'mongo-arrow' or 'libmongocrypt', the actual install path is 'bindings/python' inside it. + - Tries editable install via pip if pyproject.toml exists. + - Tries setup.py develop if setup.py exists. + - Installs from requirements.txt if available. + - Warns the user if nothing can be installed. + """ + # Special case for known subdir installs + if clone_path.endswith(("mongo-arrow", "libmongocrypt")): install_path = os.path.join(clone_path, "bindings", "python") + else: + install_path = clone_path if os.path.exists(os.path.join(install_path, "pyproject.toml")): - subprocess.run([sys.executable, "-m", "pip", "install", "-e", install_path]) + click.echo( + click.style( + f"Installing (editable) with pyproject.toml: {install_path}", fg="blue" + ) + ) + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "-e", install_path] + ) elif os.path.exists(os.path.join(install_path, "setup.py")): - subprocess.run([sys.executable, "setup.py", "develop"], cwd=install_path) + click.echo( + click.style( + f"Installing (develop) with setup.py in {install_path}", fg="blue" + ) + ) + result = subprocess.run( + [sys.executable, "setup.py", "develop"], cwd=install_path + ) elif os.path.exists(os.path.join(install_path, "requirements.txt")): - subprocess.run( + click.echo( + click.style(f"Installing requirements.txt in {install_path}", fg="blue") + ) + result = subprocess.run( [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], cwd=install_path, ) @@ -147,76 +315,160 @@ def repo_install(clone_path): f"No valid installation method found for {install_path}", fg="red" ) ) + return + + if "result" in locals(): + if result.returncode == 0: + click.echo( + click.style(f"Install successful in {install_path}.", fg="green") + ) + else: + click.echo(click.style(f"Install failed in {install_path}.", fg="red")) def repo_update(repo_entry, url_pattern, clone_path): - """Helper function to update a single repository.""" + """ + Update a single git repository at clone_path. + + Args: + repo_entry (str): The repository entry string (e.g. repo URL). + url_pattern (Pattern): Compiled regex to extract repo name from URL. + clone_path (str): Path where the repository is cloned. + + If the repo doesn't exist, skips it. Handles and reports errors. + """ url_match = url_pattern.search(repo_entry) if not url_match: + click.echo(click.style(f"Invalid repository entry: {repo_entry}", fg="red")) return repo_url = url_match.group(0) repo_name = os.path.basename(repo_url) - if os.path.exists(clone_path): - try: - click.echo(f"Updating šŸ“¦ {repo_name}") - repo = git.Repo(clone_path) - click.echo(repo.git.pull()) - except git.exc.NoSuchPathError: - click.echo(click.style("Not a valid Git repository.", fg="red")) - except git.exc.GitCommandError: - click.echo(click.style(f"Failed to update {repo_name}", fg="red")) - else: - click.echo(click.style(f"Skipping {repo_name}", fg="blue")) - + if not os.path.exists(clone_path): + click.echo( + click.style( + f"Skipping {repo_name}: {clone_path} does not exist.", fg="yellow" + ) + ) + return -def repo_status( - repo_entry, url_pattern, repo, reset=False, diff=False, branch=False, update=False + try: + # Check if clone_path is a git repo + repo = git.Repo(clone_path) + click.echo(click.style(f"Updating šŸ“¦ {repo_name}...", fg="blue")) + pull_output = repo.git.pull() + click.echo( + click.style(f"Pull result for {repo_name}:\n{pull_output}", fg="green") + ) + except git.exc.NoSuchPathError: + click.echo( + click.style(f"Not a valid Git repository at {clone_path}.", fg="red") + ) + except git.exc.GitCommandError as e: + click.echo(click.style(f"Failed to update {repo_name}: {e}", fg="red")) + except Exception as e: + click.echo(click.style(f"Unexpected error updating {repo_name}: {e}", fg="red")) + + +def get_status( + repo_entry, + url_pattern, + repo, + reset=False, + diff=False, + branch=False, + update=False, + log=False, ): - """Helper function to update a single repository.""" + """ + Show status (and optionally reset/update/log/diff/branch) for a single repo. + + Args: + repo_entry (str): The repository entry spec (from pyproject.toml). + url_pattern (Pattern): Regex to extract basename from URL. + repo (object): Repo context with .home attribute. + reset (bool): If True, hard-reset the repo. + diff (bool): If True, show git diff. + branch (bool): If True, show all branches. + update (bool): If True, run repo_update(). + log (bool): If True, show formatted log. + + Outputs status/log info with color and error reporting. + """ url_match = url_pattern.search(repo_entry) if not url_match: + click.echo(click.style(f"Invalid repository entry: {repo_entry}", fg="red")) return repo_url = url_match.group(0) repo_name = os.path.basename(repo_url) clone_path = os.path.join(repo.home, repo_name) - if os.path.exists(clone_path): - try: - repo = git.Repo(clone_path) - click.echo(f"šŸ“¦ {repo_name}") - if reset: - click.echo(click.style(f"Resetting {repo_name}", fg="red")) - click.echo(repo.git.reset("--hard")) - else: + if not os.path.exists(clone_path): + click.echo(click.style(f"Skipping šŸ“¦ {repo_name} (not cloned)", fg="yellow")) + return + + try: + repo_obj = git.Repo(clone_path) + click.echo(click.style(f"\nšŸ“¦ {repo_name}", fg="blue", bold=True)) + + if reset: + click.echo( + click.style(f"Resetting {repo_name} to HEAD (hard)...", fg="red") + ) + out = repo_obj.git.reset("--hard") + click.echo(out) + click.echo() # Space + + else: + # Print remote info + for remote in repo_obj.remotes: click.echo( - "".join( - [ - f"On remote {remote.name} at {remote.url}" - for remote in repo.remotes - ] - ), + click.style(f"Remote: {remote.name} @ {remote.url}", fg="blue") ) - click.echo(repo.git.status()) - if diff: - diff_output = repo.git.diff() - if diff_output: - click.echo(click.style(diff_output, fg="red")) - else: - click.echo(click.style("No diff output", fg="yellow")) - if branch: - click.echo(repo.git.branch("--all")) - if update: - repo_update(repo_entry, url_pattern, clone_path) - click.echo() - except git.exc.NoSuchPathError: - click.echo("Not a valid Git repository.") - else: - click.echo( - click.style( - f"Skipping šŸ“¦ {repo_name}", - fg="blue", + + # Print branch info + current_branch = ( + repo_obj.active_branch.name + if repo_obj.head.is_valid() + else "" ) + click.echo(click.style(f"On branch: {current_branch}", fg="magenta")) + + status = repo_obj.git.status() + click.echo(status) + + # Show diff if requested + if diff: + diff_output = repo_obj.git.diff() + if diff_output.strip(): + click.echo(click.style("Diff:", fg="yellow")) + click.echo(click.style(diff_output, fg="red")) + else: + click.echo(click.style("No diff output", fg="green")) + + # Show branches if requested + if branch: + click.echo(click.style("Branches:", fg="blue")) + click.echo(repo_obj.git.branch("--all")) + + # Show log if requested + if log: + click.echo(click.style("Git log:", fg="blue")) + click.echo( + repo_obj.git.log("--pretty=format:%h - %an, %ar : %s", "--graph") + ) + + # Run update if requested (AFTER showing status etc) + if update: + repo_update(repo_entry, url_pattern, clone_path) + + click.echo() # Add extra space after each repo status for legibility + + except git.exc.InvalidGitRepositoryError: + click.echo( + click.style(f"{clone_path} is not a valid Git repository.", fg="red") ) + except Exception as e: + click.echo(click.style(f"An error occurred for {repo_name}: {e}", fg="red")) diff --git a/qe.py b/qe.py index 0851113..4be1a1a 100644 --- a/qe.py +++ b/qe.py @@ -50,6 +50,6 @@ database, "encrypted_collection", encrypted_fields, "local" ) except EncryptedCollectionError as e: - print(f"Encrypted collection already exists: {e}") + print(f"Encrypted collection error: {e}") code.interact(local=locals()) diff --git a/config/allauth/allauth_apps.py b/test/apps/allauth.py similarity index 100% rename from config/allauth/allauth_apps.py rename to test/apps/allauth.py diff --git a/config/debug_toolbar/debug_toolbar_apps.py b/test/apps/debug_toolbar.py similarity index 100% rename from config/debug_toolbar/debug_toolbar_apps.py rename to test/apps/debug_toolbar.py diff --git a/config/extensions/debug_toolbar_apps.py b/test/apps/django_filter.py similarity index 100% rename from config/extensions/debug_toolbar_apps.py rename to test/apps/django_filter.py diff --git a/config/filter/filter_apps.py b/test/apps/django_mongodb_extensions.py similarity index 100% rename from config/filter/filter_apps.py rename to test/apps/django_mongodb_extensions.py diff --git a/config/rest_framework/rest_framework_apps.py b/test/apps/rest_framework.py similarity index 100% rename from config/rest_framework/rest_framework_apps.py rename to test/apps/rest_framework.py diff --git a/config/wagtail/wagtail_apps.py b/test/apps/wagtail.py similarity index 100% rename from config/wagtail/wagtail_apps.py rename to test/apps/wagtail.py diff --git a/config/allauth/allauth_settings.py b/test/settings/allauth.py similarity index 100% rename from config/allauth/allauth_settings.py rename to test/settings/allauth.py diff --git a/config/debug_toolbar/debug_toolbar_settings.py b/test/settings/debug_toolbar.py similarity index 100% rename from config/debug_toolbar/debug_toolbar_settings.py rename to test/settings/debug_toolbar.py diff --git a/config/django/django_settings.py b/test/settings/django.py similarity index 67% rename from config/django/django_settings.py rename to test/settings/django.py index c6a4279..8dd684e 100644 --- a/config/django/django_settings.py +++ b/test/settings/django.py @@ -4,17 +4,20 @@ kms_providers = encryption.get_kms_providers() -HOME = os.environ.get("HOME") - auto_encryption_opts = encryption.get_auto_encryption_opts( kms_providers=kms_providers, - crypt_shared_lib_path=f"{HOME}/Downloads/mongo_crypt_shared_v1-macos-arm64-enterprise-8.0.10/lib/mongo_crypt_v1.dylib", ) -DATABASE_URL = os.environ.get("MONGODB_URI", "mongodb://localhost:27017/djangotests") +DATABASE_URL = os.environ.get("MONGODB_URI", "mongodb://localhost:27017") DATABASES = { "default": parse_uri( - DATABASE_URL, options={"auto_encryption_opts": auto_encryption_opts} + DATABASE_URL, + db_name="djangotests", + ), + "encrypted": parse_uri( + DATABASE_URL, + options={"auto_encryption_opts": auto_encryption_opts}, + db_name="encrypted_djangotests", ), } diff --git a/config/filter/filter_settings.py b/test/settings/django_filter.py similarity index 100% rename from config/filter/filter_settings.py rename to test/settings/django_filter.py diff --git a/config/extensions/debug_toolbar_settings.py b/test/settings/django_mongodb_extensions.py similarity index 100% rename from config/extensions/debug_toolbar_settings.py rename to test/settings/django_mongodb_extensions.py diff --git a/config/rest_framework/rest_framework_settings.py b/test/settings/rest_framework.py similarity index 100% rename from config/rest_framework/rest_framework_settings.py rename to test/settings/rest_framework.py diff --git a/config/rest_framework/rest_framework_migrate.py b/test/settings/rest_framework_migrations.py similarity index 100% rename from config/rest_framework/rest_framework_migrate.py rename to test/settings/rest_framework_migrations.py diff --git a/config/wagtail/wagtail_settings.py b/test/settings/wagtail.py similarity index 100% rename from config/wagtail/wagtail_settings.py rename to test/settings/wagtail.py