diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 92789fd4ce..71700794d3 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -114,13 +114,13 @@ def assertEqual(self, a, b, msg=None): raise AssertionError("%s:\nDIFF%s:\n%s" % (msg, limit, ''.join(diff[:self.ASSERT_MAX_DIFF]))) from None def assertExists(self, path, msg=None): - """Assert the given path exists""" + """Assert that the given path exists""" if msg is None: msg = "'%s' should exist" % path self.assertTrue(os.path.exists(path), msg) def assertNotExists(self, path, msg=None): - """Assert the given path exists""" + """Assert that the given path does not exist""" if msg is None: msg = "'%s' should not exist" % path self.assertFalse(os.path.exists(path), msg) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 021d524012..5886733ff9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -90,13 +90,14 @@ from easybuild.tools.config import DATA, SOFTWARE from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256 -from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock +from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock, clean_dir from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files -from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate -from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path -from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir -from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file +from easybuild.tools.filetools import download_file, encode_class_name, extract_file +from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs +from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum +from easybuild.tools.filetools import weld_paths, write_file from easybuild.tools.hooks import ( BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, @@ -1171,10 +1172,13 @@ def make_builddir(self): # unless we're building in installation directory and we iterating over a list of (pre)config/build/installopts, # otherwise we wipe the already partially populated installation directory, # see https://github.com/easybuilders/easybuild-framework/issues/2556 - if not (self.build_in_installdir and self.iter_idx > 0): - # make sure we no longer sit in the build directory before cleaning it. + if self.build_in_installdir and self.iter_idx > 0: + pass + else: + # make sure we no longer sit in the build directory before removing it. change_dir(self.orig_workdir) - self.make_dir(self.builddir, self.cfg['cleanupoldbuild']) + # if we’re building in installation directory, clean it + self.make_dir(self.builddir, self.cfg['cleanupoldbuild'], clean_instead_of_remove=self.build_in_installdir) trace_msg("build dir: %s" % self.builddir) @@ -1220,9 +1224,10 @@ def make_installdir(self, dontcreate=None): if self.build_in_installdir: self.cfg['keeppreviousinstall'] = True dontcreate = (dontcreate is None and self.cfg['dontcreateinstalldir']) or dontcreate - self.make_dir(self.installdir, self.cfg['cleanupoldinstall'], dontcreateinstalldir=dontcreate) + self.make_dir(self.installdir, self.cfg['cleanupoldinstall'], dontcreateinstalldir=dontcreate, + clean_instead_of_remove=True) - def make_dir(self, dir_name, clean, dontcreateinstalldir=False): + def make_dir(self, dir_name, clean, dontcreateinstalldir=False, clean_instead_of_remove=False): """ Create the directory. """ @@ -1233,15 +1238,24 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): return elif build_option('module_only') or self.cfg['module_only']: self.log.info("Not touching existing directory %s in module-only mode...", dir_name) - elif clean: - remove_dir(dir_name) - self.log.info("Removed old directory %s", dir_name) else: - self.log.info("Moving existing directory %s out of the way...", dir_name) - timestamp = time.strftime("%Y%m%d-%H%M%S") - backupdir = "%s.%s" % (dir_name, timestamp) - move_file(dir_name, backupdir) - self.log.info("Moved old directory %s to %s", dir_name, backupdir) + if not clean: + self.log.info("Creating backup of directory %s...", dir_name) + timestamp = time.strftime("%Y%m%d-%H%M%S") + backupdir = "%s.%s" % (dir_name, timestamp) + if clean_instead_of_remove: + copy_dir(dir_name, backupdir) + self.log.info(f"Copied old directory {dir_name} to {backupdir}") + else: + move_file(dir_name, backupdir) + self.log.info(f"Moved old directory {dir_name} to {backupdir}") + if clean_instead_of_remove: + # clean the installation directory: first try to remove it; if that fails, empty it + clean_dir(dir_name) + self.log.info(f"Cleaned old directory {dir_name}") + elif clean: + remove_dir(dir_name) + self.log.info(f"Removed old directory {dir_name}") if dontcreateinstalldir: olddir = dir_name diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3126c1c020..5f5c9f9789 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -376,6 +376,26 @@ def remove_file(path): raise EasyBuildError("Failed to remove file %s: %s", path, err) +def empty_dir(path): + """Empty directory at specified path, keeping directory itself intact.""" + # early exit in 'dry run' mode + if build_option('extended_dry_run'): + dry_run_msg(f"directory {path} emptied", silent=build_option('silent')) + return + + if os.path.exists(path): + try: + for item in os.listdir(path): + subpath = os.path.join(path, item) + if os.path.isfile(subpath) or os.path.islink(subpath): + remove_file(subpath) + elif os.path.isdir(subpath): + remove_dir(subpath) + _log.info(f"Path {path} successfully emptied.") + except OSError as err: + raise EasyBuildError(f"Failed to empty directory {path}: {err}") + + def remove_dir(path): """Remove directory at specified path.""" # early exit in 'dry run' mode @@ -406,6 +426,18 @@ def remove_dir(path): path, max_attempts, errors) +def clean_dir(path): + """ + Try to remove directory at specified path. + If that fails, empty directory instead. + """ + try: + remove_dir(path) + except EasyBuildError as err: + _log.debug(f"Removing directory {path} failed, will try to empty it instead: {err}") + empty_dir(path) + + def remove(paths): """ Remove single file/directory or list of files and directories @@ -1952,7 +1984,7 @@ def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False if failed_paths: raise EasyBuildError("Failed to chmod/chown several paths: %s (last error: %s)", failed_paths, err_msg) - # we ignore some errors, but if there are to many, something is definitely wrong + # we ignore some errors, but if there are too many, something is definitely wrong fail_ratio = fail_cnt / float(len(allpaths)) max_fail_ratio = float(build_option('max_fail_ratio_adjust_permissions')) if fail_ratio > max_fail_ratio: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ca2b9bcb47..d16bea41fe 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2546,6 +2546,41 @@ def test_extract_file(self): self.assertFalse(stderr) self.assertFalse(stdout) + def test_empty_dir(self): + """Test empty_dir function""" + test_dir = os.path.join(self.test_prefix, 'test123') + testfile = os.path.join(test_dir, 'foo') + testfile_hidden = os.path.join(test_dir, '.foo') + test_link = os.path.join(test_dir, 'foolink') + test_subdir = os.path.join(test_dir, 'foodir') + + ft.mkdir(test_subdir, parents=True) + ft.write_file(testfile, 'bar') + ft.write_file(testfile_hidden, 'bar') + ft.symlink(testfile, test_link) + ft.empty_dir(test_dir) + self.assertExists(test_dir) + self.assertNotExists(testfile) + self.assertNotExists(testfile_hidden) + self.assertNotExists(test_link) + self.assertNotExists(test_subdir) + + # also test behaviour under --dry-run + build_options = { + 'extended_dry_run': True, + 'silent': False, + } + init_config(build_options=build_options) + + self.mock_stdout(True) + ft.mkdir(test_dir) + ft.empty_dir(test_dir) + txt = self.get_stdout() + self.mock_stdout(False) + + regex = re.compile("^directory [^ ]* emptied$") + self.assertTrue(regex.match(txt), f"Pattern '{regex.pattern}' found in: {txt}") + def test_remove(self): """Test remove_file, remove_dir and join remove functions.""" testfile = os.path.join(self.test_prefix, 'foo') @@ -2622,6 +2657,25 @@ def test_remove(self): ft.adjust_permissions(self.test_prefix, stat.S_IWUSR, add=True) + def test_clean_dir(self): + """Test clean_dir function""" + test_dir = os.path.join(self.test_prefix, 'test123') + testfile = os.path.join(test_dir, 'foo') + testfile_hidden = os.path.join(test_dir, '.foo') + test_link = os.path.join(test_dir, 'foolink') + test_subdir = os.path.join(test_dir, 'foodir') + + ft.mkdir(test_subdir, parents=True) + ft.write_file(testfile, 'bar') + ft.write_file(testfile_hidden, 'bar') + ft.symlink(testfile, test_link) + ft.clean_dir(test_dir) + ft.mkdir(test_dir, parents=True) + self.assertNotExists(testfile) + self.assertNotExists(testfile_hidden) + self.assertNotExists(test_link) + self.assertNotExists(test_subdir) + def test_index_functions(self): """Test *_index functions."""