Replace filelock with fasteners

Author: vkarak

docs/requirements.txt

@@ -2,10 +2,7 @@ archspec==0.2.5
 ClusterShell==1.9.3
 docutils==0.18.1; python_version < '3.9'
 docutils==0.21.2; python_version >= '3.9'
-filelock==3.4.1; python_version == '3.6'
-filelock==3.12.2; python_version == '3.7'
-filelock==3.16.1; python_version == '3.8'
-filelock==3.18.0; python_version > '3.8'
+fasteners==0.19
 jinja2==3.0.3; python_version == '3.6'
 jinja2==3.1.6; python_version >= '3.7'
 jsonschema==3.2.0

reframe/core/logging.py

@@ -168,6 +168,7 @@ def __init__(self, prefix, mode='a', encoding=None, fmt=None,
         self.__ignore_keys = set(ignore_keys) if ignore_keys else set()
         self.__use_locking = use_locking
         self.__lockfile_mode = lockfile_mode
+        self.__locks = {}
 
     def __generate_header(self, record):
         # Generate the header from the record and fmt
@@ -212,6 +213,14 @@ def __generate_header(self, record):
 
         return header
 
+    def __lock_file_name(self, logfile=None):
+        if logfile is None:
+            logfile = self.baseFilename
+
+        prefix = os.path.dirname(logfile)
+        basename, _ = os.path.splitext(os.path.basename(logfile))
+        return os.path.join(prefix, f'.{basename}.lock')
+
     def _emit_header(self, record):
         if self.baseFilename in self.__streams:
             return
@@ -241,14 +250,16 @@ def _emit_header(self, record):
             if self.__use_locking:
                 # When using locking, we need to  open, append and write to
                 # the file at once
-                lockfile = os.path.join(f'{self.baseFilename}.lock')
-                with osext.flock(lockfile, self.__lockfile_mode):
+                rwlock = osext.ReadWriteFileLock(self.__lock_file_name(),
+                                                 self.__lockfile_mode)
+                with rwlock.write_lock():
                     with open(self.baseFilename, mode=self.mode,
                               encoding=self.encoding) as fp:
                         if record_header != header:
                             fp.write(f'{record_header}\n')
 
                 self.__streams[self.baseFilename] = None
+                self.__locks[self.baseFilename] = rwlock
             else:
                 # Open the file for writing and write the header
                 fp = open(self.baseFilename,
@@ -272,8 +283,7 @@ def emit(self, record):
         self.baseFilename = os.path.join(dirname, f'{check_basename}.log')
         self._emit_header(record)
         if self.__use_locking:
-            lockfile = os.path.join(dirname, f'.{check_basename}.lock')
-            with osext.flock(lockfile, self.__lockfile_mode):
+            with self.__locks[self.baseFilename].write_lock():
                 with open(self.baseFilename, mode=self.mode,
                           encoding=self.encoding) as fp:
                     self.stream = fp
@@ -486,6 +496,9 @@ def _create_filelog_handler(site_config, config_prefix):
     ignore_keys = site_config.get(f'{config_prefix}/ignore_keys')
     use_locking = site_config.get(f'{config_prefix}/locking_enable')
     lockfile_mode = site_config.get(f'{config_prefix}/locking_file_mode')
+    if lockfile_mode is not None:
+        lockfile_mode = int(lockfile_mode, base=8)
+
     return MultiFileHandler(filename_patt, mode='a+' if append else 'w+',
                             fmt=format, perffmt=format_perf,
                             ignore_keys=ignore_keys,

reframe/frontend/reporting/storage.py

@@ -86,6 +86,11 @@ def __init__(self):
         else:
             self.__db_file_mode = mode
 
+        self.__db_lock = osext.ReadWriteFileLock(
+            os.path.join(os.path.dirname(self.__db_file), '.db.lock'),
+            self.__db_file_mode
+        )
+
     def _db_file(self):
         prefix = os.path.dirname(self.__db_file)
         if not os.path.exists(self.__db_file):
@@ -122,8 +127,7 @@ def _db_connect(self, *args, **kwargs):
             return sqlite3.connect(*args, **kwargs)
 
     def _db_lock(self):
-        lockfile = os.path.join(os.path.dirname(self.__db_file), '.db.lock')
-        return osext.flock(lockfile, self.__db_file_mode)
+        return self.__db_lock.write_lock()
 
     def _db_create(self):
         clsname = type(self).__name__

reframe/schemas/config.json

@@ -87,7 +87,7 @@
                             "items": {"type": "string"}
                         },
                         "locking_enable": {"type": "boolean"},
-                        "locking_file_mode": {"type": "string"}
+                        "locking_file_mode": {"type": ["string", "null"]}
                     },
                     "required": ["prefix"]
                 }
@@ -636,7 +636,7 @@
         "logging/handlers_perflog/filelog_basedir": "./perflogs",
         "logging/handlers_perflog/filelog_ignore_keys": [],
         "logging/handlers_perflog/filelog_locking_enable": false,
-        "logging/handlers_perflog/filelog_locking_file_mode": "644",
+        "logging/handlers_perflog/filelog_locking_file_mode": null,
         "logging/handlers_perflog/graylog_extras": {},
         "logging/handlers_perflog/httpjson_extras": {},
         "logging/handlers_perflog/httpjson_ignore_keys": [],

reframe/utility/osext.py

@@ -9,7 +9,7 @@
 
 import collections.abc
 import errno
-import filelock
+import fasteners
 import getpass
 import grp
 import os
@@ -864,43 +864,34 @@ def unique_abs_paths(paths, prune_children=True):
     return list(unique_paths - children)
 
 
-def flock(name, mode=None):
-    '''Obtain a file lock
+class temp_umask:
+    '''Temporarily change the umask'''
+    def __init__(self, mask):
+        self.__new_mask = mask
+        self.__old_mask = None
 
-    :arg name: the name of the lock file
-    :arg mode: the file mode of the lock file
-    :returns: a context manager that holds the lock.
-        Example usage:
-
-        .. code-block:: python
-
-            with flock('file.lock'):
-                # do something
-
-    .. versionadded:: 4.8.3
-
-    :meta private:
-    '''
-
-    if not isinstance(mode, int):
-        mode = int(mode, base=8)
+    def __enter__(self):
+        self.__old_mask = os.umask(self.__new_mask)
 
-    if mode and sys.version_info >= (3, 7):
-        kwargs = {'mode': mode}
-    else:
-        # Python 3.6 forces us to use an older filelock version that does
-        # not support file modes. File modes where introduced in
-        # filelock 3.10
-        kwargs = {}
-
-    # Create parent directories of the lock file
-    #
-    # NOTE: This is not necessary for filelock >= 3.12.3 and Python >= 3.8
-    # However, we do create it here, in order to support the older Python
-    # versions.
-    prefix = os.path.dirname(name)
-    os.makedirs(prefix, exist_ok=True)
-    return filelock.FileLock(name, **kwargs)
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        os.umask(self.__old_mask)
+
+
+class ReadWriteFileLock(fasteners.InterProcessReaderWriterLock):
+    def __init__(self, path, mode=None):
+        super().__init__(path)
+        self._mode = mode
+
+    def _do_open(self, *args, **kwargs):
+        if self._mode is not None:
+            # We create the directory structure ourselves, so that the mask
+            # applies strictly to the lock file if the parent directories do
+            # not exist
+            os.makedirs(os.path.dirname(self.path), exist_ok=True)
+            with temp_umask(0o0777 ^ self._mode):
+                return super()._do_open(*args, **kwargs)
+        else:
+            return super()._do_open(*args, **kwargs)
 
 
 def cray_cdt_version():

requirements.txt

@@ -2,10 +2,7 @@ archspec==0.2.5
 argcomplete==3.1.2; python_version < '3.8'
 argcomplete==3.6.1; python_version >= '3.8'
 ClusterShell==1.9.3
-filelock==3.4.1; python_version == '3.6'
-filelock==3.12.2; python_version == '3.7'
-filelock==3.16.1; python_version == '3.8'
-filelock==3.18.0; python_version > '3.8'
+fasteners==0.19
 importlib_metadata==4.0.1; python_version < '3.8'
 jinja2==3.0.3; python_version == '3.6'
 jinja2==3.1.6; python_version >= '3.7'