Merge pull request #843 from I2PC/release-3.23.11

Release-3.23.11 Nereus

Author: albertmena

.github/workflows/build.yml

@@ -12,9 +12,6 @@ jobs:
   build:
     # The type of runner that the job will run on
     runs-on: ubuntu-22.04
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    # Disabling shallow clone is recommended for improving relevancy of reporting
     steps:
       # Installing apt dependencies
       - name: Install dependencies
@@ -26,32 +23,34 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9.15' # Version range or exact version of a Python version to use, using SemVer's version range syntax
-          
+
+
       # Installing Python dependencies
       - name: Install python dependencies
         run: |
            python -m pip install --upgrade pip
            pip install scons numpy
-       
+
        # Installing CUDA                
-      - uses: Jimver/cuda-toolkit@v0.2.10
+      - uses: Jimver/cuda-toolkit@v0.2.11
         id: cuda-toolkit
         with:
-          cuda: '11.7.0'
-          linux-local-args: '[ "--toolkit" ]'
-      
+          cuda: '11.8.0'
+          method: network
+          sub-packages: '["nvcc", "toolkit"]'
+
       # Using CCache to speed C/C++ compilation
-      - name: Install ccache
-        uses: hendrikmuhs/ccache-action@v1.2
+      - uses: hendrikmuhs/ccache-action@v1.2
 
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
             fetch-depth: 0
 
       # Compiling Xmipp in the Pull Request branch
-      - name: Compile Xmipp
+      - name: Compile Xmipp will all dependencies in branch ${{ github.head_ref }} (if such branch exists, default is devel)
         env:
+          BRANCH_NAME: ${{ github.head_ref }}
           CIBuild: 1
           BUILD_TESTS: True
-        run: ./xmipp noAsk
+        run: ./xmipp all br=$BRANCH_NAME noAsk || (cat compileLOG.txt && false)

.github/workflows/scanBuild.yml

@@ -60,6 +60,7 @@ gmon.out
 /src/xmipp/tests/data
 *.conf
 xmippEnv.json
+compile_commands.json
 *.tgz
 
 # Ignore version file
@@ -71,3 +72,6 @@ xmippEnv.json
 compileLOG.txt
 reportInstallation.tar.gz
 xmipp_version.txt
+
+# Compilation database
+compile_commands.json

.gitignore

@@ -1,4 +1,42 @@
-   ## Release 3.23.07 - voting
+   ## Release 3.23.11 - Nereus
+   ### Xmipp Programs 
+   
+   - New programs
+      - local_volume_adjustment: This program modifies a volume in order to adjust its intensity locally to a reference volume
+      - subtomo_subtraction: New program with mpi version
+   - Programs updated
+      - tomo_extract_subtomogram: Accept target box size
+      - convert_from_pdb: Added cif support
+      - phantom_movies: Adding more options: circle and cross
+      - angular_neighbourhood: Modified to output the result in the best symmetry group
+      - batch_deep_misalignment: Detection: GPU option 
+   - Programs fixed
+      - volume_from_pdb: Bug fix in dealing with orig=0, Detection of contrast inverted maps
+      - tomo_extract_particles: Stack fixes, dose fixes
+      - image_convert: Fixed issue when providing a metadata file as input 
+      - volume_substraction: Change NaNs generated by a division between 0 by 0s.
+        
+   ### Installation and user guide
+   - Disabled downloading 380MB file not required
+   - Clarifying the requirements 
+   - More robust thread execution in the installator
+   - Fixed bug collecting versions
+   - Removed auto updater from libcifpp
+
+   ### More Xmipp 
+   - Upgraded CUDA toolkit in DLTK 1.0
+   - Fix PDB read: This fixes a bug where pdbs whose atom name start in position 12 (in base 0)
+   - CUDA 12.2 supported (not full tested)
+   - Fixed PDB reading for shorter PDBs
+   - Allow not compiling xmipp to debug changes in python scripts
+   - Xmipp will now be able to read any volume with any values of those fields correctly.
+   - Fixed pixel size not being properly stored
+   - Fixed MetaData to avoid dangling references
+   - Added compatibility for undefined axis sequences in MRC files
+   - Added option to write mirrors in angularDistance
+
+
+   ## Release 3.23.07 - Morpheus
    ### Xmipp Programs 
    - New programs
       - image_peak_high_contrast(for detecting high contrast regions in tomographic reconstruction)

CHANGELOG.md

@@ -1,7 +1,7 @@
-[![Build Status](https://github.com/I2PC/xmipp/actions/workflows/main.yml/badge.svg)](https://github.com/I2PC/xmipp/actions/workflows/main.yml)
+[![Build Status](https://github.com/I2PC/xmipp/actions/workflows/build.yml/badge.svg)](https://github.com/I2PC/xmipp/actions/workflows/build.yml)
 
 # Xmipp
-<a href="https://github.com/I2PC/xmipp"><img src="https://github.com/I2PC/scipion-em-xmipp/blob/devel/xmipp3/xmipp_logo_original.png" alt="drawing" width="330" align="left"></a>
+<a href="https://github.com/I2PC/xmipp"><img src="https://github.com/I2PC/scipion-em-xmipp/blob/devel/xmipp3/xmipp_logo.png" alt="drawing" width="330" align="left"></a>
 
 
 
@@ -17,7 +17,7 @@ The recommended way for users (not developers) to install and use Xmipp is via t
 ## Installation requirements
 ### Supported OS
 We have tested Xmipp compilation on the following operating systems:
-[Ubuntu 16.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-16.04), [Ubuntu 18.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-18.04), [Ubuntu 20.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-20.04), [Ubuntu 22.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-22.04), [Centos 7](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-CentOS-7-9.2009). Visit the OS wiki page for more details.
+[Ubuntu 18.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-18.04), [Ubuntu 20.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-20.04), [Ubuntu 22.04](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-Ubuntu-22.04), [Centos 7](https://github.com/I2PC/xmipp/wiki/Installing-Xmipp-on-CentOS-7-9.2009). Visit the OS wiki page for more details.
 
 While compilation and execution might be possible on other systems, it might not be straightforward. If you encounter a problem, please refer to known and fixed [issues](https://github.com/I2PC/xmipp/issues?q=is%3Aissue). Let us know if something is not working.
 
@@ -26,7 +26,7 @@ At least 2 processors are required to run Xmipp. In some virtual machine tools o
 
 ### Software dependencies
 #### Compiler
-Xmipp requires C++17 compatible compiler. We recommend GCC, we have good experience with GCC-8.3 in any case a version > 8 is required. If use GCC-10.3 and CUDA=11 and experience issues, [please change the compiler version](https://github.com/NVIDIA/nccl/issues/494). If use GCC-11 and experience issues, [please visit this.](https://github.com/I2PC/xmipp/issues/583). For more details about the compilation proces and installation of gcc, please visit [compiler](https://github.com/I2PC/xmipp/wiki/Compiler)
+Xmipp requires C++17 compatible compiler. We recommend GCC with G++, we have good experience with GCC/G++-8.4 in any case a version >= 8.4 is required. If use GCC/G++-10.3 and CUDA=11 and experience issues, [please change the compiler version](https://github.com/NVIDIA/nccl/issues/494). If use GCC-11 and experience issues, [please visit this](https://github.com/I2PC/xmipp/issues/583). For more details about the compilation proces and installation of gcc, please visit [compiler](https://github.com/I2PC/xmipp/wiki/Compiler)
 
 #### Cmake
 Xmipp requires Cmake 3.16 or above. To update it please [visit](https://github.com/I2PC/xmipp/wiki/Cmake-update-and-install)
@@ -47,7 +47,7 @@ We sometimes see issues regarding the HDF5 dependency. Please visit the [HDF5 Tr
 
 `pip install scons` in the enviroment that xmipp will be compiled (scipion3 if you will run Xmipp within Scipion)
 
-Also a compiler will be required (`sudo apt install gcc-8 g++-8`)
+Also a compiler will be required (`sudo apt install gcc-10 g++-10`)
 
 ## Standalone installation
 Standalone installation of Xmipp is recommended for researchers and developers. This installation allows you to use Xmipp without Scipion, however, in the next section it is explained how to link it with Scipion. Xmipp script automatically downloads several dependencies and then creates a configuration file that contains paths and flags used during the compilation. Please refer to the [Xmipp configuration](https://github.com/I2PC/xmipp/wiki/Xmipp-configuration) guide for more info.

README.md

@@ -30,7 +30,6 @@
 from .utils import *
 from.environment import Environment
 
-
 class Config:
     FILE_NAME = 'xmipp.conf'
     KEY_USE_DL = 'USE_DL'
@@ -43,9 +42,9 @@ class Config:
     OPT_NVCC = 'NVCC'
     OPT_NVCC_LINKFLAGS = 'NVCC_LINKFLAGS'
     OPT_NVCC_CXXFLAGS = 'NVCC_CXXFLAGS'
-    MINIMUM_GCC_VERSION = 8
+    MINIMUM_GCC_VERSION = '8.4.0'
     MINIMUM_CUDA_VERSION = 10.1
-    vGCC = ['12.2', '12.1',
+    vGCC = ['12.3', '12.2', '12.1',
             '11.3', '11.2', '11.1', '11',
             '10.4', '10.3', '10.2', '10.1', '10',
             '9.4', '9.3', '9.2', '9.1', '9',
@@ -55,7 +54,7 @@ class Config:
         '11.0-11.0': vGCC[vGCC.index('9.4'):],
         '11.1-11.4': vGCC[vGCC.index('10.4'):],
         '11.5-11.8': vGCC[vGCC.index('11.2'):],
-        '12.0-12.1': vGCC[vGCC.index('12.2'):],
+        '12.0-12.3': vGCC[vGCC.index('12.3'):],
     }
     CMAKE_VERSION_REQUIRED = '3.16'
 
@@ -306,8 +305,9 @@ def _config_compiler(self):
             self.configDict["DEBUG"] = "False"
         if self.configDict["CC"] == "" and checkProgram("gcc")[0]:
             self.configDict["CC"] = "gcc"
-            if get_GCC_version("gcc")[0] < Config.MINIMUM_GCC_VERSION:
-                print(red("gcc version required >= 8, detected gcc {}".format(get_GCC_version("gcc")[0])))
+            if versionToNumber(get_GCC_version("gcc")[1]) < versionToNumber(Config.MINIMUM_GCC_VERSION):
+                print(red("gcc version required >= {}, detected gcc {}".format(
+                    Config.MINIMUM_GCC_VERSION, get_GCC_version("gcc")[0])))
             else:
                 print(green('gcc {} detected'.format(get_GCC_version("gcc")[0])))
         self._set_cxx()
@@ -440,7 +440,7 @@ def _ensure_GCC_GPP_version(self, compiler):
         if gccVersion == '':
             return False, 3
         print(green('Detected ' + compiler + " in version " + fullVersion + '.'))
-        if gccVersion < Config.MINIMUM_GCC_VERSION:
+        if versionToNumber(fullVersion) < versionToNumber(Config.MINIMUM_GCC_VERSION):
             return False, 4
         return True, 0
 
@@ -564,8 +564,8 @@ def _set_nvcc_cxx(self, nvcc_version):
         candidates, resultBool = self._get_compatible_GCC(nvcc_version)
         if not resultBool:
             print(yellow('CUDA version {} not compatible with Xmipp. Please '
-                         'install CUDA>={}'.format(Config.MINIMUM_CUDA_VERSION,
-                                        nvcc_version)), *candidates, sep=", ")
+                         'install CUDA>={} and not higher than the maximum version required by the compiler'.format(nvcc_version,
+                                                   Config.MINIMUM_CUDA_VERSION)), *candidates, sep=", ")
             print('gcc candidates based on nvcc version:', *candidates, sep=", ")
             return
 

scripts/config.py

@@ -192,7 +192,7 @@ def runJob(cmd, cwd='./', show_output=True, log=None, show_command=True,
         else:
             progresL = progresLines
         line = p.stdout.readline().decode("utf-8")
-        if line != '':
+        if line:
             log.append(line)
             write_compileLog(line, COMPILE_LOG)
 
@@ -208,8 +208,8 @@ def runJob(cmd, cwd='./', show_output=True, log=None, show_command=True,
                 str2Print = line + "\n" + printProgressBar(prg)
                 print(f"{yellow(str2Print)}", end=UP)
                 n += 1
-        if not line:
-            if p.poll() == 0:
+        else:
+            if p.wait() == 0:
                 if printProgress:
                     print(printProgressBar(100))
                 return True
@@ -232,7 +232,7 @@ def runJob(cmd, cwd='./', show_output=True, log=None, show_command=True,
                 log.append(l)
     if in_parallel:
         return p
-    elif 0 == p.poll():
+    elif 0 == p.wait():
         return True
     else:
         if show_output is False and log is None:
@@ -433,6 +433,15 @@ def version_tuple(versionStr):
     # Convert the list of numerical parts to a tuple and return it
     return tuple(numerical_parts)
 
+
+def versionToNumber(strVersion):
+    listVersion = strVersion.split('.')
+    numberVersion = int(listVersion[0]) * 100 + int(listVersion[1]) * 10
+    try:
+        numberVersion = numberVersion + int(listVersion[2]) * 1
+    except Exception:
+        pass
+    return numberVersion
 def checkCMakeVersion(minimumRequired=None):
     """
     ### This function checks if the current installed version, if installed, is above the minimum required version.
@@ -502,4 +511,4 @@ def errorList(errorNum):
                                                   'and https://github.com/I2PC/xmipp/wiki/Cmake-troubleshoting ']
 
     ]
-    return errorList[errorNum]
+    return errorList[errorNum]

scripts/utils.py

@@ -1,19 +1,25 @@
-sonar.projectKey=Xmipp
-sonar.projectName=Xmipp
+projectKey=I2PC_xmipp
+sonar.organization=i2pc
+
+# This is the name and version displayed in the SonarCloud UI.
 sonar.projectVersion=3.0
 
-#sonar.log.level=DEBUG
+# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows.
+sonar.sources=src/
+sonar.tests=src/
 
-sonar.sources=./src/
-# don't use .gitignore for auto exclusion, as it doesn't work properly and exclude almost everything
+# Don't use .gitignore for auto exclusion, as it doesn't work properly and exclude almost everything
 sonar.scm.exclusions.disabled=true
+
 # NOTE - be careful with excusion rules. If you include formerly excluded folder, all files that include 
 # headers from it will be analyzed again! https://community.sonarsource.com/t/sonarscanner-cache-not-working-properly/29451 
-sonar.inclusions=**/xmipp/**, **/xmippViz/**
-sonar.exclusions=**/external/**, **/programs/**, **/tests/**, **/legacy/**
+sonar.inclusions=xmipp, scripts/**/*
+sonar.exclusions=src/**/external/**/*, src/**/legacy/**/*, src/**/resources/**/*, src/**/tests/**/*
+sonar.test.inclusions=src/**/tests/**/*
+sonar.test.exclusions=src/**/*
 
 # see https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system
-
-sonar.java.binaries=./src/xmippViz/java/build/xmipp
-sonar.java.libraries=./src/xmippViz/java/lib/*.jar
 sonar.cfamily.build-wrapper-output=bw-outputs
+
+# Python version to analyze Python code
+sonar.python.version=3

sonar-project.properties

@@ -71,6 +71,7 @@ env = Environment(ENV=os.environ,
                   BUILDERS=Environment()['BUILDERS'],
                   tools=['Make', 'AutoConfig'],
                   toolpath=[join('install', 'scons-tools')])
+env.SetOption('warn', 'no-duplicate-environment')  # Supress warnings for compilation database (due to the way we create the environments)
 # TODO: BUILDERS var added from the tricky creation of a new environment.
 # If not, they lose default builders like "Program", which are needed later
 # (by CheckLib and so on). See http://www.scons.org/doc/2.0.1/HTML/scons-user/x3516.html
@@ -296,6 +297,18 @@ def addCppLibrary(env, name, dirs=[], tars=[], untarTargets=['configure'], patte
 
     env2.Alias(name, lastTarget)
 
+    # Compilation database for libraries and shared libraries
+    env2['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1
+    env2.SetOption('warn', 'no-duplicate-environment')
+    env2.Tool('compilation_db')
+    xmipp_path = os.path.dirname(os.path.dirname(os.getcwd()))
+    cdb = env2.CompilationDatabase(os.path.join(xmipp_path, 'compile_commands.json'))
+    Alias('cdb', cdb)
+    BUILD_TARGETS.append('cdb')
+    program = env2.Program(**libraryArgs)
+    env2.Depends(program, deps)
+
+
     return lastTarget
 
 
@@ -426,6 +439,14 @@ def addProgram(env, name, src=None, pattern=None, installDir=None,
     appendUnique(libPathsCopy, external_libdirs) # This order is important, because if we should use Scipion libs, these will be before the system libs
     appendUnique(libPathsCopy, env.get('LIBPATH', '').split(os.pathsep))
     env2 = Environment()
+
+    # Compilation database for programs
+    env2.SetOption('warn', 'no-duplicate-environment')
+    env2.Tool('compilation_db')
+    xmipp_path = os.path.dirname(os.path.dirname(os.getcwd()))
+    cdb = env2.CompilationDatabase(os.path.join(xmipp_path, 'compile_commands.json'))
+    Alias('cdb', cdb)
+
     env2['ENV']['LD_LIBRARY_PATH'] = env['ENV'].get('LD_LIBRARY_PATH', '')
     env2['ENV']['PATH'] = env['ENV']['PATH']
 

src/xmipp/SConstruct

@@ -0,0 +1,28 @@
+/***************************************************************************
+ *
+ * Authors:     Estrella Fernandez Gimenez (me.fernandez@cnb.csic.es)
+ *
+ * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ * 02111-1307  USA
+ *
+ *  All comments concerning this program package may be sent to the
+ *  e-mail address 'xmipp@cnb.csic.es'
+ ***************************************************************************/
+
+#include <reconstruction/local_volume_adjust.cpp>
+
+RUN_XMIPP_PROGRAM(ProgLocalVolumeAdjust)