compiler: Switch to oneapi compiler

Author: georgebisbas

devito/arch/compiler.py

@@ -12,7 +12,7 @@
 from codepy.jit import compile_from_string
 from codepy.toolchain import GCCToolchain
 
-from devito.arch import (AMDGPUX, Cpu64, M1, NVIDIAX, SKX, POWER8, POWER9,
+from devito.arch import (AMDGPUX, NVIDIAX, M1, SKX, POWER8, POWER9, INTELGPUX,
                          get_nvidia_cc, check_cuda_runtime, get_m1_llvm_path)
 from devito.exceptions import CompilationError
 from devito.logger import debug, warning, error
@@ -515,19 +515,55 @@ def __lookup_cmds__(self):
         self.MPICXX = 'mpicxx'
 
 
-class ICXCompiler(Compiler):
+class OneapiCompiler(Compiler):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.cflags += ['-Ofast', '-xHost', '-fiopenmp']
+        language = kwargs.pop('language', configuration['language'])
+        platform = kwargs.pop('platform', configuration['platform'])
+
+        self.cflags.append("-xHost")
+        self.cflags.append("-qopt-zmm-usage=high")
+
+        if configuration['safe-math']:
+            self.cflags.append("-fp-model=strict")
+        else:
+            self.cflags.append('-fast')
+
+        if language == 'sycl':
+            self.cflags.append('-fsycl')
+            if platform is NVIDIAX:
+                self.cflags.append('-fsycl-targets=nvptx64-cuda')
+            else:
+                self.cflags.append('-fsycl-targets=spir64')
+
+        if language == 'openmp':
+            self.cflags.append('-qopenmp')
+            if platform is NVIDIAX:
+                self.cflags.append('-fopenmp-targets=nvptx64-cuda')
+            if platform is INTELGPUX:
+                self.cflags.append('-fopenmp-targets=spir64')
+                self.cflags.append('-fopenmp-target-simd')
+
+        if platform is INTELGPUX:
+            self.cflags.remove('-g')  # -g disables some optimizations in IGC
+            self.cflags.append('-gline-tables-only')
+            self.cflags.append('-fdebug-info-for-profiling')
+
+        # Make sure the MPI compiler uses `icx` underneath -- whatever the MPI distro is
+        if kwargs.get('mpi'):
+            ver = check_output([self.MPICC, "--version"]).decode("utf-8")
+            if not ver.startswith("Intel(R) oneAPI"):
+                warning("The MPI compiler `%s` doesn't use the Intel(R) oneAPI "
+                        "DPC++/C++ compiler underneath" % self.MPICC)
 
     def __lookup_cmds__(self):
-        # OneAPI Base Kit comes with dpcpp/icpx, both are clang++,
+        # OneAPI HPC ToolKit comes with icpx, which is clang++,
         # and icx, which is clang
         self.CC = 'icx'
         self.CXX = 'icpx'
-        self.MPICC = 'mpic++'
+        self.MPICC = 'mpicc'
         self.MPICXX = 'mpicxx'
 
 
@@ -795,7 +831,8 @@ def __lookup_cmds__(self):
     'intel': IntelCompiler,
     'icpc': IntelCompiler,
     'icc': IntelCompiler,
-    'icx': ICXCompiler,
+    'icx': OneapiCompiler,
+    'icpx': OneapiCompiler,
     'intel-knl': IntelKNLCompiler,
     'knl': IntelKNLCompiler,
     'dpcpp': DPCPPCompiler,

tests/conftest.py

@@ -7,7 +7,8 @@
 from devito import Eq, configuration  # noqa
 from devito.finite_differences.differentiable import EvalDerivative
 from devito.arch import Cpu64, Device, sniff_mpi_distro, Arm
-from devito.arch.compiler import compiler_registry, IntelCompiler, NvidiaCompiler
+from devito.arch.compiler import (compiler_registry, IntelCompiler, OneapiCompiler,
+                                  NvidiaCompiler)
 from devito.ir.iet import retrieve_iteration_tree, FindNodes, Iteration, ParallelBlock
 from devito.tools import as_tuple
 
@@ -23,7 +24,8 @@ def skipif(items, whole_module=False):
     # Sanity check
     accepted = set()
     accepted.update({'device', 'device-C', 'device-openmp', 'device-openacc',
-                     'device-aomp', 'cpu64-icc', 'cpu64-nvc', 'cpu64-arm'})
+                     'device-aomp', 'cpu64-icc', 'cpu64-icpx', 'cpu64-nvc',
+                     'cpu64-arm'})
     accepted.update({'nompi', 'nodevice'})
     unknown = sorted(set(items) - accepted)
     if unknown:
@@ -69,6 +71,12 @@ def skipif(items, whole_module=False):
            isinstance(configuration['platform'], Cpu64):
             skipit = "`icc+cpu64` won't work with this test"
             break
+        # Skip if it won't run with OneAPICompiler
+        if i == 'cpu64-icpx' and \
+           isinstance(configuration['compiler'], OneapiCompiler) and \
+           isinstance(configuration['platform'], Cpu64):
+            skipit = "`icpx+cpu64` won't work with this test"
+            break
         # Skip if it won't run on Arm
         if i == 'cpu64-arm' and isinstance(configuration['platform'], Arm):
             skipit = "Arm doesn't support x86-specific instructions"

tests/test_dimension.py

@@ -1336,6 +1336,8 @@ def test_affiness(self):
         iterations = [i for i in FindNodes(Iteration).visit(op) if i.dim is not time]
         assert all(i.is_Affine for i in iterations)
 
+    # Skipping this test with icpx, as it requires safe-math
+    @skipif('cpu64-icpx')
     def test_sparse_time_function(self):
         nt = 20
 

tests/test_dle.py

@@ -793,6 +793,7 @@ def test_incs_no_atomic(self):
         assert 'collapse(1)' in str(op1)
         assert 'atomic' not in str(op1)
 
+    @skipif('cpu64-icpx')
     @pytest.mark.parametrize('exprs,simd_level,expected', [
         (['Eq(y.symbolic_max, g[0, x], implicit_dims=(t, x))',
          'Inc(h1[0, 0], 1, implicit_dims=(t, x, y))'],

tests/test_dse.py

@@ -317,6 +317,7 @@ def test_time_dependent_split(opt):
 
 class TestLifting(object):
 
+    @skipif('cpu64-icpx')
     @pytest.mark.parametrize('exprs,expected', [
         # none (different distance)
         (['Eq(y.symbolic_max, g[0, x], implicit_dims=(t, x))',