diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index e1962f765fb1..6984264c4572 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -4545,6 +4545,11 @@ def from_pretrained(
         if gguf_file is not None and not is_accelerate_available():
             raise ValueError("accelerate is required when loading a GGUF file `pip install accelerate`.")
 
+        if kernel_config is not None and not use_kernels:
+            raise ValueError(
+                "kernel_config is only supported when use_kernels is set to True, please set use_kernels to True or remove kernel_config."
+            )
+
         if commit_hash is None:
             if not isinstance(config, PreTrainedConfig):
                 # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible