diff --git a/doc/whats_new/v0.24.rst b/doc/whats_new/v0.24.rst
index c6ef6e5994cdf..9294bab242432 100644
--- a/doc/whats_new/v0.24.rst
+++ b/doc/whats_new/v0.24.rst
@@ -112,7 +112,15 @@ Changelog
 - |Enhancement| :func:`datasets.fetch_covtype` now now supports the optional
   argument `as_frame`; when it is set to True, the returned Bunch object's
   `data` and `frame` members are pandas DataFrames, and the `target` member is
-  a pandas Series. :pr:`17491` by :user:`Alex Liang <tianchuliang>`.
+  a pandas Series.
+  :pr:`17491` by :user:`Alex Liang <tianchuliang>`.
+
+- |Enhancement| :func:`datasets.fetch_kddcup99` now now supports the optional
+  argument `as_frame`; when it is set to True, the returned Bunch object's
+  `data` and `frame` members are pandas DataFrames, and the `target` member is
+  a pandas Series.
+  :pr:`18280` by :user:`Alex Liang <tianchuliang>` and
+  `Guillaume Lemaitre`_.

 - |API| The default value of `as_frame` in :func:`datasets.fetch_openml` is
   changed from False to 'auto'.
diff --git a/sklearn/datasets/_kddcup99.py b/sklearn/datasets/_kddcup99.py
index 2c9263701e0c4..e5c8bb2f298de 100644
--- a/sklearn/datasets/_kddcup99.py
+++ b/sklearn/datasets/_kddcup99.py
@@ -18,6 +18,7 @@
 import joblib

 from ._base import _fetch_remote
+from ._base import _convert_data_dataframe
 from . import get_data_home
 from ._base import RemoteFileMetadata
 from ..utils import Bunch
@@ -48,7 +49,8 @@
 @_deprecate_positional_args
 def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False,
                    random_state=None,
-                   percent10=True, download_if_missing=True, return_X_y=False):
+                   percent10=True, download_if_missing=True, return_X_y=False,
+                   as_frame=False):
     """Load the kddcup99 dataset (classification).

     Download it if necessary.
@@ -97,29 +99,48 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False,

         .. versionadded:: 0.20

+    as_frame : bool, default=False
+        If `True`, returns a pandas Dataframe for the ``data`` and ``target``
+        objects in the `Bunch` returned object; `Bunch` return object will also
+        have a ``frame`` member.
+
+        .. versionadded:: 0.24
+
     Returns
     -------
     data : :class:`~sklearn.utils.Bunch`
         Dictionary-like object, with the following attributes.

-        data : ndarray of shape (494021, 41)
-            The data matrix to learn.
-        target : ndarray of shape (494021,)
-            The regression target for each sample.
+        data : {ndarray, dataframe} of shape (494021, 41)
+            The data matrix to learn. If `as_frame=True`, `data` will be a
+            pandas DataFrame.
+        target : {ndarray, series} of shape (494021,)
+            The regression target for each sample. If `as_frame=True`, `target`
+            will be a pandas Series.
+        frame : dataframe of shape (494021, 42)
+            Only present when `as_frame=True`. Contains `data` and `target`.
         DESCR : str
             The full description of the dataset.
+        feature_names : list
+            The names of the dataset columns
+        target_names: list
+            The names of the target columns

     (data, target) : tuple if ``return_X_y`` is True

         .. versionadded:: 0.20
     """
     data_home = get_data_home(data_home=data_home)
-    kddcup99 = _fetch_brute_kddcup99(data_home=data_home,
-                                     percent10=percent10,
-                                     download_if_missing=download_if_missing)
+    kddcup99 = _fetch_brute_kddcup99(
+        data_home=data_home,
+        percent10=percent10,
+        download_if_missing=download_if_missing
+    )

     data = kddcup99.data
     target = kddcup99.target
+    feature_names = kddcup99.feature_names
+    target_names = kddcup99.target_names

     if subset == 'SA':
         s = target == b'normal.'
@@ -143,6 +164,7 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False,
         # select all samples with positive logged_in attribute:
         s = data[:, 11] == 1
         data = np.c_[data[s, :11], data[s, 12:]]
+        feature_names = feature_names[:11] + feature_names[12:]
         target = target[s]

         data[:, 0] = np.log((data[:, 0] + 0.1).astype(float, copy=False))
@@ -154,15 +176,21 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False,
             data = data[s]
             target = target[s]
             data = np.c_[data[:, 0], data[:, 4], data[:, 5]]
+            feature_names = [feature_names[0], feature_names[4],
+                             feature_names[5]]

         if subset == 'smtp':
             s = data[:, 2] == b'smtp'
             data = data[s]
             target = target[s]
             data = np.c_[data[:, 0], data[:, 4], data[:, 5]]
+            feature_names = [feature_names[0], feature_names[4],
+                             feature_names[5]]

         if subset == 'SF':
             data = np.c_[data[:, 0], data[:, 2], data[:, 4], data[:, 5]]
+            feature_names = [feature_names[0], feature_names[2],
+                             feature_names[4], feature_names[5]]

     if shuffle:
         data, target = shuffle_method(data, target, random_state=random_state)
@@ -174,7 +202,20 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False,
     if return_X_y:
         return data, target

-    return Bunch(data=data, target=target, DESCR=fdescr)
+    frame = None
+    if as_frame:
+        frame, data, target = _convert_data_dataframe(
+            "fetch_kddcup99", data, target, feature_names, target_names
+        )
+
+    return Bunch(
+        data=data,
+        target=target,
+        frame=frame,
+        target_names=target_names,
+        feature_names=feature_names,
+        DESCR=fdescr,
+    )

 def _fetch_brute_kddcup99(data_home=None,
@@ -205,6 +246,10 @@ def _fetch_brute_kddcup99(data_home=None,
         target : ndarray of shape (494021,)
             Each value corresponds to one of the 21 attack types or to the
             label 'normal.'.
+        feature_names : list
+            The names of the dataset columns
+        target_names: list
+            The names of the target columns
         DESCR : str
             Description of the kddcup99 dataset.

@@ -224,52 +269,56 @@ def _fetch_brute_kddcup99(data_home=None,
     targets_path = join(kddcup_dir, "targets")
     available = exists(samples_path)

+    dt = [('duration', int),
+          ('protocol_type', 'S4'),
+          ('service', 'S11'),
+          ('flag', 'S6'),
+          ('src_bytes', int),
+          ('dst_bytes', int),
+          ('land', int),
+          ('wrong_fragment', int),
+          ('urgent', int),
+          ('hot', int),
+          ('num_failed_logins', int),
+          ('logged_in', int),
+          ('num_compromised', int),
+          ('root_shell', int),
+          ('su_attempted', int),
+          ('num_root', int),
+          ('num_file_creations', int),
+          ('num_shells', int),
+          ('num_access_files', int),
+          ('num_outbound_cmds', int),
+          ('is_host_login', int),
+          ('is_guest_login', int),
+          ('count', int),
+          ('srv_count', int),
+          ('serror_rate', float),
+          ('srv_serror_rate', float),
+          ('rerror_rate', float),
+          ('srv_rerror_rate', float),
+          ('same_srv_rate', float),
+          ('diff_srv_rate', float),
+          ('srv_diff_host_rate', float),
+          ('dst_host_count', int),
+          ('dst_host_srv_count', int),
+          ('dst_host_same_srv_rate', float),
+          ('dst_host_diff_srv_rate', float),
+          ('dst_host_same_src_port_rate', float),
+          ('dst_host_srv_diff_host_rate', float),
+          ('dst_host_serror_rate', float),
+          ('dst_host_srv_serror_rate', float),
+          ('dst_host_rerror_rate', float),
+          ('dst_host_srv_rerror_rate', float),
+          ('labels', 'S16')]
+
+    column_names = [c[0] for c in dt]
+    target_names = column_names[-1]
+    feature_names = column_names[:-1]
     if download_if_missing and not available:
         _mkdirp(kddcup_dir)
         logger.info("Downloading %s" % archive.url)
         _fetch_remote(archive, dirname=kddcup_dir)
-        dt = [('duration', int),
-              ('protocol_type', 'S4'),
-              ('service', 'S11'),
-              ('flag', 'S6'),
-              ('src_bytes', int),
-              ('dst_bytes', int),
-              ('land', int),
-              ('wrong_fragment', int),
-              ('urgent', int),
-              ('hot', int),
-              ('num_failed_logins', int),
-              ('logged_in', int),
-              ('num_compromised', int),
-              ('root_shell', int),
-              ('su_attempted', int),
-              ('num_root', int),
-              ('num_file_creations', int),
-              ('num_shells', int),
-              ('num_access_files', int),
-              ('num_outbound_cmds', int),
-              ('is_host_login', int),
-              ('is_guest_login', int),
-              ('count', int),
-              ('srv_count', int),
-              ('serror_rate', float),
-              ('srv_serror_rate', float),
-              ('rerror_rate', float),
-              ('srv_rerror_rate', float),
-              ('same_srv_rate', float),
-              ('diff_srv_rate', float),
-              ('srv_diff_host_rate', float),
-              ('dst_host_count', int),
-              ('dst_host_srv_count', int),
-              ('dst_host_same_srv_rate', float),
-              ('dst_host_diff_srv_rate', float),
-              ('dst_host_same_src_port_rate', float),
-              ('dst_host_srv_diff_host_rate', float),
-              ('dst_host_serror_rate', float),
-              ('dst_host_srv_serror_rate', float),
-              ('dst_host_rerror_rate', float),
-              ('dst_host_srv_rerror_rate', float),
-              ('labels', 'S16')]
         DT = np.dtype(dt)
         logger.debug("extracting archive")
         archive_path = join(kddcup_dir, archive.filename)
@@ -304,7 +353,12 @@ def _fetch_brute_kddcup99(data_home=None,
         X = joblib.load(samples_path)
         y = joblib.load(targets_path)

-    return Bunch(data=X, target=y)
+    return Bunch(
+        data=X,
+        target=y,
+        feature_names=feature_names,
+        target_names=[target_names],
+    )

 def _mkdirp(d):
diff --git a/sklearn/datasets/descr/kddcup99.rst b/sklearn/datasets/descr/kddcup99.rst
index 00427ac08b748..8bdcccf7973ea 100644
--- a/sklearn/datasets/descr/kddcup99.rst
+++ b/sklearn/datasets/descr/kddcup99.rst
@@ -78,8 +78,9 @@ General KDD structure :

 :func:`sklearn.datasets.fetch_kddcup99` will load the kddcup99 dataset; it
 returns a dictionary-like object with the feature matrix in the ``data`` member
-and the target values in ``target``. The dataset will be downloaded from the
-web if necessary.
+and the target values in ``target``. The "as_frame" optional argument converts
+``data`` into a pandas DataFrame and ``target`` into a pandas Series. The
+dataset will be downloaded from the web if necessary.

 .. topic: References

@@ -92,4 +93,3 @@ web if necessary.
            discounting learning algorithms. In Proceedings of the sixth
            ACM SIGKDD international conference on Knowledge discovery
            and data mining, pages 320-324. ACM Press, 2000.
-

diff --git a/sklearn/datasets/tests/test_kddcup99.py b/sklearn/datasets/tests/test_kddcup99.py
index 414c1bab1acd5..11adaacfaae20 100644
--- a/sklearn/datasets/tests/test_kddcup99.py
+++ b/sklearn/datasets/tests/test_kddcup99.py
@@ -6,41 +6,56 @@
 is too big to use in unit-testing.
 """

-from sklearn.datasets.tests.test_common import check_return_X_y
 from functools import partial
+import pytest

+from sklearn.datasets import fetch_kddcup99
+from sklearn.datasets.tests.test_common import check_as_frame
+from sklearn.datasets.tests.test_common import check_pandas_dependency_message
+from sklearn.datasets.tests.test_common import check_return_X_y

-def test_percent10(fetch_kddcup99_fxt):
-    data = fetch_kddcup99_fxt()
-
-    assert data.data.shape == (494021, 41)
-    assert data.target.shape == (494021,)
-
-    data_shuffled = fetch_kddcup99_fxt(shuffle=True, random_state=0)
-    assert data.data.shape == data_shuffled.data.shape
-    assert data.target.shape == data_shuffled.target.shape

-    data = fetch_kddcup99_fxt(subset='SA')
-    assert data.data.shape == (100655, 41)
-    assert data.target.shape == (100655,)
+@pytest.mark.parametrize("as_frame", [True, False])
+@pytest.mark.parametrize(
+    "subset, n_samples, n_features",
+    [(None, 494021, 41),
+     ("SA", 100655, 41),
+     ("SF", 73237, 4),
+     ("http", 58725, 3),
+     ("smtp", 9571, 3)]
+)
+def test_fetch_kddcup99_percent10(
+    fetch_kddcup99_fxt, as_frame, subset, n_samples, n_features
+):
+    data = fetch_kddcup99_fxt(subset=subset, as_frame=as_frame)
+    assert data.data.shape == (n_samples, n_features)
+    assert data.target.shape == (n_samples,)
+    if as_frame:
+        assert data.frame.shape == (n_samples, n_features + 1)
+
+
+def test_fetch_kddcup99_return_X_y(fetch_kddcup99_fxt):
+    fetch_func = partial(fetch_kddcup99_fxt, subset='smtp')
+    data = fetch_func()
+    check_return_X_y(data, fetch_func)

-    data = fetch_kddcup99_fxt(subset='SF')
-    assert data.data.shape == (73237, 4)
-    assert data.target.shape == (73237,)

-    data = fetch_kddcup99_fxt(subset='http')
-    assert data.data.shape == (58725, 3)
-    assert data.target.shape == (58725,)
+def test_fetch_kddcup99_as_frame(fetch_kddcup99_fxt):
+    bunch = fetch_kddcup99_fxt()
+    check_as_frame(bunch, fetch_kddcup99_fxt)

-    data = fetch_kddcup99_fxt(subset='smtp')
-    assert data.data.shape == (9571, 3)
-    assert data.target.shape == (9571,)

-    fetch_func = partial(fetch_kddcup99_fxt, subset='smtp')
-    check_return_X_y(data, fetch_func)
+def test_fetch_kddcup99_shuffle(fetch_kddcup99_fxt):
+    dataset = fetch_kddcup99_fxt(
+        random_state=0, subset='SA', percent10=True,
+    )
+    dataset_shuffled = fetch_kddcup99_fxt(
+        random_state=0, subset='SA', shuffle=True, percent10=True,
+    )
+    assert set(dataset['target']) == set(dataset_shuffled['target'])
+    assert dataset_shuffled.data.shape == dataset.data.shape
+    assert dataset_shuffled.target.shape == dataset.target.shape

-def test_shuffle(fetch_kddcup99_fxt):
-    dataset = fetch_kddcup99_fxt(random_state=0, subset='SA', shuffle=True,
-                                 percent10=True)
-    assert(any(dataset.target[-100:] == b'normal.'))
+def test_pandas_dependency_message(fetch_kddcup99_fxt, hide_available_pandas):
+    check_pandas_dependency_message(fetch_kddcup99)

diff --git a/doc/whats_new/v0.24.rst b/doc/whats_new/v0.24.rst
index c6ef6e599..03ec5788e 100644
--- a/doc/whats_new/v0.24.rst
+++ b/doc/whats_new/v0.24.rst
@@ -112,7 +112,13 @@ Changelog
 - |Enhancement| :func:`datasets.fetch_covtype` now now supports the optional
   argument `as_frame`; when it is set to True, the returned Bunch object's
   `data` and `frame` members are pandas DataFrames, and the `target` member is
-  a pandas Series. :pr:`17491` by :user:`Alex Liang <tianchuliang>`.
+  a pandas Series.
+  :pr:`17491` by :user:`Alex Liang <tianchuliang>`.
+
+- |Enhancement| :func:`datasets.fetch_kddcup99` now now supports the optional
+  argument `as_frame`; when it is set to True, the returned Bunch object's
+  `data` and `frame` members are pandas DataFrames, and the `target` member is
+  a pandas Series.

 - |API| The default value of `as_frame` in :func:`datasets.fetch_openml` is
   changed from False to 'auto'.


diff --git a/benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py b/benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py
new file mode 100644
index 0000000000000..d871967ad1327
--- /dev/null
+++ b/benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py
@@ -0,0 +1,148 @@
+"""
+=============================================================
+Kernel PCA Solvers comparison benchmark: time vs n_components
+=============================================================
+
+This benchmark shows that the approximate solvers provided in Kernel PCA can
+help significantly improve its execution speed when an approximate solution
+(small `n_components`) is acceptable. In many real-world datasets a few
+hundreds of principal components are indeed sufficient enough to capture the
+underlying distribution.
+
+Description:
+------------
+A fixed number of training (default: 2000) and test (default: 1000) samples
+with 2 features is generated using the `make_circles` helper method.
+
+KernelPCA models are trained on the training set with an increasing number of
+principal components, between 1 and `max_n_compo` (default: 1999), with
+`n_compo_grid_size` positions (default: 10). For each value of `n_components`
+to try, KernelPCA models are trained for the various possible `eigen_solver`
+values. The execution times are displayed in a plot at the end of the
+experiment.
+
+What you can observe:
+---------------------
+When the number of requested principal components is small, the dense solver
+takes more time to complete, while the randomized method returns similar
+results with shorter execution times.
+
+Going further:
+--------------
+You can adjust `max_n_compo` and `n_compo_grid_size` if you wish to explore a
+different range of values for `n_components`.
+
+You can also set `arpack_all=True` to activate arpack solver for large number
+of components (this takes more time).
+"""
+# Authors: Sylvain MARIE, Schneider Electric
+
+import time
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+from numpy.testing import assert_array_almost_equal
+from sklearn.decomposition import KernelPCA
+from sklearn.datasets import make_circles
+
+
+print(__doc__)
+
+
+# 1- Design the Experiment
+# ------------------------
+n_train, n_test = 2000, 1000            # the sample sizes to use
+max_n_compo = 1999                      # max n_components to try
+n_compo_grid_size = 10                  # nb of positions in the grid to try
+# generate the grid
+n_compo_range = [np.round(np.exp((x / (n_compo_grid_size - 1))
+                                 * np.log(max_n_compo)))
+                 for x in range(0, n_compo_grid_size)]
+
+n_iter = 3          # the number of times each experiment will be repeated
+arpack_all = False  # set to True if you wish to run arpack for all n_compo
+
+
+# 2- Generate random data
+# -----------------------
+n_features = 2
+X, y = make_circles(n_samples=(n_train + n_test), factor=.3, noise=.05,
+                    random_state=0)
+X_train, X_test = X[:n_train, :], X[n_train:, :]
+
+
+# 3- Benchmark
+# ------------
+# init
+ref_time = np.empty((len(n_compo_range), n_iter)) * np.nan
+a_time = np.empty((len(n_compo_range), n_iter)) * np.nan
+r_time = np.empty((len(n_compo_range), n_iter)) * np.nan
+# loop
+for j, n_components in enumerate(n_compo_range):
+
+    n_components = int(n_components)
+    print("Performing kPCA with n_components = %i" % n_components)
+
+    # A- reference (dense)
+    print("  - dense solver")
+    for i in range(n_iter):
+        start_time = time.perf_counter()
+        ref_pred = KernelPCA(n_components, eigen_solver="dense") \
+            .fit(X_train).transform(X_test)
+        ref_time[j, i] = time.perf_counter() - start_time
+
+    # B- arpack (for small number of components only, too slow otherwise)
+    if arpack_all or n_components < 100:
+        print("  - arpack solver")
+        for i in range(n_iter):
+            start_time = time.perf_counter()
+            a_pred = KernelPCA(n_components, eigen_solver="arpack") \
+                .fit(X_train).transform(X_test)
+            a_time[j, i] = time.perf_counter() - start_time
+            # check that the result is still correct despite the approx
+            assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred))
+
+    # C- randomized
+    print("  - randomized solver")
+    for i in range(n_iter):
+        start_time = time.perf_counter()
+        r_pred = KernelPCA(n_components, eigen_solver="randomized") \
+            .fit(X_train).transform(X_test)
+        r_time[j, i] = time.perf_counter() - start_time
+        # check that the result is still correct despite the approximation
+        assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred))
+
+# Compute statistics for the 3 methods
+avg_ref_time = ref_time.mean(axis=1)
+std_ref_time = ref_time.std(axis=1)
+avg_a_time = a_time.mean(axis=1)
+std_a_time = a_time.std(axis=1)
+avg_r_time = r_time.mean(axis=1)
+std_r_time = r_time.std(axis=1)
+
+
+# 4- Plots
+# --------
+fig, ax = plt.subplots(figsize=(12, 8))
+
+# Display 1 plot with error bars per method
+ax.errorbar(n_compo_range, avg_ref_time, yerr=std_ref_time,
+            marker='x', linestyle='', color='r', label='full')
+ax.errorbar(n_compo_range, avg_a_time, yerr=std_a_time, marker='x',
+            linestyle='', color='g', label='arpack')
+ax.errorbar(n_compo_range, avg_r_time, yerr=std_r_time, marker='x',
+            linestyle='', color='b', label='randomized')
+ax.legend(loc='upper left')
+
+# customize axes
+ax.set_xscale('log')
+ax.set_xlim(1, max(n_compo_range) * 1.1)
+ax.set_ylabel("Execution time (s)")
+ax.set_xlabel("n_components")
+
+ax.set_title("kPCA Execution time comparison on %i samples with %i "
+             "features, according to the choice of `eigen_solver`"
+             "" % (n_train, n_features))
+
+plt.show()
diff --git a/benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py b/benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py
new file mode 100644
index 0000000000000..d238802a68d64
--- /dev/null
+++ b/benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py
@@ -0,0 +1,153 @@
+"""
+==========================================================
+Kernel PCA Solvers comparison benchmark: time vs n_samples
+==========================================================
+
+This benchmark shows that the approximate solvers provided in Kernel PCA can
+help significantly improve its execution speed when an approximate solution
+(small `n_components`) is acceptable. In many real-world datasets the number of
+samples is very large, but a few hundreds of principal components are
+sufficient enough to capture the underlying distribution.
+
+Description:
+------------
+An increasing number of examples is used to train a KernelPCA, between
+`min_n_samples` (default: 101) and `max_n_samples` (default: 4000) with
+`n_samples_grid_size` positions (default: 4). Samples have 2 features, and are
+generated using `make_circles`. For each training sample size, KernelPCA models
+are trained for the various possible `eigen_solver` values. All of them are
+trained to obtain `n_components` principal components (default: 100). The
+execution times are displayed in a plot at the end of the experiment.
+
+What you can observe:
+---------------------
+When the number of samples provided gets large, the dense solver takes a lot
+of time to complete, while the randomized method returns similar results in
+much shorter execution times.
+
+Going further:
+--------------
+You can increase `max_n_samples` and `nb_n_samples_to_try` if you wish to
+explore a wider range of values for `n_samples`.
+
+You can also set `include_arpack=True` to add this other solver in the
+experiments (much slower).
+
+Finally you can have a look at the second example of this series, "Kernel PCA
+Solvers comparison benchmark: time vs n_components", where this time the number
+of examples is fixed, and the desired number of components varies.
+"""
+# Author: Sylvain MARIE, Schneider Electric
+
+import time
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+from numpy.testing import assert_array_almost_equal
+from sklearn.decomposition import KernelPCA
+from sklearn.datasets import make_circles
+
+
+print(__doc__)
+
+
+# 1- Design the Experiment
+# ------------------------
+min_n_samples, max_n_samples = 101, 4000  # min and max n_samples to try
+n_samples_grid_size = 4                   # nb of positions in the grid to try
+# generate the grid
+n_samples_range = [min_n_samples + np.floor((x / (n_samples_grid_size - 1))
+                                            * (max_n_samples - min_n_samples))
+                   for x in range(0, n_samples_grid_size)]
+
+n_components = 100      # the number of principal components we want to use
+n_iter = 3              # the number of times each experiment will be repeated
+include_arpack = False  # set this to True to include arpack solver (slower)
+
+
+# 2- Generate random data
+# -----------------------
+n_features = 2
+X, y = make_circles(n_samples=max_n_samples, factor=.3, noise=.05,
+                    random_state=0)
+
+
+# 3- Benchmark
+# ------------
+# init
+ref_time = np.empty((len(n_samples_range), n_iter)) * np.nan
+a_time = np.empty((len(n_samples_range), n_iter)) * np.nan
+r_time = np.empty((len(n_samples_range), n_iter)) * np.nan
+
+# loop
+for j, n_samples in enumerate(n_samples_range):
+
+    n_samples = int(n_samples)
+    print("Performing kPCA with n_samples = %i" % n_samples)
+
+    X_train = X[:n_samples, :]
+    X_test = X_train
+
+    # A- reference (dense)
+    print("  - dense")
+    for i in range(n_iter):
+        start_time = time.perf_counter()
+        ref_pred = KernelPCA(n_components, eigen_solver="dense") \
+            .fit(X_train).transform(X_test)
+        ref_time[j, i] = time.perf_counter() - start_time
+
+    # B- arpack
+    if include_arpack:
+        print("  - arpack")
+        for i in range(n_iter):
+            start_time = time.perf_counter()
+            a_pred = KernelPCA(n_components, eigen_solver="arpack") \
+                .fit(X_train).transform(X_test)
+            a_time[j, i] = time.perf_counter() - start_time
+            # check that the result is still correct despite the approx
+            assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred))
+
+    # C- randomized
+    print("  - randomized")
+    for i in range(n_iter):
+        start_time = time.perf_counter()
+        r_pred = KernelPCA(n_components, eigen_solver="randomized") \
+            .fit(X_train).transform(X_test)
+        r_time[j, i] = time.perf_counter() - start_time
+        # check that the result is still correct despite the approximation
+        assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred))
+
+# Compute statistics for the 3 methods
+avg_ref_time = ref_time.mean(axis=1)
+std_ref_time = ref_time.std(axis=1)
+avg_a_time = a_time.mean(axis=1)
+std_a_time = a_time.std(axis=1)
+avg_r_time = r_time.mean(axis=1)
+std_r_time = r_time.std(axis=1)
+
+
+# 4- Plots
+# --------
+fig, ax = plt.subplots(figsize=(12, 8))
+
+# Display 1 plot with error bars per method
+ax.errorbar(n_samples_range, avg_ref_time, yerr=std_ref_time,
+            marker='x', linestyle='', color='r', label='full')
+if include_arpack:
+    ax.errorbar(n_samples_range, avg_a_time, yerr=std_a_time, marker='x',
+                linestyle='', color='g', label='arpack')
+ax.errorbar(n_samples_range, avg_r_time, yerr=std_r_time, marker='x',
+            linestyle='', color='b', label='randomized')
+ax.legend(loc='upper left')
+
+# customize axes
+ax.set_xlim(min(n_samples_range) * 0.9, max(n_samples_range) * 1.1)
+ax.set_ylabel("Execution time (s)")
+ax.set_xlabel("n_samples")
+
+ax.set_title("Execution time comparison of kPCA with %i components on samples "
+             "with %i features, according to the choice of `eigen_solver`"
+             "" % (n_components, n_features))
+
+plt.show()
diff --git a/doc/modules/decomposition.rst b/doc/modules/decomposition.rst
index e971d784c63d6..fd51f60d8bfc6 100644
--- a/doc/modules/decomposition.rst
+++ b/doc/modules/decomposition.rst
@@ -166,32 +166,16 @@ Note: the implementation of ``inverse_transform`` in :class:`PCA` with

 .. topic:: References:

-    * `"Finding structure with randomness: Stochastic algorithms for
+    * Algorithm 4.3 in
+      `"Finding structure with randomness: Stochastic algorithms for
       constructing approximate matrix decompositions"
       <https://arxiv.org/abs/0909.4061>`_
       Halko, et al., 2009

-
-.. _kernel_PCA:
-
-Kernel PCA
-----------
-
-:class:`KernelPCA` is an extension of PCA which achieves non-linear
-dimensionality reduction through the use of kernels (see :ref:`metrics`). It
-has many applications including denoising, compression and structured
-prediction (kernel dependency estimation). :class:`KernelPCA` supports both
-``transform`` and ``inverse_transform``.
-
-.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_kernel_pca_001.png
-    :target: ../auto_examples/decomposition/plot_kernel_pca.html
-    :align: center
-    :scale: 75%
-
-.. topic:: Examples:
-
-    * :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py`
-
+    * `"An implementation of a randomized algorithm for principal component
+      analysis"
+      <https://arxiv.org/pdf/1412.3510.pdf>`_
+      A. Szlam et al. 2014

 .. _SparsePCA:

@@ -278,6 +262,100 @@ factorization, while larger values shrink many coefficients to zero.
      R. Jenatton, G. Obozinski, F. Bach, 2009

+.. _kernel_PCA:
+
+Kernel Principal Component Analysis (kPCA)
+==========================================
+
+Exact Kernel PCA
+----------------
+
+:class:`KernelPCA` is an extension of PCA which achieves non-linear
+dimensionality reduction through the use of kernels (see :ref:`metrics`). It
+has many applications including denoising, compression and structured
+prediction (kernel dependency estimation). :class:`KernelPCA` supports both
+``transform`` and ``inverse_transform``.
+
+.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_kernel_pca_001.png
+    :target: ../auto_examples/decomposition/plot_kernel_pca.html
+    :align: center
+    :scale: 75%
+
+.. topic:: Examples:
+
+    * :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py`
+
+.. topic:: References:
+
+    * Kernel PCA was introduced in "Kernel principal component analysis"
+      Bernhard Schoelkopf, Alexander J. Smola, and Klaus-Robert Mueller. 1999.
+      In Advances in kernel methods, MIT Press, Cambridge, MA, USA 327-352.
+
+
+.. _kPCA_Solvers:
+
+Choice of solver for Kernel PCA
+-------------------------------
+
+While in :class:`PCA` the number of components is bounded by the number of
+features, in :class:`KernelPCA` the number of components is bounded by the
+number of samples. Many real-world datasets have large number of samples! In
+these cases finding *all* the components with a full kPCA is a waste of
+computation time, as data is mostly described by the first few components
+(e.g. ``n_components<=100``). In other words, the centered Gram matrix that
+is eigendecomposed in the Kernel PCA fitting process has an effective rank that
+is much smaller than its size. This is a situation where approximate
+eigensolvers can provide speedup with very low precision loss.
+
+The optional parameter ``eigen_solver='randomized'`` can be used to
+*significantly* reduce the computation time when the number of requested
+``n_components`` is small compared with the number of samples. It relies on
+randomized decomposition methods to find an approximate solution in a shorter
+time.
+
+The time complexity of the randomized :class:`KernelPCA` is
+:math:`O(n_{\mathrm{samples}}^2 \cdot n_{\mathrm{components}})`
+instead of :math:`O(n_{\mathrm{samples}}^3)` for the exact method
+implemented with ``eigen_solver='dense'``.
+
+The memory footprint of randomized :class:`KernelPCA` is also proportional to
+:math:`2 \cdot n_{\mathrm{samples}} \cdot n_{\mathrm{components}}` instead of
+:math:`n_{\mathrm{samples}}^2` for the exact method.
+
+Note: this technique is the same as in :ref:`RandomizedPCA`.
+
+In addition to the above two solvers, ``eigen_solver='arpack'`` can be used as
+an alternate way to get an approximate decomposition. In practice, this method
+only provides reasonable execution times when the number of components to find
+is extremely small. It is enabled by default when the desired number of
+components is less than 10 (strict) and the number of samples is more than 200
+(strict). See :class:`KernelPCA` for details.
+
+.. topic:: References:
+
+    * *dense* solver:
+      `scipy.linalg.eigh documentation
+      <https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.eigh.html>`_
+
+    * *randomized* solver:
+
+        - Algorithm 4.3 in
+          `"Finding structure with randomness: Stochastic algorithms for
+          constructing approximate matrix decompositions"
+          <https://arxiv.org/abs/0909.4061>`_
+          Halko, et al., 2009
+
+        - `"An implementation of a randomized algorithm for principal component
+          analysis"
+          <https://arxiv.org/pdf/1412.3510.pdf>`_
+          A. Szlam et al. 2014
+
+    * *arpack* solver:
+      `scipy.sparse.linalg.eigsh documentation
+      <https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.eigsh.html>`_
+      R. B. Lehoucq, D. C. Sorensen, and C. Yang, 1998
+
+
 .. _LSA:

 Truncated singular value decomposition and latent semantic analysis
diff --git a/doc/whats_new/v1.0.rst b/doc/whats_new/v1.0.rst
index 3b3884e68e185..e89eecfe0874c 100644
--- a/doc/whats_new/v1.0.rst
+++ b/doc/whats_new/v1.0.rst
@@ -159,14 +159,17 @@ Changelog
 - |Fix| Fixes incorrect multiple data-conversion warnings when clustering
   boolean data. :pr:`19046` by :user:`Surya Prakash <jdsurya>`.

-:mod:`sklearn.decomposition`
-............................
-
 - |Fix| Fixed :func:`dict_learning`, used by :class:`DictionaryLearning`, to
   ensure determinism of the output. Achieved by flipping signs of the SVD
   output which is used to initialize the code.
   :pr:`18433` by :user:`Bruno Charron <brcharron>`.

+- |Enhancement| added a new approximate solver (randomized SVD, available with
+  `eigen_solver='randomized'`) to :class:`decomposition.KernelPCA`. This
+  significantly accelerates computation when the number of samples is much
+  larger than the desired number of components.
+  :pr:`12069` by :user:`Sylvain Marié <smarie>`.
+
 - |Fix| Fixed a bug in :class:`MiniBatchDictionaryLearning`,
   :class:`MiniBatchSparsePCA` and :func:`dict_learning_online` where the
   update of the dictionary was incorrect. :pr:`19198` by
@@ -389,8 +392,8 @@ Changelog
   supporting sparse matrix and raise the appropriate error message.
   :pr:`19879` by :user:`Guillaume Lemaitre <glemaitre>`.

-- |Efficiency| Changed ``algorithm`` argument for :class:`cluster.KMeans` in
-  :class:`preprocessing.KBinsDiscretizer` from ``auto`` to ``full``.
+- |Efficiency| Changed ``algorithm`` argument for :class:`cluster.KMeans` in
+  :class:`preprocessing.KBinsDiscretizer` from ``auto`` to ``full``.
   :pr:`19934` by :user:`Gleb Levitskiy <GLevV>`.

 :mod:`sklearn.tree`
diff --git a/sklearn/decomposition/_kernel_pca.py b/sklearn/decomposition/_kernel_pca.py
index 415ee034c1769..8663193a8383e 100644
--- a/sklearn/decomposition/_kernel_pca.py
+++ b/sklearn/decomposition/_kernel_pca.py
@@ -1,6 +1,7 @@
 """Kernel Principal Components Analysis."""

 # Author: Mathieu Blondel <mathieu@mblondel.org>
+#         Sylvain Marie <sylvain.marie@schneider-electric.com>
 # License: BSD 3 clause

 import numpy as np
@@ -8,7 +9,7 @@
 from scipy.sparse.linalg import eigsh

 from ..utils._arpack import _init_arpack_v0
-from ..utils.extmath import svd_flip
+from ..utils.extmath import svd_flip, _randomized_eigsh
 from ..utils.validation import check_is_fitted, _check_psd_eigenvalues
 from ..utils.deprecation import deprecated
 from ..exceptions import NotFittedError
@@ -24,6 +25,12 @@ class KernelPCA(TransformerMixin, BaseEstimator):
     Non-linear dimensionality reduction through the use of kernels (see
     :ref:`metrics`).

+    It uses the `scipy.linalg.eigh` LAPACK implementation of the full SVD or
+    the `scipy.sparse.linalg.eigsh` ARPACK implementation of the truncated SVD,
+    depending on the shape of the input data and the number of components to
+    extract. It can also use a randomized truncated SVD by the method of
+    Halko et al. 2009, see `eigen_solver`.
+
     Read more in the :ref:`User Guide <kernel_PCA>`.

     Parameters
@@ -59,10 +66,37 @@ class KernelPCA(TransformerMixin, BaseEstimator):
         Learn the inverse transform for non-precomputed kernels.
         (i.e. learn to find the pre-image of a point)

-    eigen_solver : {'auto', 'dense', 'arpack'}, default='auto'
-        Select eigensolver to use. If n_components is much less than
-        the number of training samples, arpack may be more efficient
-        than the dense eigensolver.
+    eigen_solver : {'auto', 'dense', 'arpack', 'randomized'}, \
+        default='auto'
+        Select eigensolver to use. If `n_components` is much
+        less than the number of training samples, randomized (or arpack to a
+        smaller extend) may be more efficient than the dense eigensolver.
+        Randomized SVD is performed according to the method of Halko et al.
+
+        auto :
+            the solver is selected by a default policy based on n_samples
+            (the number of training samples) and `n_components`:
+            if the number of components to extract is less than 10 (strict) and
+            the number of samples is more than 200 (strict), the 'arpack'
+            method is enabled. Otherwise the exact full eigenvalue
+            decomposition is computed and optionally truncated afterwards
+            ('dense' method).
+        dense :
+            run exact full eigenvalue decomposition calling the standard
+            LAPACK solver via `scipy.linalg.eigh`, and select the components
+            by postprocessing
+        arpack :
+            run SVD truncated to n_components calling ARPACK solver using
+            `scipy.sparse.linalg.eigsh`. It requires strictly
+            0 < n_components < n_samples
+        randomized :
+            run randomized SVD by the method of Halko et al. The current
+            implementation selects eigenvalues based on their module; therefore
+            using this method can lead to unexpected results if the kernel is
+            not positive semi-definite.
+
+        .. versionchanged:: 1.0
+           `'randomized'` was added.

     tol : float, default=0
         Convergence tolerance for arpack.
@@ -72,6 +106,13 @@ class KernelPCA(TransformerMixin, BaseEstimator):
         Maximum number of iterations for arpack.
         If None, optimal value will be chosen by arpack.

+    iterated_power : int >= 0, or 'auto', default='auto'
+        Number of iterations for the power method computed by
+        svd_solver == 'randomized'. When 'auto', it is set to 7 when
+        `n_components < 0.1 * min(X.shape)`, other it is set to 4.
+
+        .. versionadded:: 1.0
+
     remove_zero_eig : bool, default=False
         If True, then all components with zero eigenvalues are removed, so
         that the number of components in the output may be < n_components
@@ -80,8 +121,8 @@ class KernelPCA(TransformerMixin, BaseEstimator):
         with zero eigenvalues are removed regardless.

     random_state : int, RandomState instance or None, default=None
-        Used when ``eigen_solver`` == 'arpack'. Pass an int for reproducible
-        results across multiple function calls.
+        Used when ``eigen_solver`` == 'arpack' or 'randomized'. Pass an int
+        for reproducible results across multiple function calls.
         See :term:`Glossary <random_state>`.

         .. versionadded:: 0.18
@@ -141,12 +182,22 @@ class KernelPCA(TransformerMixin, BaseEstimator):
         and Klaus-Robert Mueller. 1999. Kernel principal
         component analysis. In Advances in kernel methods,
         MIT Press, Cambridge, MA, USA 327-352.
+
+    For eigen_solver == 'arpack', refer to `scipy.sparse.linalg.eigsh`.
+
+    For eigen_solver == 'randomized', see:
+        Finding structure with randomness: Stochastic algorithms
+        for constructing approximate matrix decompositions Halko, et al., 2009
+        (arXiv:909)
+        A randomized algorithm for the decomposition of matrices
+        Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert
     """
     @_deprecate_positional_args
     def __init__(self, n_components=None, *, kernel="linear",
                  gamma=None, degree=3, coef0=1, kernel_params=None,
                  alpha=1.0, fit_inverse_transform=False, eigen_solver='auto',
-                 tol=0, max_iter=None, remove_zero_eig=False,
+                 tol=0, max_iter=None, iterated_power='auto',
+                 remove_zero_eig=False,
                  random_state=None, copy_X=True, n_jobs=None):
         if fit_inverse_transform and kernel == 'precomputed':
             raise ValueError(
@@ -160,9 +211,10 @@ def __init__(self, n_components=None, *, kernel="linear",
         self.alpha = alpha
         self.fit_inverse_transform = fit_inverse_transform
         self.eigen_solver = eigen_solver
-        self.remove_zero_eig = remove_zero_eig
         self.tol = tol
         self.max_iter = max_iter
+        self.iterated_power = iterated_power
+        self.remove_zero_eig = remove_zero_eig
         self.random_state = random_state
         self.n_jobs = n_jobs
         self.copy_X = copy_X
@@ -191,9 +243,14 @@ def _fit_transform(self, K):
         # center kernel
         K = self._centerer.fit_transform(K)

+        # adjust n_components according to user inputs
         if self.n_components is None:
-            n_components = K.shape[0]
+            n_components = K.shape[0]  # use all dimensions
         else:
+            if self.n_components < 1:
+                raise ValueError(
+                    f"`n_components` should be >= 1, got: {self.n_component}"
+                )
             n_components = min(K.shape[0], self.n_components)

         # compute eigenvectors
@@ -206,6 +263,7 @@ def _fit_transform(self, K):
             eigen_solver = self.eigen_solver

         if eigen_solver == 'dense':
+            # Note: eigvals specifies the indices of smallest/largest to return
             self.lambdas_, self.alphas_ = linalg.eigh(
                 K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1))
         elif eigen_solver == 'arpack':
@@ -215,6 +273,14 @@ def _fit_transform(self, K):
                                                 tol=self.tol,
                                                 maxiter=self.max_iter,
                                                 v0=v0)
+        elif eigen_solver == 'randomized':
+            self.lambdas_, self.alphas_ = _randomized_eigsh(
+                K, n_components=n_components, n_iter=self.iterated_power,
+                random_state=self.random_state, selection='module'
+            )
+        else:
+            raise ValueError("Unsupported value for `eigen_solver`: %r"
+                             % eigen_solver)

         # make sure that the eigenvalues are ok and fix numerical issues
         self.lambdas_ = _check_psd_eigenvalues(self.lambdas_,
diff --git a/sklearn/utils/extmath.py b/sklearn/utils/extmath.py
index add8c5883a751..c72c54bd1aa4d 100644
--- a/sklearn/utils/extmath.py
+++ b/sklearn/utils/extmath.py
@@ -249,6 +249,9 @@ def randomized_svd(M, n_components, *, n_oversamples=10, n_iter='auto',
                    flip_sign=True, random_state='warn'):
     """Computes a truncated randomized SVD.

+    This method solves the fixed-rank approximation problem described in the
+    Halko et al paper (problem (1.5), p5).
+
     Parameters
     ----------
     M : {ndarray, sparse matrix}
@@ -262,13 +265,23 @@ def randomized_svd(M, n_components, *, n_oversamples=10, n_iter='auto',
         to ensure proper conditioning. The total number of random vectors
         used to find the range of M is n_components + n_oversamples. Smaller
         number can improve speed but can negatively impact the quality of
-        approximation of singular vectors and singular values.
+        approximation of singular vectors and singular values. Users might wish
+        to increase this parameter up to `2*k - n_components` where k is the
+        effective rank, for large matrices, noisy problems, matrices with
+        slowly decaying spectrums, or to increase precision accuracy. See Halko
+        et al (pages 5, 23 and 26).

     n_iter : int or 'auto', default='auto'
         Number of power iterations. It can be used to deal with very noisy
         problems. When 'auto', it is set to 4, unless `n_components` is small
-        (< .1 * min(X.shape)) `n_iter` in which case is set to 7.
-        This improves precision with few components.
+        (< .1 * min(X.shape)) in which case `n_iter` is set to 7.
+        This improves precision with few components. Note that in general
+        users should rather increase `n_oversamples` before increasing `n_iter`
+        as the principle of the randomized method is to avoid usage of these
+        more costly power iterations steps. When `n_components` is equal
+        or greater to the effective matrix rank and the spectrum does not
+        present a slow decay, `n_iter=0` or `1` should even work fine in theory
+        (see Halko et al paper, page 9).

         .. versionchanged:: 0.18

@@ -316,12 +329,15 @@ def randomized_svd(M, n_components, *, n_oversamples=10, n_iter='auto',
     computations. It is particularly fast on large matrices on which
     you wish to extract only a small number of components. In order to
     obtain further speed up, `n_iter` can be set <=2 (at the cost of
-    loss of precision).
+    loss of precision). To increase the precision it is recommended to
+    increase `n_oversamples`, up to `2*k-n_components` where k is the
+    effective rank. Usually, `n_components` is chosen to be greater than k
+    so increasing `n_oversamples` up to `n_components` should be enough.

     References
     ----------
     * Finding structure with randomness: Stochastic algorithms for constructing
-      approximate matrix decompositions
+      approximate matrix decompositions (Algorithm 4.3)
       Halko, et al., 2009 https://arxiv.org/abs/0909.4061

     * A randomized algorithm for the decomposition of matrices
@@ -393,6 +409,152 @@ def randomized_svd(M, n_components, *, n_oversamples=10, n_iter='auto',
         return U[:, :n_components], s[:n_components], Vt[:n_components, :]

+@_deprecate_positional_args
+def _randomized_eigsh(M, n_components, *, n_oversamples=10, n_iter='auto',
+                      power_iteration_normalizer='auto',
+                      selection='module', random_state=None):
+    """Computes a truncated eigendecomposition using randomized methods
+
+    This method solves the fixed-rank approximation problem described in the
+    Halko et al paper.
+
+    The choice of which components to select can be tuned with the `selection`
+    parameter.
+
+    .. versionadded:: 0.24
+
+    Parameters
+    ----------
+    M : ndarray or sparse matrix
+        Matrix to decompose, it should be real symmetric square or complex
+        hermitian
+
+    n_components : int
+        Number of eigenvalues and vectors to extract.
+
+    n_oversamples : int, default=10
+        Additional number of random vectors to sample the range of M so as
+        to ensure proper conditioning. The total number of random vectors
+        used to find the range of M is n_components + n_oversamples. Smaller
+        number can improve speed but can negatively impact the quality of
+        approximation of eigenvectors and eigenvalues. Users might wish
+        to increase this parameter up to `2*k - n_components` where k is the
+        effective rank, for large matrices, noisy problems, matrices with
+        slowly decaying spectrums, or to increase precision accuracy. See Halko
+        et al (pages 5, 23 and 26).
+
+    n_iter : int or 'auto', default='auto'
+        Number of power iterations. It can be used to deal with very noisy
+        problems. When 'auto', it is set to 4, unless `n_components` is small
+        (< .1 * min(X.shape)) in which case `n_iter` is set to 7.
+        This improves precision with few components. Note that in general
+        users should rather increase `n_oversamples` before increasing `n_iter`
+        as the principle of the randomized method is to avoid usage of these
+        more costly power iterations steps. When `n_components` is equal
+        or greater to the effective matrix rank and the spectrum does not
+        present a slow decay, `n_iter=0` or `1` should even work fine in theory
+        (see Halko et al paper, page 9).
+
+    power_iteration_normalizer : {'auto', 'QR', 'LU', 'none'}, default='auto'
+        Whether the power iterations are normalized with step-by-step
+        QR factorization (the slowest but most accurate), 'none'
+        (the fastest but numerically unstable when `n_iter` is large, e.g.
+        typically 5 or larger), or 'LU' factorization (numerically stable
+        but can lose slightly in accuracy). The 'auto' mode applies no
+        normalization if `n_iter` <= 2 and switches to LU otherwise.
+
+    selection : {'value', 'module'}, default='module'
+        Strategy used to select the n components. When `selection` is `'value'`
+        (not yet implemented, will become the default when implemented), the
+        components corresponding to the n largest eigenvalues are returned.
+        When `selection` is `'module'`, the components corresponding to the n
+        eigenvalues with largest modules are returned.
+
+    random_state : int, RandomState instance, default=None
+        The seed of the pseudo random number generator to use when shuffling
+        the data, i.e. getting the random vectors to initialize the algorithm.
+        Pass an int for reproducible results across multiple function calls.
+        See :term:`Glossary <random_state>`.
+
+    Notes
+    -----
+    This algorithm finds a (usually very good) approximate truncated
+    eigendecomposition using randomized methods to speed up the computations.
+
+    This method is particularly fast on large matrices on which
+    you wish to extract only a small number of components. In order to
+    obtain further speed up, `n_iter` can be set <=2 (at the cost of
+    loss of precision). To increase the precision it is recommended to
+    increase `n_oversamples`, up to `2*k-n_components` where k is the
+    effective rank. Usually, `n_components` is chosen to be greater than k
+    so increasing `n_oversamples` up to `n_components` should be enough.
+
+    Strategy 'value': not implemented yet.
+    Algorithms 5.3, 5.4 and 5.5 in the Halko et al paper should provide good
+    condidates for a future implementation.
+
+    Strategy 'module':
+    The principle is that for diagonalizable matrices, the singular values and
+    eigenvalues are related: if t is an eigenvalue of A, then :math:`|t|` is a
+    singular value of A. This method relies on a randomized SVD to find the n
+    singular components corresponding to the n singular values with largest
+    modules, and then uses the signs of the singular vectors to find the true
+    sign of t: if the sign of left and right singular vectors are different
+    then the corresponding eigenvalue is negative.
+
+    Returns
+    -------
+    eigvals : 1D array of shape (n_components,) containing the `n_components`
+        eigenvalues selected (see ``selection`` parameter).
+    eigvecs : 2D array of shape (M.shape[0], n_components) containing the
+        `n_components` eigenvectors corresponding to the `eigvals`, in the
+        corresponding order. Note that this follows the `scipy.linalg.eigh`
+        convention.
+
+    See Also
+    --------
+    :func:`randomized_svd`
+
+    References
+    ----------
+    * Finding structure with randomness: Stochastic algorithms for constructing
+      approximate matrix decompositions (Algorithm 4.3 for strategy 'module')
+      Halko, et al., 2009 https://arxiv.org/abs/0909.4061
+
+    """
+    if selection == 'value':  # pragma: no cover
+        # to do : an algorithm can be found in the Halko et al reference
+        raise NotImplementedError()
+
+    elif selection == 'module':
+        # Note: no need for deterministic U and Vt (flip_sign=True),
+        # as we only use the dot product UVt afterwards
+        U, S, Vt = randomized_svd(
+            M, n_components=n_components, n_oversamples=n_oversamples,
+            n_iter=n_iter,
+            power_iteration_normalizer=power_iteration_normalizer,
+            flip_sign=False, random_state=random_state)
+
+        eigvecs = U[:, :n_components]
+        eigvals = S[:n_components]
+
+        # Conversion of Singular values into Eigenvalues:
+        # For any eigenvalue t, the corresponding singular value is |t|.
+        # So if there is a negative eigenvalue t, the corresponding singular
+        # value will be -t, and the left (U) and right (V) singular vectors
+        # will have opposite signs.
+        # Fastest way: see <https://stackoverflow.com/a/61974002/7262247>
+        diag_VtU = np.einsum('ji,ij->j',
+                             Vt[:n_components, :], U[:, :n_components])
+        signs = np.sign(diag_VtU)
+        eigvals = eigvals * signs
+
+    else:  # pragma: no cover
+        raise ValueError("Invalid `selection`: %r" % selection)
+
+    return eigvals, eigvecs
+
+
 @_deprecate_positional_args
 def weighted_mode(a, w, *, axis=0):
     """Returns an array of the weighted modal (most common) value in a.

diff --git a/sklearn/decomposition/tests/test_kernel_pca.py b/sklearn/decomposition/tests/test_kernel_pca.py
index adf68f1db1a6c..5c8d052a7aa14 100644
--- a/sklearn/decomposition/tests/test_kernel_pca.py
+++ b/sklearn/decomposition/tests/test_kernel_pca.py
@@ -3,11 +3,13 @@
 import pytest

 from sklearn.utils._testing import (assert_array_almost_equal,
-                                   assert_allclose)
+                                    assert_array_equal,
+                                    assert_allclose)

 from sklearn.decomposition import PCA, KernelPCA
 from sklearn.datasets import make_circles
 from sklearn.datasets import make_blobs
+from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import Perceptron
 from sklearn.pipeline import Pipeline
 from sklearn.preprocessing import StandardScaler
@@ -17,6 +19,12 @@

 def test_kernel_pca():
+    """Nominal test for all solvers and all known kernels + a custom one
+
+    It tests
+     - that fit_transform is equivalent to fit+transform
+     - that the shapes of transforms and inverse transforms are correct
+    """
     rng = np.random.RandomState(0)
     X_fit = rng.random_sample((5, 4))
     X_pred = rng.random_sample((2, 4))
@@ -26,7 +34,7 @@ def histogram(x, y, **kwargs):
         assert kwargs == {}    # no kernel_params that we didn't ask for
         return np.minimum(x, y).sum()

-    for eigen_solver in ("auto", "dense", "arpack"):
+    for eigen_solver in ("auto", "dense", "arpack", "randomized"):
         for kernel in ("linear", "rbf", "poly", histogram):
             # histogram kernel produces singular matrix inside linalg.solve
             # XXX use a least-squares approximation?
@@ -55,12 +63,31 @@ def histogram(x, y, **kwargs):
                 assert X_pred2.shape == X_pred.shape

+def test_kernel_pca_invalid_solver():
+    """Check that kPCA raises an error if the solver parameter is invalid
+
+    """
+    with pytest.raises(ValueError):
+        KernelPCA(eigen_solver="unknown").fit(np.random.randn(10, 10))
+
+
 def test_kernel_pca_invalid_parameters():
+    """Check that kPCA raises an error if the parameters are invalid
+
+    Tests fitting inverse transform with a precomputed kernel raises a
+    ValueError.
+    """
     with pytest.raises(ValueError):
         KernelPCA(10, fit_inverse_transform=True, kernel='precomputed')

 def test_kernel_pca_consistent_transform():
+    """Check robustness to mutations in the original training array
+
+    Test that after fitting a kPCA model, it stays independent of any
+    mutation of the values of the original data object by relying on an
+    internal copy.
+    """
     # X_fit_ needs to retain the old, unmodified copy of X
     state = np.random.RandomState(0)
     X = state.rand(10, 10)
@@ -74,6 +101,10 @@ def test_kernel_pca_consistent_transform():

 def test_kernel_pca_deterministic_output():
+    """Test that Kernel PCA produces deterministic output
+
+    Tests that the same inputs and random state produce the same output.
+    """
     rng = np.random.RandomState(0)
     X = rng.rand(10, 10)
     eigen_solver = ('arpack', 'dense')
@@ -89,15 +120,20 @@ def test_kernel_pca_deterministic_output():

 def test_kernel_pca_sparse():
+    """Test that kPCA works on a sparse data input.
+
+    Same test as ``test_kernel_pca except inverse_transform`` since it's not
+    implemented for sparse matrices.
+    """
     rng = np.random.RandomState(0)
     X_fit = sp.csr_matrix(rng.random_sample((5, 4)))
     X_pred = sp.csr_matrix(rng.random_sample((2, 4)))

-    for eigen_solver in ("auto", "arpack"):
+    for eigen_solver in ("auto", "arpack", "randomized"):
         for kernel in ("linear", "rbf", "poly"):
             # transform fit data
             kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver,
-                             fit_inverse_transform=False)
+                             fit_inverse_transform=False, random_state=0)
             X_fit_transformed = kpca.fit_transform(X_fit)
             X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
             assert_array_almost_equal(np.abs(X_fit_transformed),
@@ -108,31 +144,47 @@ def test_kernel_pca_sparse():
             assert (X_pred_transformed.shape[1] ==
                          X_fit_transformed.shape[1])

-            # inverse transform
-            # X_pred2 = kpca.inverse_transform(X_pred_transformed)
-            # assert X_pred2.shape == X_pred.shape)
+            # inverse transform: not available for sparse matrices
+            # XXX: should we raise another exception type here? For instance:
+            # NotImplementedError.
+            with pytest.raises(NotFittedError):
+                kpca.inverse_transform(X_pred_transformed)

-def test_kernel_pca_linear_kernel():
+@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"])
+@pytest.mark.parametrize("n_features", [4, 10])
+def test_kernel_pca_linear_kernel(solver, n_features):
+    """Test that kPCA with linear kernel is equivalent to PCA for all solvers.
+
+    KernelPCA with linear kernel should produce the same output as PCA.
+    """
     rng = np.random.RandomState(0)
-    X_fit = rng.random_sample((5, 4))
-    X_pred = rng.random_sample((2, 4))
+    X_fit = rng.random_sample((5, n_features))
+    X_pred = rng.random_sample((2, n_features))

     # for a linear kernel, kernel PCA should find the same projection as PCA
     # modulo the sign (direction)
     # fit only the first four components: fifth is near zero eigenvalue, so
     # can be trimmed due to roundoff error
+    n_comps = 3 if solver == "arpack" else 4
     assert_array_almost_equal(
-        np.abs(KernelPCA(4).fit(X_fit).transform(X_pred)),
-        np.abs(PCA(4).fit(X_fit).transform(X_pred)))
+        np.abs(KernelPCA(n_comps, eigen_solver=solver).fit(X_fit)
+               .transform(X_pred)),
+        np.abs(PCA(n_comps, svd_solver=solver if solver != "dense" else "full")
+               .fit(X_fit).transform(X_pred)))

 def test_kernel_pca_n_components():
+    """Test that `n_components` is correctly taken into account for projections
+
+    For all solvers this tests that the output has the correct shape depending
+    on the selected number of components.
+    """
     rng = np.random.RandomState(0)
     X_fit = rng.random_sample((5, 4))
     X_pred = rng.random_sample((2, 4))

-    for eigen_solver in ("dense", "arpack"):
+    for eigen_solver in ("dense", "arpack", "randomized"):
         for c in [1, 2, 4]:
             kpca = KernelPCA(n_components=c, eigen_solver=eigen_solver)
             shape = kpca.fit(X_fit).transform(X_pred).shape
@@ -141,6 +193,11 @@ def test_kernel_pca_n_components():

 def test_remove_zero_eig():
+    """Check that the ``remove_zero_eig`` parameter works correctly.
+
+    Tests that the null-space (Zero) eigenvalues are removed when
+    remove_zero_eig=True, whereas they are not by default.
+    """
     X = np.array([[1 - 1e-30, 1], [1, 1], [1, 1 - 1e-20]])

     # n_components=None (default) => remove_zero_eig is True
@@ -158,9 +215,11 @@ def test_remove_zero_eig():

 def test_leave_zero_eig():
-    """This test checks that fit().transform() returns the same result as
+    """Non-regression test for issue #12141 (PR #12143)
+
+    This test checks that fit().transform() returns the same result as
     fit_transform() in case of non-removed zero eigenvalue.
-    Non-regression test for issue #12141 (PR #12143)"""
+    """
     X_fit = np.array([[1, 1], [0, 0]])

     # Assert that even with all np warnings on, there is no div by zero warning
@@ -184,23 +243,29 @@ def test_leave_zero_eig():

 def test_kernel_pca_precomputed():
+    """Test that kPCA works with a precomputed kernel, for all solvers
+
+    """
     rng = np.random.RandomState(0)
     X_fit = rng.random_sample((5, 4))
     X_pred = rng.random_sample((2, 4))

-    for eigen_solver in ("dense", "arpack"):
-        X_kpca = KernelPCA(4, eigen_solver=eigen_solver).\
-            fit(X_fit).transform(X_pred)
+    for eigen_solver in ("dense", "arpack", "randomized"):
+        X_kpca = KernelPCA(
+            4, eigen_solver=eigen_solver, random_state=0
+        ).fit(X_fit).transform(X_pred)
+
         X_kpca2 = KernelPCA(
-            4, eigen_solver=eigen_solver, kernel='precomputed').fit(
-                np.dot(X_fit, X_fit.T)).transform(np.dot(X_pred, X_fit.T))
+            4, eigen_solver=eigen_solver, kernel='precomputed', random_state=0
+        ).fit(np.dot(X_fit, X_fit.T)).transform(np.dot(X_pred, X_fit.T))

         X_kpca_train = KernelPCA(
-            4, eigen_solver=eigen_solver,
-            kernel='precomputed').fit_transform(np.dot(X_fit, X_fit.T))
+            4, eigen_solver=eigen_solver, kernel='precomputed', random_state=0
+        ).fit_transform(np.dot(X_fit, X_fit.T))
+
         X_kpca_train2 = KernelPCA(
-            4, eigen_solver=eigen_solver, kernel='precomputed').fit(
-                np.dot(X_fit, X_fit.T)).transform(np.dot(X_fit, X_fit.T))
+            4, eigen_solver=eigen_solver, kernel='precomputed', random_state=0
+        ).fit(np.dot(X_fit, X_fit.T)).transform(np.dot(X_fit, X_fit.T))

         assert_array_almost_equal(np.abs(X_kpca),
                                   np.abs(X_kpca2))
@@ -209,7 +274,42 @@ def test_kernel_pca_precomputed():
                                   np.abs(X_kpca_train2))

+@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"])
+def test_kernel_pca_precomputed_non_symmetric(solver):
+    """Check that the kernel centerer works.
+
+    Tests that a non symmetric precomputed kernel is actually accepted
+    because the kernel centerer does its job correctly.
+    """
+
+    # a non symmetric gram matrix
+    K = [
+        [1, 2],
+        [3, 40]
+    ]
+    kpca = KernelPCA(kernel="precomputed", eigen_solver=solver,
+                     n_components=1, random_state=0)
+    kpca.fit(K)  # no error
+
+    # same test with centered kernel
+    Kc = [
+        [9, -9],
+        [-9, 9]
+    ]
+    kpca_c = KernelPCA(kernel="precomputed", eigen_solver=solver,
+                       n_components=1, random_state=0)
+    kpca_c.fit(Kc)
+
+    # comparison between the non-centered and centered versions
+    assert_array_equal(kpca.alphas_, kpca_c.alphas_)
+    assert_array_equal(kpca.lambdas_, kpca_c.lambdas_)
+
+
 def test_kernel_pca_invalid_kernel():
+    """Tests that using an invalid kernel name raises a ValueError
+
+    An invalid kernel name should raise a ValueError at fit time.
+    """
     rng = np.random.RandomState(0)
     X_fit = rng.random_sample((2, 4))
     kpca = KernelPCA(kernel="tototiti")
@@ -218,8 +318,11 @@ def test_kernel_pca_invalid_kernel():

 def test_gridsearch_pipeline():
-    # Test if we can do a grid-search to find parameters to separate
-    # circles with a perceptron model.
+    """Check that kPCA works as expected in a grid search pipeline
+
+    Test if we can do a grid-search to find parameters to separate
+    circles with a perceptron model.
+    """
     X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                         random_state=0)
     kpca = KernelPCA(kernel="rbf", n_components=2)
@@ -232,8 +335,11 @@ def test_gridsearch_pipeline():

 def test_gridsearch_pipeline_precomputed():
-    # Test if we can do a grid-search to find parameters to separate
-    # circles with a perceptron model using a precomputed kernel.
+    """Check that kPCA works as expected in a grid search pipeline (2)
+
+    Test if we can do a grid-search to find parameters to separate
+    circles with a perceptron model. This test uses a precomputed kernel.
+    """
     X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                         random_state=0)
     kpca = KernelPCA(kernel="precomputed", n_components=2)
@@ -247,7 +353,12 @@ def test_gridsearch_pipeline_precomputed():

 def test_nested_circles():
-    # Test the linear separability of the first 2D KPCA transform
+    """Check that kPCA projects in a space where nested circles are separable
+
+    Tests that 2D nested circles become separable with a perceptron when
+    projected in the first 2 kPCA using an RBF kernel, while raw samples
+    are not directly separable in the original space.
+    """
     X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                         random_state=0)

@@ -270,8 +381,10 @@ def test_nested_circles():

 def test_kernel_conditioning():
-    """ Test that ``_check_psd_eigenvalues`` is correctly called
-    Non-regression test for issue #12140 (PR #12145)"""
+    """Check that ``_check_psd_eigenvalues`` is correctly called in kPCA
+
+    Non-regression test for issue #12140 (PR #12145).
+    """

     # create a pathological X leading to small non-zero eigenvalue
     X = [[5, 1],
@@ -286,11 +399,93 @@ def test_kernel_conditioning():
     assert np.all(kpca.lambdas_ == _check_psd_eigenvalues(kpca.lambdas_))

+@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"])
+def test_precomputed_kernel_not_psd(solver):
+    """Check how KernelPCA works with non-PSD kernels depending on n_components
+
+    Tests for all methods what happens with a non PSD gram matrix (this
+    can happen in an isomap scenario, or with custom kernel functions, or
+    maybe with ill-posed datasets).
+
+    When ``n_component`` is large enough to capture a negative eigenvalue, an
+    error should be raised. Otherwise, KernelPCA should run without error
+    since the negative eigenvalues are not selected.
+    """
+
+    # a non PSD kernel with large eigenvalues, already centered
+    # it was captured from an isomap call and multiplied by 100 for compacity
+    K = [
+        [4.48, -1., 8.07, 2.33, 2.33, 2.33, -5.76, -12.78],
+        [-1., -6.48, 4.5, -1.24, -1.24, -1.24, -0.81, 7.49],
+        [8.07, 4.5, 15.48, 2.09, 2.09, 2.09, -11.1, -23.23],
+        [2.33, -1.24, 2.09, 4., -3.65, -3.65, 1.02, -0.9],
+        [2.33, -1.24, 2.09, -3.65, 4., -3.65, 1.02, -0.9],
+        [2.33, -1.24, 2.09, -3.65, -3.65, 4., 1.02, -0.9],
+        [-5.76, -0.81, -11.1, 1.02, 1.02, 1.02, 4.86, 9.75],
+        [-12.78, 7.49, -23.23, -0.9, -0.9, -0.9, 9.75, 21.46]
+    ]
+    # this gram matrix has 5 positive eigenvalues and 3 negative ones
+    # [ 52.72,   7.65,   7.65,   5.02,   0.  ,  -0.  ,  -6.13, -15.11]
+
+    # 1. ask for enough components to get a significant negative one
+    kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=7)
+    # make sure that the appropriate error is raised
+    with pytest.raises(ValueError,
+                       match="There are significant negative eigenvalues"):
+        kpca.fit(K)
+
+    # 2. ask for a small enough n_components to get only positive ones
+    kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=2)
+    if solver == 'randomized':
+        # the randomized method is still inconsistent with the others on this
+        # since it selects the eigenvalues based on the largest 2 modules, not
+        # on the largest 2 values.
+        #
+        # At least we can ensure that we return an error instead of returning
+        # the wrong eigenvalues
+        with pytest.raises(ValueError,
+                           match="There are significant negative eigenvalues"):
+            kpca.fit(K)
+    else:
+        # general case: make sure that it works
+        kpca.fit(K)
+
+
+@pytest.mark.parametrize("n_components", [4, 10, 20])
+def test_kernel_pca_solvers_equivalence(n_components):
+    """Check that 'dense' 'arpack' & 'randomized' solvers give similar results
+    """
+
+    # Generate random data
+    n_train, n_test = 2000, 100
+    X, _ = make_circles(n_samples=(n_train + n_test), factor=.3, noise=.05,
+                        random_state=0)
+    X_fit, X_pred = X[:n_train, :], X[n_train:, :]
+
+    # reference (full)
+    ref_pred = KernelPCA(n_components, eigen_solver="dense", random_state=0
+                         ).fit(X_fit).transform(X_pred)
+
+    # arpack
+    a_pred = KernelPCA(n_components, eigen_solver="arpack", random_state=0
+                       ).fit(X_fit).transform(X_pred)
+    # check that the result is still correct despite the approx
+    assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred))
+
+    # randomized
+    r_pred = KernelPCA(n_components, eigen_solver="randomized", random_state=0
+                       ).fit(X_fit).transform(X_pred)
+    # check that the result is still correct despite the approximation
+    assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred))
+
+
 def test_kernel_pca_inverse_transform_reconstruction():
-    # Test if the reconstruction is a good approximation.
-    # Note that in general it is not possible to get an arbitrarily good
-    # reconstruction because of kernel centering that does not
-    # preserve all the information of the original data.
+    """Test if the reconstruction is a good approximation.
+
+    Note that in general it is not possible to get an arbitrarily good
+    reconstruction because of kernel centering that does not
+    preserve all the information of the original data.
+    """
     X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0)

     kpca = KernelPCA(
@@ -302,8 +497,11 @@ def test_kernel_pca_inverse_transform_reconstruction():

 def test_32_64_decomposition_shape():
-    """ Test that the decomposition is similar for 32 and 64 bits data """
-    # see https://github.com/scikit-learn/scikit-learn/issues/18146
+    """Test that the decomposition is similar for 32 and 64 bits data
+
+    Non regression test for
+    https://github.com/scikit-learn/scikit-learn/issues/18146
+    """
     X, y = make_blobs(
         n_samples=30,
         centers=[[0, 0, 0], [1, 1, 1]],
@@ -321,6 +519,10 @@ def test_32_64_decomposition_shape():

 # TODO: Remove in 1.1
 def test_kernel_pcc_pairwise_is_deprecated():
+    """Check that `_pairwise` is correctly marked with deprecation warning
+
+    Tests that a `FutureWarning` is issued when `_pairwise` is accessed.
+    """
     kp = KernelPCA(kernel='precomputed')
     msg = r"Attribute _pairwise was deprecated in version 0\.24"
     with pytest.warns(FutureWarning, match=msg):
diff --git a/sklearn/utils/tests/test_extmath.py b/sklearn/utils/tests/test_extmath.py
index 8e53d94d911f0..1a77d08b12388 100644
--- a/sklearn/utils/tests/test_extmath.py
+++ b/sklearn/utils/tests/test_extmath.py
@@ -8,11 +8,12 @@
 from scipy import sparse
 from scipy import linalg
 from scipy import stats
+from scipy.sparse.linalg import eigsh
 from scipy.special import expit

 import pytest
 from sklearn.utils import gen_batches
-
+from sklearn.utils._arpack import _init_arpack_v0
 from sklearn.utils._testing import assert_almost_equal
 from sklearn.utils._testing import assert_allclose
 from sklearn.utils._testing import assert_allclose_dense_sparse
@@ -23,7 +24,7 @@
 from sklearn.utils._testing import skip_if_32bit

 from sklearn.utils.extmath import density, _safe_accumulator_op
-from sklearn.utils.extmath import randomized_svd
+from sklearn.utils.extmath import randomized_svd, _randomized_eigsh
 from sklearn.utils.extmath import row_norms
 from sklearn.utils.extmath import weighted_mode
 from sklearn.utils.extmath import cartesian
@@ -34,7 +35,7 @@
 from sklearn.utils.extmath import softmax
 from sklearn.utils.extmath import stable_cumsum
 from sklearn.utils.extmath import safe_sparse_dot
-from sklearn.datasets import make_low_rank_matrix
+from sklearn.datasets import make_low_rank_matrix, make_sparse_spd_matrix

 def test_density():
@@ -161,6 +162,128 @@ def test_randomized_svd_low_rank_all_dtypes(dtype):
     check_randomized_svd_low_rank(dtype)

+@pytest.mark.parametrize('dtype',
+                         (np.int32, np.int64, np.float32, np.float64))
+def test_randomized_eigsh(dtype):
+    """Test that `_randomized_eigsh` returns the appropriate components"""
+
+    rng = np.random.RandomState(42)
+    X = np.diag(np.array([1., -2., 0., 3.], dtype=dtype))
+    # random rotation that preserves the eigenvalues of X
+    rand_rot = np.linalg.qr(rng.normal(size=X.shape))[0]
+    X = rand_rot @ X @ rand_rot.T
+
+    # with 'module' selection method, the negative eigenvalue shows up
+    eigvals, eigvecs = _randomized_eigsh(X, n_components=2, selection='module')
+    # eigenvalues
+    assert eigvals.shape == (2,)
+    assert_array_almost_equal(eigvals, [3., -2.])  # negative eigenvalue here
+    # eigenvectors
+    assert eigvecs.shape == (4, 2)
+
+    # with 'value' selection method, the negative eigenvalue does not show up
+    with pytest.raises(NotImplementedError):
+        _randomized_eigsh(X, n_components=2, selection='value')
+
+
+@pytest.mark.parametrize('k', (10, 50, 100, 199, 200))
+def test_randomized_eigsh_compared_to_others(k):
+    """Check that `_randomized_eigsh` is similar to other `eigsh`
+
+    Tests that for a random PSD matrix, `_randomized_eigsh` provides results
+    comparable to LAPACK (scipy.linalg.eigh) and ARPACK
+    (scipy.sparse.linalg.eigsh).
+
+    Note: some versions of ARPACK do not support k=n_features.
+    """
+
+    # make a random PSD matrix
+    n_features = 200
+    X = make_sparse_spd_matrix(n_features, random_state=0)
+
+    # compare two versions of randomized
+    # rough and fast
+    eigvals, eigvecs = _randomized_eigsh(X, n_components=k, selection='module',
+                                         n_iter=25, random_state=0)
+    # more accurate but slow (TODO find realistic settings here)
+    eigvals_qr, eigvecs_qr = _randomized_eigsh(
+        X, n_components=k, n_iter=25, n_oversamples=20, random_state=0,
+        power_iteration_normalizer="QR", selection='module'
+    )
+
+    # with LAPACK
+    eigvals_lapack, eigvecs_lapack = linalg.eigh(X, eigvals=(n_features - k,
+                                                             n_features - 1))
+    indices = eigvals_lapack.argsort()[::-1]
+    eigvals_lapack = eigvals_lapack[indices]
+    eigvecs_lapack = eigvecs_lapack[:, indices]
+
+    # -- eigenvalues comparison
+    assert eigvals_lapack.shape == (k,)
+    # comparison precision
+    assert_array_almost_equal(eigvals, eigvals_lapack, decimal=6)
+    assert_array_almost_equal(eigvals_qr, eigvals_lapack, decimal=6)
+
+    # -- eigenvectors comparison
+    assert eigvecs_lapack.shape == (n_features, k)
+    # flip eigenvectors' sign to enforce deterministic output
+    dummy_vecs = np.zeros_like(eigvecs).T
+    eigvecs, _ = svd_flip(eigvecs, dummy_vecs)
+    eigvecs_qr, _ = svd_flip(eigvecs_qr, dummy_vecs)
+    eigvecs_lapack, _ = svd_flip(eigvecs_lapack, dummy_vecs)
+    assert_array_almost_equal(eigvecs, eigvecs_lapack, decimal=4)
+    assert_array_almost_equal(eigvecs_qr, eigvecs_lapack, decimal=6)
+
+    # comparison ARPACK ~ LAPACK (some ARPACK implems do not support k=n)
+    if k < n_features:
+        v0 = _init_arpack_v0(n_features, random_state=0)
+        # "LA" largest algebraic <=> selection="value" in randomized_eigsh
+        eigvals_arpack, eigvecs_arpack = eigsh(X, k, which="LA", tol=0,
+                                               maxiter=None, v0=v0)
+        indices = eigvals_arpack.argsort()[::-1]
+        # eigenvalues
+        eigvals_arpack = eigvals_arpack[indices]
+        assert_array_almost_equal(eigvals_lapack, eigvals_arpack, decimal=10)
+        # eigenvectors
+        eigvecs_arpack = eigvecs_arpack[:, indices]
+        eigvecs_arpack, _ = svd_flip(eigvecs_arpack, dummy_vecs)
+        assert_array_almost_equal(eigvecs_arpack, eigvecs_lapack, decimal=8)
+
+
+@pytest.mark.parametrize("n,rank", [
+    (10, 7),
+    (100, 10),
+    (100, 80),
+    (500, 10),
+    (500, 250),
+    (500, 400),
+])
+def test_randomized_eigsh_reconst_low_rank(n, rank):
+    """Check that randomized_eigsh is able to reconstruct a low rank psd matrix
+
+    Tests that the decomposition provided by `_randomized_eigsh` leads to
+    orthonormal eigenvectors, and that a low rank PSD matrix can be effectively
+    reconstructed with good accuracy using it.
+    """
+    assert rank < n
+
+    # create a low rank PSD
+    rng = np.random.RandomState(69)
+    X = rng.randn(n, rank)
+    A = X @ X.T
+
+    # approximate A with the "right" number of components
+    S, V = _randomized_eigsh(A, n_components=rank, random_state=rng)
+    # orthonormality checks
+    assert_array_almost_equal(np.linalg.norm(V, axis=0), np.ones(S.shape))
+    assert_array_almost_equal(V.T @ V, np.diag(np.ones(S.shape)))
+    # reconstruction
+    A_reconstruct = V @ np.diag(S) @ V.T
+
+    # test that the approximation is good
+    assert_array_almost_equal(A_reconstruct, A, decimal=6)
+
+
 @pytest.mark.parametrize('dtype',
                          (np.float32, np.float64))
 def test_row_norms(dtype):

diff --git a/doc/modules/decomposition.rst b/doc/modules/decomposition.rst
index e971d784c..fd51f60d8 100644
--- a/doc/modules/decomposition.rst
+++ b/doc/modules/decomposition.rst
@@ -166,32 +166,16 @@ Note: the implementation of ``inverse_transform`` in :class:`PCA` with

 .. topic:: References:

-    * `"Finding structure with randomness: Stochastic algorithms for
+    * Algorithm 4.3 in
+      `"Finding structure with randomness: Stochastic algorithms for
       constructing approximate matrix decompositions"
       <https://arxiv.org/abs/0909.4061>`_
       Halko, et al., 2009

-
-.. _kernel_PCA:
-
-Kernel PCA
-----------
-
-:class:`KernelPCA` is an extension of PCA which achieves non-linear
-dimensionality reduction through the use of kernels (see :ref:`metrics`). It
-has many applications including denoising, compression and structured
-prediction (kernel dependency estimation). :class:`KernelPCA` supports both
-``transform`` and ``inverse_transform``.
-
-.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_kernel_pca_001.png
-    :target: ../auto_examples/decomposition/plot_kernel_pca.html
-    :align: center
-    :scale: 75%
-
-.. topic:: Examples:
-
-    * :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py`
-
+    * `"An implementation of a randomized algorithm for principal component
+      analysis"
+      <https://arxiv.org/pdf/1412.3510.pdf>`_
+      A. Szlam et al. 2014

 .. _SparsePCA:

@@ -278,6 +262,100 @@ factorization, while larger values shrink many coefficients to zero.
      R. Jenatton, G. Obozinski, F. Bach, 2009

+.. _kernel_PCA:
+
+Kernel Principal Component Analysis (kPCA)
+==========================================
+
+Exact Kernel PCA
+----------------
+
+:class:`KernelPCA` is an extension of PCA which achieves non-linear
+dimensionality reduction through the use of kernels (see :ref:`metrics`). It
+has many applications including denoising, compression and structured
+prediction (kernel dependency estimation). :class:`KernelPCA` supports both
+``transform`` and ``inverse_transform``.
+
+.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_kernel_pca_001.png
+    :target: ../auto_examples/decomposition/plot_kernel_pca.html
+    :align: center
+    :scale: 75%
+
+.. topic:: Examples:
+
+    * :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py`
+
+.. topic:: References:
+
+    * Kernel PCA was introduced in "Kernel principal component analysis"
+      Bernhard Schoelkopf, Alexander J. Smola, and Klaus-Robert Mueller. 1999.
+      In Advances in kernel methods, MIT Press, Cambridge, MA, USA 327-352.
+
+
+.. _kPCA_Solvers:
+
+Choice of solver for Kernel PCA
+-------------------------------
+
+While in :class:`PCA` the number of components is bounded by the number of
+features, in :class:`KernelPCA` the number of components is bounded by the
+number of samples. Many real-world datasets have large number of samples! In
+these cases finding *all* the components with a full kPCA is a waste of
+computation time, as data is mostly described by the first few components
+(e.g. ``n_components<=100``). In other words, the centered Gram matrix that
+is eigendecomposed in the Kernel PCA fitting process has an effective rank that
+is much smaller than its size. This is a situation where approximate
+eigensolvers can provide speedup with very low precision loss.
+
+The optional parameter ``eigen_solver='randomized'`` can be used to
+*significantly* reduce the computation time when the number of requested
+``n_components`` is small compared with the number of samples. It relies on
+randomized decomposition methods to find an approximate solution in a shorter
+time.
+
+The time complexity of the randomized :class:`KernelPCA` is
+:math:`O(n_{\mathrm{samples}}^2 \cdot n_{\mathrm{components}})`
+instead of :math:`O(n_{\mathrm{samples}}^3)` for the exact method
+implemented with ``eigen_solver='dense'``.
+
+The memory footprint of randomized :class:`KernelPCA` is also proportional to
+:math:`2 \cdot n_{\mathrm{samples}} \cdot n_{\mathrm{components}}` instead of
+:math:`n_{\mathrm{samples}}^2` for the exact method.
+
+Note: this technique is the same as in :ref:`RandomizedPCA`.
+
+In addition to the above two solvers, ``eigen_solver='arpack'`` can be used as
+an alternate way to get an approximate decomposition. In practice, this method
+only provides reasonable execution times when the number of components to find
+is extremely small. It is enabled by default when the desired number of
+components is less than 10 (strict) and the number of samples is more than 200
+(strict). See :class:`KernelPCA` for details.
+
+.. topic:: References:
+
+    * *dense* solver:
+      `scipy.linalg.eigh documentation
+      <https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.eigh.html>`_
+
+    * *randomized* solver:
+
+        - Algorithm 4.3 in
+          `"Finding structure with randomness: Stochastic algorithms for
+          constructing approximate matrix decompositions"
+          <https://arxiv.org/abs/0909.4061>`_
+          Halko, et al., 2009
+
+        - `"An implementation of a randomized algorithm for principal component
+          analysis"
+          <https://arxiv.org/pdf/1412.3510.pdf>`_
+          A. Szlam et al. 2014
+
+    * *arpack* solver:
+      `scipy.sparse.linalg.eigsh documentation
+      <https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.eigsh.html>`_
+      R. B. Lehoucq, D. C. Sorensen, and C. Yang, 1998
+
+
 .. _LSA:

 Truncated singular value decomposition and latent semantic analysis

diff --git a/doc/whats_new/v1.0.rst b/doc/whats_new/v1.0.rst
index 3b3884e68..7faf6bdf6 100644
--- a/doc/whats_new/v1.0.rst
+++ b/doc/whats_new/v1.0.rst
@@ -159,14 +159,16 @@ Changelog
 - |Fix| Fixes incorrect multiple data-conversion warnings when clustering
   boolean data. :pr:`19046` by :user:`Surya Prakash <jdsurya>`.

-:mod:`sklearn.decomposition`
-............................
-
 - |Fix| Fixed :func:`dict_learning`, used by :class:`DictionaryLearning`, to
   ensure determinism of the output. Achieved by flipping signs of the SVD
   output which is used to initialize the code.
   :pr:`18433` by :user:`Bruno Charron <brcharron>`.

+- |Enhancement| added a new approximate solver (randomized SVD, available with
+  `eigen_solver='randomized'`) to :class:`decomposition.KernelPCA`. This
+  significantly accelerates computation when the number of samples is much
+  larger than the desired number of components.
+
 - |Fix| Fixed a bug in :class:`MiniBatchDictionaryLearning`,
   :class:`MiniBatchSparsePCA` and :func:`dict_learning_online` where the
   update of the dictionary was incorrect. :pr:`19198` by
@@ -389,8 +391,8 @@ Changelog
   supporting sparse matrix and raise the appropriate error message.
   :pr:`19879` by :user:`Guillaume Lemaitre <glemaitre>`.

-- |Efficiency| Changed ``algorithm`` argument for :class:`cluster.KMeans` in
-  :class:`preprocessing.KBinsDiscretizer` from ``auto`` to ``full``.
+- |Efficiency| Changed ``algorithm`` argument for :class:`cluster.KMeans` in
+  :class:`preprocessing.KBinsDiscretizer` from ``auto`` to ``full``.
   :pr:`19934` by :user:`Gleb Levitskiy <GLevV>`.

 :mod:`sklearn.tree`

If completing this task requires creating new files, classes, fields, or error messages, you may consider using the following suggested entity names:
[{'type': 'file', 'name': 'benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py'}, {'type': 'file', 'name': 'benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py'}]