From 552bf028ded3839f78a906e65ac8535ce210387a Mon Sep 17 00:00:00 2001
From: Santos Gallegos <stsewd@proton.me>
Date: Wed, 26 Jul 2023 10:09:52 -0500
Subject: [PATCH] Search: unify Sphinx and generic parser  (#10520)

The current differences from the generic parser and the Sphinx parser are:

- The Sphinx parser doesn't need to find the main content node, since that's given by the `.fjson` file.
- The Sphinx parser removes nodes that have the following classes:
   - .toctree-wrapper (toc from the `toctree` directive)
   - .contents.local.topic (local toc from the `contents` directive)

How can we unify both?

- I think we handle finding the main node for Sphinx projects correctly with the generic parser.
- Removing elements that have the `.contents.local.topic` class is redundant, since that's inside a `nav` tag (we remove those).
- Elements generated by the `toctree` directive are plain divs with a  `.toctree-wrapper` class, so we can't have a generic way of removing them. This is the only impediment for merging both parsers, we have two options

a) Just don't remove those elements, and index them, maybe suggest upstream to enclose them inside a nav tag.
    The downside is that search results will have some duplicates (matching the toctree and the actual document), for example, here you can see two pages matching, the actual page and the index pages that has the ToC.
![Screenshot 2023-07-06 at 17-25-07 Search project dev Contributing to Read the Docs Read the Docs](https://github.com/readthedocs/readthedocs.org/assets/4975310/04379572-2247-426a-8f99-c04d9254cf27)

b) Just add that special case to our generic parser (we kind of already do this for `linenos`, `lineno`, and `headerlink`). Maybe suggest upstream to use a nav tag as well, so one day we may be able to stop especial casing that :D

I think I'm fine with option b.
---
 docs/dev/search-integration.rst               |   1 +
 readthedocs/search/parsers.py                 |  31 +-
 .../tests/data/sphinx/in/local-toc.html       | 446 ++++++++++++++++++
 .../search/tests/data/sphinx/in/toctree.html  |  89 ++++
 .../tests/data/sphinx/out/local-toc.json      |  46 ++
 .../search/tests/data/sphinx/out/toctree.json |  11 +
 readthedocs/search/tests/test_parsers.py      |  60 ++-
 7 files changed, 654 insertions(+), 30 deletions(-)
 create mode 100644 readthedocs/search/tests/data/sphinx/in/local-toc.html
 create mode 100644 readthedocs/search/tests/data/sphinx/in/toctree.html
 create mode 100644 readthedocs/search/tests/data/sphinx/out/local-toc.json
 create mode 100644 readthedocs/search/tests/data/sphinx/out/toctree.json

diff --git a/docs/dev/search-integration.rst b/docs/dev/search-integration.rst
index f0616519299..984e803a4f1 100644
--- a/docs/dev/search-integration.rst
+++ b/docs/dev/search-integration.rst
@@ -123,6 +123,7 @@ Special rules that are derived from specific documentation tools applied in the
 
 - ``.linenos``, ``.lineno`` (line numbers in code-blocks, comes from both MkDocs and Sphinx)
 - ``.headerlink`` (added by Sphinx to links in headers)
+- ``.toctree-wrapper`` (added by Sphinx to the table of contents generated from the ``toctree`` directive)
 
 Example:
 
diff --git a/readthedocs/search/parsers.py b/readthedocs/search/parsers.py
index 28d6108815f..83393e1251a 100644
--- a/readthedocs/search/parsers.py
+++ b/readthedocs/search/parsers.py
@@ -269,12 +269,15 @@ def _clean_body(self, body):
             body.css('nav'),
             body.css('[role=navigation]'),
             body.css('[role=search]'),
-            # Permalinks
+            # Permalinks, this is a Sphinx convention.
             body.css('.headerlink'),
             # Line numbers from code blocks, they are very noisy in contents.
             # This convention is popular in Sphinx.
             body.css(".linenos"),
             body.css(".lineno"),
+            # Sphinx doesn't wrap the result from the `toctree` directive
+            # in a nav tag. so we need to manually remove that content.
+            body.css(".toctree-wrapper"),
         )
         for node in nodes_to_be_removed:
             node.decompose()
@@ -516,29 +519,3 @@ def _process_fjson(self, fjson_path):
             "title": title,
             "sections": sections,
         }
-
-    def _clean_body(self, body):
-        """
-        Removes nodes in Sphinx-generated HTML structures.
-
-        This method is overridden to remove contents that are likely
-        to be useless for search indexing.
-
-        Currently: TOC elements.
-        """
-        body = super()._clean_body(body)
-
-        # TODO: see if we really need to remove TOC elements like below?
-        # benjaoming: I didn't see this in sphinx-rtd-theme, however since it wraps the menu in
-        # a <nav>, it's already covered. I didn't see this match local table of contents, neither
-        # and they are also wrapped in a <nav> so covered by _clean_body as well.
-        nodes_to_be_removed = itertools.chain(
-            body.css(".toctree-wrapper"),
-            body.css(".contents.local.topic"),
-        )
-
-        # removing all nodes in list
-        for node in nodes_to_be_removed:
-            node.decompose()
-
-        return body
diff --git a/readthedocs/search/tests/data/sphinx/in/local-toc.html b/readthedocs/search/tests/data/sphinx/in/local-toc.html
new file mode 100644
index 00000000000..f9781525e10
--- /dev/null
+++ b/readthedocs/search/tests/data/sphinx/in/local-toc.html
@@ -0,0 +1,446 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en">
+
+<head>
+  <meta charset="utf-8" />
+  <meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />
+  <meta property="og:title" content="Security reports" />
+  <meta property="og:type" content="website" />
+  <meta property="og:url" content="https://docs.readthedocs.io/en/stable/security.html" />
+  <meta property="og:site_name" content="Read the Docs Documentation" />
+  <meta property="og:description"
+    content="Security is very important to us at Read the Docs. We follow generally accepted industry standards to protect the personal information submitted to us, both during transmission and once we receive it. In the spirit of transparency, we are committed to responsible reporting and disclosure of secur..." />
+  <meta property="og:image" content="https://docs.readthedocs.io/en/latest/_static/img/logo-opengraph.png" />
+  <meta property="og:image:alt" content="Read the Docs Documentation" />
+  <meta name="description"
+    content="Security is very important to us at Read the Docs. We follow generally accepted industry standards to protect the personal information submitted to us, both during transmission and once we receive it. In the spirit of transparency, we are committed to responsible reporting and disclosure of secur..." />
+  <meta name="twitter:card" content="summary_large_image" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Security reports &mdash; Read the Docs user documentation 9.15.0 documentation</title>
+  <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster.custom.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster.bundle.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster-sideTip-shadow.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster-sideTip-punk.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster-sideTip-noir.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster-sideTip-light.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/tooltipster-sideTip-borderless.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/micromodal.css" type="text/css" />
+  <link rel="stylesheet" href="_static/copybutton.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/sphinx_rtd_theme.css" type="text/css" />
+  <link rel="stylesheet" href="_static/tabs.css" type="text/css" />
+  <link rel="stylesheet" href="_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/rtd_sphinx_search.min.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/custom.css" type="text/css" />
+  <link rel="stylesheet" href="_static/css/sphinx_prompt_css.css" type="text/css" />
+  <link rel="canonical" href="https://docs.readthedocs.io/en/stable/security.html" />
+  <!--[if lt IE 9]>
+    <script src="_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+
+  <script src="_static/jquery.js"></script>
+  <script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
+  <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
+  <script src="_static/doctools.js"></script>
+  <script src="_static/sphinx_highlight.js"></script>
+  <script src="_static/js/hoverxref.js"></script>
+  <script src="_static/js/tooltipster.bundle.min.js"></script>
+  <script src="_static/js/micromodal.min.js"></script>
+  <script src="_static/clipboard.min.js"></script>
+  <script src="_static/copybutton.js"></script>
+  <script src="_static/tabs.js"></script>
+  <script src="_static/design-tabs.js"></script>
+  <script src="_static/js/rtd_search_config.js"></script>
+  <script src="_static/js/rtd_sphinx_search.min.js"></script>
+  <script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+  <script async="async" src="/_/static/javascript/readthedocs-doc-embed.js"></script>
+  <script src="_static/js/expand_tabs.js"></script>
+  <script src="_static/js/theme.js"></script>
+  <link rel="index" title="Index" href="genindex.html" />
+  <link rel="search" title="Search" href="search.html" />
+  <link rel="next" title="Read the Docs Terms of Service" href="terms-of-service.html" />
+  <link rel="prev" title="Security policy" href="legal/security-policy.html" />
+
+
+
+  <script defer data-domain="docs.readthedocs.io" src="https://plausible.io/js/script.js"></script>
+
+
+
+  <!-- RTD Extra Head -->
+
+  <link rel="stylesheet" href="/_/static/css/readthedocs-doc-embed.css" type="text/css" />
+
+  <script type="application/json"
+    id="READTHEDOCS_DATA">{"ad_free": false, "api_host": "https://readthedocs.org", "builder": "sphinx", "canonical_url": null, "docroot": "/docs/user/", "features": {"docsearch_disabled": false}, "global_analytics_code": "UA-17997319-1", "language": "en", "page": "security", "programming_language": "py", "project": "docs", "proxied_api_host": "/_", "source_suffix": ".rst", "subprojects": {}, "theme": "sphinx_rtd_theme", "user_analytics_code": "UA-17997319-6", "version": "stable"}</script>
+
+  <!--
+Using this variable directly instead of using `JSON.parse` is deprecated.
+The READTHEDOCS_DATA global variable will be removed in the future.
+-->
+  <script type="text/javascript">
+    READTHEDOCS_DATA = JSON.parse(document.getElementById('READTHEDOCS_DATA').innerHTML);
+  </script>
+
+  <script type="text/javascript" src="/_/static/javascript/readthedocs-analytics.js" async="async"></script>
+
+  <!-- end RTD <extrahead> -->
+</head>
+
+<body class="wy-body-for-nav">
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+
+          <a href="index.html">
+
+            <img src="_static/logo.svg" class="logo" alt="Logo" />
+          </a>
+          <div role="search">
+            <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+              <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+              <input type="hidden" name="check_keywords" value="yes" />
+              <input type="hidden" name="area" value="default" />
+            </form>
+          </div>
+        </div>
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+          <p class="caption" role="heading"><span class="caption-text">🚀 Tutorials</span></p>
+          <ul>
+            <li class="toctree-l1"><a class="reference internal" href="tutorial/index.html">Read the Docs tutorial</a>
+            </li>
+            <li class="toctree-l1"><a class="reference internal" href="intro/getting-started-with-sphinx.html">Getting
+                started with Sphinx</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="intro/getting-started-with-mkdocs.html">Getting
+                started with MkDocs</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="intro/import-guide.html">Importing your
+                documentation</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="examples.html">Example projects</a></li>
+          </ul>
+          <p class="caption" role="heading"><span class="caption-text">💡 Explanation</span></p>
+          <ul>
+            <li class="toctree-l1"><a class="reference internal" href="choosing-a-site.html">Choosing between our two
+                platforms</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="integrations.html">Continuous Documentation
+                Deployment</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="explanation/advanced.html">Deep dive into Read
+                the Docs</a></li>
+          </ul>
+          <p class="caption" role="heading"><span class="caption-text">🪄 How-to guides</span></p>
+          <ul>
+            <li class="toctree-l1"><a class="reference internal" href="guides/setup/index.html">Project setup and
+                configuration</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="guides/build/index.html">Build process</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="guides/maintenance/index.html">Upgrading and
+                maintaining projects</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="guides/content/index.html">Content, themes and
+                SEO</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="guides/access/index.html">Security and access</a>
+            </li>
+            <li class="toctree-l1"><a class="reference internal" href="guides/management/index.html">Account
+                management</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="guides/best-practice/index.html">Best
+                practice</a></li>
+            <li class="toctree-l1"><a class="reference internal"
+                href="guides/troubleshooting/index.html">Troubleshooting problems</a></li>
+          </ul>
+          <p class="caption" role="heading"><span class="caption-text">📚 Reference</span></p>
+          <ul class="current">
+            <li class="toctree-l1"><a class="reference internal" href="reference/features.html">Feature reference</a>
+            </li>
+            <li class="toctree-l1"><a class="reference internal" href="config-file/v2.html">Configuration file v2
+                (.readthedocs.yaml)</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="builds.html">Build process overview</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="build-customization.html">Build process
+                customization</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="server-side-search/syntax.html">Search query
+                syntax</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="faq.html">Frequently asked questions</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="api/index.html">Public API</a></li>
+            <li class="toctree-l1"><a class="reference internal" href="changelog.html">Changelog</a></li>
+            <li class="toctree-l1 current"><a class="reference internal" href="about/index.html">About Read the Docs</a>
+              <ul class="current">
+                <li class="toctree-l2"><a class="reference internal" href="commercial/index.html">About Read the Docs
+                    for Business</a></li>
+                <li class="toctree-l2 current"><a class="reference internal" href="reference/policies.html">Policies and
+                    legal documents</a>
+                  <ul class="current">
+                    <li class="toctree-l3"><a class="reference internal" href="abandoned-projects.html">Abandoned
+                        projects policy</a></li>
+                    <li class="toctree-l3"><a class="reference internal" href="unofficial-projects.html">Unofficial and
+                        unmaintained projects policy</a></li>
+                    <li class="toctree-l3"><a class="reference internal" href="privacy-policy.html">Privacy Policy</a>
+                    </li>
+                    <li class="toctree-l3"><a class="reference internal" href="legal/security-policy.html">Security
+                        policy</a></li>
+                    <li class="toctree-l3 current"><a class="current reference internal" href="#">Security reports</a>
+                      <ul>
+                        <li class="toctree-l4"><a class="reference internal" href="#supported-versions">Supported
+                            versions</a></li>
+                        <li class="toctree-l4"><a class="reference internal"
+                            href="#reporting-a-security-issue">Reporting a security issue</a></li>
+                        <li class="toctree-l4"><a class="reference internal" href="#pgp-key">PGP key</a></li>
+                        <li class="toctree-l4"><a class="reference internal" href="#bug-bounties">Bug bounties</a></li>
+                        <li class="toctree-l4"><a class="reference internal" href="#security-issue-archive">Security
+                            issue archive</a></li>
+                      </ul>
+                    </li>
+                    <li class="toctree-l3"><a class="reference internal" href="terms-of-service.html">Read the Docs
+                        Terms of Service</a></li>
+                    <li class="toctree-l3"><a class="reference internal" href="dmca/index.html">DMCA takedown policy</a>
+                    </li>
+                    <li class="toctree-l3"><a class="reference internal" href="legal/dpa/index.html">Data Processing
+                        Addendum (DPA)</a></li>
+                  </ul>
+                </li>
+                <li class="toctree-l2"><a class="reference internal" href="advertising/index.html">Advertising</a></li>
+                <li class="toctree-l2"><a class="reference internal" href="story.html">The story of Read the Docs</a>
+                </li>
+                <li class="toctree-l2"><a class="reference internal" href="sponsors.html">Sponsors of Read the Docs</a>
+                </li>
+                <li class="toctree-l2"><a class="reference internal" href="open-source-philosophy.html">Read the Docs
+                    open source philosophy</a></li>
+                <li class="toctree-l2"><a class="reference internal" href="team.html">Read the Docs team</a></li>
+                <li class="toctree-l2"><a class="reference internal" href="support.html">Site support</a></li>
+                <li class="toctree-l2"><a class="reference internal" href="glossary.html">Glossary</a></li>
+              </ul>
+            </li>
+            <li class="toctree-l1"><a class="reference external" href="https://dev.readthedocs.io">Developer
+                Documentation</a></li>
+          </ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" aria-label="Mobile navigation menu">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="index.html">Read the Docs user documentation</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+            <ul class="wy-breadcrumbs">
+              <li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
+              <li class="breadcrumb-item"><a href="about/index.html">About Read the Docs</a></li>
+              <li class="breadcrumb-item"><a href="reference/policies.html">Policies and legal documents</a></li>
+              <li class="breadcrumb-item active">Security reports</li>
+              <li class="wy-breadcrumbs-aside">
+                <a href="https://github.com/readthedocs/readthedocs.org/blob/b34b0c6001e84bd84f583816ab4561139bd09fea/docs/user/security.rst"
+                  class="fa fa-github"> Edit on GitHub</a>
+              </li>
+            </ul>
+            <hr />
+          </div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+
+            <div itemprop="articleBody">
+
+              <section id="security-reports">
+                <h1>Security reports<a class="headerlink" href="#security-reports"
+                    title="Permalink to this heading"></a></h1>
+                <p>Security is very important to us at Read the Docs.
+                  We follow generally accepted industry standards to protect the personal information
+                  submitted to us, both during transmission and once we receive it.
+                  In the spirit of transparency,
+                  we are committed to responsible reporting and disclosure of security issues.</p>
+                <nav class="contents local" id="contents">
+                  <p class="topic-title">Contents</p>
+                  <ul class="simple">
+                    <li>
+                      <p><a class="reference internal" href="#supported-versions" id="id1">Supported versions</a></p>
+                    </li>
+                    <li>
+                      <p><a class="reference internal" href="#reporting-a-security-issue" id="id2">Reporting a security
+                          issue</a></p>
+                    </li>
+                    <li>
+                      <p><a class="reference internal" href="#pgp-key" id="id3">PGP key</a></p>
+                    </li>
+                    <li>
+                      <p><a class="reference internal" href="#bug-bounties" id="id4">Bug bounties</a></p>
+                    </li>
+                    <li>
+                      <p><a class="reference internal" href="#security-issue-archive" id="id5">Security issue
+                          archive</a></p>
+                    </li>
+                  </ul>
+                </nav>
+                <div class="admonition seealso">
+                  <p class="admonition-title">See also</p>
+                  <dl class="simple">
+                    <dt><a class="reference internal" href="legal/security-policy.html"><span class="doc">Security
+                          policy</span></a></dt>
+                    <dd>
+                      <p>Read our policy for security, which we base our security handling and reporting on.</p>
+                    </dd>
+                  </dl>
+                </div>
+                <section id="supported-versions">
+                  <h2>Supported versions<a class="headerlink" href="#supported-versions"
+                      title="Permalink to this heading"></a></h2>
+                  <p>Only the latest version of Read the Docs will receive security updates.
+                    We don’t support security updates for <a class="reference internal"
+                      href="open-source-philosophy.html"><span class="doc">custom installations</span></a> of Read the
+                    Docs.</p>
+                </section>
+                <section id="reporting-a-security-issue">
+                  <h2>Reporting a security issue<a class="headerlink" href="#reporting-a-security-issue"
+                      title="Permalink to this heading"></a></h2>
+                  <p>If you believe you’ve discovered a security issue at Read the Docs,
+                    please contact us at <strong>security&#64;readthedocs.org</strong> (optionally using our <a
+                      class="hxr-hoverxref hxr-tooltip reference internal" href="#pgp-key"><span class="std std-ref">PGP
+                        key</span></a>).
+                    We request that you please not publicly disclose the issue until it has been addressed by us.</p>
+                  <p>You can expect:</p>
+                  <ul class="simple">
+                    <li>
+                      <p>We will respond acknowledging your email typically within one business day.</p>
+                    </li>
+                    <li>
+                      <p>We will follow up if and when we have confirmed the issue with a timetable for the fix.</p>
+                    </li>
+                    <li>
+                      <p>We will notify you when the issue is fixed.</p>
+                    </li>
+                    <li>
+                      <p>We will create a <a class="reference external"
+                          href="https://github.com/readthedocs/readthedocs.org/security/advisories">GitHub advisory</a>
+                        and publish it when the issue has been fixed
+                        and deployed in our platforms.</p>
+                    </li>
+                  </ul>
+                </section>
+                <section id="pgp-key">
+                  <h2>PGP key<a class="headerlink" href="#pgp-key" title="Permalink to this heading"></a></h2>
+                  <p>You may use this <a class="reference download internal" download=""
+                      href="_downloads/37e127f3b4c52c9a970f3607dce9f440/pgpkey.txt"><code
+                        class="xref download docutils literal notranslate"><span class="pre">PGP</span> <span class="pre">key</span></code></a>
+                    to securely communicate with us and to verify signed messages you receive from us.</p>
+                </section>
+                <section id="bug-bounties">
+                  <h2>Bug bounties<a class="headerlink" href="#bug-bounties" title="Permalink to this heading"></a>
+                  </h2>
+                  <p>While we sincerely appreciate and encourage reports of suspected security problems,
+                    please note that the Read the Docs is an open source project, and <strong>does not run any bug
+                      bounty programs</strong>.</p>
+                </section>
+                <section id="security-issue-archive">
+                  <h2>Security issue archive<a class="headerlink" href="#security-issue-archive"
+                      title="Permalink to this heading"></a></h2>
+                  <p>You can see all past reports at <a class="reference external"
+                      href="https://github.com/readthedocs/readthedocs.org/security/advisories">https://github.com/readthedocs/readthedocs.org/security/advisories</a>.
+                  </p>
+                  <section id="version-3-2-0">
+                    <h3>Version 3.2.0<a class="headerlink" href="#version-3-2-0" title="Permalink to this heading"></a>
+                    </h3>
+                    <p><a class="hxr-hoverxref hxr-tooltip reference internal" href="changelog.html#version-3-2-0"><span
+                          class="std std-ref">Version 3.2.0</span></a> resolved an issue where a specially crafted
+                      request
+                      could result in a DNS query to an arbitrary domain.</p>
+                    <p>This issue was found by <a class="reference external"
+                        href="https://www.cybersmartdefence.com/">Cyber Smart Defence</a>
+                      who reported it as part of a security audit to a firm running a local installation
+                      of Read the Docs.</p>
+                  </section>
+                  <section id="release-2-3-0">
+                    <h3>Release 2.3.0<a class="headerlink" href="#release-2-3-0" title="Permalink to this heading"></a>
+                    </h3>
+                    <p><a class="hxr-hoverxref hxr-tooltip reference internal" href="changelog.html#version-2-3-0"><span
+                          class="std std-ref">Version 2.3.0</span></a> resolves a security issue with translations on
+                      our community
+                      hosting site that allowed users to modify the hosted path of a target project by
+                      adding it as a translation project of their own project. A check was added to
+                      ensure project ownership before adding the project as a translation.</p>
+                    <p>In order to add a project as a translation now, users must now first be granted
+                      ownership in the translation project.</p>
+                  </section>
+                </section>
+              </section>
+
+
+            </div>
+          </div>
+
+          <div id="rtd-stickybox-container">
+            <div class="raised" data-ea-publisher="readthedocs" data-ea-type="image" data-ea-style="stickybox"></div>
+          </div>
+
+          <footer>
+            <div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+              <a href="legal/security-policy.html" class="btn btn-neutral float-left" title="Security policy"
+                accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+              <a href="terms-of-service.html" class="btn btn-neutral float-right" title="Read the Docs Terms of Service"
+                accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+            </div>
+
+            <hr />
+
+            <div role="contentinfo">
+              <p>&#169; Copyright Read the Docs, Inc &amp; contributors.
+                <span class="commit">Revision <code>b34b0c60</code>.
+                </span>
+              </p>
+            </div>
+
+            Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+            <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+            provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+
+          </footer>
+        </div>
+      </div>
+    </section>
+  </div>
+
+
+  <div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="Versions">
+    <span class="rst-current-version" data-toggle="rst-current-version">
+      <span class="fa fa-book"> Read the Docs</span>
+      v: stable
+      <span class="fa fa-caret-down"></span>
+    </span>
+    <div class="rst-other-versions">
+      <dl>
+        <dt>Versions</dt>
+
+        <dd><a href="/en/latest/">latest</a></dd>
+
+        <dd><a href="/en/stable/">stable</a></dd>
+
+      </dl>
+      <dl>
+        <dt>Downloads</dt>
+
+        <dd><a href="//docs.readthedocs.io/_/downloads/en/stable/htmlzip/">html</a></dd>
+
+        <dd><a href="//docs.readthedocs.io/_/downloads/en/stable/epub/">epub</a></dd>
+
+      </dl>
+      <dl>
+
+        <dt>On Read the Docs</dt>
+        <dd>
+          <a href="//readthedocs.org/projects/docs/?fromdocs=docs">Project Home</a>
+        </dd>
+        <dd>
+          <a href="//readthedocs.org/builds/docs/?fromdocs=docs">Builds</a>
+        </dd>
+      </dl>
+    </div>
+  </div>
+  <script>
+    jQuery(function () {
+      SphinxRtdTheme.Navigation.enable(true);
+    });
+  </script>
+
+</body>
+
+</html>
diff --git a/readthedocs/search/tests/data/sphinx/in/toctree.html b/readthedocs/search/tests/data/sphinx/in/toctree.html
new file mode 100644
index 00000000000..0d1fb99b542
--- /dev/null
+++ b/readthedocs/search/tests/data/sphinx/in/toctree.html
@@ -0,0 +1,89 @@
+<section id="public-api">
+  <h1>Public API<a class="headerlink" href="#public-api" title="Permalink to this heading"></a></h1>
+  <p>This section of the documentation details the public API
+    usable to get details of projects, builds, versions and other details
+    from Read the Docs.</p>
+  <div class="toctree-wrapper compound">
+    <ul>
+      <li class="toctree-l1"><a class="reference internal" href="v3.html">API v3</a>
+        <ul>
+          <li class="toctree-l2"><a class="reference internal"
+              href="v3.html#authentication-and-authorization">Authentication and authorization</a>
+            <ul>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#token">Token</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#session">Session</a></li>
+            </ul>
+          </li>
+          <li class="toctree-l2"><a class="reference internal" href="v3.html#resources">Resources</a>
+            <ul>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#projects">Projects</a>
+              </li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#versions">Versions</a>
+              </li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#builds">Builds</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#subprojects">Subprojects</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#translations">Translations</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#redirects">Redirects</a>
+              </li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#environment-variables">Environment
+                  variables</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#organizations">Organizations</a>
+              </li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#remote-organizations">Remote
+                  organizations</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v3.html#remote-repositories">Remote
+                  repositories</a></li>
+            </ul>
+          </li>
+          <li class="toctree-l2"><a class="reference internal" href="v3.html#embed">Embed</a></li>
+          <li class="toctree-l2"><a class="reference internal" href="v3.html#additional-apis">Additional
+              APIs</a></li>
+        </ul>
+      </li>
+      <li class="toctree-l1"><a class="reference internal" href="v2.html">API v2</a>
+        <ul>
+          <li class="toctree-l2"><a class="reference internal"
+              href="v2.html#authentication-and-authorization">Authentication and authorization</a></li>
+          <li class="toctree-l2"><a class="reference internal" href="v2.html#resources">Resources</a>
+            <ul>
+              <li class="toctree-l3"><a class="reference internal" href="v2.html#projects">Projects</a>
+              </li>
+              <li class="toctree-l3"><a class="reference internal" href="v2.html#versions">Versions</a>
+              </li>
+              <li class="toctree-l3"><a class="reference internal" href="v2.html#builds">Builds</a></li>
+              <li class="toctree-l3"><a class="reference internal" href="v2.html#embed">Embed</a></li>
+              <li class="toctree-l3"><a class="reference internal"
+                  href="v2.html#undocumented-resources-and-endpoints">Undocumented resources and
+                  endpoints</a></li>
+            </ul>
+          </li>
+        </ul>
+      </li>
+      <li class="toctree-l1"><a class="reference internal" href="../server-side-search/api.html">Server
+          side search API</a>
+        <ul>
+          <li class="toctree-l2"><a class="reference internal" href="../server-side-search/api.html#api-v3">API
+              v3</a>
+            <ul>
+              <li class="toctree-l3"><a class="reference internal"
+                  href="../server-side-search/api.html#migrating-from-api-v2">Migrating from API v2</a>
+              </li>
+            </ul>
+          </li>
+          <li class="toctree-l2"><a class="reference internal"
+              href="../server-side-search/api.html#authentication-and-authorization">Authentication and
+              authorization</a></li>
+          <li class="toctree-l2"><a class="reference internal"
+              href="../server-side-search/api.html#api-v2-deprecated">API v2 (deprecated)</a></li>
+        </ul>
+      </li>
+      <li class="toctree-l1"><a class="reference internal" href="cross-site-requests.html">Cross-site
+          requests</a>
+        <ul>
+          <li class="toctree-l2"><a class="reference internal" href="cross-site-requests.html#cookies">Cookies</a>
+          </li>
+        </ul>
+      </li>
+    </ul>
+  </div>
+</section>
diff --git a/readthedocs/search/tests/data/sphinx/out/local-toc.json b/readthedocs/search/tests/data/sphinx/out/local-toc.json
new file mode 100644
index 00000000000..c6604b64cb3
--- /dev/null
+++ b/readthedocs/search/tests/data/sphinx/out/local-toc.json
@@ -0,0 +1,46 @@
+{
+  "path": "local-toc.html",
+  "title": "Security reports",
+  "sections": [
+    {
+      "id": "security-reports",
+      "title": "Security reports",
+      "content": "Security is very important to us at Read the Docs. We follow generally accepted industry standards to protect the personal information submitted to us, both during transmission and once we receive it. In the spirit of transparency, we are committed to responsible reporting and disclosure of security issues. See also Security policy Read our policy for security, which we base our security handling and reporting on."
+    },
+    {
+      "id": "supported-versions",
+      "title": "Supported versions",
+      "content": "Only the latest version of Read the Docs will receive security updates. We don’t support security updates for custom installations of Read the Docs."
+    },
+    {
+      "id": "reporting-a-security-issue",
+      "title": "Reporting a security issue",
+      "content": "If you believe you’ve discovered a security issue at Read the Docs, please contact us at security@readthedocs.org (optionally using our PGP key). We request that you please not publicly disclose the issue until it has been addressed by us. You can expect: We will respond acknowledging your email typically within one business day. We will follow up if and when we have confirmed the issue with a timetable for the fix. We will notify you when the issue is fixed. We will create a GitHub advisory and publish it when the issue has been fixed and deployed in our platforms."
+    },
+    {
+      "id": "pgp-key",
+      "title": "PGP key",
+      "content": "You may use this PGP key to securely communicate with us and to verify signed messages you receive from us."
+    },
+    {
+      "id": "bug-bounties",
+      "title": "Bug bounties",
+      "content": "While we sincerely appreciate and encourage reports of suspected security problems, please note that the Read the Docs is an open source project, and does not run any bug bounty programs."
+    },
+    {
+      "id": "security-issue-archive",
+      "title": "Security issue archive",
+      "content": "You can see all past reports at https://github.com/readthedocs/readthedocs.org/security/advisories."
+    },
+    {
+      "id": "version-3-2-0",
+      "title": "Version 3.2.0",
+      "content": "Version 3.2.0 resolved an issue where a specially crafted request could result in a DNS query to an arbitrary domain. This issue was found by Cyber Smart Defence who reported it as part of a security audit to a firm running a local installation of Read the Docs."
+    },
+    {
+      "id": "release-2-3-0",
+      "title": "Release 2.3.0",
+      "content": "Version 2.3.0 resolves a security issue with translations on our community hosting site that allowed users to modify the hosted path of a target project by adding it as a translation project of their own project. A check was added to ensure project ownership before adding the project as a translation. In order to add a project as a translation now, users must now first be granted ownership in the translation project."
+    }
+  ]
+}
diff --git a/readthedocs/search/tests/data/sphinx/out/toctree.json b/readthedocs/search/tests/data/sphinx/out/toctree.json
new file mode 100644
index 00000000000..87c8eb9bf8c
--- /dev/null
+++ b/readthedocs/search/tests/data/sphinx/out/toctree.json
@@ -0,0 +1,11 @@
+{
+  "path": "",
+  "title": "",
+  "sections": [
+    {
+      "id": "public-api",
+      "title": "Public API",
+      "content": "This section of the documentation details the public API usable to get details of projects, builds, versions and other details from Read the Docs."
+    }
+  ]
+}
diff --git a/readthedocs/search/tests/test_parsers.py b/readthedocs/search/tests/test_parsers.py
index 1e129f752e7..b56b8cdae2d 100644
--- a/readthedocs/search/tests/test_parsers.py
+++ b/readthedocs/search/tests/test_parsers.py
@@ -266,16 +266,70 @@ def test_sphinx_autodoc(self, storage_open, storage_exists):
         expected_json = json.load(open(data_path / "sphinx/out/autodoc.json"))
         assert parsed_json == expected_json
 
+    @mock.patch.object(BuildMediaFileSystemStorage, "exists")
+    @mock.patch.object(BuildMediaFileSystemStorage, "open")
+    def test_sphinx_local_toc(self, storage_open, storage_exists):
+        """
+        Test that the local table of contents from the ``contents``
+        directive is not included in the indexed content.
+        """
+        # Source:
+        # https://docs.readthedocs.io/en/stable/security.html
+        html_content = data_path / "sphinx/in/local-toc.html"
+        storage_open.side_effect = self._mock_open(html_content.open().read())
+        storage_exists.return_value = True
+
+        self.project.feature_set.add(self.feature)
+        self.version.documentation_type = SPHINX
+        self.version.save()
+
+        page_file = get(
+            HTMLFile,
+            project=self.project,
+            version=self.version,
+            path="local-toc.html",
+        )
+
+        parsed_json = page_file.processed_json
+        expected_json = json.load(open(data_path / "sphinx/out/local-toc.json"))
+        assert parsed_json == expected_json
+
+    @mock.patch.object(BuildMediaFileSystemStorage, "exists")
+    @mock.patch.object(BuildMediaFileSystemStorage, "open")
+    def test_sphinx_toctree(self, storage_open, storage_exists):
+        """
+        Test that the table of contents from the ``toctree``
+        directive is not included in the indexed content.
+        """
+        # Source:
+        # https://docs.readthedocs.io/en/stable/api/index.html
+        html_content = data_path / "sphinx/in/toctree.html"
+        json_content = {"body": html_content.open().read()}
+        storage_open.side_effect = self._mock_open(json.dumps(json_content))
+        storage_exists.return_value = True
+
+        self.version.documentation_type = SPHINX
+        self.version.save()
+
+        page_file = get(
+            HTMLFile,
+            project=self.project,
+            version=self.version,
+            path="toctree.html",
+        )
+
+        parsed_json = page_file.processed_json
+        expected_json = json.load(open(data_path / "sphinx/out/toctree.json"))
+        assert parsed_json == expected_json
+
     @mock.patch.object(BuildMediaFileSystemStorage, "exists")
     @mock.patch.object(BuildMediaFileSystemStorage, "open")
     def test_sphinx_requests(self, storage_open, storage_exists):
         # Source:
         # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-automodule
-        json_file = data_path / "sphinx/in/requests.json"
         html_content = data_path / "sphinx/in/requests.html"
 
-        json_content = json.load(json_file.open())
-        json_content["body"] = html_content.open().read()
+        json_content = {"body": html_content.open().read()}
         storage_open.side_effect = self._mock_open(json.dumps(json_content))
         storage_exists.return_value = True