chore: make Sphinx cookbook extension pip-installable (#2447)

Fixes #2432.

docs/source/conf.py

@@ -19,6 +19,7 @@
 import sys
 from pathlib import Path
 
+sys.path.append(str(Path("./ext/sphinx_recipe").resolve()))
 sys.path.append(str(Path("./ext").resolve()))
 
 # -- Project information -----------------------------------------------------
@@ -40,7 +41,7 @@
 exclude_patterns = []
 extensions = [
     # recipe directive
-    "adbc_cookbook",
+    "sphinx_recipe",
     # generic directives to enable intersphinx for java
     "adbc_java_domain",
     "numpydoc",
@@ -114,6 +115,12 @@
     "source_directory": "docs/source/",
 }
 
+# -- Options for sphinx-recipe -----------------------------------------------
+
+recipe_repo_url_template = (
+    "https://github.com/apache/arrow-adbc/blob/main/docs/source/{rel_filename}"
+)
+
 # -- Options for Intersphinx -------------------------------------------------
 
 intersphinx_mapping = {

docs/source/ext/sphinx_recipe/README.markdown

@@ -0,0 +1,84 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# sphinx-recipe: Not Literate Programming, For Sphinx
+
+[Literate Programming][literate] is a style of programming introduced by the
+venerable Donald Knuth where a program is developed as a prose document
+interspersed with code.  The code is then extracted from the document for
+actual use.
+
+This isn't that.
+
+sphinx-recipe is an extension for the [Sphinx][sphinx] documentation generator
+that lets you write code interspersed with special comments.  The extension
+then separates the code and comments to generate a prose document interspersed
+with code.  Meanwhile, since the source document is still just code, it can be
+run and/or tested without special documentation-specific tooling.  And Sphinx
+doesn't need to understand anything about the source language other than what
+comment delimiter it should look for.
+
+This extension makes it easy to write self-contained "cookbook" documentation
+pages that can be run and tested as real code while still being nicely
+formatted in documentation.
+
+## Usage
+
+1. Install this extension.
+
+   This extension can be installed via pip from this repo:
+
+   ```bash
+   pip install 'git+https://github.com/apache/arrow-adbc@main#subdirectory=docs/source/ext/sphinx_recipe'
+   pip install 'https://github.com/apache/arrow-adbc/archive/main.zip#subdirectory=docs/source/ext/sphinx_recipe'
+   ```
+
+   The latter is likely a bit faster (it downloads the source archive vs
+   having to actually perform a Git clone).
+
+1. Add `sphinx_recipe` to `extensions` in your `conf.py`.
+1. Optionally, set the config value `recipe_repo_url_template` to a URL
+   with a `{rel_filename}` placeholder.  This will be used to link to the full
+   recipe source on GitHub or your favorite code hosting platform.
+1. Write your cookbook recipe.  For example:
+
+    ```python
+
+    # This is my license header. It's boring, so I don't want it to show up
+    # in the documentation.
+
+    # RECIPE STARTS HERE
+    #: Some prose describing what I'm about to do.
+    #: **reStructuredText syntax works here.**
+    print("Hello, world!")
+
+    #: I can have more prose now.
+    #:
+    #: - I can even write lists.
+    print("Goodbye, world!")
+    ```
+
+1. Include the recipe in your documentation via the `recipe` directive:
+
+    ```rst
+    .. recipe:: helloworld.py
+    ```
+
+[literate]: https://en.wikipedia.org/wiki/Literate_programming
+[sphinx]: https://www.sphinx-doc.org/en/master/

docs/source/ext/sphinx_recipe/pyproject.toml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[project]
+name = "sphinx-recipe"
+description = "A Sphinx directive for not-quite-literate programming in the form of code recipes."
+authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
+license = {text = "Apache-2.0"}
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = ["sphinx"]
+version = "0.1.0"
+
+[project.urls]
+homepage = "https://arrow.apache.org/adbc/"
+repository = "https://github.com/apache/arrow-adbc"
+
+[build-system]
+requires = ["setuptools >= 61.0.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+xfail_strict = true
+
+[tool.setuptools]
+packages = ["sphinx_recipe"]
+py-modules = ["sphinx_recipe"]

docs/source/ext/adbc_cookbook.py → docs/source/ext/sphinx_recipe/sphinx_recipe/__init__.py

@@ -27,6 +27,8 @@
 from sphinx.util.nodes import nested_parse_with_titles
 from sphinx.util.typing import OptionSpec
 
+__all__ = ["setup"]
+
 
 class SourceLine(typing.NamedTuple):
     content: str
@@ -107,17 +109,22 @@ def run(self):
         # --- Generate the final reST as a whole and parse it
         # That way, section hierarchy works properly
 
+        generated_lines = []
+
         # Link to the source on GitHub
-        github_url = (
-            f"https://github.com/apache/arrow-adbc/blob/main/docs/source/{rel_filename}"
-        )
-        generated_lines = [
-            PREAMBLE.format(
-                name=Path(rel_filename).name,
-                url=github_url,
-            ),
-            "",
-        ]
+        repo_url_template = self.env.config.recipe_repo_url_template
+        if repo_url_template is not None:
+            repo_url = repo_url_template.format(rel_filename=rel_filename)
+            generated_lines.append(
+                PREAMBLE.format(
+                    name=Path(rel_filename).name,
+                    url=repo_url,
+                )
+            )
+
+        # Paragraph break
+        generated_lines.append("")
+
         for fragment in fragments:
             if fragment.kind == "prose":
                 generated_lines.extend([line.content for line in fragment.lines])
@@ -147,6 +154,12 @@ def run(self):
 
 
 def setup(app) -> None:
+    app.add_config_value(
+        "recipe_repo_url_template",
+        default=None,
+        rebuild="html",
+        types=str,
+    )
     app.add_directive("recipe", RecipeDirective)
 
     return {