From c259495c5e8fb51fee6be8685bdbededb84d6091 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Sun, 17 Dec 2023 22:13:20 -0500 Subject: [PATCH 1/4] remove unnecessary liquid directives; add toc --- docs/sql-ref-functions-builtin.md | 103 +----------------------------- 1 file changed, 3 insertions(+), 100 deletions(-) diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md index 0ff1432fabf8e..81fc27c31195a 100644 --- a/docs/sql-ref-functions-builtin.md +++ b/docs/sql-ref-functions-builtin.md @@ -17,202 +17,105 @@ license: | limitations under the License. --- -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-agg-funcs-table.html' %} +* Table of contents +{:toc} + ### Aggregate Functions {% include_relative generated-agg-funcs-table.html %} #### Examples {% include_relative generated-agg-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-window-funcs-table.html' %} ### Window Functions {% include_relative generated-window-funcs-table.html %} #### Examples {% include_relative generated-window-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-array-funcs-table.html' %} ### Array Functions {% include_relative generated-array-funcs-table.html %} #### Examples {% include_relative generated-array-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-collection-funcs-table.html' %} ### Collection Functions {% include_relative generated-collection-funcs-table.html %} #### Examples {% include_relative generated-collection-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-struct-funcs-table.html' %} ### STRUCT Functions {% include_relative generated-struct-funcs-table.html %} #### Examples {% include_relative generated-struct-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-map-funcs-table.html' %} ### Map Functions {% include_relative generated-map-funcs-table.html %} #### Examples {% include_relative generated-map-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-datetime-funcs-table.html' %} ### Date and Timestamp Functions {% include_relative generated-datetime-funcs-table.html %} #### Examples {% include_relative generated-datetime-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-math-funcs-table.html' %} ### Mathematical Functions {% include_relative generated-math-funcs-table.html %} #### Examples {% include_relative generated-math-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-string-funcs-table.html' %} ### String Functions {% include_relative generated-string-funcs-table.html %} #### Examples {% include_relative generated-string-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-conditional-funcs-table.html' %} ### Conditional Functions {% include_relative generated-conditional-funcs-table.html %} #### Examples {% include_relative generated-conditional-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-hash-funcs-table.html' %} ### Hash Functions {% include_relative generated-hash-funcs-table.html %} #### Examples {% include_relative generated-hash-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-csv-funcs-table.html' %} ### CSV Functions {% include_relative generated-csv-funcs-table.html %} #### Examples {% include_relative generated-csv-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-json-funcs-table.html' %} ### JSON Functions {% include_relative generated-json-funcs-table.html %} #### Examples {% include_relative generated-json-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-xml-funcs-table.html' %} ### XML Functions {% include_relative generated-xml-funcs-table.html %} #### Examples {% include_relative generated-xml-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-url-funcs-table.html' %} ### URL Functions {% include_relative generated-url-funcs-table.html %} #### Examples {% include_relative generated-url-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-bitwise-funcs-table.html' %} ### Bitwise Functions {% include_relative generated-bitwise-funcs-table.html %} #### Examples {% include_relative generated-bitwise-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-conversion-funcs-table.html' %} ### Conversion Functions {% include_relative generated-conversion-funcs-table.html %} #### Examples {% include_relative generated-conversion-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-predicate-funcs-table.html' %} ### Predicate Functions {% include_relative generated-predicate-funcs-table.html %} #### Examples {% include_relative generated-predicate-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-misc-funcs-table.html' %} ### Misc Functions {% include_relative generated-misc-funcs-table.html %} #### Examples {% include_relative generated-misc-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-generator-funcs-table.html' %} ### Generator Functions {% include_relative generated-generator-funcs-table.html %} #### Examples {% include_relative generated-generator-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} From ed95998c59e51c8cb1d640fc3caa9aa74c3dda2d Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Sun, 17 Dec 2023 22:31:20 -0500 Subject: [PATCH 2/4] move generated function html into subdir --- docs/.gitignore | 1 + docs/sql-ref-functions-builtin.md | 80 +++++++++++++++---------------- sql/gen-sql-functions-docs.py | 15 +++--- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index 9df83f37815b7..bcfdcbf5dccfb 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1 +1,2 @@ generated-*.html +_generated_function_html/ diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md index 81fc27c31195a..8688e6e98e96b 100644 --- a/docs/sql-ref-functions-builtin.md +++ b/docs/sql-ref-functions-builtin.md @@ -21,101 +21,101 @@ license: | {:toc} ### Aggregate Functions -{% include_relative generated-agg-funcs-table.html %} +{% include_relative _generated_function_html/agg-funcs-table.html %} #### Examples -{% include_relative generated-agg-funcs-examples.html %} +{% include_relative _generated_function_html/agg-funcs-examples.html %} ### Window Functions -{% include_relative generated-window-funcs-table.html %} +{% include_relative _generated_function_html/window-funcs-table.html %} #### Examples -{% include_relative generated-window-funcs-examples.html %} +{% include_relative _generated_function_html/window-funcs-examples.html %} ### Array Functions -{% include_relative generated-array-funcs-table.html %} +{% include_relative _generated_function_html/array-funcs-table.html %} #### Examples -{% include_relative generated-array-funcs-examples.html %} +{% include_relative _generated_function_html/array-funcs-examples.html %} ### Collection Functions -{% include_relative generated-collection-funcs-table.html %} +{% include_relative _generated_function_html/collection-funcs-table.html %} #### Examples -{% include_relative generated-collection-funcs-examples.html %} +{% include_relative _generated_function_html/collection-funcs-examples.html %} ### STRUCT Functions -{% include_relative generated-struct-funcs-table.html %} +{% include_relative _generated_function_html/struct-funcs-table.html %} #### Examples -{% include_relative generated-struct-funcs-examples.html %} +{% include_relative _generated_function_html/struct-funcs-examples.html %} ### Map Functions -{% include_relative generated-map-funcs-table.html %} +{% include_relative _generated_function_html/map-funcs-table.html %} #### Examples -{% include_relative generated-map-funcs-examples.html %} +{% include_relative _generated_function_html/map-funcs-examples.html %} ### Date and Timestamp Functions -{% include_relative generated-datetime-funcs-table.html %} +{% include_relative _generated_function_html/datetime-funcs-table.html %} #### Examples -{% include_relative generated-datetime-funcs-examples.html %} +{% include_relative _generated_function_html/datetime-funcs-examples.html %} ### Mathematical Functions -{% include_relative generated-math-funcs-table.html %} +{% include_relative _generated_function_html/math-funcs-table.html %} #### Examples -{% include_relative generated-math-funcs-examples.html %} +{% include_relative _generated_function_html/math-funcs-examples.html %} ### String Functions -{% include_relative generated-string-funcs-table.html %} +{% include_relative _generated_function_html/string-funcs-table.html %} #### Examples -{% include_relative generated-string-funcs-examples.html %} +{% include_relative _generated_function_html/string-funcs-examples.html %} ### Conditional Functions -{% include_relative generated-conditional-funcs-table.html %} +{% include_relative _generated_function_html/conditional-funcs-table.html %} #### Examples -{% include_relative generated-conditional-funcs-examples.html %} +{% include_relative _generated_function_html/conditional-funcs-examples.html %} ### Hash Functions -{% include_relative generated-hash-funcs-table.html %} +{% include_relative _generated_function_html/hash-funcs-table.html %} #### Examples -{% include_relative generated-hash-funcs-examples.html %} +{% include_relative _generated_function_html/hash-funcs-examples.html %} ### CSV Functions -{% include_relative generated-csv-funcs-table.html %} +{% include_relative _generated_function_html/csv-funcs-table.html %} #### Examples -{% include_relative generated-csv-funcs-examples.html %} +{% include_relative _generated_function_html/csv-funcs-examples.html %} ### JSON Functions -{% include_relative generated-json-funcs-table.html %} +{% include_relative _generated_function_html/json-funcs-table.html %} #### Examples -{% include_relative generated-json-funcs-examples.html %} +{% include_relative _generated_function_html/json-funcs-examples.html %} ### XML Functions -{% include_relative generated-xml-funcs-table.html %} +{% include_relative _generated_function_html/xml-funcs-table.html %} #### Examples -{% include_relative generated-xml-funcs-examples.html %} +{% include_relative _generated_function_html/xml-funcs-examples.html %} ### URL Functions -{% include_relative generated-url-funcs-table.html %} +{% include_relative _generated_function_html/url-funcs-table.html %} #### Examples -{% include_relative generated-url-funcs-examples.html %} +{% include_relative _generated_function_html/url-funcs-examples.html %} ### Bitwise Functions -{% include_relative generated-bitwise-funcs-table.html %} +{% include_relative _generated_function_html/bitwise-funcs-table.html %} #### Examples -{% include_relative generated-bitwise-funcs-examples.html %} +{% include_relative _generated_function_html/bitwise-funcs-examples.html %} ### Conversion Functions -{% include_relative generated-conversion-funcs-table.html %} +{% include_relative _generated_function_html/conversion-funcs-table.html %} #### Examples -{% include_relative generated-conversion-funcs-examples.html %} +{% include_relative _generated_function_html/conversion-funcs-examples.html %} ### Predicate Functions -{% include_relative generated-predicate-funcs-table.html %} +{% include_relative _generated_function_html/predicate-funcs-table.html %} #### Examples -{% include_relative generated-predicate-funcs-examples.html %} +{% include_relative _generated_function_html/predicate-funcs-examples.html %} ### Misc Functions -{% include_relative generated-misc-funcs-table.html %} +{% include_relative _generated_function_html/misc-funcs-table.html %} #### Examples -{% include_relative generated-misc-funcs-examples.html %} +{% include_relative _generated_function_html/misc-funcs-examples.html %} ### Generator Functions -{% include_relative generated-generator-funcs-table.html %} +{% include_relative _generated_function_html/generator-funcs-table.html %} #### Examples -{% include_relative generated-generator-funcs-examples.html %} +{% include_relative _generated_function_html/generator-funcs-examples.html %} diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py index 053e11d10295b..82522478140e2 100644 --- a/sql/gen-sql-functions-docs.py +++ b/sql/gen-sql-functions-docs.py @@ -16,9 +16,10 @@ # import itertools -import os import re +import shutil from collections import namedtuple +from pathlib import Path # To avoid adding a new direct dependency, we import markdown from within mkdocs. from mkdocs.structure.pages import markdown @@ -26,6 +27,8 @@ from pyspark.java_gateway import launch_gateway +SPARK_PROJECT_ROOT = Path(__file__).parents[1] + ExpressionInfo = namedtuple("ExpressionInfo", "name usage examples group") groups = { @@ -210,7 +213,7 @@ def generate_functions_table_html(jvm, html_output_dir): for key, infos in _list_grouped_function_infos(jvm): function_table = _make_pretty_usage(infos) key = key.replace("_", "-") - with open("%s/generated-%s-table.html" % (html_output_dir, key), 'w') as table_html: + with open(html_output_dir / f"{key}-table.html", 'w') as table_html: table_html.write(function_table) @@ -233,8 +236,7 @@ def generate_functions_examples_html(jvm, jspark, html_output_dir): examples = _make_pretty_examples(jspark, infos) key = key.replace("_", "-") if examples is not None: - with open("%s/generated-%s-examples.html" % ( - html_output_dir, key), 'w') as examples_html: + with open(html_output_dir / f"{key}-examples.html", 'w') as examples_html: examples_html.write(examples) @@ -242,7 +244,8 @@ def generate_functions_examples_html(jvm, jspark, html_output_dir): jvm = launch_gateway().jvm jspark = jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate() jspark.sparkContext().setLogLevel("ERROR") # Make it less noisy. - spark_root_dir = os.path.dirname(os.path.dirname(__file__)) - html_output_dir = os.path.join(spark_root_dir, "docs") + html_output_dir = SPARK_PROJECT_ROOT / "docs" / "_generated_function_html" + shutil.rmtree(html_output_dir, ignore_errors=True) + html_output_dir.mkdir() generate_functions_table_html(jvm, html_output_dir) generate_functions_examples_html(jvm, jspark, html_output_dir) From ebe3b05ae5b450c6d02c9ceb95b6f4d8372f45c6 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Wed, 20 Dec 2023 19:05:01 -0500 Subject: [PATCH 3/4] h4 -> bold --- docs/sql-ref-functions-builtin.md | 60 ++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md index 8688e6e98e96b..88ed309a883a3 100644 --- a/docs/sql-ref-functions-builtin.md +++ b/docs/sql-ref-functions-builtin.md @@ -22,100 +22,120 @@ license: | ### Aggregate Functions {% include_relative _generated_function_html/agg-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/agg-funcs-examples.html %} ### Window Functions {% include_relative _generated_function_html/window-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/window-funcs-examples.html %} ### Array Functions {% include_relative _generated_function_html/array-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/array-funcs-examples.html %} ### Collection Functions {% include_relative _generated_function_html/collection-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/collection-funcs-examples.html %} ### STRUCT Functions {% include_relative _generated_function_html/struct-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/struct-funcs-examples.html %} ### Map Functions {% include_relative _generated_function_html/map-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/map-funcs-examples.html %} ### Date and Timestamp Functions {% include_relative _generated_function_html/datetime-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/datetime-funcs-examples.html %} ### Mathematical Functions {% include_relative _generated_function_html/math-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/math-funcs-examples.html %} ### String Functions {% include_relative _generated_function_html/string-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/string-funcs-examples.html %} ### Conditional Functions {% include_relative _generated_function_html/conditional-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/conditional-funcs-examples.html %} ### Hash Functions {% include_relative _generated_function_html/hash-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/hash-funcs-examples.html %} ### CSV Functions {% include_relative _generated_function_html/csv-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/csv-funcs-examples.html %} ### JSON Functions {% include_relative _generated_function_html/json-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/json-funcs-examples.html %} ### XML Functions {% include_relative _generated_function_html/xml-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/xml-funcs-examples.html %} ### URL Functions {% include_relative _generated_function_html/url-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/url-funcs-examples.html %} ### Bitwise Functions {% include_relative _generated_function_html/bitwise-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/bitwise-funcs-examples.html %} ### Conversion Functions {% include_relative _generated_function_html/conversion-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/conversion-funcs-examples.html %} ### Predicate Functions {% include_relative _generated_function_html/predicate-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/predicate-funcs-examples.html %} ### Misc Functions {% include_relative _generated_function_html/misc-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/misc-funcs-examples.html %} ### Generator Functions {% include_relative _generated_function_html/generator-funcs-table.html %} -#### Examples + +**Examples** {% include_relative _generated_function_html/generator-funcs-examples.html %} From a9ae743eb66004dce9a9dd7e7ca24ef17823d5aa Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Wed, 20 Dec 2023 19:06:19 -0500 Subject: [PATCH 4/4] fix broken link --- docs/sql-ref-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md index cc9edd61f41ee..b4891fe72eb35 100644 --- a/docs/sql-ref-functions.md +++ b/docs/sql-ref-functions.md @@ -20,7 +20,7 @@ license: | --- Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs). -Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task. +Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/index.html) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task. ### Built-in Functions