From e0d75f582e485a7de509e2aa79bcbda05d1f571e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Tue, 19 Nov 2024 16:57:12 +0100 Subject: [PATCH] fix: sturdier `base64` encoding in `application/typedoc-data;base64` (#26) Also, fixes an omission from before (packs the Python scripts into the NPM package) --- packages/plugin/package.json | 3 +- .../docspec-gen/__init__.py | 0 .../docspec-gen/generate_ast.py | 0 .../docspec-gen/google_docstring_processor.py | 0 .../type-parsing/parse_types.py | 0 packages/plugin/src/components/ApiItem.tsx | 37 +++++++++++++------ packages/plugin/src/plugin/python/index.ts | 8 +++- .../src/plugin/python/type-parsing/index.ts | 2 +- packages/plugin/src/utils/reexports.ts | 8 +++- playground/python/src/foo.py | 2 + playground/website/docusaurus.config.ts | 1 - 11 files changed, 43 insertions(+), 18 deletions(-) rename packages/plugin/{src/plugin/python => python-scripts}/docspec-gen/__init__.py (100%) rename packages/plugin/{src/plugin/python => python-scripts}/docspec-gen/generate_ast.py (100%) rename packages/plugin/{src/plugin/python => python-scripts}/docspec-gen/google_docstring_processor.py (100%) rename packages/plugin/{src/plugin/python => python-scripts}/type-parsing/parse_types.py (100%) diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 20ee5f0..d5af243 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@apify/docusaurus-plugin-typedoc-api", - "version": "4.2.11-1", + "version": "4.2.11-2", "description": "Docusaurus plugin that provides source code API documentation powered by TypeDoc. ", "keywords": [ "docusaurus", @@ -18,6 +18,7 @@ "types": "./lib/index.d.ts", "files": [ "assets/**/*", + "python-scripts/**/*", "lib/**/*", "src/**/*", "styles.css" diff --git a/packages/plugin/src/plugin/python/docspec-gen/__init__.py b/packages/plugin/python-scripts/docspec-gen/__init__.py similarity index 100% rename from packages/plugin/src/plugin/python/docspec-gen/__init__.py rename to packages/plugin/python-scripts/docspec-gen/__init__.py diff --git a/packages/plugin/src/plugin/python/docspec-gen/generate_ast.py b/packages/plugin/python-scripts/docspec-gen/generate_ast.py similarity index 100% rename from packages/plugin/src/plugin/python/docspec-gen/generate_ast.py rename to packages/plugin/python-scripts/docspec-gen/generate_ast.py diff --git a/packages/plugin/src/plugin/python/docspec-gen/google_docstring_processor.py b/packages/plugin/python-scripts/docspec-gen/google_docstring_processor.py similarity index 100% rename from packages/plugin/src/plugin/python/docspec-gen/google_docstring_processor.py rename to packages/plugin/python-scripts/docspec-gen/google_docstring_processor.py diff --git a/packages/plugin/src/plugin/python/type-parsing/parse_types.py b/packages/plugin/python-scripts/type-parsing/parse_types.py similarity index 100% rename from packages/plugin/src/plugin/python/type-parsing/parse_types.py rename to packages/plugin/python-scripts/type-parsing/parse_types.py diff --git a/packages/plugin/src/components/ApiItem.tsx b/packages/plugin/src/components/ApiItem.tsx index 3283b78..35c1fa2 100644 --- a/packages/plugin/src/components/ApiItem.tsx +++ b/packages/plugin/src/components/ApiItem.tsx @@ -122,6 +122,16 @@ function deepCopy(obj: any): any { return copy; } +function base64Encode(message: string): string { + const bytes = new TextEncoder().encode(message); + + const binString = Array.from(bytes, (byte) => + String.fromCodePoint(byte), + ).join(""); + + return btoa(binString); +} + export default function ApiItem({ readme: Readme, route }: ApiItemProps) { const [hideInherited, setHideInherited] = useState(false); const apiOptions = useMemo( @@ -200,18 +210,21 @@ export default function ApiItem({ readme: Readme, route }: ApiItemProps) { {/* The `application/json+typedoc-data;base64` is an base64 encoded JSON object that contains the machine-readable API item data. */} - + ); diff --git a/packages/plugin/src/plugin/python/index.ts b/packages/plugin/src/plugin/python/index.ts index 415c997..45d5ffa 100644 --- a/packages/plugin/src/plugin/python/index.ts +++ b/packages/plugin/src/plugin/python/index.ts @@ -21,13 +21,17 @@ export function processPythonDocs({ }) { const pydocMarkdownDumpPath = path.join(__dirname, './pydoc-markdown-dump.json'); - childProcess.spawnSync('python', [ - path.join(__dirname, './docspec-gen/generate_ast.py'), + const out = childProcess.spawnSync('python', [ + path.join(__dirname, '..', '..', '..', './python-scripts/docspec-gen/generate_ast.py'), '-i', pythonModulePath, pydocMarkdownDumpPath, ]); + if (out.error) { + throw new Error(out.error.message); + } + const moduleShortcuts = JSON.parse(fs.readFileSync(moduleShortcutsPath, 'utf8')) as Record< string, string diff --git a/packages/plugin/src/plugin/python/type-parsing/index.ts b/packages/plugin/src/plugin/python/type-parsing/index.ts index 81bae24..297a65f 100644 --- a/packages/plugin/src/plugin/python/type-parsing/index.ts +++ b/packages/plugin/src/plugin/python/type-parsing/index.ts @@ -6,7 +6,7 @@ import type { DocspecType, TypeDocType } from '../types'; const RAW_TYPES_JSON_FILEPATH = path.join(__dirname, 'typedoc-types.raw'); const PARSED_TYPES_JSON_FILEPATH = path.join(__dirname, 'typedoc-types-parsed.json'); -const PYTHON_SCRIPT_FILEPATH = path.join(__dirname, 'parse_types.py'); +const PYTHON_SCRIPT_FILEPATH = path.join(__dirname, '..', '..', '..', '..', 'python-scripts/type-parsing/parse_types.py'); /** * Keeps track of Typedoc type objects. When `resolveTypes` is called, it tries to parse diff --git a/packages/plugin/src/utils/reexports.ts b/packages/plugin/src/utils/reexports.ts index c82865d..db1a67f 100644 --- a/packages/plugin/src/utils/reexports.ts +++ b/packages/plugin/src/utils/reexports.ts @@ -14,6 +14,12 @@ interface ExternalApiItem { groups: string[]; } +function decodeBase64UTF8(base64: string): string { + const binString = atob(base64); + const bytes = Uint8Array.from(binString, (m) => m.codePointAt(0)); + return new TextDecoder('utf8').decode(bytes); +} + async function loadExternalApiItem(url: string): Promise { console.log(`Loading external API item from ${url}...`); const response = await fetch(url); @@ -23,7 +29,7 @@ async function loadExternalApiItem(url: string): Promise { if(!base64encoded) return null; - const jsonData = atob(base64encoded); + const jsonData = decodeBase64UTF8(base64encoded); return JSON.parse(jsonData) as ExternalApiItem; } diff --git a/playground/python/src/foo.py b/playground/python/src/foo.py index 46830c7..aa7e4c8 100644 --- a/playground/python/src/foo.py +++ b/playground/python/src/foo.py @@ -19,6 +19,8 @@ def bar(self): def bar_param(self, param): """ The bar method of the foo class, prints "bar" and the given parameter. + + There are more-than-1-byte characters here, look: žžžžžžž! """ print("bar", param) diff --git a/playground/website/docusaurus.config.ts b/playground/website/docusaurus.config.ts index 0a4bc97..2382039 100644 --- a/playground/website/docusaurus.config.ts +++ b/playground/website/docusaurus.config.ts @@ -54,7 +54,6 @@ const config: Config = { moduleShortcutsPath: __dirname + '/../python/module_shortcuts.json', pythonModulePath: __dirname + '/../python/src', }, - reexports: ['https://crawlee.dev/python/api/class/BasicCrawler'], }), ],