fix: sturdier base64 encoding in application/typedoc-data;base64 (#…

…26)

Also, fixes an omission from before (packs the Python scripts into the
NPM package)

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apify/docusaurus-plugin-typedoc-api",
-  "version": "4.2.11-1",
+  "version": "4.2.11-2",
   "description": "Docusaurus plugin that provides source code API documentation powered by TypeDoc. ",
   "keywords": [
     "docusaurus",
@@ -18,6 +18,7 @@
   "types": "./lib/index.d.ts",
   "files": [
     "assets/**/*",
+    "python-scripts/**/*",
     "lib/**/*",
     "src/**/*",
     "styles.css"

packages/plugin/src/components/ApiItem.tsx

@@ -122,6 +122,16 @@ function deepCopy(obj: any): any {
 	return copy;
 }
 
+function base64Encode(message: string): string {
+	const bytes = new TextEncoder().encode(message);
+
+	const binString = Array.from(bytes, (byte) =>
+		String.fromCodePoint(byte),
+	  ).join("");
+
+	return btoa(binString);
+}
+
 export default function ApiItem({ readme: Readme, route }: ApiItemProps) {
 	const [hideInherited, setHideInherited] = useState(false);
 	const apiOptions = useMemo(
@@ -200,18 +210,21 @@ export default function ApiItem({ readme: Readme, route }: ApiItemProps) {
 
 				<Reflection reflection={item} />
 				{/* The `application/json+typedoc-data;base64` is an base64 encoded JSON object that contains the machine-readable API item data. */}
-				<script type="application/typedoc-data;base64">{btoa(JSON.stringify(
-						{
-							// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-							item: {
-								...apiItem,
-								nextId: undefined,
-								previousId: undefined,
-								parentId: undefined,
-							},
-							groups: getOwnGroupNames(item, reflections),
-						}, 
-					))}</script>
+				<script type="application/typedoc-data;base64">{
+					base64Encode(
+						JSON.stringify(
+							{
+								// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+								item: {
+									...apiItem,
+									nextId: undefined,
+									previousId: undefined,
+									parentId: undefined,
+								},
+								groups: getOwnGroupNames(item, reflections),
+							}, 
+						)
+					)}</script>
 			</ApiItemLayout>
 		</ApiOptionsContext.Provider>
 	);

packages/plugin/src/plugin/python/index.ts

@@ -21,13 +21,17 @@ export function processPythonDocs({
 }) {
 	const pydocMarkdownDumpPath = path.join(__dirname, './pydoc-markdown-dump.json');
 
-	childProcess.spawnSync('python', [
-		path.join(__dirname, './docspec-gen/generate_ast.py'),
+	const out = childProcess.spawnSync('python', [
+		path.join(__dirname,  '..', '..', '..', './python-scripts/docspec-gen/generate_ast.py'),
 		'-i',
 		pythonModulePath,
 		pydocMarkdownDumpPath,
 	]);
 
+	if (out.error) {
+		throw new Error(out.error.message);
+	}
+
 	const moduleShortcuts = JSON.parse(fs.readFileSync(moduleShortcutsPath, 'utf8')) as Record<
 		string,
 		string

packages/plugin/src/plugin/python/type-parsing/index.ts

@@ -6,7 +6,7 @@ import type { DocspecType, TypeDocType } from '../types';
 const RAW_TYPES_JSON_FILEPATH = path.join(__dirname, 'typedoc-types.raw');
 const PARSED_TYPES_JSON_FILEPATH = path.join(__dirname, 'typedoc-types-parsed.json');
 
-const PYTHON_SCRIPT_FILEPATH = path.join(__dirname, 'parse_types.py');
+const PYTHON_SCRIPT_FILEPATH = path.join(__dirname, '..', '..', '..', '..', 'python-scripts/type-parsing/parse_types.py');
 
 /**
  * Keeps track of Typedoc type objects. When `resolveTypes` is called, it tries to parse

packages/plugin/src/utils/reexports.ts

@@ -14,6 +14,12 @@ interface ExternalApiItem {
     groups: string[];
 }
 
+function decodeBase64UTF8(base64: string): string {
+    const binString = atob(base64);
+    const bytes = Uint8Array.from(binString, (m) => m.codePointAt(0));
+    return new TextDecoder('utf8').decode(bytes);
+}
+
 async function loadExternalApiItem(url: string): Promise<ExternalApiItem> {
     console.log(`Loading external API item from ${url}...`);
     const response = await fetch(url);
@@ -23,7 +29,7 @@ async function loadExternalApiItem(url: string): Promise<ExternalApiItem> {
 
     if(!base64encoded) return null;
 
-    const jsonData = atob(base64encoded);
+    const jsonData = decodeBase64UTF8(base64encoded);
     return JSON.parse(jsonData) as ExternalApiItem;
 }
 

playground/python/src/foo.py

@@ -19,6 +19,8 @@ def bar(self):
     def bar_param(self, param):
         """
         The bar method of the foo class, prints "bar" and the given parameter.
+
+        There are more-than-1-byte characters here, look: žžžžžžž!
         """
 
         print("bar", param)

playground/website/docusaurus.config.ts

@@ -54,7 +54,6 @@ const config: Config = {
 					moduleShortcutsPath: __dirname + '/../python/module_shortcuts.json',
 					pythonModulePath: __dirname + '/../python/src',
 				},
-				reexports: ['https://crawlee.dev/python/api/class/BasicCrawler'],
 			}),
 	],