Voodoo fixes following the change in layout

src/driver/cmd_outputs.ml

@@ -25,3 +25,12 @@ let submit log_dest desc cmd output_file =
       maybe_log log_dest x;
       String.split_on_char '\n' x.output
   | Error exn -> raise exn
+
+let submit_ignore_failures log_dest desc cmd output_file =
+  match Worker_pool.submit desc cmd output_file with
+  | Ok x ->
+      maybe_log log_dest x;
+      ()
+  | Error exn ->
+      Logs.err (fun m -> m "Error: %s" (Printexc.to_string exn));
+      ()

src/driver/compile.ml

@@ -4,6 +4,8 @@ open Bos
 
 type compiled = Odoc_unit.t
 
+let odoc_partial_filename = "__odoc_partial.m"
+
 let mk_byhash (pkgs : Odoc_unit.t list) =
   List.fold_left
     (fun acc (u : Odoc_unit.t) ->
@@ -76,7 +78,7 @@ let find_partials odoc_dir :
   let hashes_result =
     OS.Dir.fold_contents ~dotfiles:false ~elements:`Dirs
       (fun p hashes ->
-        let index_m = Fpath.( / ) p "index.m" in
+        let index_m = Fpath.( / ) p odoc_partial_filename in
         match OS.File.exists index_m with
         | Ok true ->
             let tbl', hashes' = unmarshal index_m in
@@ -133,7 +135,12 @@ let compile ?partial ~partial_dir (all : Odoc_unit.t list) =
                            None)
                      deps
                  in
-                 let includes = unit.include_dirs in
+                 let includes =
+                   List.fold_left
+                     (fun acc (_lib, path) -> Fpath.Set.add path acc)
+                     Fpath.Set.empty
+                     (Odoc_unit.Pkg_args.compiled_libs unit.pkg_args)
+                 in
                  Odoc.compile ~output_dir:unit.output_dir
                    ~input_file:unit.input_file ~includes
                    ~parent_id:unit.parent_id;
@@ -164,7 +171,12 @@ let compile ?partial ~partial_dir (all : Odoc_unit.t list) =
     | `Intf _ as kind ->
         (compile_mod { unit with kind } :> (Odoc_unit.t list, _) Result.t)
     | `Impl src ->
-        let includes = unit.include_dirs in
+        let includes =
+          List.fold_left
+            (fun acc (_lib, path) -> Fpath.Set.add path acc)
+            Fpath.Set.empty
+            (Odoc_unit.Pkg_args.compiled_libs unit.pkg_args)
+        in
         let source_id = src.src_id in
         Odoc.compile_impl ~output_dir:unit.output_dir
           ~input_file:unit.input_file ~includes ~parent_id:unit.parent_id
@@ -177,7 +189,7 @@ let compile ?partial ~partial_dir (all : Odoc_unit.t list) =
         Atomic.incr Stats.stats.compiled_assets;
         Ok [ unit ]
     | `Mld ->
-        let includes = unit.include_dirs in
+        let includes = Fpath.Set.empty in
         Odoc.compile ~output_dir:unit.output_dir ~input_file:unit.input_file
           ~includes ~parent_id:unit.parent_id;
         Atomic.incr Stats.stats.compiled_mlds;
@@ -208,7 +220,7 @@ let compile ?partial ~partial_dir (all : Odoc_unit.t list) =
       res
   in
   (match partial with
-  | Some l -> marshal (zipped, hashes) Fpath.(l / "index.m")
+  | Some l -> marshal (zipped, hashes) Fpath.(l / odoc_partial_filename)
   | None -> ());
   all
 
@@ -221,8 +233,7 @@ let link : compiled list -> _ =
     let link input_file output_file enable_warnings =
       let libs = Odoc_unit.Pkg_args.compiled_libs c.pkg_args in
       let pages = Odoc_unit.Pkg_args.compiled_pages c.pkg_args in
-      let includes = c.include_dirs in
-      Odoc.link ~input_file ~output_file ~includes ~libs ~docs:pages
+      Odoc.link ~input_file ~output_file ~libs ~docs:pages
         ~ignore_output:(not enable_warnings) ?current_package:c.pkgname ()
     in
     match c.kind with

src/driver/compile.mli

@@ -1,4 +1,4 @@
-type compiled
+type compiled = Odoc_unit.t
 
 val init_stats : Odoc_unit.t list -> unit
 

src/driver/landing_pages.ml

@@ -17,7 +17,6 @@ let make_index ~dirs ~rel_dir ?index ~content () =
     input_file;
     odoc_file;
     odocl_file;
-    include_dirs = Fpath.Set.empty;
     enable_warnings = false;
     kind = `Mld;
     index;

src/driver/library_names.ml

@@ -29,7 +29,7 @@ let read_libraries_from_pkg_defs ~library_name pkg_defs =
     in
 
     let deps_str = Fl_metascanner.lookup "requires" [] pkg_defs in
-    let deps = Astring.String.fields deps_str in
+    let deps = Astring.String.fields ~empty:false deps_str in
     let dir =
       List.find_opt (fun d -> d.Fl_metascanner.def_var = "directory") pkg_defs
     in

src/driver/odoc.ml

@@ -130,17 +130,12 @@ let lib_args libs =
       v "-L" % s %% acc)
     Cmd.empty libs
 
-let link ?(ignore_output = false) ~input_file:file ?output_file ~includes ~docs
-    ~libs ?current_package () =
+let link ?(ignore_output = false) ~input_file:file ?output_file ~docs ~libs
+    ?current_package () =
   let open Cmd in
   let output_file =
     match output_file with Some f -> f | None -> Fpath.set_ext "odocl" file
   in
-  let includes =
-    Fpath.Set.fold
-      (fun path acc -> Cmd.(acc % "-I" % p path))
-      includes Cmd.empty
-  in
   let docs = doc_args docs in
   let libs = lib_args libs in
   let current_package =
@@ -149,7 +144,7 @@ let link ?(ignore_output = false) ~input_file:file ?output_file ~includes ~docs
     | Some c -> Cmd.(v "--current-package" % c)
   in
   let cmd =
-    !odoc % "link" % p file % "-o" % p output_file %% includes %% docs %% libs
+    !odoc % "link" % p file % "-o" % p output_file %% docs %% libs
     %% current_package % "--enable-missing-root-warning"
   in
   let cmd =

src/driver/odoc.mli

@@ -34,7 +34,6 @@ val link :
   ?ignore_output:bool ->
   input_file:Fpath.t ->
   ?output_file:Fpath.t ->
-  includes:Fpath.set ->
   docs:(string * Fpath.t) list ->
   libs:(string * Fpath.t) list ->
   ?current_package:string ->

src/driver/odoc_driver.ml

@@ -111,8 +111,10 @@ let remap_virtual_interfaces duplicate_hashes pkgs =
       })
     pkgs
 
+type action_mode = CompileOnly | LinkAndGen | All
+
 type mode =
-  | Voodoo of { package_name : string; blessed : bool }
+  | Voodoo of { package_name : string; blessed : bool; actions : action_mode }
   | Dune of { path : Fpath.t }
   | OpamLibs of { libs : string list }
   | OpamPackages of { packages : string list }
@@ -141,18 +143,22 @@ let run mode
   Stats.init_nprocs nb_workers;
   let () = Worker_pool.start_workers env sw nb_workers in
 
-  let all, extra_libs_paths =
+  let all, extra_paths, actions =
     match mode with
-    | Voodoo { package_name = p; blessed } ->
+    | Voodoo { package_name = p; blessed; actions } ->
         let all = Voodoo.of_voodoo p ~blessed in
-        let extra_libs_paths = Voodoo.extra_libs_paths odoc_dir in
-        (all, extra_libs_paths)
-    | Dune { path } -> (Dune_style.of_dune_build path, Util.StringMap.empty)
+        let extra_paths = Voodoo.extra_paths odoc_dir in
+        (all, extra_paths, actions)
+    | Dune { path } ->
+        (Dune_style.of_dune_build path, Voodoo.empty_extra_paths, All)
     | OpamLibs { libs } ->
         ( Packages.of_libs ~packages_dir:None (Util.StringSet.of_list libs),
-          Util.StringMap.empty )
+          Voodoo.empty_extra_paths,
+          All )
     | OpamPackages { packages } ->
-        (Packages.of_packages ~packages_dir:None packages, Util.StringMap.empty)
+        ( Packages.of_packages ~packages_dir:None packages,
+          Voodoo.empty_extra_paths,
+          All )
   in
 
   let virtual_check =
@@ -192,27 +198,40 @@ let run mode
   let () =
     Eio.Fiber.both
       (fun () ->
-        let all =
+        let units =
           let all = Util.StringMap.bindings all |> List.map snd in
           let dirs =
             let odocl_dir = Option.value odocl_dir ~default:odoc_dir in
             { Odoc_unit.odoc_dir; odocl_dir; index_dir; mld_dir }
           in
-          Odoc_units_of.packages ~dirs ~extra_libs_paths all
+          Odoc_units_of.packages ~dirs ~extra_paths all
         in
-        Compile.init_stats all;
-        let compiled = Compile.compile ?partial ~partial_dir:odoc_dir all in
-        let linked = Compile.link compiled in
-        let occurrence_file =
-          let output =
-            Fpath.( / ) odoc_dir "occurrences-all.odoc-occurrences"
-          in
-          let () = Odoc.count_occurrences ~input:[ odoc_dir ] ~output in
-          output
+        Compile.init_stats units;
+        let compiled =
+          match actions with
+          | LinkAndGen -> units
+          | CompileOnly | All ->
+              Compile.compile ?partial ~partial_dir:odoc_dir units
         in
-        let () = Compile.html_generate ~occurrence_file html_dir linked in
-        let _ = Odoc.support_files html_dir in
-        ())
+        let () =
+          match mode with
+          | Voodoo _ -> Voodoo.write_lib_markers odoc_dir all
+          | Dune _ | OpamLibs _ | OpamPackages _ -> ()
+        in
+        match actions with
+        | CompileOnly -> ()
+        | LinkAndGen | All ->
+            let linked = Compile.link compiled in
+            let occurrence_file =
+              let output =
+                Fpath.( / ) odoc_dir "occurrences-all.odoc-occurrences"
+              in
+              let () = Odoc.count_occurrences ~input:[ odoc_dir ] ~output in
+              output
+            in
+            let () = Compile.html_generate ~occurrence_file html_dir linked in
+            let _ = Odoc.support_files html_dir in
+            ())
       (fun () -> render_stats env nb_workers)
   in
 
@@ -253,7 +272,8 @@ let run mode
 open Cmdliner
 
 module Voodoo_mode = struct
-  let run package_name blessed = run (Voodoo { package_name; blessed })
+  let run package_name blessed actions =
+    run (Voodoo { package_name; blessed; actions })
 
   let package_name =
     let doc = "Name of package to process with voodoo" in
@@ -263,10 +283,32 @@ module Voodoo_mode = struct
     let doc = "Blessed" in
     Arg.(value & flag & info [ "blessed" ] ~doc)
 
+  let action_of_string = function
+    | "compile-only" -> Ok CompileOnly
+    | "link-and-gen" -> Ok LinkAndGen
+    | "all" -> Ok All
+    | _ ->
+        Error
+          (`Msg
+            "Invalid action. Options are 'compile-only', 'link-and-gen' or \
+             'all'")
+
+  let string_of_action fmt = function
+    | CompileOnly -> Format.fprintf fmt "compile-only"
+    | LinkAndGen -> Format.fprintf fmt "link-and-gen"
+    | All -> Format.fprintf fmt "all"
+
+  let action_conv = Arg.conv (action_of_string, string_of_action)
+
+  let actions =
+    let doc = "Actions to perform" in
+    Arg.(value & opt action_conv All & info [ "actions" ] ~doc)
+
   let cmd =
     let doc = "Process output from voodoo-prep" in
     let info = Cmd.info "voodoo" ~doc in
-    Cmd.v info Term.(const run $ package_name $ blessed $ Common_args.term)
+    Cmd.v info
+      Term.(const run $ package_name $ blessed $ actions $ Common_args.term)
 end
 
 module Dune_mode = struct

src/driver/odoc_unit.ml

@@ -56,7 +56,6 @@ type 'a unit = {
   odocl_file : Fpath.t;
   pkg_args : Pkg_args.t;
   pkgname : string option;
-  include_dirs : Fpath.Set.t;
   index : index option;
   enable_warnings : bool;
   kind : 'a;
@@ -105,15 +104,12 @@ and pp : all_kinds unit Fmt.t =
      odocl_file: %a@;\
      pkg_args: %a@;\
      pkgname: %a@;\
-     include_dirs: [%a]@;\
      index: %a@;\
      kind:%a@;\
      @]"
     (Odoc.Id.to_string x.parent_id)
     Fpath.pp x.input_file Fpath.pp x.output_dir Fpath.pp x.odoc_file Fpath.pp
     x.odocl_file Pkg_args.pp x.pkg_args (Fmt.option Fmt.string) x.pkgname
-    Fmt.(list ~sep:comma Fpath.pp)
-    (Fpath.Set.to_list x.include_dirs)
     (Fmt.option pp_index) x.index pp_kind
     (x.kind :> all_kinds)
 

src/driver/odoc_unit.mli

@@ -31,7 +31,6 @@ type 'a unit = {
   odocl_file : Fpath.t;
   pkg_args : Pkg_args.t;
   pkgname : string option;
-  include_dirs : Fpath.Set.t;
   index : index option;
   enable_warnings : bool;
   kind : 'a;