V10/ordered nodes (#547)

* Allow - ordered node precaching on report/import. * optimized the clear key lookup
KevinJump · Oct 2, 2023 · e6b61eb · e6b61eb
1 parent c2de644
commit e6b61eb
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 17 deletions.
diff --git a/uSync.BackOffice/Services/uSyncService_Single.cs b/uSync.BackOffice/Services/uSyncService_Single.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Xml.Linq;
@@ -27,11 +28,18 @@ public partial class uSyncService
         public IEnumerable<uSyncAction> ReportPartial(string folder, uSyncPagedImportOptions options, out int total)
         {
             var orderedNodes = LoadOrderedNodes(folder);
+            return ReportPartial(orderedNodes, options, out total); 
+        }
+
+        public IEnumerable<uSyncAction> ReportPartial(IList<OrderedNodeInfo> orderedNodes, uSyncPagedImportOptions options, out int total)
+        {
             total = orderedNodes.Count;
 
             var actions = new List<uSyncAction>();
             var lastType = string.Empty;
 
+            var folder = Path.GetDirectoryName(orderedNodes.FirstOrDefault()?.FileName ?? options.RootFolder);
+
             SyncHandlerOptions syncHandlerOptions = HandlerOptionsFromPaged(options);
 
             HandlerConfigPair handlerPair = null;
@@ -53,6 +61,8 @@ public IEnumerable<uSyncAction> ReportPartial(string folder, uSyncPagedImportOpt
                     continue;
                 }
 
+                handlerPair.Handler.PreCacheFolderKeys(folder, orderedNodes.Select(x => x.Key).ToList());
+
                 options.Callbacks?.Update.Invoke(item.Node.GetAlias(),
                     CalculateProgress(index, total, options.ProgressMin, options.ProgressMax), 100);
 
@@ -71,12 +81,17 @@ public IEnumerable<uSyncAction> ReportPartial(string folder, uSyncPagedImportOpt
         ///  Peform a paged Import against a given folder 
         /// </summary>
         public IEnumerable<uSyncAction> ImportPartial(string folder, uSyncPagedImportOptions options, out int total)
+        {
+            var orderedNodes = LoadOrderedNodes(folder);
+            return ImportPartial(orderedNodes, options, out total);
+        }
+
+        public IEnumerable<uSyncAction> ImportPartial(IList<OrderedNodeInfo> orderedNodes, uSyncPagedImportOptions options, out int total)
         {
             lock (_importLock)
             {
                 using (var pause = _mutexService.ImportPause(options.PauseDuringImport))
                 {
-                    var orderedNodes = LoadOrderedNodes(folder);
 
                     total = orderedNodes.Count;
 
@@ -282,7 +297,7 @@ private SyncHandlerOptions HandlerOptionsFromPaged(uSyncPagedImportOptions optio
         /// <summary>
         ///  Load the xml in a folder in level order so we process the higher level items first.
         /// </summary>
-        private IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
+        public IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
         {
             var files = _syncFileService.GetFiles(folder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true);
 
@@ -298,19 +313,6 @@ private IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
                 .ToList();
         }
 
-        private class OrderedNodeInfo
-        {
-            public OrderedNodeInfo(string filename, XElement node)
-            {
-                this.FileName = filename;
-                this.Node = node;
-            }
-
-            public XElement Node { get; set; }
-            public string FileName { get; set; }
-        }
-
-
         /// <summary>
         ///  calculate the percentage progress we are making between a range. 
         /// </summary>
@@ -319,6 +321,35 @@ public OrderedNodeInfo(string filename, XElement node)
         /// </remarks>
         private int CalculateProgress(int value, int total, int min, int max)
             => (int)(min + (((float)value / total) * (max - min)));
+    }
 
+    /// <summary>
+    ///  detail for a usync file that can be ordered
+    /// </summary>
+    public class OrderedNodeInfo
+    {
+        /// <summary>
+        ///  constructor
+        /// </summary>
+        public OrderedNodeInfo(string filename, XElement node)
+        {
+            FileName = filename;
+            Node = node;
+            Key = node.GetKey();
+        }
+
+        /// <summary>
+        ///  xml element of the node
+        /// </summary>
+        public XElement Node { get; set; }
+
+        public Guid Key { get; set; }
+
+        /// <summary>
+        ///  path to the physical file 
+        /// </summary>
+        public string FileName { get; set; }
     }
+
+
 }
diff --git a/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncHandler.cs
@@ -163,5 +163,13 @@ public interface ISyncHandler
         /// </summary>
         ChangeType GetItemStatus(XElement node) => ChangeType.NoChange;
 
+
+        /// <summary>
+        ///  precaches the keys of a folder
+        /// </summary>
+        /// <param name="folder"></param>
+        /// <param name="keys"></param>
+        void PreCacheFolderKeys(string folder, IList<Guid> keys)
+        { }
     }
 }
diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs
@@ -605,6 +605,20 @@ protected virtual IEnumerable<uSyncAction> CleanFolder(string cleanFile, bool re
             }
         }
 
+        /// <summary>
+        ///  pre-populates the cache folder key list. 
+        /// </summary>
+        /// <remarks>
+        ///  this means if we are calling the process multiple times, 
+        ///  we can optimise the key code and only load it once. 
+        /// </remarks>
+        public void PreCacheFolderKeys(string folder, IList<Guid> folderKeys)
+        {
+            var cacheKey = $"{GetCacheKeyBase()}_{folder.GetHashCode()}";
+            runtimeCache.ClearByKey(cacheKey) ; 
+            runtimeCache.GetCacheItem(cacheKey, () => folderKeys);
+        }
+
         /// <summary>
         ///  Get the GUIDs for all items in a folder
         /// </summary>
@@ -620,10 +634,11 @@ protected IList<Guid> GetFolderKeys(string folder, bool flat)
 
             var cacheKey = $"{GetCacheKeyBase()}_{folderKey}";
 
-            logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey);
 
             return runtimeCache.GetCacheItem(cacheKey, () =>
             {
+                logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey);
+
                 // when it's not flat structure we also get the sub folders. (extra defensive get them all)
                 var keys = new List<Guid>();
                 var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat).ToList();