- fixing SORAX-related crashes by moving all SORAX actions into the U…

…I thread; the lib uses COM under the hood, which requires a working and accessible Windows message pipe, something which only the UI thread can provide.

- littered the code with WPFDoEvents UI/not-UI assertions -- which caught the above scenario in a Dispose() for a page image render. And that was the hint the needed to progress a little further towards stibility: it was SORAX which caused a *lot* of the out-of-memory failures due to crazy COM/WPF/UI failures, even for smaller libraries under test.
- fix bit of an odd crash in the Lucene flush/cleanup during shutdown, where Lucene kept busy with 'optimizing the index' while a quick application termination was happening in the background, resulting in lockup and then a crash.
- this MAY be a fix for the reported "number of documents reported not matching reality": added update/refresh code to update the library list panel when PDF documents are added in the background via FolderWatcher or other means (async library loading). WARNING: this code is still incomplete/buggy!
- most UI assertions have been covered now. Keeping them anyway as this is hairy stuff and should be tested more.

Addresses (but is not guaranteed to fix) jimmejardine#290, jimmejardine#283, jimmejardine#281, jimmejardine#280, jimmejardine#243

Qiqqa/DocumentLibrary/FolderWatching/FolderWatcher.cs

@@ -161,6 +161,7 @@ internal class WatchStatistics
             public int processed_file_count;
             public int scanned_file_count;
             public int skipped_file_count;
+            public int files_added_since_last_sleep;
 
             // Store the *hashes* of the files we have processed during this run.
             //
@@ -183,6 +184,7 @@ public void Reset(Daemon d)
                 processed_file_count = 0;
                 scanned_file_count = 0;
                 skipped_file_count = 0;
+                files_added_since_last_sleep = 0;
 
                 file_hashes_added = new Dictionary<string, string>();
 
@@ -194,6 +196,46 @@ public void Reset(Daemon d)
 
         private WatchStatistics watch_stats;
 
+
+        public class GlobalWatchStatistics
+        {
+            private double scanned_file_count;
+            private long previous_promillage;
+            private object _lock = new object();
+
+            // Return true when there's a significant change in promillage (one or more)
+            public void Inc(double step = 1.0)
+            {
+                lock (_lock)
+                {
+                    scanned_file_count += Math.Max(0.001, step);
+
+                    long rv = (long)scanned_file_count;
+                    if (previous_promillage != rv)
+                    {
+                        previous_promillage = rv;
+
+                        // When we hit the upper bound, we RESET the promillage counter:
+                        if (rv >= 1000)
+                        {
+                            scanned_file_count = 0;
+                        }
+
+                        StatusManager.Instance.UpdateStatus("FolderWatcher", "📂👀", rv, 1000);
+                    }
+                }
+            }
+
+            public GlobalWatchStatistics()
+            {
+                scanned_file_count = 0;
+                previous_promillage = -1;
+            }
+        }
+
+        // Track global statistics for UI display
+        public static GlobalWatchStatistics global_watch_stats = new GlobalWatchStatistics();
+
         /// <summary>
         /// The daemon code calls this occasionally to poke it into action to do work
         /// </summary>
@@ -314,6 +356,8 @@ public void ExecuteBackgroundProcess(Daemon daemon)
                 //     in a log, so that the enumeration continues as in the case of Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.ContinueOnException
                 //     option but the user will be informed about errors.
                 //
+                global_watch_stats.Inc();
+
                 DirectoryEnumerationFilters filter = new DirectoryEnumerationFilters();
                 filter.ErrorFilter = DecideIfErrorDuringDirScan;
                 filter.InclusionFilter = DecideIfIncludeDuringDirScan;
@@ -364,6 +408,8 @@ public void ExecuteBackgroundProcess(Daemon daemon)
                     }
                 }
 
+                Logging.Debug特("Directory.EnumerateFiles took {0} ms", clk.ElapsedMilliseconds);
+
                 // Create the import records
                 List<FilenameWithMetadataImport> filename_with_metadata_imports = new List<FilenameWithMetadataImport>();
                 foreach (var filename in filenames_that_are_new)
@@ -456,9 +502,11 @@ internal bool DecideIfIncludeDuringDirScan(FileSystemEntryInfo obj)
                 throw new OperationCanceledException("FolderWatcher: Breaking out of inner processing loop due to disposed library and/or watch manager");
             }
 
+            global_watch_stats.Inc(0.1);
+
             bool have_we_slept = false;
 
-            if (watch_stats.index_processing_clock.ElapsedMilliseconds > MAX_SECONDS_PER_ITERATION)
+            if (watch_stats.index_processing_clock.ElapsedMilliseconds > MAX_SECONDS_PER_ITERATION && watch_stats.files_added_since_last_sleep > 42)
             {
                 Logging.Info("FolderWatcher: Taking a nap due to MAX_SECONDS_PER_ITERATION: {0} seconds consumed, {1} threads pending", watch_stats.index_processing_clock.ElapsedMilliseconds / 1E3, SafeThreadPool.QueuedThreadCount);
 
@@ -471,7 +519,7 @@ internal bool DecideIfIncludeDuringDirScan(FileSystemEntryInfo obj)
 
                 int duration = 1 * 1000 + thr_cnt * 250 + queued_cnt * 20 + textify_count * 50 + ocr_count * 500;
 
-                watch_stats.daemon.Sleep(Math.Min(60 * 1000, duration));
+                watch_stats.daemon.Sleep(Math.Min(5 * 1000, duration));
 
                 // Relinquish control to the UI thread to make sure responsiveness remains tolerable at 100% CPU load.
                 WPFDoEvents.WaitForUIThreadActivityDone();
@@ -556,6 +604,7 @@ internal bool DecideIfIncludeDuringDirScan(FileSystemEntryInfo obj)
                 }
 
                 watch_stats.file_hashes_added.Add(fingerprint, obj.FullPath);
+                watch_stats.files_added_since_last_sleep++;
 
                 return true;
             }

Qiqqa/DocumentLibrary/ImportingIntoLibrary.cs

@@ -59,6 +59,8 @@ public static void AddNewPDFDocumentsToLibrary_ASYNCHRONOUS(WebLibraryDetail web
 
         public static PDFDocument AddNewPDFDocumentsToLibrary_SYNCHRONOUS(WebLibraryDetail web_library_detail, bool suppress_notifications, bool suppress_signal_that_docs_have_changed, params string[] filenames)
         {
+            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
+
             FilenameWithMetadataImport[] filename_with_metadata_imports = new FilenameWithMetadataImport[filenames.Length];
             for (int i = 0; i < filenames.Length; ++i)
             {

Qiqqa/DocumentLibrary/Library.cs

@@ -418,6 +418,8 @@ private PDFDocument AddNewDocumentToLibrary(string filename, WebLibraryDetail we
                 last_pdf_add_time.Restart();
             }
 
+            FolderWatcher.global_watch_stats.Inc(0.1);
+
             if (String.IsNullOrEmpty(filename) || filename.EndsWith(".vanilla_reference"))
             {
                 return AddVanillaReferenceDocumentToLibrary(bibtex, web_library_detail, tags, comments, suppressDialogs, suppress_signal_that_docs_have_changed);
@@ -569,6 +571,8 @@ private PDFDocument AddNewDocumentToLibrary(string filename, WebLibraryDetail we
                 SignalThatDocumentsHaveChanged(pdf_document);
             }
 
+            FolderWatcher.global_watch_stats.Inc();
+
             return pdf_document;
         }
 
@@ -647,6 +651,8 @@ public PDFDocument AddVanillaReferenceDocumentToLibrary(string bibtex, WebLibrar
                 SignalThatDocumentsHaveChanged(pdf_document);
             }
 
+            FolderWatcher.global_watch_stats.Inc();
+
             return pdf_document;
         }
 

Qiqqa/DocumentLibrary/LibraryCatalog/LibraryCatalogControl.xaml.cs

@@ -10,6 +10,7 @@
 using Qiqqa.Documents.PDF;
 using Utilities;
 using Utilities.GUI;
+using Utilities.Misc;
 using Utilities.Reflection;
 
 namespace Qiqqa.DocumentLibrary.LibraryCatalog
@@ -36,6 +37,13 @@ public LibraryCatalogControl()
             ListPDFDocuments.MouseDoubleClick += ListPDFDocuments_MouseDoubleClick;
             ListPDFDocuments.IsVisibleChanged += ListPDFDocuments_IsVisibleChanged;
             ReconsiderPDFDocumentDetail();
+
+#if DEBUG
+            if (Runtime.IsRunningInVisualStudioDesigner)
+            {
+                //DataContext = dummy;
+            }
+#endif
         }
 
         private void ListPDFDocuments_IsVisibleChanged(object sender, DependencyPropertyChangedEventArgs e)
@@ -125,15 +133,49 @@ public WebLibraryDetail LibraryRef
 
                 drag_to_library_manager = new DragToLibraryManager(web_library_detail);
                 drag_to_library_manager.RegisterControl(this);
+
+                web_library_detail = DataContext as WebLibraryDetail;
+                if (null != web_library_detail)
+                {
+                    // WEAK EVENT HANDLER FOR: web_library_detail.library.OnDocumentsChanged += library_OnDocumentsChanged;
+                    WeakEventHandler<Library.PDFDocumentEventArgs>.Register<WebLibraryDetail, LibraryCatalogControl>(
+                        web_library_detail,
+                        registerWeakEvent,
+                        deregisterWeakEvent,
+                        this,
+                        forwardWeakEvent
+                    );
+
+                    drag_to_library_manager.DefaultLibrary = web_library_detail;
+                }
             }
         }
 
-        public void SetPDFDocuments(IEnumerable<PDFDocument> pdf_documents, PDFDocument pdf_document_to_focus_on)
+        private static void registerWeakEvent(WebLibraryDetail sender, EventHandler<Library.PDFDocumentEventArgs> eh)
+        {
+            sender.Xlibrary.OnDocumentsChanged += eh;
+        }
+        private static void deregisterWeakEvent(WebLibraryDetail sender, EventHandler<Library.PDFDocumentEventArgs> eh)
         {
-            SetPDFDocuments(pdf_documents, pdf_document_to_focus_on, null, null);
+            sender.Xlibrary.OnDocumentsChanged -= eh;
+        }
+        private static void forwardWeakEvent(LibraryCatalogControl me, object event_sender, Library.PDFDocumentEventArgs args)
+        {
+            me.library_OnDocumentsChanged();
+        }
+
+        private void library_OnDocumentsChanged()
+        {
+            WPFDoEvents.InvokeAsyncInUIThread(() =>
+            {
+                if (ListPDFDocuments.IsVisible)
+                {
+                    ListPDFDocuments.UpdateLayout();
+                }
+            });
         }
 
-        public void SetPDFDocuments(IEnumerable<PDFDocument> pdf_documents, PDFDocument pdf_document_to_focus_on, string filter_terms, Dictionary<string, double> search_scores)
+        public void SetPDFDocuments(IEnumerable<PDFDocument> pdf_documents, PDFDocument pdf_document_to_focus_on = null, string filter_terms = null, Dictionary<string, double> search_scores = null)
         {
             this.filter_terms = filter_terms;
             this.search_scores = search_scores;

Qiqqa/DocumentLibrary/LibraryCatalog/LibraryCatalogOverviewControl.xaml.cs

@@ -69,6 +69,30 @@ public LibraryCatalogOverviewControl()
             MouseLeave += LibraryCatalogOverviewControl_MouseLeave;
 
             DataContextChanged += LibraryCatalogOverviewControl_DataContextChanged;
+
+#if DEBUG
+            if (Runtime.IsRunningInVisualStudioDesigner)
+            {
+                if (PDFDocumentBindable == null || true)
+                {
+                    PDFDocument fake_doc = PDFDocument.CreateFakeForDesigner();
+                    fake_doc.BibTex = "@booklet{fubar42, title={The Qiqqa Sample for Desginer View}, author={The Qiqqa Team and Hobbelt, Ger}, year={2013}}";
+
+                    fake_doc.Comments = "comments";
+                    fake_doc.YearCombined = "2021";
+                    //fake_doc.TitleCombined = "Sample for Designer View";
+                    //fake_doc.AuthorsCombined = "Ger Hobbelt";
+                    fake_doc.Publication = "Moir Brandts Honk";
+                    fake_doc.Tags = "bloo, blub";
+                    fake_doc.ReadingStage = "mid-way";
+                    fake_doc.Rating = "Superb";
+                    //fake_doc.Id = "foobar42";
+
+                    AugmentedBindable<PDFDocument> dummy = new AugmentedBindable<PDFDocument>(fake_doc);
+                    DataContext = dummy;
+                }
+            }
+#endif
         }
 
         private void ButtonThemeSwatch_Click(object sender, RoutedEventArgs e)
@@ -98,26 +122,28 @@ private void LibraryCatalogOverviewControl_MouseEnter(object sender, MouseEventA
             c.Background = ThemeColours.Background_Brush_Blue_LightToDark;
         }
 
-        private LibraryCatalogControl _library_catalog_control;
+        private WeakReference<LibraryCatalogControl> _library_catalog_control = new WeakReference<LibraryCatalogControl>(null);
         private LibraryCatalogControl LibraryCatalogControl
         {
             get
             {
                 try
                 {
-                    if (null == _library_catalog_control)
+                    LibraryCatalogControl rv = null;
+                    if (!_library_catalog_control.TryGetTarget(out rv) || rv == null)
                     {
-                        _library_catalog_control = GUITools.GetParentControl<LibraryCatalogControl>(this);
+                        rv = GUITools.GetParentControl<LibraryCatalogControl>(this);
+                        _library_catalog_control.SetTarget(rv);
                     }
+                    return rv;
                 }
-
                 catch (Exception ex)
                 {
                     Logging.Warn(ex, "There was a problem while trying to detemine the parent of the library catalog overview control");
-                    _library_catalog_control = null;
+                    _library_catalog_control.SetTarget(null);
                 }
 
-                return _library_catalog_control;
+                return null;
             }
         }
 

Qiqqa/DocumentLibrary/LibraryFilter/AITagExplorerStuff/AITagExplorerControl.xaml.cs

@@ -147,6 +147,8 @@ public void Reset()
 
         internal static MultiMapSet<string, string> GetNodeItems(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
         {
+            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
+
             Logging.Info("+Getting node items for AutoTags");
 
             if (null == web_library_detail.Xlibrary.AITagManager.AITags)

Qiqqa/DocumentLibrary/LibraryFilter/AuthorExplorerStuff/AuthorExplorerControl.xaml.cs

@@ -7,6 +7,7 @@
 using Qiqqa.Documents.PDF;
 using Utilities;
 using Utilities.Collections;
+using Utilities.GUI;
 using Utilities.Language;
 
 namespace Qiqqa.DocumentLibrary.LibraryFilter.AuthorExplorerStuff
@@ -55,6 +56,8 @@ public void Reset()
 
         internal static MultiMapSet<string, string> GetNodeItems(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
         {
+            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
+
             Logging.Info("+Getting node items for Authors");
 
             List<PDFDocument> pdf_documents = null;

Qiqqa/DocumentLibrary/LibraryFilter/GeneralExplorers/RatingExplorerControl.xaml.cs

@@ -6,6 +6,7 @@
 using Qiqqa.Documents.PDF;
 using Utilities;
 using Utilities.Collections;
+using Utilities.GUI;
 
 namespace Qiqqa.DocumentLibrary.LibraryFilter.GeneralExplorers
 {
@@ -53,6 +54,8 @@ public void Reset()
 
         internal static MultiMapSet<string, string> GetNodeItems(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
         {
+            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
+
             List<PDFDocument> pdf_documents = null;
             if (null == parent_fingerprints)
             {

Qiqqa/DocumentLibrary/LibraryFilter/GeneralExplorers/ReadingStageExplorerControl.xaml.cs

@@ -6,6 +6,7 @@
 using Qiqqa.Documents.PDF;
 using Utilities;
 using Utilities.Collections;
+using Utilities.GUI;
 
 namespace Qiqqa.DocumentLibrary.LibraryFilter.GeneralExplorers
 {
@@ -53,6 +54,8 @@ public void Reset()
 
         internal static MultiMapSet<string, string> GetNodeItems(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
         {
+            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
+
             List<PDFDocument> pdf_documents = null;
             if (null == parent_fingerprints)
             {

Qiqqa/DocumentLibrary/LibraryFilter/GeneralExplorers/ThemeExplorerControl.xaml.cs

@@ -9,6 +9,7 @@
 using Qiqqa.Expedition;
 using Utilities;
 using Utilities.Collections;
+using Utilities.GUI;
 
 namespace Qiqqa.DocumentLibrary.LibraryFilter.GeneralExplorers
 {
@@ -61,27 +62,17 @@ public void Reset()
 
         // -----------------------------
 
-        private MultiMapSet<string, string> GetNodeItems(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
+        private void UpdateVisibility(bool has_themes)
         {
-            MultiMapSet<string, string> results = GetNodeItems_STATIC(web_library_detail, parent_fingerprints);
+            WPFDoEvents.AssertThisCodeIsRunningInTheUIThread();
 
-            // Show the no themes message
-            {
-                bool have_topics =
-                    true
-                    && null != web_library_detail.Xlibrary.ExpeditionManager
-                    && null != web_library_detail.Xlibrary.ExpeditionManager.ExpeditionDataSource
-                    && 0 < web_library_detail.Xlibrary.ExpeditionManager.ExpeditionDataSource.LDAAnalysis.NUM_TOPICS
-                    ;
-
-                TxtNoThemesMessage.Visibility = 0 == results.Count ? Visibility.Visible : Visibility.Collapsed;
-            }
-
-            return results;
+            TxtNoThemesMessage.Visibility = !has_themes ? Visibility.Visible : Visibility.Collapsed;
         }
 
-        public static MultiMapSet<string, string> GetNodeItems_STATIC(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
+        public static MultiMapSet<string, string> GetNodeItems(WebLibraryDetail web_library_detail, HashSet<string> parent_fingerprints)
         {
+            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
+
             MultiMapSet<string, string> results = new MultiMapSet<string, string>();
 
             try