Implement persistent subscription cache

src/constants.js

@@ -25,11 +25,13 @@ const IpcChannels = {
   DB_HISTORY: 'db-history',
   DB_PROFILES: 'db-profiles',
   DB_PLAYLISTS: 'db-playlists',
+  DB_SUBSCRIPTION_CACHE: 'db-subscription-cache',
 
   SYNC_SETTINGS: 'sync-settings',
   SYNC_HISTORY: 'sync-history',
   SYNC_PROFILES: 'sync-profiles',
   SYNC_PLAYLISTS: 'sync-playlists',
+  SYNC_SUBSCRIPTION_CACHE: 'sync-subscription-cache',
 
   GET_REPLACE_HTTP_CACHE: 'get-replace-http-cache',
   TOGGLE_REPLACE_HTTP_CACHE: 'toggle-replace-http-cache',
@@ -66,7 +68,15 @@ const DBActions = {
     DELETE_VIDEO_ID: 'db-action-playlists-delete-video-by-playlist-name',
     DELETE_VIDEO_IDS: 'db-action-playlists-delete-video-ids',
     DELETE_ALL_VIDEOS: 'db-action-playlists-delete-all-videos',
-  }
+  },
+
+  SUBSCRIPTION_CACHE: {
+    UPDATE_VIDEOS_BY_CHANNEL: 'db-action-subscriptions-update-videos-by-channel',
+    UPDATE_LIVE_STREAMS_BY_CHANNEL: 'db-action-subscriptions-update-live-streams-by-channel',
+    UPDATE_SHORTS_BY_CHANNEL: 'db-action-subscriptions-update-shorts-by-channel',
+    UPDATE_SHORTS_WITH_CHANNEL_PAGE_SHORTS_BY_CHANNEL: 'db-action-subscriptions-update-shorts-with-channel-page-shorts-by-channel',
+    UPDATE_COMMUNITY_POSTS_BY_CHANNEL: 'db-action-subscriptions-update-community-posts-by-channel',
+  },
 }
 
 const SyncEvents = {
@@ -90,7 +100,15 @@ const SyncEvents = {
   PLAYLISTS: {
     UPSERT_VIDEO: 'sync-playlists-upsert-video',
     DELETE_VIDEO: 'sync-playlists-delete-video',
-  }
+  },
+
+  SUBSCRIPTION_CACHE: {
+    UPDATE_VIDEOS_BY_CHANNEL: 'sync-subscriptions-update-videos-by-channel',
+    UPDATE_LIVE_STREAMS_BY_CHANNEL: 'sync-subscriptions-update-live-streams-by-channel',
+    UPDATE_SHORTS_BY_CHANNEL: 'sync-subscriptions-update-shorts-by-channel',
+    UPDATE_SHORTS_WITH_CHANNEL_PAGE_SHORTS_BY_CHANNEL: 'sync-subscriptions-update-shorts-with-channel-page-shorts-by-channel',
+    UPDATE_COMMUNITY_POSTS_BY_CHANNEL: 'sync-subscriptions-update-community-posts-by-channel',
+  },
 }
 
 // Utils

src/datastores/handlers/base.js

@@ -197,12 +197,92 @@ class Playlists {
   }
 }
 
+class SubscriptionCache {
+  static find() {
+    return db.subscriptionCache.findAsync({})
+  }
+
+  static updateVideosByChannelId({ channelId, entries, timestamp }) {
+    return db.subscriptionCache.updateAsync(
+      { _id: channelId },
+      { $set: { videos: entries, videosTimestamp: timestamp } },
+      { upsert: true }
+    )
+  }
+
+  static updateLiveStreamsByChannelId({ channelId, entries, timestamp }) {
+    return db.subscriptionCache.updateAsync(
+      { _id: channelId },
+      { $set: { liveStreams: entries, liveStreamsTimestamp: timestamp } },
+      { upsert: true }
+    )
+  }
+
+  static updateShortsByChannelId({ channelId, entries, timestamp }) {
+    return db.subscriptionCache.updateAsync(
+      { _id: channelId },
+      { $set: { shorts: entries, shortsTimestamp: timestamp } },
+      { upsert: true }
+    )
+  }
+
+  static updateShortsWithChannelPageShortsByChannelId({ channelId, entries }) {
+    return db.subscriptionCache.findOneAsync({ _id: channelId }, { shorts: 1 }).then((doc) => {
+      if (doc == null) { return }
+
+      const shorts = doc.shorts
+      const cacheShorts = Array.isArray(shorts) ? shorts : []
+
+      cacheShorts.forEach(cachedVideo => {
+        const channelVideo = entries.find(short => cachedVideo.videoId === short.videoId)
+        if (!channelVideo) { return }
+
+        // authorId probably never changes, so we don't need to update that
+        cachedVideo.title = channelVideo.title
+        cachedVideo.author = channelVideo.author
+
+        // as the channel shorts page only has compact view counts for numbers above 1000 e.g. 12k
+        // and the RSS feeds include an exact value, we only want to overwrite it when the number is larger than the cached value
+        // 12345 vs 12000 => 12345
+        // 12345 vs 15000 => 15000
+
+        if (channelVideo.viewCount > cachedVideo.viewCount) {
+          cachedVideo.viewCount = channelVideo.viewCount
+        }
+      })
+
+      return db.subscriptionCache.updateAsync(
+        { _id: channelId },
+        { $set: { shorts: cacheShorts } },
+        { upsert: true }
+      )
+    })
+  }
+
+  static updateCommunityPostsByChannelId({ channelId, entries, timestamp }) {
+    return db.subscriptionCache.updateAsync(
+      { _id: channelId },
+      { $set: { communityPosts: entries, communityPostsTimestamp: timestamp } },
+      { upsert: true }
+    )
+  }
+
+  static deleteMultipleChannels(channelIds) {
+    return db.subscriptionCache.removeAsync({ _id: { $in: channelIds } }, { multi: true })
+  }
+
+  static deleteAll() {
+    return db.subscriptionCache.removeAsync({}, { multi: true })
+  }
+}
+
 function compactAllDatastores() {
   return Promise.allSettled([
     db.settings.compactDatafileAsync(),
     db.history.compactDatafileAsync(),
     db.profiles.compactDatafileAsync(),
     db.playlists.compactDatafileAsync(),
+    db.subscriptionCache.compactDatafileAsync(),
   ])
 }
 
@@ -211,6 +291,7 @@ export {
   History as history,
   Profiles as profiles,
   Playlists as playlists,
+  SubscriptionCache as subscriptionCache,
 
   compactAllDatastores,
 }

src/datastores/handlers/electron.js

@@ -211,9 +211,83 @@ class Playlists {
   }
 }
 
+class SubscriptionCache {
+  static find() {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      { action: DBActions.GENERAL.FIND }
+    )
+  }
+
+  static updateVideosByChannelId({ channelId, entries, timestamp }) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      {
+        action: DBActions.SUBSCRIPTION_CACHE.UPDATE_VIDEOS_BY_CHANNEL,
+        data: { channelId, entries, timestamp },
+      }
+    )
+  }
+
+  static updateLiveStreamsByChannelId({ channelId, entries, timestamp }) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      {
+        action: DBActions.SUBSCRIPTION_CACHE.UPDATE_LIVE_STREAMS_BY_CHANNEL,
+        data: { channelId, entries, timestamp },
+      }
+    )
+  }
+
+  static updateShortsByChannelId({ channelId, entries, timestamp }) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      {
+        action: DBActions.SUBSCRIPTION_CACHE.UPDATE_SHORTS_BY_CHANNEL,
+        data: { channelId, entries, timestamp },
+      }
+    )
+  }
+
+  static updateShortsWithChannelPageShortsByChannelId({ channelId, entries }) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      {
+        action: DBActions.SUBSCRIPTION_CACHE.UPDATE_SHORTS_WITH_CHANNEL_PAGE_SHORTS_BY_CHANNEL,
+        data: { channelId, entries },
+      }
+    )
+  }
+
+  static updateCommunityPostsByChannelId({ channelId, entries, timestamp }) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      {
+        action: DBActions.SUBSCRIPTION_CACHE.UPDATE_COMMUNITY_POSTS_BY_CHANNEL,
+        data: { channelId, entries, timestamp },
+      }
+    )
+  }
+
+  static deleteMultipleChannels(channelIds) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      { action: DBActions.GENERAL.DELETE_MULTIPLE, data: channelIds }
+    )
+  }
+
+  static deleteAll() {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_SUBSCRIPTION_CACHE,
+      { action: DBActions.GENERAL.DELETE_ALL }
+    )
+  }
+}
+
 export {
   Settings as settings,
   History as history,
   Profiles as profiles,
-  Playlists as playlists
+  Playlists as playlists,
+  SubscriptionCache as subscriptionCache,
 }

src/datastores/handlers/index.js

@@ -2,5 +2,6 @@ export {
   settings as DBSettingHandlers,
   history as DBHistoryHandlers,
   profiles as DBProfileHandlers,
-  playlists as DBPlaylistHandlers
+  playlists as DBPlaylistHandlers,
+  subscriptionCache as DBSubscriptionCacheHandlers,
 } from 'DB_HANDLERS_ELECTRON_RENDERER_OR_WEB'

src/datastores/handlers/web.js

@@ -118,9 +118,63 @@ class Playlists {
   }
 }
 
+class SubscriptionCache {
+  static find() {
+    return baseHandlers.subscriptionCache.find()
+  }
+
+  static updateVideosByChannelId({ channelId, entries, timestamp }) {
+    return baseHandlers.subscriptionCache.updateVideosByChannelId({
+      channelId,
+      entries,
+      timestamp,
+    })
+  }
+
+  static updateLiveStreamsByChannelId({ channelId, entries, timestamp }) {
+    return baseHandlers.subscriptionCache.updateLiveStreamsByChannelId({
+      channelId,
+      entries,
+      timestamp,
+    })
+  }
+
+  static updateShortsByChannelId({ channelId, entries, timestamp }) {
+    return baseHandlers.subscriptionCache.updateShortsByChannelId({
+      channelId,
+      entries,
+      timestamp,
+    })
+  }
+
+  static updateShortsWithChannelPageShortsByChannelId({ channelId, entries }) {
+    return baseHandlers.subscriptionCache.updateShortsWithChannelPageShortsByChannelId({
+      channelId,
+      entries,
+    })
+  }
+
+  static updateCommunityPostsByChannelId({ channelId, entries, timestamp }) {
+    return baseHandlers.subscriptionCache.updateCommunityPostsByChannelId({
+      channelId,
+      entries,
+      timestamp,
+    })
+  }
+
+  static deleteMultipleChannels(channelIds) {
+    return baseHandlers.subscriptionCache.deleteMultipleChannels(channelIds)
+  }
+
+  static deleteAll() {
+    return baseHandlers.subscriptionCache.deleteAll()
+  }
+}
+
 export {
   Settings as settings,
   History as history,
   Profiles as profiles,
-  Playlists as playlists
+  Playlists as playlists,
+  SubscriptionCache as subscriptionCache,
 }

src/datastores/index.js

@@ -26,3 +26,4 @@ export const settings = new Datastore({ filename: dbPath('settings'), autoload:
 export const profiles = new Datastore({ filename: dbPath('profiles'), autoload: true })
 export const playlists = new Datastore({ filename: dbPath('playlists'), autoload: true })
 export const history = new Datastore({ filename: dbPath('history'), autoload: true })
+export const subscriptionCache = new Datastore({ filename: dbPath('subscription-cache'), autoload: true })