Add prefix search functionality to the search store (#632)

* Add prefix search functionality to the search store

***Description:*** This commit adds the `prefix` function to the `Lexical.RemoteControl.Search.Store` module, allowing for searching entries by prefix. It also implements the `find_by_prefix` callback in the `Lexical.RemoteControl.Search.Store.Backends.Ets` module.

***Changes:***

- Added `prefix` function to `Lexical.RemoteControl.Search.Store`
- Implemented `find_by_prefix` callback in `Lexical.RemoteControl.Search.Store.Backends.Ets`

* Revert `to_subject_charlist` to `to_subject`

* Add a comment for `to_prefix`

* Refine the implementation logic for prefix search.

Utilize List.pop_at directly.

* Revert the `Schemas.V2` changes, use V3 to do the force migiration

* Update apps/remote_control/lib/lexical/remote_control/search/store/backends/ets/state.ex

Co-authored-by: Steve Cohen <[email protected]>

---------

Co-authored-by: Steve Cohen <[email protected]>

apps/remote_control/lib/lexical/remote_control/search/store.ex

@@ -56,6 +56,10 @@ defmodule Lexical.RemoteControl.Search.Store do
     GenServer.call(__MODULE__, {:exact, subject, constraints})
   end
 
+  def prefix(prefix, constraints) do
+    GenServer.call(__MODULE__, {:prefix, prefix, constraints})
+  end
+
   def parent(%Entry{} = entry) do
     GenServer.call(__MODULE__, {:parent, entry})
   end
@@ -184,6 +188,10 @@ defmodule Lexical.RemoteControl.Search.Store do
     {:reply, State.exact(state, subject, constraints), {ref, state}}
   end
 
+  def handle_call({:prefix, prefix, constraints}, _from, {ref, %State{} = state}) do
+    {:reply, State.prefix(state, prefix, constraints), {ref, state}}
+  end
+
   def handle_call({:fuzzy, subject, constraints}, _from, {ref, %State{} = state}) do
     {:reply, State.fuzzy(state, subject, constraints), {ref, state}}
   end

apps/remote_control/lib/lexical/remote_control/search/store/backend.ex

@@ -83,6 +83,11 @@ defmodule Lexical.RemoteControl.Search.Store.Backend do
   """
   @callback find_by_subject(subject_query(), type_query(), subtype_query()) :: [Entry.t()]
 
+  @doc """
+  Finds all entries by prefix
+  """
+  @callback find_by_prefix(subject_query(), type_query(), subtype_query()) :: [Entry.t()]
+
   @doc """
   Finds entries whose ref attribute is in the given list
   """

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets.ex

@@ -64,6 +64,11 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets do
     GenServer.call(genserver_name(), {:find_by_subject, [subject, type, subtype]})
   end
 
+  @impl Backend
+  def find_by_prefix(prefix, type, subtype) do
+    GenServer.call(genserver_name(), {:find_by_prefix, [prefix, type, subtype]})
+  end
+
   @impl Backend
   def find_by_ids(ids, type, subtype) do
     GenServer.call(genserver_name(), {:find_by_ids, [ids, type, subtype]})

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets/schemas/v3.ex

@@ -0,0 +1,78 @@
+defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas.V3 do
+  alias Lexical.RemoteControl.Search.Indexer.Entry
+  alias Lexical.RemoteControl.Search.Store.Backends.Ets.Schema
+
+  require Entry
+  use Schema, version: 3
+
+  defkey :by_id, [:id, :type, :subtype]
+
+  defkey :by_subject, [
+    :subject,
+    :type,
+    :subtype,
+    :path
+  ]
+
+  defkey :by_path, [:path]
+  defkey :by_block_id, [:block_id, :path]
+  defkey :structure, [:path]
+
+  def migrate(entries) do
+    migrated =
+      entries
+      |> Stream.filter(fn
+        {query_by_subject(), %Entry{}} -> true
+        _ -> false
+      end)
+      |> Stream.map(fn {_, entry} -> entry end)
+      |> Schema.entries_to_rows(__MODULE__)
+
+    {:ok, migrated}
+  end
+
+  def to_rows(%Entry{} = entry) when Entry.is_structure(entry) do
+    structure_key = structure(path: entry.path)
+    [{structure_key, entry.subject}]
+  end
+
+  def to_rows(%Entry{} = entry) do
+    subject_key =
+      by_subject(
+        subject: to_subject(entry.subject),
+        type: entry.type,
+        subtype: entry.subtype,
+        path: entry.path
+      )
+
+    id_key =
+      by_id(
+        id: entry.id,
+        type: entry.type,
+        subtype: entry.subtype
+      )
+
+    path_key = by_path(path: entry.path)
+    block_key = by_block_id(path: entry.path, block_id: entry.block_id)
+
+    [{id_key, entry}, {subject_key, id_key}, {path_key, id_key}, {block_key, id_key}]
+  end
+
+  # This case will handle any namespaced entries
+  def to_rows(%{type: _, subtype: _, id: _} = entry) do
+    map = Map.delete(entry, :__struct__)
+
+    Entry
+    |> struct(map)
+    |> to_rows()
+  end
+
+  def table_options do
+    [:named_table, :ordered_set, :compressed]
+  end
+
+  def to_subject(charlist) when is_list(charlist), do: charlist
+  def to_subject(:_), do: :_
+  def to_subject(atom) when is_atom(atom), do: atom |> inspect() |> to_charlist()
+  def to_subject(other), do: to_charlist(other)
+end

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets/state.ex

@@ -14,20 +14,22 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.State do
   @schema_order [
     Schemas.LegacyV0,
     Schemas.V1,
-    Schemas.V2
+    Schemas.V2,
+    Schemas.V3
   ]
 
   import Wal, only: :macros
   import Entry, only: :macros
 
-  import Schemas.V2,
+  import Schemas.V3,
     only: [
       by_block_id: 1,
       query_by_id: 1,
       query_by_path: 1,
       query_structure: 1,
       query_by_subject: 1,
-      structure: 1
+      structure: 1,
+      to_subject: 1
     ]
 
   defstruct [:project, :table_name, :leader?, :leader_pid, :wal_state]
@@ -97,6 +99,31 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.State do
     |> Enum.flat_map(&:ets.lookup_element(state.table_name, &1, 2))
   end
 
+  def find_by_prefix(%__MODULE__{} = state, subject, type, subtype) do
+    match_pattern =
+      query_by_subject(
+        subject: to_prefix(subject),
+        type: type,
+        subtype: subtype
+      )
+
+    state.table_name
+    |> :ets.select([{{match_pattern, :_}, [], [:"$_"]}])
+    |> Stream.flat_map(fn {_, id_keys} -> id_keys end)
+    |> Stream.uniq()
+    |> Enum.flat_map(&:ets.lookup_element(state.table_name, &1, 2))
+  end
+
+  @dialyzer {:nowarn_function, to_prefix: 1}
+
+  defp to_prefix(prefix) when is_binary(prefix) do
+    # what we really want to do here is convert the prefix to a improper list
+    # like this: `'abc' -> [97, 98, 99 | :_]`, it's different from `'abc' ++ [:_]`
+    # this is the required format for the `:ets.select` function.
+    {last_char, others} = prefix |> String.to_charlist() |> List.pop_at(-1)
+    others ++ [last_char | :_]
+  end
+
   def siblings(%__MODULE__{} = state, %Entry{} = entry) do
     key = by_block_id(block_id: entry.block_id, path: entry.path)
 
@@ -264,11 +291,6 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.State do
     {query_by_id(id: id, type: type, subtype: subtype), :_}
   end
 
-  defp to_subject(binary) when is_binary(binary), do: binary
-  defp to_subject(:_), do: :_
-  defp to_subject(atom) when is_atom(atom), do: inspect(atom)
-  defp to_subject(other), do: to_string(other)
-
   defp current_schema do
     List.last(@schema_order)
   end

apps/remote_control/lib/lexical/remote_control/search/store/state.ex

@@ -88,6 +88,12 @@ defmodule Lexical.RemoteControl.Search.Store.State do
     state.backend.find_by_subject(subject, type, subtype)
   end
 
+  def prefix(%__MODULE__{} = state, prefix, constraints) do
+    type = Keyword.get(constraints, :type, :_)
+    subtype = Keyword.get(constraints, :subtype, :_)
+    state.backend.find_by_prefix(prefix, type, subtype)
+  end
+
   def fuzzy(%__MODULE__{} = state, subject, constraints) do
     case Fuzzy.match(state.fuzzy, subject) do
       [] ->

apps/remote_control/test/lexical/remote_control/search/store_test.exs

@@ -97,6 +97,23 @@ defmodule Lexical.RemoteControl.Search.StoreTest do
         assert entry.id == 1
       end
 
+      test "matching prefix tokens should work" do
+        Store.replace([
+          definition(id: 1, subject: Foo.Bar),
+          definition(id: 2, subject: Foo.Baa.Baa),
+          definition(id: 3, subject: Foo.Bar.Baz)
+        ])
+
+        assert [entry1, entry3] =
+                 Store.prefix("Foo.Bar", type: :module, subtype: :definition)
+
+        assert entry1.subject == Foo.Bar
+        assert entry3.subject == Foo.Bar.Baz
+
+        assert entry1.id == 1
+        assert entry3.id == 3
+      end
+
       test "matching fuzzy tokens works" do
         Store.replace([
           definition(id: 1, subject: Foo.Bar.Baz),