Advanced searching with elasticsearch 🔥🚀 - PHASE 1 (basic) (#350)

This PR adds
- elasticsearch integration for product search
- updates product search API.
- updates docker build to work with webpack.

.dockerignore

@@ -0,0 +1,11 @@
+.git
+.scripts
+_build
+cover
+erl_crash.dump
+uploads
+.elixir_ls
+.github
+deps
+vendor
+dist

.gitignore

@@ -25,3 +25,6 @@ erl_crash.dump
 
 /env/local.env
 /env/prod.env
+
+/vendor/
+/uploads/images/

apps/snitch_api/lib/snitch_api/products_context/products_context.ex

@@ -3,28 +3,18 @@ defmodule SnitchApi.ProductsContext do
   The JSON-API context.
   """
   alias Snitch.Core.Tools.MultiTenancy.Repo
-  alias Snitch.Data.Schema.{Product, Review, Variation}
+  alias Snitch.Data.Schema.{Product, Review}
+  alias Snitch.Tools.ElasticSearch.ProductSearch
 
-  import Ecto.Query, only: [from: 2, order_by: 2]
-
-  @filter_allowables ~w(taxon_id brand_id)a
-  @partial_search_allowables ~w(name)a
-  @default_filter [state: 1]
+  import Ecto.Query
+  # @filter_allowables ~w(taxon_id brand_id)a
+  # @partial_search_allowables ~w(name)a
 
   @doc """
   List out all the products
   """
   def list_products(conn, params) do
-    # TODO Here we are skipping the products that are child product but
-    # this can be easily handled by product types once it is introduced
-    child_product_ids = from(c in Variation, select: c.child_product_id) |> Repo.all()
-
-    query = define_query(params)
-    query = from(p in query, where: p.id not in ^child_product_ids)
-
-    page = create_page(query, %{}, conn)
-    products = paginate_collection(query, params)
-    {products, page}
+    ProductSearch.run(conn, params)
   end
 
   @doc """
@@ -52,148 +42,4 @@ defmodule SnitchApi.ProductsContext do
         {:ok, product}
     end
   end
-
-  @doc """
-  Creates the page. The page comprises all the related pagination links
-  """
-  def create_page(query, params, conn) do
-    query
-    |> Repo.all()
-    |> gen_page_links(params, conn)
-  end
-
-  @doc """
-  Executes the given `query` applying the page in params.
-  """
-  def paginate_collection(query, params) do
-    {page_number, size} = extract_page_params(params)
-
-    review_query = from(c in Review, limit: 5, preload: [rating_option_vote: :rating_option])
-
-    query
-    |> paginate(page_number, size)
-    |> Repo.all()
-    |> Repo.preload(
-      reviews: review_query,
-      variants: [:images, options: :option_type, theme: [:option_types]],
-      theme: [:option_types],
-      options: :option_type
-    )
-  end
-
-  @doc """
-  Generates the pagination links like `prev` `self` `next` `last` using
-  data-collection and params-page
-  """
-  def gen_page_links(collection, params, conn) do
-    {number, size} = extract_page_params(params)
-
-    JaSerializer.Builder.PaginationLinks.build(
-      %{
-        number: number,
-        size: size,
-        total: div(Enum.count(collection), size),
-        base_url: "http://" <> get_base_url(conn) <> "/api/v1/products"
-      },
-      conn
-    )
-  end
-
-  @doc """
-  This develops the query according to the page parameters.
-  """
-  def paginate(query, page_number, size) do
-    from(
-      query,
-      limit: ^size,
-      offset: ^((page_number - 1) * size)
-    )
-  end
-
-  @doc """
-  This extracts the page parameters and converts to integer
-  """
-  def extract_page_params(params) do
-    case params do
-      %{"page" => page} ->
-        {String.to_integer(page["offset"]), String.to_integer(page["limit"])}
-
-      _ ->
-        {1, Application.get_env(:ja_serializer, :page_size, 2)}
-    end
-  end
-
-  def get_base_url(conn) do
-    case conn.req_headers
-         |> Enum.filter(fn {x, _y} -> x == "host" end)
-         |> Enum.at(0) do
-      {_, base_url} -> base_url
-      _ -> ""
-    end
-  end
-
-  def extend_query(query, keyword_list) do
-    from(q in query, where: ^keyword_list)
-  end
-
-  @doc """
-  Develops the query based on the giver params. At least sorting the
-  products in Ascending orders of their names is considered as priority.
-  This supports the following api calling...
-  - /products
-  - /products/slug
-  - /products?sort=Z-A
-  - /products?sort=A-Z
-  - /products?sort=date
-  - /products?sort=A-Z&filter[name]=Hill's
-  - /products?sort=A-Z&filter[name]=Hill's&page[limit]=2&page[offset]=2
-  - /products?sort=A-Z&filter[name]=Hill's&page[limit]=2
-  """
-
-  def define_query(params) do
-    query =
-      case params["sort"] do
-        "Z-A" ->
-          order_by(Product, desc: :name)
-
-        "date" ->
-          order_by(Product, desc: :inserted_at)
-
-        _ ->
-          order_by(Product, asc: :name)
-      end
-
-    filter_query(query, params["filter"], @filter_allowables)
-    |> like_query(params["filter"], @partial_search_allowables)
-  end
-
-  defp filter_query(query, nil, _allowables), do: extend_query(query, @default_filter)
-
-  defp filter_query(query, filter_params, allowables) do
-    filter_params = @default_filter ++ make_filter_params_list(filter_params, allowables)
-
-    extend_query(query, filter_params)
-  end
-
-  defp like_query(query, nil, _allowables), do: query
-
-  defp like_query(query, filter_params, allowables) do
-    filter_params = make_filter_params_list(filter_params, allowables)
-
-    Enum.reduce(filter_params, query, fn {key, value}, nquery ->
-      from(q in nquery, where: ilike(fragment("CAST(? AS TEXT)", field(q, ^key)), ^"%#{value}%"))
-    end)
-  end
-
-  defp get_value("true"), do: true
-
-  defp get_value("false"), do: false
-
-  defp get_value(value), do: value
-
-  defp make_filter_params_list(filter_params, allowables) do
-    filter_params
-    |> Enum.into([], fn x -> {String.to_atom(elem(x, 0)), get_value(elem(x, 1))} end)
-    |> Enum.reject(fn x -> elem(x, 0) not in allowables end)
-  end
 end

apps/snitch_api/lib/snitch_api_web/controllers/product_controller.ex

@@ -4,6 +4,7 @@ defmodule SnitchApiWeb.ProductController do
   alias Snitch.Data.Model.{Product, ProductReview}
   alias Snitch.Core.Tools.MultiTenancy.Repo
   alias SnitchApi.ProductsContext, as: Context
+  alias SnitchApiWeb.Elasticsearch.ProductView, as: ESPView
 
   plug(SnitchApiWeb.Plug.DataToAttributes)
   action_fallback(SnitchApiWeb.FallbackController)
@@ -37,13 +38,14 @@ defmodule SnitchApiWeb.ProductController do
   variants.options,variants.options.option_type,options,options.option_type,
   theme,theme.option_types,reviews.rating_option_vote.rating_option)
   def index(conn, params) do
-    {products, page} = Context.list_products(conn, params)
+    {products, page, aggregations, total} = Context.list_products(conn, params)
 
-    render(
+    json(
       conn,
-      "index.json-api",
-      data: products,
-      opts: [page: page]
+      JaSerializer.format(ESPView, products, conn,
+        page: page,
+        meta: %{aggregations: aggregations, total: total}
+      )
     )
   end
 

apps/snitch_api/lib/snitch_api_web/router.ex

@@ -1,7 +1,7 @@
 defmodule SnitchApiWeb.Router do
   use SnitchApiWeb, :router
   use Plug.ErrorHandler
-  use Sentry.Plug
+  # use Sentry.Plug
 
   alias SnitchApiWeb.Guardian
 
@@ -21,6 +21,7 @@ defmodule SnitchApiWeb.Router do
     post("/login", UserController, :login)
     post("/orders/blank", OrderController, :guest_order)
     get("/variants/favorites", VariantController, :favorite_variants)
+    post("/products", ProductController, :index)
     resources("/products", ProductController, except: [:new, :edit], param: "product_slug")
     get("/orders/:order_number", OrderController, :fetch_guest_order)
     post("/hosted-payment/:payment_source/success", HostedPaymentController, :payment_success)

apps/snitch_api/lib/snitch_api_web/views/elasticsearch/product_view.ex

@@ -0,0 +1,46 @@
+defmodule SnitchApiWeb.Elasticsearch.ProductView do
+  use SnitchApiWeb, :view
+  use JaSerializer.PhoenixView
+
+  attributes([
+    :id,
+    # :parent_id,
+    # :has_parent,
+    :slug,
+    :name,
+    :updated_at,
+    :images,
+    :rating_summary,
+    :selling_price,
+    :max_retail_price,
+    :brand
+  ])
+
+  defp source(product), do: product["_source"]
+
+
+  defp id(product) do
+    [_, id] = String.split(product["_id"], "_")
+    String.to_integer(id)
+  end
+
+  # defp parent_id(product), do: source(product)["parent_id"]
+
+  # defp has_parent(product), do: parent_id(product) == id(product)
+
+  defp slug(product), do: source(product)["slug"]
+
+  defp name(product), do: source(product)["name"]
+
+  defp updated_at(product), do: source(product)["updated_at"]
+
+  defp images(product), do: source(product)["images"]
+
+  defp rating_summary(product), do: source(product)["rating_summary"]
+
+  defp selling_price(product), do: source(product)["selling_price"]
+
+  defp max_retail_price(product), do: source(product)["max_retail_price"]
+
+  defp brand(product), do: source(product)["brand"]
+end

apps/snitch_api/lib/snitch_api_web/views/product_view.ex

@@ -33,6 +33,19 @@ defmodule SnitchApiWeb.ProductView do
     :display_max_retail_price
   ])
 
+  def name(product), do: append_option_value_in_name(product)
+
+  defp append_option_value_in_name(%{options: [], name: name}), do: name
+
+  defp append_option_value_in_name(%{options: options, name: name}) do
+    postfix =
+      options
+      |> Enum.map(&String.capitalize(&1.value))
+      |> Enum.join(",")
+
+    name <> " (" <> postfix <> ")"
+  end
+
   def selling_price(product) do
     Money.round(product.selling_price, currency_digits: :cash)
   end

apps/snitch_core/.gitignore

@@ -15,3 +15,4 @@ erl_crash.dump
 # secrets files as long as you replace their contents by environment
 # variables.
 /config/*.secret.exs
+/uploads/

apps/snitch_core/config/config.exs

@@ -32,3 +32,5 @@ config :triplex,
   migrations_path: "migrations"
 
 import_config "#{Mix.env()}.exs"
+
+import_config("elasticsearch.exs")

apps/snitch_core/config/elasticsearch.exs

@@ -0,0 +1,56 @@
+use Mix.Config
+
+config :snitch_core, Snitch.Tools.ElasticsearchCluster,
+  # The URL where Elasticsearch is hosted on your system
+  url:  System.get_env("ELASTIC_HOST"),
+
+  # If your Elasticsearch cluster uses HTTP basic authentication,
+  # specify the username and password here:
+  # username: "username",
+  # password: "password",
+
+  # If you want to mock the responses of the Elasticsearch JSON API
+  # for testing or other purposes, you can inject a different module
+  # here. It must implement the Elasticsearch.API behaviour.
+  api: Elasticsearch.API.HTTP,
+
+  # Customize the library used for JSON encoding/decoding.
+  # or Jason
+  json_library: Poison,
+
+  # You should configure each index which you maintain in Elasticsearch here.
+  # This configuration will be read by the `mix elasticsearch.build` task,
+  # described below.
+  indexes: %{
+    # This is the base name of the Elasticsearch index. Each index will be
+    # built with a timestamp included in the name, like "products-5902341238".
+    # It will then be aliased to "products" for easy querying.
+    products: %{
+      # This file describes the mappings and settings for your index. It will
+      # be posted as-is to Elasticsearch when you create your index, and
+      # therefore allows all the settings you could post directly.
+      settings: "priv/elasticsearch/products.json",
+
+      # This store module must implement a store behaviour. It will be used to
+      # fetch data for each source in each indexes' `sources` list, below:
+      store: Snitch.Tools.ElasticSearch.ProductStore,
+
+      # This is the list of data sources that should be used to populate this
+      # index. The `:store` module above will be passed each one of these
+      # sources for fetching.
+      #
+      # Each piece of data that is returned by the store must implement the
+      # Elasticsearch.Document protocol.
+      sources: [Snitch.Data.Schema.Product],
+
+      # When indexing data using the `mix elasticsearch.build` task,
+      # control the data ingestion rate by raising or lowering the number
+      # of items to send in each bulk request.
+      bulk_page_size: 5000,
+
+      # Likewise, wait a given period between posting pages to give
+      # Elasticsearch time to catch up.
+      # 15 seconds
+      bulk_wait_interval: 15_000
+    }
+  }

apps/snitch_core/lib/core/application.ex

@@ -12,7 +12,9 @@ defmodule Snitch.Application do
     {:ok, pid} =
       Supervisor.start_link(
         [
-          supervisor(Snitch.Repo, [])
+          supervisor(Snitch.Repo, []),
+          supervisor(Snitch.Tools.ElasticsearchCluster, []),
+          worker(Cachex, [:snitch_cache, []])
         ],
         strategy: :one_for_one,
         name: Snitch.Supervisor