improve tests

config/.env.dev

@@ -28,4 +28,3 @@ S3_REGION=us-east-1
 S3_ENDPOINT=http://localhost:10000
 S3_EXPORTS_BUCKET=dev-exports
 S3_IMPORTS_BUCKET=dev-imports
-S3_CLICKHOUSE_HOST=172.17.0.2

config/runtime.exs

@@ -786,6 +786,5 @@ unless s3_disabled? do
 
   config :plausible, Plausible.S3,
     exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET"),
-    imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET"),
-    clickhouse_host: get_var_from_path_or_env(config_dir, "S3_CLICKHOUSE_HOST")
+    imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET")
 end

lib/plausible/imported/csv_importer.ex

@@ -100,10 +100,13 @@ defmodule Plausible.Imported.CSVImporter do
 
       iex> date_range([
       ...>   %{"filename" => "imported_devices_20190101_20210101.csv"},
-      ...>   %{"filename" => "imported_pages_20200101_20220101.csv"}
+      ...>   "imported_pages_20200101_20220101.csv"
       ...> ])
       Date.range(~D[2019-01-01], ~D[2022-01-01])
 
+      iex> date_range([])
+      ** (ArgumentError) empty uploads
+
   """
   @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()
   def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)

lib/plausible/s3.ex

@@ -43,16 +43,12 @@ defmodule Plausible.S3 do
 
   Example:
 
-      import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv")
-      %{
-        filename: "imported_browsers.csv",
-        s3_url: "http://localhost:9000/imports/123/imported_browsers.csv",
-        presigned_url: "http://localhost:9000/imports/123/imported_browsers.csv?X-Aws-Signature=XXX&..."
-      }
+      iex> %{
+      ...>   s3_url:  "http://localhost:10000/test-imports/123/imported_browsers.csv",
+      ...>   presigned_url: "http://localhost:10000/test-imports/123/imported_browsers.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin" <> _
+      ...> } = import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv")
 
   """
-  @spec import_presign_upload(pos_integer, String.t()) ::
-          %{s3_url: :uri_string.uri_string(), pregisned_url: :uri_string.uri_string()}
   def import_presign_upload(site_id, filename) do
     config = ExAws.Config.new(:s3)
     s3_path = Path.join(to_string(site_id), filename)
@@ -61,12 +57,21 @@ defmodule Plausible.S3 do
     %{s3_url: extract_s3_url(presigned_url), presigned_url: presigned_url}
   end
 
-  defp extract_s3_url(presigned_url) do
-    [s3_url, _] = String.split(presigned_url, "?")
-
-    if ch_host = Keyword.get(config(), :clickhouse_host) do
-      URI.to_string(%URI{URI.parse(s3_url) | host: ch_host})
-    else
+  # to make ClickHouse see MinIO in dev and test envs we replace
+  # the host in the S3 URL with whatever's set in S3_CLICKHOUSE_HOST env var
+  if Mix.env() in [:dev, :test, :small_dev, :small_test] do
+    defp extract_s3_url(presigned_url) do
+      [s3_url, _] = String.split(presigned_url, "?")
+
+      if ch_host = System.get_env("S3_CLICKHOUSE_HOST") do
+        URI.to_string(%URI{URI.parse(s3_url) | host: ch_host})
+      else
+        s3_url
+      end
+    end
+  else
+    defp extract_s3_url(presigned_url) do
+      [s3_url, _] = String.split(presigned_url, "?")
       s3_url
     end
   end
@@ -127,6 +132,12 @@ defmodule Plausible.S3 do
 
   @doc """
   Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3.
+
+  Example:
+
+      iex> import_clickhouse_credentials()
+      %{access_key_id: "minioadmin", secret_access_key: "minioadmin"}
+
   """
   @spec import_clickhouse_credentials ::
           %{access_key_id: String.t(), secret_access_key: String.t()}

test/plausible/config_test.exs

@@ -273,8 +273,7 @@ defmodule Plausible.ConfigTest do
 
       assert get_in(runtime_config(env), [:plausible, Plausible.S3]) == [
                exports_bucket: "my-exports",
-               imports_bucket: "my-imports",
-               clickhouse_host: nil
+               imports_bucket: "my-imports"
              ]
     end
   end

test/plausible/imported/csv_importer_test.exs

@@ -538,7 +538,13 @@ defmodule Plausible.Imported.CSVImporterTest do
   end
 
   defp minio_url(minio, bucket, key) do
-    port = minio |> MinioContainer.connection_opts() |> Keyword.fetch!(:port)
-    Path.join(["http://172.17.0.1:#{port}", bucket, key])
+    arch = to_string(:erlang.system_info(:system_architecture))
+
+    if String.contains?(arch, "darwin") do
+      Path.join(["http://#{minio.ip_address}:9000", bucket, key])
+    else
+      port = minio |> MinioContainer.connection_opts() |> Keyword.fetch!(:port)
+      Path.join(["http://172.17.0.1:#{port}", bucket, key])
+    end
   end
 end