From a06c7ff85b0127085d2d6a3c2603c810152f93ca Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Sun, 18 Feb 2024 23:03:41 +0800 Subject: [PATCH 01/17] add csv importer --- config/.env.dev | 8 +++ config/.env.test | 8 +++ config/runtime.exs | 66 +++++++++++++++++++++++ lib/plausible/imported/csv_importer.ex | 75 ++++++++++++++++++++++++-- lib/plausible/s3.ex | 31 +++++++++++ lib/plausible/s3/client.ex | 17 ++++++ mix.exs | 5 +- mix.lock | 3 ++ 8 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 lib/plausible/s3.ex create mode 100644 lib/plausible/s3/client.ex diff --git a/config/.env.dev b/config/.env.dev index db52322d91f6..5641f20b00e2 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -20,3 +20,11 @@ GOOGLE_CLIENT_SECRET=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 PROMEX_DISABLED=false SITE_DEFAULT_INGEST_THRESHOLD=1000000 + +S3_DISABLED=false +S3_ACCESS_KEY_ID=minioadmin +S3_SECRET_ACCESS_KEY=minioadmin +S3_REGION=us-east-1 +S3_ENDPOINT=http://localhost:9000 +S3_IMPORTS_BUCKET=imports +S3_HOST_FOR_CLICKHOUSE=172.19.0.4 diff --git a/config/.env.test b/config/.env.test index 050d6f879965..2ebba9cc21ac 100644 --- a/config/.env.test +++ b/config/.env.test @@ -15,3 +15,11 @@ IP_GEOLOCATION_DB=test/priv/GeoLite2-City-Test.mmdb SITE_DEFAULT_INGEST_THRESHOLD=1000000 GOOGLE_CLIENT_ID=fake_client_id GOOGLE_CLIENT_SECRET=fake_client_secret + +S3_DISABLED=false +S3_ACCESS_KEY_ID=minioadmin +S3_SECRET_ACCESS_KEY=minioadmin +S3_REGION=us-east-1 +S3_ENDPOINT=http://localhost:9000 +S3_IMPORTS_BUCKET=imports +S3_HOST_FOR_CLICKHOUSE=172.19.0.4 diff --git a/config/runtime.exs b/config/runtime.exs index b771a9e223c2..e14dfeb1b3a1 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -704,3 +704,69 @@ if not is_selfhost do config :plausible, Plausible.Site, default_ingest_threshold: site_default_ingest_threshold end + +s3_disabled? = + config_dir + |> get_var_from_path_or_env("S3_DISABLED", "true") + |> String.to_existing_atom() + +unless s3_disabled? do + s3_env = [ + %{ + name: "S3_ACCESS_KEY_ID", + example: "AKIAIOSFODNN7EXAMPLE" + }, + %{ + name: "S3_SECRET_ACCESS_KEY", + example: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + }, + %{ + name: "S3_REGION", + example: "us-east-1" + }, + %{ + name: "S3_ENDPOINT", + example: "https://.r2.cloudflarestorage.com" + }, + %{ + name: "S3_IMPORTS_BUCKET", + example: "my-imports-bucket" + } + ] + + s3_env = + Enum.map(s3_env, fn var -> + Map.put(var, :value, get_var_from_path_or_env(config_dir, var.name)) + end) + + s3_missing_env = Enum.filter(s3_env, &is_nil(&1.value)) + + unless s3_missing_env == [] do + raise """ + Missing S3 configuration. Please set #{s3_missing_env |> Enum.map(& &1.name) |> Enum.join(", ")} environment variable(s): + + #{s3_missing_env |> Enum.map(fn %{name: name, example: example} -> "\t#{name}=#{example}" end) |> Enum.join("\n")} + """ + end + + s3_env_value = fn name -> + s3_env |> Enum.find(&(&1.name == name)) |> Map.fetch!(:value) + end + + config :ex_aws, + http_client: Plausible.S3.Client, + access_key_id: s3_env_value.("S3_ACCESS_KEY_ID"), + secret_access_key: s3_env_value.("S3_SECRET_ACCESS_KEY"), + region: s3_env_value.("S3_REGION") + + %URI{scheme: s3_scheme, host: s3_host, port: s3_port} = URI.parse(s3_env_value.("S3_ENDPOINT")) + + config :ex_aws, :s3, + scheme: s3_scheme <> "://", + host: s3_host, + port: s3_port + + config :plausible, Plausible.S3, + imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET"), + host_for_clickhouse: get_var_from_path_or_env(config_dir, "S3_HOST_FOR_CLICKHOUSE") +end diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 8c7844fe304b..208c0d7dfcba 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -1,6 +1,6 @@ defmodule Plausible.Imported.CSVImporter do @moduledoc """ - CSV importer stub. + CSV importer from S3. """ use Plausible.Imported.Importer @@ -16,10 +16,77 @@ defmodule Plausible.Imported.CSVImporter do def email_template(), do: "google_analytics_import.html" @impl true - def parse_args(%{"s3_path" => s3_path}), do: [s3_path: s3_path] + def parse_args(%{"uploads" => uploads}), do: [uploads: uploads] @impl true - def import_data(_site_import, _opts) do - :ok + def import_data(site_import, opts) do + %{id: import_id, site_id: site_id} = site_import + uploads = Keyword.fetch!(opts, :uploads) + + %{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} = + Plausible.S3.import_clickhouse_credentials() + + {:ok, ch} = + Plausible.IngestRepo.config() + |> Keyword.replace!(:pool_size, 1) + |> Ch.start_link() + + Enum.each(uploads, fn upload -> + %{"filename" => filename, "s3_path" => s3_path} = upload + + ".csv" = Path.extname(filename) + table = Path.rootname(filename) + + s3_structure = input_structure(table) + s3_url = Plausible.S3.import_clickhouse_url(s3_path) + + statement = + """ + INSERT INTO {table:Identifier} \ + SELECT {site_id:UInt64} AS site_id, {import_id:UInt64} AS import_id, * \ + FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\ + """ + + params = %{ + "table" => table, + "site_id" => site_id, + "import_id" => import_id, + "s3_url" => s3_url, + "s3_access_key_id" => s3_access_key_id, + "s3_secret_access_key" => s3_secret_access_key, + "s3_format" => "CSVWithNames", + "s3_structure" => s3_structure + } + + Ch.query!(ch, statement, params, timeout: :infinity) + end) + end + + input_structures = %{ + "imported_browsers" => + "date Date, browser String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", + "imported_devices" => + "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", + "imported_entry_pages" => + "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt64", + "imported_exit_pages" => "date Date, exit_page String, visitors UInt64, exits UInt64", + "imported_locations" => + "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", + "imported_operating_systems" => + "date Date, operating_system String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", + "imported_pages" => + "date Date, hostname String, page String, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64", + "imported_sources" => + "date Date, source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", + "imported_visitors" => + "date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64" + } + + for {table, input_structure} <- input_structures do + defp input_structure(unquote(table)), do: unquote(input_structure) + end + + defp input_structure(table) do + raise ArgumentError, "table #{table} is not supported for data import" end end diff --git a/lib/plausible/s3.ex b/lib/plausible/s3.ex new file mode 100644 index 000000000000..ecfaa33df2a2 --- /dev/null +++ b/lib/plausible/s3.ex @@ -0,0 +1,31 @@ +defmodule Plausible.S3 do + @moduledoc """ + Helper functions for S3 exports/imports. + """ + + defp config, do: Application.fetch_env!(:plausible, __MODULE__) + defp config(key), do: Keyword.fetch!(config(), key) + + @doc """ + Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3. + """ + @spec import_clickhouse_credentials :: + %{access_key_id: String.t(), secret_access_key: String.t()} + def import_clickhouse_credentials do + %{access_key_id: access_key_id, secret_access_key: secret_access_key} = ExAws.Config.new(:s3) + %{access_key_id: access_key_id, secret_access_key: secret_access_key} + end + + @doc """ + Returns S3 URL for an object to be used by ClickHouse during imports from S3. + + In the current implementation the bucket always goes into the path component. + """ + @spec import_clickhouse_url(Path.t()) :: :uri_string.uri_string() + def import_clickhouse_url(s3_path) do + %{scheme: scheme, host: host} = config = ExAws.Config.new(:s3) + host = Keyword.get(config(), :host_for_clickhouse) || host + port = ExAws.S3.Utils.sanitized_port_component(config) + Path.join(["#{scheme}#{host}#{port}", config(:imports_bucket), s3_path]) + end +end diff --git a/lib/plausible/s3/client.ex b/lib/plausible/s3/client.ex new file mode 100644 index 000000000000..54817c8da41c --- /dev/null +++ b/lib/plausible/s3/client.ex @@ -0,0 +1,17 @@ +defmodule Plausible.S3.Client do + @moduledoc false + @behaviour ExAws.Request.HttpClient + + @impl true + def request(method, url, body, headers, opts) do + req = Finch.build(method, url, headers, body) + + case Finch.request(req, Plausible.Finch, opts) do + {:ok, %Finch.Response{status: status, headers: headers, body: body}} -> + {:ok, %{status_code: status, headers: headers, body: body}} + + {:error, reason} -> + {:error, %{reason: reason}} + end + end +end diff --git a/mix.exs b/mix.exs index 4b253148c854..38f7967dcfd8 100644 --- a/mix.exs +++ b/mix.exs @@ -135,7 +135,10 @@ defmodule Plausible.MixProject do {:esbuild, "~> 0.7", runtime: Mix.env() in [:dev, :small_dev]}, {:tailwind, "~> 0.2.0", runtime: Mix.env() in [:dev, :small_dev]}, {:ex_json_logger, "~> 1.4.0"}, - {:ecto_network, "~> 1.5.0"} + {:ecto_network, "~> 1.5.0"}, + {:ex_aws, "~> 2.5"}, + {:ex_aws_s3, "~> 2.5"}, + {:sweet_xml, "~> 0.7.4"} ] end diff --git a/mix.lock b/mix.lock index 8a108e96e2c6..f10c50458123 100644 --- a/mix.lock +++ b/mix.lock @@ -41,6 +41,8 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, + "ex_aws": {:hex, :ex_aws, "2.5.1", "7418917974ea42e9e84b25e88b9f3d21a861d5f953ad453e212f48e593d8d39f", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1b95431f70c446fa1871f0eb9b183043c5a625f75f9948a42d25f43ae2eff12b"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"}, "ex_cldr": {:hex, :ex_cldr, "2.37.5", "9da6d97334035b961d2c2de167dc6af8cd3e09859301a5b8f49f90bd8b034593", [:mix], [{:cldr_utils, "~> 2.21", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.19", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: true]}], "hexpm", "74ad5ddff791112ce4156382e171a5f5d3766af9d5c4675e0571f081fe136479"}, "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.15.1", "e92ba17c41e7405b7784e0e65f406b5f17cfe313e0e70de9befd653e12854822", [:mix], [{:ex_cldr, "~> 2.34", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "31df8bd37688340f8819bdd770eb17d659652078d34db632b85d4a32864d6a25"}, "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.32.3", "b631ff94c982ec518e46bf4736000a30a33d6b58facc085d5f240305f512ad4a", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 0.3 or ~> 1.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.37", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, ">= 2.14.2", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "7b626ff1e59a0ec9c3c5db5ce9ca91a6995e2ab56426b71f3cbf67181ea225f5"}, @@ -131,6 +133,7 @@ "siphash": {:hex, :siphash, "3.2.0", "ec03fd4066259218c85e2a4b8eec4bb9663bc02b127ea8a0836db376ba73f2ed", [:make, :mix], [], "hexpm", "ba3810701c6e95637a745e186e8a4899087c3b079ba88fb8f33df054c3b0b7c3"}, "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"}, "tailwind": {:hex, :tailwind, "0.2.2", "9e27288b568ede1d88517e8c61259bc214a12d7eed271e102db4c93fcca9b2cd", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "ccfb5025179ea307f7f899d1bb3905cd0ac9f687ed77feebc8f67bdca78565c4"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, From d7ebb18145a2e3e01766437cd4e64be3e3eab7fd Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:30:28 +0800 Subject: [PATCH 02/17] make table validation explicit --- lib/plausible/imported/csv_importer.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 208c0d7dfcba..4656c3c58f94 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -36,8 +36,9 @@ defmodule Plausible.Imported.CSVImporter do ".csv" = Path.extname(filename) table = Path.rootname(filename) + ensure_importable_table!(table) - s3_structure = input_structure(table) + s3_structure = input_structure!(table) s3_url = Plausible.S3.import_clickhouse_url(s3_path) statement = @@ -83,10 +84,11 @@ defmodule Plausible.Imported.CSVImporter do } for {table, input_structure} <- input_structures do - defp input_structure(unquote(table)), do: unquote(input_structure) + defp input_structure!(unquote(table)), do: unquote(input_structure) + defp ensure_importable_table!(unquote(table)), do: :ok end - defp input_structure(table) do + defp ensure_importable_table!(table) do raise ArgumentError, "table #{table} is not supported for data import" end end From 2925e89d51652b26d53e2229e633c1d45a69e1da Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:32:28 +0800 Subject: [PATCH 03/17] update some docs --- lib/plausible/imported.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex index 82623fbd2d10..f1417802881c 100644 --- a/lib/plausible/imported.ex +++ b/lib/plausible/imported.ex @@ -7,8 +7,7 @@ defmodule Plausible.Imported do * `Plausible.Imported.UniversalAnalytics` - existing mechanism, for legacy Google analytics formerly known as "Google Analytics" * `Plausible.Imported.NoopImporter` - importer stub, used mainly for testing purposes - * `Plausible.Imported.CSVImporter` - a placeholder stub for CSV importer that will - be added soon + * `Plausible.Imported.CSVImporter` - CSV importer from S3 For more information on implementing importers, see `Plausible.Imported.Importer`. """ From 54de24851876872f0dadf88a447af6438c93f87a Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:43:31 +0800 Subject: [PATCH 04/17] improve docs --- lib/plausible/s3.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/plausible/s3.ex b/lib/plausible/s3.ex index ecfaa33df2a2..dfefc896f2a7 100644 --- a/lib/plausible/s3.ex +++ b/lib/plausible/s3.ex @@ -19,7 +19,12 @@ defmodule Plausible.S3 do @doc """ Returns S3 URL for an object to be used by ClickHouse during imports from S3. - In the current implementation the bucket always goes into the path component. + In the current implementation the bucket goes into the path component: + + ${S3_ENDPOINT}/${S3_IMPORTS_BUCKET}/${S3_PATH} + + https://s3.us-east-1.amazonaws.com/my-plausible-imports/1/imported_browsers.csv + """ @spec import_clickhouse_url(Path.t()) :: :uri_string.uri_string() def import_clickhouse_url(s3_path) do From 30f3ca63bc0ae7a4947fda1ed61c5ce3450a3365 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 19 Feb 2024 18:01:42 +0800 Subject: [PATCH 05/17] add minio container to ci --- .github/workflows/elixir.yml | 10 ++++++++++ test/test_helper.exs | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index ac3c344f5651..7beae7400a8e 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -77,6 +77,16 @@ jobs: run: mix test --include slow --max-failures 1 --warnings-as-errors - name: Run tests (small build) run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors + - name: Start MinIO (for S3 tests) + run: docker run -d -p 9000:9000 minio/minio server /data + - name: Create s3://imports bucket + run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://imports + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_EC2_METADATA_DISABLED: true + - name: Run tests (S3) + run: mix test --only minio --max-failures 1 --warnings-as-errors - name: Check Dialyzer run: mix dialyzer env: diff --git a/test/test_helper.exs b/test/test_helper.exs index 7cbd300df15f..8a58cb071183 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -7,8 +7,8 @@ Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual) if Mix.env() == :small_test do IO.puts("Test mode: SMALL") - ExUnit.configure(exclude: [:slow, :full_build_only]) + ExUnit.configure(exclude: [:slow, :minio, :full_build_only]) else IO.puts("Test mode: FULL") - ExUnit.configure(exclude: [:slow, :small_build_only]) + ExUnit.configure(exclude: [:slow, :minio, :small_build_only]) end From b30069a8d563811ef46cfcbfa141c8f5b3e8a37e Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 19 Feb 2024 20:58:59 +0800 Subject: [PATCH 06/17] more tests --- .github/workflows/elixir.yml | 2 + lib/plausible/imported/csv_importer.ex | 51 ++++++++++++ test/plausible/imported/csv_importer_test.exs | 78 +++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 test/plausible/imported/csv_importer_test.exs diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 7beae7400a8e..617dc62d858d 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -87,6 +87,8 @@ jobs: AWS_EC2_METADATA_DISABLED: true - name: Run tests (S3) run: mix test --only minio --max-failures 1 --warnings-as-errors + env: + S3_HOST_FOR_CLICKHOUSE: 172.17.0.1 - name: Check Dialyzer run: mix dialyzer env: diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 4656c3c58f94..14d5ef1b6fb5 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -15,6 +15,41 @@ defmodule Plausible.Imported.CSVImporter do @impl true def email_template(), do: "google_analytics_import.html" + @doc ~S""" + Extracts min/max dates from the date ranges specified in the upload files. + + Examples: + + iex> extract_min_max_date_range([]) + ** (ArgumentError) filenames cannot be empty + + iex> extract_min_max_date_range(["my_data.csv"]) + ** (ArgumentError) filename "my_data.csv" does not conform to the expected "#{table}_#{start_date}_#{end_date}.csv" format + + iex> extract_min_max_date_range(["imported_devices_00010101_20250101.csv"]) + Date.range(~D[0001-01-01], ~D[2025-01-01]) + + iex> extract_min_max_date_range(["imported_devices_00010101_09990101.csv", "imported_browsers_10010101_20250101.csv"]) + Date.range(~D[0001-01-01], ~D[2025-01-01]) + + """ + @spec extract_min_max_date_range([String.t()]) :: Date.Range.t() + def extract_min_max_date_range(filenames) do + if Enum.empty?(filenames) do + raise ArgumentError, message: "filenames cannot be empty" + end + + ranges = + Enum.map(filenames, fn filename -> + [_table, start_date, end_date] = parse_filename!(filename) + Date.range(start_date, end_date) + end) + + min = Enum.min_by(ranges, & &1.first) + max = Enum.max_by(ranges, & &1.last) + Date.range(min.first, max.last) + end + @impl true def parse_args(%{"uploads" => uploads}), do: [uploads: uploads] @@ -86,9 +121,25 @@ defmodule Plausible.Imported.CSVImporter do for {table, input_structure} <- input_structures do defp input_structure!(unquote(table)), do: unquote(input_structure) defp ensure_importable_table!(unquote(table)), do: :ok + + defp parse_filename!( + <> + ) do + [ + unquote(table), + Timex.parse!(start_date, "{YYYY}{0M}{0D}"), + Timex.parse!(end_date, "{YYYY}{0M}{0D}") + ] + end end defp ensure_importable_table!(table) do raise ArgumentError, "table #{table} is not supported for data import" end + + defp parse_filename!(filename) do + raise ArgumentError, + "filename #{inspect(filename)} " <> + ~S[ does not conform to the expected "#{table}_#{start_date}_#{end_date}.csv" format] + end end diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs new file mode 100644 index 000000000000..64086b282647 --- /dev/null +++ b/test/plausible/imported/csv_importer_test.exs @@ -0,0 +1,78 @@ +defmodule Plausible.Imported.CSVImporterTest do + use Plausible.DataCase, async: true + + doctest Plausible.Imported.CSVImporter, import: true + + @moduletag :minio + + # uses https://min.io + # docker run -d --rm -p 9000:9000 -p 9001:9001 --name minio minio/minio server /data --console-address ":9001" + # docker exec minio mc alias set local http://localhost:9000 minioadmin minioadmin + # docker exec minio mc mb local/imports + + alias Plausible.Imported.{CSVImporter, SiteImport} + require SiteImport + + describe "new_import/3 and parse_args/1" do + setup [:create_user, :create_new_site] + + test "parses job args properly", %{user: user, site: site} do + tables = [ + "imported_browsers", + "imported_devices", + "imported_entry_pages", + "imported_exit_pages", + "imported_locations", + "imported_operating_systems", + "imported_pages", + "imported_sources", + "imported_visitors" + ] + + start_date = "20231001" + end_date = "20240102" + + uploads = + Enum.map(tables, fn table -> + filename = "#{table}_#{start_date}_#{end_date}.csv" + s3_path = "#{site.id}/#{filename}" + %{"filename" => filename, "s3_path" => s3_path} + end) + + min_max_date_range = + uploads + |> Enum.map(& &1["filename"]) + |> CSVImporter.extract_min_max_date_range() + + assert {:ok, job} = + CSVImporter.new_import(site, user, + start_date: min_max_date_range.first, + end_date: min_max_date_range.last, + uploads: uploads + ) + + assert %Oban.Job{args: %{"import_id" => import_id, "uploads" => ^uploads} = args} = + Repo.reload!(job) + + assert [ + %{ + id: ^import_id, + source: :csv, + start_date: ~D[2023-10-01], + end_date: ~D[2024-01-02], + status: SiteImport.pending() + } + ] = Plausible.Imported.list_all_imports(site) + + assert %{imported_data: nil} = Repo.reload!(site) + assert CSVImporter.parse_args(args) == [uploads: uploads] + end + end + + describe "import_data/2" do + setup [:create_user, :create_new_site] + + test "imports tables from S3" + test "invalid CSV" + end +end From 6eab42f90409ce4e5c113c4af5602eea1b0db033 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 19 Feb 2024 23:14:59 +0800 Subject: [PATCH 07/17] eh --- lib/plausible/imported/csv_importer.ex | 113 ++++++++---------- test/plausible/imported/csv_importer_test.exs | 21 +--- 2 files changed, 52 insertions(+), 82 deletions(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 14d5ef1b6fb5..de28ec6ccd8d 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -15,41 +15,6 @@ defmodule Plausible.Imported.CSVImporter do @impl true def email_template(), do: "google_analytics_import.html" - @doc ~S""" - Extracts min/max dates from the date ranges specified in the upload files. - - Examples: - - iex> extract_min_max_date_range([]) - ** (ArgumentError) filenames cannot be empty - - iex> extract_min_max_date_range(["my_data.csv"]) - ** (ArgumentError) filename "my_data.csv" does not conform to the expected "#{table}_#{start_date}_#{end_date}.csv" format - - iex> extract_min_max_date_range(["imported_devices_00010101_20250101.csv"]) - Date.range(~D[0001-01-01], ~D[2025-01-01]) - - iex> extract_min_max_date_range(["imported_devices_00010101_09990101.csv", "imported_browsers_10010101_20250101.csv"]) - Date.range(~D[0001-01-01], ~D[2025-01-01]) - - """ - @spec extract_min_max_date_range([String.t()]) :: Date.Range.t() - def extract_min_max_date_range(filenames) do - if Enum.empty?(filenames) do - raise ArgumentError, message: "filenames cannot be empty" - end - - ranges = - Enum.map(filenames, fn filename -> - [_table, start_date, end_date] = parse_filename!(filename) - Date.range(start_date, end_date) - end) - - min = Enum.min_by(ranges, & &1.first) - max = Enum.max_by(ranges, & &1.last) - Date.range(min.first, max.last) - end - @impl true def parse_args(%{"uploads" => uploads}), do: [uploads: uploads] @@ -66,36 +31,54 @@ defmodule Plausible.Imported.CSVImporter do |> Keyword.replace!(:pool_size, 1) |> Ch.start_link() - Enum.each(uploads, fn upload -> - %{"filename" => filename, "s3_path" => s3_path} = upload - - ".csv" = Path.extname(filename) - table = Path.rootname(filename) - ensure_importable_table!(table) - - s3_structure = input_structure!(table) - s3_url = Plausible.S3.import_clickhouse_url(s3_path) - - statement = - """ - INSERT INTO {table:Identifier} \ - SELECT {site_id:UInt64} AS site_id, {import_id:UInt64} AS import_id, * \ - FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\ - """ - - params = %{ - "table" => table, - "site_id" => site_id, - "import_id" => import_id, - "s3_url" => s3_url, - "s3_access_key_id" => s3_access_key_id, - "s3_secret_access_key" => s3_secret_access_key, - "s3_format" => "CSVWithNames", - "s3_structure" => s3_structure - } - - Ch.query!(ch, statement, params, timeout: :infinity) - end) + # TODO what if it succeeds for some tables and then fails? Should the other tables be dropped? + ranges = + Enum.each(uploads, fn upload -> + %{"filename" => filename, "s3_path" => s3_path} = upload + + ".csv" = Path.extname(filename) + table = Path.rootname(filename) + ensure_importable_table!(table) + + s3_structure = input_structure!(table) + s3_url = Plausible.S3.import_clickhouse_url(s3_path) + + statement = + """ + INSERT INTO {table:Identifier} \ + SELECT {site_id:UInt64} AS site_id, {import_id:UInt64} AS import_id, * \ + FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\ + """ + + params = %{ + "table" => table, + "site_id" => site_id, + "import_id" => import_id, + "s3_url" => s3_url, + "s3_access_key_id" => s3_access_key_id, + "s3_secret_access_key" => s3_secret_access_key, + "s3_format" => "CSVWithNames", + "s3_structure" => s3_structure + } + + Ch.query!(ch, statement, params, timeout: :infinity) + + %Ch.Result{rows: [[min_date, max_date]]} + + Ch.query!( + ch, + "SELECT min(date), max(date) FROM {table:Identifier} WHERE site_id = {site_id:UInt64} AND import_id = {import_id:UInt64}", + %{"table" => table, "site_id" => site_id, "import_id" => import_id} + ) + + Date.range(min_date, max_date) + end) + + {:ok, + %{ + start_date: Enum.min_by(ranges, & &1.first), + end_date: Enum.max_by(ranges, & &1.last) + }} end input_structures = %{ diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index 64086b282647..91faa74702ce 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -29,27 +29,14 @@ defmodule Plausible.Imported.CSVImporterTest do "imported_visitors" ] - start_date = "20231001" - end_date = "20240102" - uploads = Enum.map(tables, fn table -> - filename = "#{table}_#{start_date}_#{end_date}.csv" + filename = "#{table}.csv" s3_path = "#{site.id}/#{filename}" %{"filename" => filename, "s3_path" => s3_path} end) - min_max_date_range = - uploads - |> Enum.map(& &1["filename"]) - |> CSVImporter.extract_min_max_date_range() - - assert {:ok, job} = - CSVImporter.new_import(site, user, - start_date: min_max_date_range.first, - end_date: min_max_date_range.last, - uploads: uploads - ) + assert {:ok, job} = CSVImporter.new_import(site, user, uploads: uploads) assert %Oban.Job{args: %{"import_id" => import_id, "uploads" => ^uploads} = args} = Repo.reload!(job) @@ -58,8 +45,8 @@ defmodule Plausible.Imported.CSVImporterTest do %{ id: ^import_id, source: :csv, - start_date: ~D[2023-10-01], - end_date: ~D[2024-01-02], + start_date: ~D[0001-01-01], + end_date: ~D[0001-01-01], status: SiteImport.pending() } ] = Plausible.Imported.list_all_imports(site) From c83de0a1d48a0b63aef3ec1172ac15b01abbca43 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Tue, 20 Feb 2024 23:22:51 +0800 Subject: [PATCH 08/17] continue --- lib/plausible/imported/csv_importer.ex | 32 ++++--------------- test/plausible/imported/csv_importer_test.exs | 12 +++++-- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index de28ec6ccd8d..34151be61b32 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -31,9 +31,8 @@ defmodule Plausible.Imported.CSVImporter do |> Keyword.replace!(:pool_size, 1) |> Ch.start_link() - # TODO what if it succeeds for some tables and then fails? Should the other tables be dropped? ranges = - Enum.each(uploads, fn upload -> + Enum.map(uploads, fn upload -> %{"filename" => filename, "s3_path" => s3_path} = upload ".csv" = Path.extname(filename) @@ -63,13 +62,12 @@ defmodule Plausible.Imported.CSVImporter do Ch.query!(ch, statement, params, timeout: :infinity) - %Ch.Result{rows: [[min_date, max_date]]} - - Ch.query!( - ch, - "SELECT min(date), max(date) FROM {table:Identifier} WHERE site_id = {site_id:UInt64} AND import_id = {import_id:UInt64}", - %{"table" => table, "site_id" => site_id, "import_id" => import_id} - ) + %Ch.Result{rows: [[min_date, max_date]]} = + Ch.query!( + ch, + "SELECT min(date), max(date) FROM {table:Identifier} WHERE site_id = {site_id:UInt64} AND import_id = {import_id:UInt64}", + %{"table" => table, "site_id" => site_id, "import_id" => import_id} + ) Date.range(min_date, max_date) end) @@ -104,25 +102,9 @@ defmodule Plausible.Imported.CSVImporter do for {table, input_structure} <- input_structures do defp input_structure!(unquote(table)), do: unquote(input_structure) defp ensure_importable_table!(unquote(table)), do: :ok - - defp parse_filename!( - <> - ) do - [ - unquote(table), - Timex.parse!(start_date, "{YYYY}{0M}{0D}"), - Timex.parse!(end_date, "{YYYY}{0M}{0D}") - ] - end end defp ensure_importable_table!(table) do raise ArgumentError, "table #{table} is not supported for data import" end - - defp parse_filename!(filename) do - raise ArgumentError, - "filename #{inspect(filename)} " <> - ~S[ does not conform to the expected "#{table}_#{start_date}_#{end_date}.csv" format] - end end diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index 91faa74702ce..c14fcb7782da 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -36,7 +36,13 @@ defmodule Plausible.Imported.CSVImporterTest do %{"filename" => filename, "s3_path" => s3_path} end) - assert {:ok, job} = CSVImporter.new_import(site, user, uploads: uploads) + assert {:ok, job} = + CSVImporter.new_import(site, user, + # to satisfy the non null constraints on the table I'm providing "0" dates (according to ClickHouse) + start_date: ~D[1970-01-01], + end_date: ~D[1970-01-01], + uploads: uploads + ) assert %Oban.Job{args: %{"import_id" => import_id, "uploads" => ^uploads} = args} = Repo.reload!(job) @@ -45,8 +51,8 @@ defmodule Plausible.Imported.CSVImporterTest do %{ id: ^import_id, source: :csv, - start_date: ~D[0001-01-01], - end_date: ~D[0001-01-01], + start_date: ~D[1970-01-01], + end_date: ~D[1970-01-01], status: SiteImport.pending() } ] = Plausible.Imported.list_all_imports(site) From 558dd117437ee075c136fc8b9ba30c8063d4ee0e Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Wed, 21 Feb 2024 19:57:49 +0800 Subject: [PATCH 09/17] add passing test --- config/.env.dev | 2 +- config/.env.test | 2 +- lib/plausible/imported/csv_importer.ex | 6 +- test/plausible/imported/csv_importer_test.exs | 915 +++++++++++++++++- 4 files changed, 919 insertions(+), 6 deletions(-) diff --git a/config/.env.dev b/config/.env.dev index 5641f20b00e2..d3f39d92a44c 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -27,4 +27,4 @@ S3_SECRET_ACCESS_KEY=minioadmin S3_REGION=us-east-1 S3_ENDPOINT=http://localhost:9000 S3_IMPORTS_BUCKET=imports -S3_HOST_FOR_CLICKHOUSE=172.19.0.4 +S3_HOST_FOR_CLICKHOUSE=172.17.0.1 diff --git a/config/.env.test b/config/.env.test index 2ebba9cc21ac..699a90bb70ad 100644 --- a/config/.env.test +++ b/config/.env.test @@ -22,4 +22,4 @@ S3_SECRET_ACCESS_KEY=minioadmin S3_REGION=us-east-1 S3_ENDPOINT=http://localhost:9000 S3_IMPORTS_BUCKET=imports -S3_HOST_FOR_CLICKHOUSE=172.19.0.4 +S3_HOST_FOR_CLICKHOUSE=172.17.0.1 diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 34151be61b32..b23c4f98e04a 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -45,7 +45,7 @@ defmodule Plausible.Imported.CSVImporter do statement = """ INSERT INTO {table:Identifier} \ - SELECT {site_id:UInt64} AS site_id, {import_id:UInt64} AS import_id, * \ + SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \ FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\ """ @@ -74,8 +74,8 @@ defmodule Plausible.Imported.CSVImporter do {:ok, %{ - start_date: Enum.min_by(ranges, & &1.first), - end_date: Enum.max_by(ranges, & &1.last) + start_date: Enum.min_by(ranges, & &1.first, Date).first, + end_date: Enum.max_by(ranges, & &1.last, Date).last }} end diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index c14fcb7782da..14136a9280e5 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -65,7 +65,920 @@ defmodule Plausible.Imported.CSVImporterTest do describe "import_data/2" do setup [:create_user, :create_new_site] - test "imports tables from S3" + test "imports tables from S3", %{site: site, user: user} do + csvs = [ + %{ + name: "imported_browsers.csv", + body: """ + "date","browser","visitors","visits","visit_duration","bounces" + "2021-12-30","Amazon Silk",2,2,0,2 + "2021-12-30","Chrome",31,32,329,29 + "2021-12-30","Edge",3,3,0,3 + "2021-12-30","Firefox",1,1,0,1 + "2021-12-30","Internet Explorer",1,1,0,1 + "2021-12-30","Mobile App",2,2,0,2 + "2021-12-30","Mobile App",4,4,0,4 + "2021-12-30","Mobile App",1,1,0,1 + "2021-12-30","Safari",32,36,0,36 + "2021-12-30","Samsung Internet",2,2,0,2 + "2021-12-30","UC Browser",1,1,0,1 + "2021-12-31","Chrome",24,25,75,23 + "2021-12-31","Edge",3,3,0,3 + "2021-12-31","Firefox",1,1,466,0 + "2021-12-31","Mobile App",4,5,0,5 + "2021-12-31","Mobile App",4,4,0,4 + "2021-12-31","Mobile App",1,1,85,0 + "2021-12-31","Safari",37,45,1957,42 + "2021-12-31","Samsung Internet",1,1,199,0 + "2022-01-01","'DuckDuckBot-Https",17,18,0,18 + "2022-01-01","Chrome",30,33,1149,29 + "2022-01-01","Edge",1,1,0,1 + "2022-01-01","Firefox",1,1,0,1 + "2022-01-01","Mobile App",1,1,0,1 + "2022-01-01","Mobile App",5,5,0,5 + "2022-01-01","Mobile App",2,2,0,2 + "2022-01-01","Opera",1,1,0,1 + "2022-01-01","Safari",36,38,693,33 + "2022-01-01","Samsung Internet",2,2,0,2 + "2022-01-02","Chrome",38,38,1411,34 + "2022-01-02","Firefox",2,2,0,2 + "2022-01-02","Mobile App",3,3,36,2 + "2022-01-02","Mobile App",9,9,0,9 + "2022-01-02","Mobile App",1,1,0,1 + "2022-01-02","Safari",50,57,1089,52 + "2022-01-03","Chrome",35,40,2067,34 + "2022-01-03","Edge",3,3,0,3 + "2022-01-03","Firefox",1,1,105,0 + "2022-01-03","Mobile App",3,3,124,2 + "2022-01-03","Mobile App",10,10,137,9 + "2022-01-03","Mobile App",1,1,0,1 + "2022-01-03","Safari",53,55,5786,45 + "2022-01-03","Samsung Internet",2,2,0,2 + "2022-01-04","Amazon Silk",1,1,0,1 + "2022-01-04","Chrome",22,23,250,21 + "2022-01-04","Edge",3,3,0,3 + "2022-01-04","Firefox",1,1,0,1 + "2022-01-04","Mobile App",1,1,0,1 + "2022-01-04","Mobile App",12,12,248,9 + "2022-01-04","Mobile App",1,1,0,1 + "2022-01-04","Mobile App",1,1,0,1 + "2022-01-04","Opera",1,1,0,1 + "2022-01-04","Safari",43,47,1806,43 + "2022-01-04","Samsung Internet",1,1,0,1 + "2022-01-05","Chrome",36,39,1133,34 + "2022-01-05","Edge",3,3,0,3 + "2022-01-05","Firefox",2,2,0,2 + "2022-01-05","Mobile App",1,1,147,0 + "2022-01-05","Mobile App",1,1,0,1 + "2022-01-05","Safari",38,38,1125,36 + "2022-01-05","Samsung Internet",1,1,0,1 + "2022-01-06","'DuckDuckBot-Https",18,18,0,18 + "2022-01-06","Chrome",36,37,1897,32 + "2022-01-06","Edge",2,2,10,1 + "2022-01-06","Firefox",1,1,0,1 + "2022-01-06","Internet Explorer",1,1,0,1 + "2022-01-06","Mobile App",1,1,0,1 + "2022-01-06","Mobile App",8,8,0,8 + "2022-01-06","Mobile App",2,2,0,2 + "2022-01-06","Safari",40,45,1206,42 + "2022-01-06","iubenda-radar",5,5,0,5 + "2022-01-07","Chrome",25,26,630,23 + "2022-01-07","Edge",1,1,0,1 + "2022-01-07","Firefox",1,1,0,1 + "2022-01-07","Safari",32,33,856,30 + "2022-01-07","Samsung Internet",3,3,0,3 + "2022-01-07","YaBrowser",1,1,0,1 + "2022-01-08","Chrome",33,33,229,31 + "2022-01-08","Edge",3,3,1700,2 + "2022-01-08","Firefox",1,1,94,0 + "2022-01-08","Mobile App",1,1,0,1 + "2022-01-08","Mobile App",4,4,10,3 + "2022-01-08","Mobile App",2,2,0,2 + "2022-01-08","Opera",1,1,0,1 + "2022-01-08","Safari",47,51,4070,45 + "2022-01-09","Amazon Silk",1,1,0,1 + "2022-01-09","Chrome",23,25,1012,20 + "2022-01-09","Edge",4,4,0,4 + "2022-01-09","Mobile App",3,4,16,3 + "2022-01-09","Mobile App",11,12,203,9 + "2022-01-09","Mobile App",1,1,0,1 + "2022-01-09","Safari",42,46,636,44 + "2022-01-09","Samsung Internet",2,2,0,2 + "2022-01-10","Amazon Silk",1,1,0,1 + "2022-01-10","Chrome",18,19,0,19 + "2022-01-10","Edge",1,1,0,1 + "2022-01-10","Firefox",2,2,0,2 + "2022-01-10","Mobile App",1,1,0,1 + "2022-01-10","Mobile App",4,4,0,4 + "2022-01-10","Safari",41,44,160,41 + """ + }, + %{ + name: "imported_devices.csv", + body: """ + "date","device","visitors","visits","visit_duration","bounces" + "2021-12-30","Desktop",25,28,75,27 + "2021-12-30","Mobile",49,51,254,49 + "2021-12-30","Tablet",6,6,0,6 + "2021-12-31","Desktop",20,26,496,24 + "2021-12-31","Mobile",50,54,1842,49 + "2021-12-31","Tablet",5,5,444,4 + "2022-01-01","Desktop",33,34,1117,32 + "2022-01-01","Mobile",55,60,306,54 + "2022-01-01","Tablet",8,8,419,7 + "2022-01-02","Desktop",28,28,86,26 + "2022-01-02","Mobile",66,73,2450,65 + "2022-01-02","Tablet",9,9,0,9 + "2022-01-03","Desktop",37,37,1855,30 + "2022-01-03","Mobile",64,68,6364,56 + "2022-01-03","Tablet",7,10,0,10 + "2022-01-04","Desktop",26,26,1330,22 + "2022-01-04","Mobile",58,63,954,59 + "2022-01-04","Tablet",3,3,20,2 + "2022-01-05","Desktop",30,32,1162,29 + "2022-01-05","Mobile",45,46,1084,43 + "2022-01-05","Tablet",7,7,159,5 + "2022-01-06","Desktop",53,53,114,51 + "2022-01-06","Mobile",55,61,2999,54 + "2022-01-06","Tablet",6,6,0,6 + "2022-01-07","Desktop",15,16,247,15 + "2022-01-07","Mobile",47,48,1239,43 + "2022-01-07","Tablet",1,1,0,1 + "2022-01-08","Desktop",27,28,2171,23 + "2022-01-08","Mobile",59,63,3932,57 + "2022-01-08","Tablet",5,5,0,5 + "2022-01-09","Desktop",31,33,1415,26 + "2022-01-09","Mobile",48,54,167,51 + "2022-01-09","Tablet",8,8,285,7 + "2022-01-10","Desktop",22,24,0,24 + "2022-01-10","Mobile",46,49,160,46 + "2022-01-10","Tablet",4,4,0,4 + "2022-01-11","Desktop",29,30,1835,23 + "2022-01-11","Mobile",39,40,185,35 + "2022-01-11","Tablet",10,10,0,10 + "2022-01-12","Desktop",16,16,1543,14 + "2022-01-12","Mobile",61,66,2113,57 + "2022-01-12","Tablet",7,7,18,6 + "2022-01-13","Desktop",34,35,3000,29 + "2022-01-13","Mobile",47,51,262,48 + "2022-01-13","Tablet",5,5,0,5 + "2022-01-14","Desktop",21,21,5,20 + "2022-01-14","Mobile",52,60,178,58 + "2022-01-14","Tablet",3,3,0,3 + "2022-01-15","Desktop",22,22,347,21 + "2022-01-15","Mobile",73,76,3633,72 + "2022-01-15","Tablet",4,4,231,3 + "2022-01-16","Desktop",30,34,829,30 + "2022-01-16","Mobile",64,65,2438,56 + "2022-01-16","Tablet",9,10,1449,9 + "2022-01-17","Desktop",41,42,550,39 + "2022-01-17","Mobile",59,65,2177,59 + "2022-01-17","Tablet",4,4,0,4 + "2022-01-18","Desktop",18,21,176,20 + "2022-01-18","Mobile",46,49,3004,42 + "2022-01-18","Tablet",4,4,117,3 + "2022-01-19","Desktop",15,15,0,15 + "2022-01-19","Mobile",49,53,3145,44 + "2022-01-19","Tablet",3,3,0,3 + "2022-01-20","Desktop",25,25,2734,20 + "2022-01-20","Mobile",41,45,1585,43 + "2022-01-20","Tablet",3,3,0,3 + "2022-01-21","Desktop",43,44,3041,39 + "2022-01-21","Mobile",47,52,2183,46 + "2022-01-21","Tablet",4,4,0,4 + "2022-01-22","Desktop",24,25,682,19 + "2022-01-22","Mobile",63,67,3209,55 + "2022-01-22","Tablet",3,3,591,2 + "2022-01-23","Desktop",20,20,594,18 + "2022-01-23","Mobile",64,70,2059,65 + "2022-01-23","Tablet",2,2,0,2 + "2022-01-24","Desktop",27,27,2004,25 + "2022-01-24","Mobile",58,62,659,56 + "2022-01-24","Tablet",5,5,0,5 + "2022-01-25","Desktop",32,37,280,34 + "2022-01-25","Mobile",34,36,467,30 + "2022-01-25","Tablet",4,4,0,4 + "2022-01-26","Desktop",14,15,0,15 + "2022-01-26","Mobile",57,61,4114,55 + "2022-01-26","Tablet",3,3,0,3 + "2022-01-27","Desktop",38,39,692,37 + "2022-01-27","Mobile",52,56,1237,49 + "2022-01-27","Tablet",2,2,0,2 + "2022-01-28","Desktop",18,20,161,18 + "2022-01-28","Mobile",56,62,0,62 + "2022-01-28","Tablet",2,3,0,3 + "2022-01-29","Desktop",32,32,946,29 + "2022-01-29","Mobile",66,69,2403,63 + "2022-01-29","Tablet",2,2,0,2 + "2022-01-30","Desktop",30,31,1189,28 + "2022-01-30","Mobile",56,59,2253,54 + "2022-01-30","Tablet",5,5,283,4 + "2022-01-31","Desktop",31,32,0,32 + "2022-01-31","Mobile",46,52,1515,46 + "2022-01-31","Tablet",2,2,382,1 + "2022-02-01","Desktop",26,26,211,23 + """ + }, + %{ + name: "imported_entry_pages.csv", + body: """ + "date","visitors","entrances","visit_duration","bounces","entry_page" + "2021-12-30",6,6,0,6,"/14776416252794997127" + "2021-12-30",1,1,0,1,"/15455127321321119046" + "2021-12-30",1,1,43,0,"/10399835914295020763" + "2021-12-30",1,1,0,1,"/9102354072466236765" + "2021-12-30",1,1,0,1,"/4457889102355683190" + "2021-12-30",1,1,0,1,"/12105301321223776356" + "2021-12-30",1,2,0,2,"/1526239929864936398" + "2021-12-30",3,3,0,3,"/12574817671425987843" + "2021-12-30",1,1,0,1,"/18226692671132987727" + "2021-12-30",1,1,0,1,"/13530392472417087202" + "2021-12-30",3,3,0,3,"/14629996869565295116" + "2021-12-30",2,2,0,2,"/16500758973580097263" + "2021-12-30",3,3,0,3,"/12300842990999856228" + "2021-12-30",2,2,75,1,"/17859765562822550514" + "2021-12-30",1,1,0,1,"/1250717791477281255" + "2021-12-30",1,1,0,1,"/1586391735863371077" + "2021-12-30",1,1,0,1,"/3457026921000639206" + "2021-12-30",2,3,0,3,"/6077502147861556415" + "2021-12-30",1,1,0,1,"/14280570555317344651" + "2021-12-30",3,3,0,3,"/5284268072698982201" + "2021-12-30",1,1,0,1,"/7478911940502018071" + "2021-12-30",1,1,0,1,"/6402607186523575652" + "2021-12-30",2,2,0,2,"/9962503789684934900" + "2021-12-30",8,10,0,10,"/13595620304963848161" + "2021-12-30",2,2,0,2,"/17019199732013993436" + "2021-12-30",31,31,211,30,"/9874837495456455794" + "2021-12-31",4,4,0,4,"/14776416252794997127" + "2021-12-31",1,1,0,1,"/8738789417178304429" + "2021-12-31",1,1,0,1,"/7445073500314667742" + "2021-12-31",1,1,0,1,"/4897404798407749335" + "2021-12-31",1,1,45,0,"/11263893625781431659" + "2021-12-31",1,1,0,1,"/16478773157730928089" + "2021-12-31",1,1,0,1,"/1710995203264225236" + "2021-12-31",2,3,0,3,"/12574817671425987843" + "2021-12-31",1,2,0,2,"/4072002299714740082" + "2021-12-31",1,1,0,1,"/13544703054662457518" + "2021-12-31",1,1,0,1,"/3363270934034278688" + "2021-12-31",4,4,0,4,"/14629996869565295116" + "2021-12-31",2,2,284,0,"/5918158559063394909" + "2021-12-31",1,1,0,1,"/16500758973580097263" + "2021-12-31",1,1,0,1,"/15335700069940448722" + "2021-12-31",7,7,0,7,"/1250717791477281255" + "2021-12-31",3,3,0,3,"/3457026921000639206" + "2021-12-31",1,1,0,1,"/6077502147861556415" + "2021-12-31",1,1,0,1,"/14280570555317344651" + "2021-12-31",4,5,444,4,"/5284268072698982201" + "2021-12-31",2,2,466,1,"/7478911940502018071" + "2021-12-31",9,16,1455,15,"/13595620304963848161" + "2021-12-31",25,25,88,23,"/9874837495456455794" + "2022-01-01",2,3,0,3,"/14776416252794997127" + "2022-01-01",1,1,0,1,"/18394065791342797546" + "2022-01-01",2,2,0,2,"/9102354072466236765" + "2022-01-01",2,2,0,2,"/2528868181315148597" + "2022-01-01",1,1,0,1,"/4457889102355683190" + "2022-01-01",1,1,0,1,"/12105301321223776356" + "2022-01-01",2,2,0,2,"/6318438003888425693" + "2022-01-01",2,2,0,2,"/5878724061840196349" + "2022-01-01",1,1,0,1,"/1526239929864936398" + "2022-01-01",2,2,0,2,"/7692634448754428624" + "2022-01-01",4,4,0,4,"/12574817671425987843" + "2022-01-01",1,1,0,1,"/14307781859600070983" + "2022-01-01",2,2,0,2,"/7110820102771013606" + "2022-01-01",3,3,0,3,"/14629996869565295116" + "2022-01-01",2,2,0,2,"/64779230489549655" + "2022-01-01",1,1,0,1,"/5551134200879446973" + "2022-01-01",1,1,25,0,"/11888590162960019765" + "2022-01-01",1,1,0,1,"/12567804068906971541" + "2022-01-01",1,1,0,1,"/14033373606203782378" + "2022-01-01",2,2,0,2,"/10341536794299767967" + "2022-01-01",1,1,0,1,"/12766075726240916242" + "2022-01-01",1,1,0,1,"/4362292817884281301" + "2022-01-01",1,2,0,2,"/1250717791477281255" + "2022-01-01",2,2,0,2,"/3457026921000639206" + "2022-01-01",1,1,0,1,"/6077502147861556415" + "2022-01-01",2,2,0,2,"/18342609412020394218" + "2022-01-01",1,1,0,1,"/12189381298883635575" + "2022-01-01",2,2,0,2,"/14280570555317344651" + "2022-01-01",5,6,435,4,"/5284268072698982201" + "2022-01-01",4,4,0,4,"/7478911940502018071" + "2022-01-01",1,1,0,1,"/6402607186523575652" + "2022-01-01",1,1,0,1,"/8587605562711759914" + "2022-01-01",1,1,0,1,"/9962503789684934900" + "2022-01-01",7,8,187,6,"/13595620304963848161" + "2022-01-01",1,1,0,1,"/17019199732013993436" + "2022-01-01",32,33,1195,29,"/9874837495456455794" + "2022-01-02",3,3,81,2,"/14776416252794997127" + "2022-01-02",3,6,33,5,"/9102354072466236765" + "2022-01-02",1,1,0,1,"/17912324809189526683" + "2022-01-02",1,1,0,1,"/2528868181315148597" + "2022-01-02",1,1,0,1,"/4457889102355683190" + "2022-01-02",1,1,0,1,"/4897404798407749335" + "2022-01-02",1,1,0,1,"/5290432771315151230" + "2022-01-02",1,1,0,1,"/3861893315241731546" + "2022-01-02",1,1,0,1,"/12571626520250599954" + "2022-01-02",11,11,832,10,"/12574817671425987843" + "2022-01-02",2,2,0,2,"/15791285625100101971" + "2022-01-02",2,2,0,2,"/16602440950175620202" + "2022-01-02",1,1,0,1,"/3363270934034278688" + "2022-01-02",6,6,122,5,"/14629996869565295116" + "2022-01-02",1,2,0,2,"/17952015439202551610" + """ + }, + %{ + name: "imported_exit_pages.csv", + body: """ + "date","visitors","exits","exit_page" + "2021-12-30",6,6,"/14776416252794997127" + "2021-12-30",1,1,"/15455127321321119046" + "2021-12-30",1,1,"/9102354072466236765" + "2021-12-30",1,1,"/4457889102355683190" + "2021-12-30",1,1,"/12105301321223776356" + "2021-12-30",1,2,"/1526239929864936398" + "2021-12-30",3,3,"/12574817671425987843" + "2021-12-30",1,1,"/18226692671132987727" + "2021-12-30",1,1,"/13530392472417087202" + "2021-12-30",3,3,"/14629996869565295116" + "2021-12-30",2,2,"/16500758973580097263" + "2021-12-30",3,3,"/12300842990999856228" + "2021-12-30",2,2,"/17859765562822550514" + "2021-12-30",1,1,"/1250717791477281255" + "2021-12-30",1,1,"/1586391735863371077" + "2021-12-30",1,1,"/3457026921000639206" + "2021-12-30",2,3,"/6077502147861556415" + "2021-12-30",1,1,"/14280570555317344651" + "2021-12-30",3,3,"/5284268072698982201" + "2021-12-30",1,1,"/7478911940502018071" + "2021-12-30",1,1,"/6402607186523575652" + "2021-12-30",2,2,"/9962503789684934900" + "2021-12-30",8,10,"/13595620304963848161" + "2021-12-30",2,2,"/17019199732013993436" + "2021-12-30",32,32,"/9874837495456455794" + "2021-12-31",4,4,"/14776416252794997127" + "2021-12-31",1,1,"/8738789417178304429" + "2021-12-31",1,1,"/7445073500314667742" + "2021-12-31",1,1,"/4897404798407749335" + "2021-12-31",1,1,"/11263893625781431659" + "2021-12-31",1,1,"/16478773157730928089" + "2021-12-31",1,1,"/1710995203264225236" + "2021-12-31",2,3,"/12574817671425987843" + "2021-12-31",1,2,"/4072002299714740082" + "2021-12-31",1,1,"/13544703054662457518" + "2021-12-31",1,1,"/3363270934034278688" + "2021-12-31",4,4,"/14629996869565295116" + "2021-12-31",2,2,"/5918158559063394909" + "2021-12-31",1,1,"/16500758973580097263" + "2021-12-31",1,1,"/15335700069940448722" + "2021-12-31",7,7,"/1250717791477281255" + "2021-12-31",3,3,"/3457026921000639206" + "2021-12-31",1,1,"/6077502147861556415" + "2021-12-31",1,1,"/14280570555317344651" + "2021-12-31",4,5,"/5284268072698982201" + "2021-12-31",2,2,"/7478911940502018071" + "2021-12-31",9,16,"/13595620304963848161" + "2021-12-31",25,25,"/9874837495456455794" + "2022-01-01",2,3,"/14776416252794997127" + "2022-01-01",1,1,"/18394065791342797546" + "2022-01-01",2,2,"/9102354072466236765" + "2022-01-01",2,2,"/2528868181315148597" + "2022-01-01",1,1,"/4457889102355683190" + "2022-01-01",1,1,"/12105301321223776356" + "2022-01-01",2,2,"/6318438003888425693" + "2022-01-01",2,2,"/5878724061840196349" + "2022-01-01",1,1,"/1526239929864936398" + "2022-01-01",2,2,"/7692634448754428624" + "2022-01-01",4,4,"/12574817671425987843" + "2022-01-01",1,1,"/14307781859600070983" + "2022-01-01",2,2,"/7110820102771013606" + "2022-01-01",3,3,"/14629996869565295116" + "2022-01-01",2,2,"/64779230489549655" + "2022-01-01",1,1,"/5551134200879446973" + "2022-01-01",1,1,"/11888590162960019765" + "2022-01-01",1,1,"/12567804068906971541" + "2022-01-01",1,1,"/14033373606203782378" + "2022-01-01",2,2,"/10341536794299767967" + "2022-01-01",1,1,"/12766075726240916242" + "2022-01-01",1,1,"/4362292817884281301" + "2022-01-01",1,2,"/1250717791477281255" + "2022-01-01",2,2,"/3457026921000639206" + "2022-01-01",1,1,"/6077502147861556415" + "2022-01-01",2,2,"/18342609412020394218" + "2022-01-01",1,1,"/12189381298883635575" + "2022-01-01",2,2,"/14280570555317344651" + "2022-01-01",5,6,"/5284268072698982201" + "2022-01-01",4,4,"/7478911940502018071" + "2022-01-01",1,1,"/6402607186523575652" + "2022-01-01",1,1,"/8587605562711759914" + "2022-01-01",1,1,"/9962503789684934900" + "2022-01-01",7,8,"/13595620304963848161" + "2022-01-01",1,1,"/17019199732013993436" + "2022-01-01",32,33,"/9874837495456455794" + "2022-01-02",4,4,"/14776416252794997127" + "2022-01-02",3,5,"/9102354072466236765" + "2022-01-02",1,1,"/17912324809189526683" + "2022-01-02",1,1,"/2528868181315148597" + "2022-01-02",1,1,"/4457889102355683190" + "2022-01-02",1,1,"/4897404798407749335" + "2022-01-02",1,1,"/5290432771315151230" + "2022-01-02",1,1,"/3861893315241731546" + "2022-01-02",1,1,"/12571626520250599954" + "2022-01-02",10,10,"/12574817671425987843" + "2022-01-02",1,1,"/15302856869047719385" + "2022-01-02",2,2,"/15791285625100101971" + "2022-01-02",2,2,"/16602440950175620202" + "2022-01-02",1,1,"/3363270934034278688" + "2022-01-02",6,6,"/14629996869565295116" + "2022-01-02",1,2,"/17952015439202551610" + """ + }, + %{ + name: "imported_locations.csv", + body: """ + "date","country","region","city","visitors","visits","visit_duration","bounces" + "2021-12-30","AU","",0,1,1,43,0 + "2021-12-30","AU","",2078025,3,4,211,3 + "2021-12-30","AU","",2147714,2,2,0,2 + "2021-12-30","AU","",2158177,2,2,0,2 + "2021-12-30","AU","",2174003,1,1,0,1 + "2021-12-30","BE","",0,1,1,0,1 + "2021-12-30","BE","",2792196,1,1,0,1 + "2021-12-30","BR","",0,1,1,0,1 + "2021-12-30","CA","",0,1,1,0,1 + "2021-12-30","CA","",5907364,1,1,0,1 + "2021-12-30","CA","",5918118,1,1,0,1 + "2021-12-30","CA","",5946768,1,1,0,1 + "2021-12-30","CA","",6066513,2,2,0,2 + "2021-12-30","CA","",6122091,1,1,0,1 + "2021-12-30","CA","",6141256,1,1,0,1 + "2021-12-30","CA","",6167865,1,1,0,1 + "2021-12-30","CN","",1784658,1,1,0,1 + "2021-12-30","CN","",1796236,1,1,0,1 + "2021-12-30","DE","",2865716,1,1,0,1 + "2021-12-30","DE","",2874225,1,1,0,1 + "2021-12-30","DE","",2950159,1,1,0,1 + "2021-12-30","DO","",3492908,2,2,0,2 + "2021-12-30","FR","",2996944,1,1,0,1 + "2021-12-30","GB","",2641523,1,1,0,1 + "2021-12-30","GB","",2643179,1,1,0,1 + "2021-12-30","GB","",2643743,1,1,0,1 + "2021-12-30","GB","",2644688,1,1,0,1 + "2021-12-30","GB","",2646504,1,1,0,1 + "2021-12-30","GB","",2653822,1,1,0,1 + "2021-12-30","IT","",3173435,1,1,0,1 + "2021-12-30","NL","",2747373,2,2,75,1 + "2021-12-30","PL","",0,1,1,0,1 + "2021-12-30","PL","",756135,1,1,0,1 + "2021-12-30","US","",0,1,1,0,1 + "2021-12-30","US","",0,1,1,0,1 + "2021-12-30","US","",0,1,1,0,1 + "2021-12-30","US","",0,1,1,0,1 + "2021-12-30","US","",4063926,1,1,0,1 + "2021-12-30","US","",4074013,1,3,0,3 + "2021-12-30","US","",4159077,1,1,0,1 + "2021-12-30","US","",4170688,1,1,0,1 + "2021-12-30","US","",4180183,1,1,0,1 + "2021-12-30","US","",4193699,1,1,0,1 + "2021-12-30","US","",4212992,1,1,0,1 + "2021-12-30","US","",4234436,1,1,0,1 + "2021-12-30","US","",4255836,1,1,0,1 + "2021-12-30","US","",4373238,1,1,0,1 + "2021-12-30","US","",4374798,1,1,0,1 + "2021-12-30","US","",4390705,1,1,0,1 + "2021-12-30","US","",4532846,1,1,0,1 + "2021-12-30","US","",4682665,1,1,0,1 + "2021-12-30","US","",4685987,1,1,0,1 + "2021-12-30","US","",4726311,1,1,0,1 + "2021-12-30","US","",4786619,1,1,0,1 + "2021-12-30","US","",4898015,1,1,0,1 + "2021-12-30","US","",4984247,1,1,0,1 + "2021-12-30","US","",5028537,2,2,0,2 + "2021-12-30","US","",5079991,1,1,0,1 + "2021-12-30","US","",5089478,1,1,0,1 + "2021-12-30","US","",5118743,1,1,0,1 + "2021-12-30","US","",5178713,1,1,0,1 + "2021-12-30","US","",5337561,1,1,0,1 + "2021-12-30","US","",5368361,1,1,0,1 + "2021-12-30","US","",5393049,1,1,0,1 + "2021-12-30","US","",5809844,2,3,0,3 + "2021-12-30","US","",5814916,1,2,0,2 + "2021-12-30","ZA","",0,1,1,0,1 + "2021-12-30","ZA","",0,1,1,0,1 + "2021-12-30","ZA","",936374,1,1,0,1 + "2021-12-30","ZA","",967106,1,1,0,1 + "2021-12-30","ZA","",1105777,1,1,0,1 + "2021-12-31","AU","",2078025,1,1,0,1 + "2021-12-31","AU","",2147714,3,3,0,3 + "2021-12-31","AU","",2158177,2,2,0,2 + "2021-12-31","CA","",0,1,1,0,1 + "2021-12-31","CA","",5907364,2,2,0,2 + "2021-12-31","CA","",5937615,1,1,0,1 + "2021-12-31","CA","",5990579,1,1,0,1 + "2021-12-31","CA","",6050610,1,1,0,1 + "2021-12-31","CA","",6087892,1,2,0,2 + "2021-12-31","CA","",6167865,3,3,0,3 + "2021-12-31","FR","",2988507,1,1,0,1 + "2021-12-31","GB","",0,1,1,0,1 + "2021-12-31","GB","",2640194,1,2,0,2 + "2021-12-31","GB","",2643743,2,2,0,2 + "2021-12-31","GB","",2644688,1,1,0,1 + "2021-12-31","GB","",2646274,1,1,0,1 + "2021-12-31","GB","",2654675,1,1,45,0 + "2021-12-31","GB","",2657562,1,1,0,1 + "2021-12-31","IE","",2964574,1,1,0,1 + "2021-12-31","IT","",3176959,1,1,85,0 + "2021-12-31","KR","",1835848,1,1,0,1 + "2021-12-31","LV","",456172,1,1,0,1 + "2021-12-31","MX","",3530757,2,3,0,3 + "2021-12-31","NL","",0,1,1,0,1 + "2021-12-31","NL","",0,1,2,0,2 + "2021-12-31","NL","",2745321,1,1,0,1 + "2021-12-31","NO","",0,1,1,199,0 + "2021-12-31","SE","",0,1,1,0,1 + "2021-12-31","SG","",1880252,1,1,0,1 + """ + }, + %{ + name: "imported_operating_systems.csv", + body: """ + "date","operating_system","visitors","visits","visit_duration","bounces" + "2021-12-30","Android",25,26,254,24 + "2021-12-30","Mac",13,16,0,16 + "2021-12-30","Windows",12,12,75,11 + "2021-12-30","iOS",30,31,0,31 + "2021-12-31","Android",15,16,329,13 + "2021-12-31","Mac",13,19,0,19 + "2021-12-31","Windows",7,7,496,5 + "2021-12-31","iOS",40,43,1957,40 + "2022-01-01","",17,18,0,18 + "2022-01-01","Android",25,28,32,26 + "2022-01-01","Chrome OS",1,1,0,1 + "2022-01-01","Mac",6,6,0,6 + "2022-01-01","Windows",9,9,1117,7 + "2022-01-01","iOS",38,40,693,35 + "2022-01-02","Android",21,21,1406,18 + "2022-01-02","Chrome OS",2,2,0,2 + "2022-01-02","Mac",18,18,86,16 + "2022-01-02","Windows",8,8,0,8 + "2022-01-02","iOS",54,61,1044,56 + "2022-01-03","Android",26,31,2009,26 + "2022-01-03","Chrome OS",1,1,58,0 + "2022-01-03","GNU/Linux",3,3,0,3 + "2022-01-03","Mac",25,25,1692,20 + "2022-01-03","Windows",8,8,105,7 + "2022-01-03","iOS",45,47,4355,40 + "2022-01-04","Android",21,22,250,20 + "2022-01-04","GNU/Linux",1,1,0,1 + "2022-01-04","Mac",18,18,1330,14 + "2022-01-04","Windows",7,7,0,7 + "2022-01-04","iOS",40,44,724,41 + "2022-01-05","Android",18,19,775,16 + "2022-01-05","Mac",12,12,1125,10 + "2022-01-05","Windows",18,20,37,19 + "2022-01-05","iOS",34,34,468,32 + "2022-01-06","",23,23,0,23 + "2022-01-06","Android",26,27,1793,23 + "2022-01-06","GNU/Linux",1,1,0,1 + "2022-01-06","Mac",16,16,0,16 + "2022-01-06","Windows",13,13,114,11 + "2022-01-06","iOS",35,40,1206,37 + "2022-01-07","Android",21,21,383,19 + "2022-01-07","Chrome OS",1,1,0,1 + "2022-01-07","Mac",6,6,0,6 + "2022-01-07","Windows",8,9,247,8 + "2022-01-07","iOS",27,28,856,25 + "2022-01-08","Android",26,26,163,25 + "2022-01-08","Chrome OS",1,1,0,1 + "2022-01-08","GNU/Linux",2,2,0,2 + "2022-01-08","Mac",17,18,405,15 + "2022-01-08","Windows",7,7,1766,5 + "2022-01-08","iOS",38,42,3769,37 + "2022-01-09","Android",15,16,151,14 + "2022-01-09","GNU/Linux",1,1,0,1 + "2022-01-09","Mac",19,20,554,16 + "2022-01-09","Windows",11,12,861,9 + "2022-01-09","iOS",41,46,301,44 + "2022-01-10","Android",11,13,0,13 + "2022-01-10","GNU/Linux",4,4,0,4 + "2022-01-10","Mac",13,15,0,15 + "2022-01-10","Windows",5,5,0,5 + "2022-01-10","iOS",39,40,160,37 + "2022-01-11","Android",19,19,68,16 + "2022-01-11","GNU/Linux",1,1,12,0 + "2022-01-11","Mac",14,14,700,10 + "2022-01-11","Windows",14,15,1123,13 + "2022-01-11","iOS",30,31,117,29 + "2022-01-12","Android",29,31,1735,26 + "2022-01-12","Chrome OS",1,1,1431,0 + "2022-01-12","GNU/Linux",1,1,0,1 + "2022-01-12","Mac",10,10,0,10 + "2022-01-12","Windows",4,4,112,3 + "2022-01-12","iOS",39,42,396,37 + "2022-01-13","Android",22,22,262,19 + "2022-01-13","Chrome OS",1,1,0,1 + "2022-01-13","Mac",18,19,2845,16 + "2022-01-13","Windows",15,15,155,12 + "2022-01-13","iOS",30,34,0,34 + "2022-01-14","Android",20,25,0,25 + "2022-01-14","Chrome OS",1,1,0,1 + "2022-01-14","GNU/Linux",1,1,0,1 + "2022-01-14","Mac",11,11,0,11 + "2022-01-14","Windows",8,8,5,7 + "2022-01-14","iOS",35,38,178,36 + "2022-01-15","Android",27,28,48,27 + "2022-01-15","Mac",15,15,347,14 + "2022-01-15","Windows",7,7,0,7 + "2022-01-15","iOS",50,52,3816,48 + "2022-01-16","Android",22,24,2292,20 + "2022-01-16","Chrome OS",1,1,0,1 + "2022-01-16","Mac",17,21,737,20 + "2022-01-16","Windows",12,12,92,9 + "2022-01-16","iOS",51,51,1595,45 + "2022-01-17","Android",25,29,1576,25 + "2022-01-17","GNU/Linux",2,2,0,2 + "2022-01-17","Mac",20,21,81,20 + "2022-01-17","Windows",19,19,469,17 + "2022-01-17","iOS",38,40,601,38 + "2022-01-18","Android",17,18,68,17 + "2022-01-18","Chrome OS",1,1,0,1 + "2022-01-18","GNU/Linux",3,3,0,3 + """ + }, + %{ + name: "imported_pages.csv", + body: """ + "date","visitors","pageviews","exits","time_on_page","hostname","page" + "2021-12-30",1,1,0,43,"lucky.numbers.com","/14776416252794997127" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/14776416252794997127" + "2021-12-30",6,6,6,0,"lucky.numbers.com","/14776416252794997127" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/9102354072466236765" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/4457889102355683190" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/12105301321223776356" + "2021-12-30",1,2,2,0,"lucky.numbers.com","/1526239929864936398" + "2021-12-30",3,3,3,0,"lucky.numbers.com","/12574817671425987843" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/18226692671132987727" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/13530392472417087202" + "2021-12-30",3,3,3,0,"lucky.numbers.com","/14629996869565295116" + "2021-12-30",2,2,2,0,"lucky.numbers.com","/16500758973580097263" + "2021-12-30",3,3,3,0,"lucky.numbers.com","/12300842990999856228" + "2021-12-30",2,3,2,75,"lucky.numbers.com","/17859765562822550514" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/1250717791477281255" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/1586391735863371077" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/3457026921000639206" + "2021-12-30",2,3,3,0,"lucky.numbers.com","/6077502147861556415" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/14280570555317344651" + "2021-12-30",3,3,3,0,"lucky.numbers.com","/5284268072698982201" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/7478911940502018071" + "2021-12-30",1,1,1,0,"lucky.numbers.com","/6402607186523575652" + "2021-12-30",2,2,2,0,"lucky.numbers.com","/9962503789684934900" + "2021-12-30",8,10,10,0,"lucky.numbers.com","/13595620304963848161" + "2021-12-30",2,2,2,0,"lucky.numbers.com","/17019199732013993436" + "2021-12-30",32,33,32,211,"lucky.numbers.com","/9874837495456455794" + "2021-12-31",4,4,4,0,"lucky.numbers.com","/14776416252794997127" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/8738789417178304429" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/7445073500314667742" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/4897404798407749335" + "2021-12-31",1,2,1,29,"lucky.numbers.com","/11263893625781431659" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/16478773157730928089" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/1710995203264225236" + "2021-12-31",2,3,3,0,"lucky.numbers.com","/12574817671425987843" + "2021-12-31",1,2,2,0,"lucky.numbers.com","/4072002299714740082" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/13544703054662457518" + "2021-12-31",1,1,0,16,"lucky.numbers.com","/11454211114661615085" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/3363270934034278688" + "2021-12-31",4,4,4,0,"lucky.numbers.com","/14629996869565295116" + "2021-12-31",1,1,0,60,"lucky.numbers.com","/11880955414137016346" + "2021-12-31",2,4,2,224,"lucky.numbers.com","/5918158559063394909" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/16500758973580097263" + "2021-12-31",1,1,0,72,"lucky.numbers.com","/13683424886884591498" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/15335700069940448722" + "2021-12-31",7,7,7,0,"lucky.numbers.com","/1250717791477281255" + "2021-12-31",3,3,3,0,"lucky.numbers.com","/3457026921000639206" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/6077502147861556415" + "2021-12-31",1,1,1,0,"lucky.numbers.com","/14280570555317344651" + "2021-12-31",4,6,5,444,"lucky.numbers.com","/5284268072698982201" + "2021-12-31",2,3,2,394,"lucky.numbers.com","/7478911940502018071" + "2021-12-31",9,17,16,1455,"lucky.numbers.com","/13595620304963848161" + "2021-12-31",25,27,25,88,"lucky.numbers.com","/9874837495456455794" + "2022-01-01",2,3,3,0,"lucky.numbers.com","/14776416252794997127" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/18394065791342797546" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/9102354072466236765" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/2528868181315148597" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/4457889102355683190" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/12105301321223776356" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/6318438003888425693" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/5878724061840196349" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/1526239929864936398" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/7692634448754428624" + "2022-01-01",4,4,4,0,"lucky.numbers.com","/12574817671425987843" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/14307781859600070983" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/7110820102771013606" + "2022-01-01",3,3,3,0,"lucky.numbers.com","/14629996869565295116" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/64779230489549655" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/5551134200879446973" + "2022-01-01",1,2,0,189,"lucky.numbers.com","/8000643558843134787" + "2022-01-01",1,2,1,25,"lucky.numbers.com","/11888590162960019765" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/12567804068906971541" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/14033373606203782378" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/10341536794299767967" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/12766075726240916242" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/4362292817884281301" + "2022-01-01",1,2,2,0,"lucky.numbers.com","/1250717791477281255" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/3457026921000639206" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/6077502147861556415" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/18342609412020394218" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/12189381298883635575" + "2022-01-01",2,2,2,0,"lucky.numbers.com","/14280570555317344651" + "2022-01-01",5,12,6,246,"lucky.numbers.com","/5284268072698982201" + "2022-01-01",4,4,4,0,"lucky.numbers.com","/7478911940502018071" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/6402607186523575652" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/8587605562711759914" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/9962503789684934900" + "2022-01-01",7,10,8,187,"lucky.numbers.com","/13595620304963848161" + "2022-01-01",1,1,1,0,"lucky.numbers.com","/17019199732013993436" + "2022-01-01",32,37,33,1195,"lucky.numbers.com","/9874837495456455794" + "2022-01-02",4,5,4,81,"lucky.numbers.com","/14776416252794997127" + "2022-01-02",3,6,5,33,"lucky.numbers.com","/9102354072466236765" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/17912324809189526683" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/2528868181315148597" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/4457889102355683190" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/4897404798407749335" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/5290432771315151230" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/3861893315241731546" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/12571626520250599954" + "2022-01-02",11,12,10,660,"lucky.numbers.com","/12574817671425987843" + "2022-01-02",1,1,1,0,"lucky.numbers.com","/15302856869047719385" + """ + }, + %{ + name: "imported_sources.csv", + body: """ + "date","source","utm_medium","utm_campaign","utm_content","utm_term","visitors","visits","visit_duration","bounces" + "2021-12-30","","","","","",25,26,254,24 + "2021-12-30","Hacker News","referral","","","",2,2,0,2 + "2021-12-30","Google","organic","","","",20,22,75,21 + "2021-12-30","Pinterest","referral","","","",25,26,0,26 + "2021-12-30","baidu","organic","","","",1,1,0,1 + "2021-12-30","yahoo","organic","","","",3,3,0,3 + "2021-12-31","","","","","",16,16,199,15 + "2021-12-31","Bing","organic","","","",1,1,0,1 + "2021-12-31","DuckDuckGo","organic","","","",1,1,0,1 + "2021-12-31","Hacker News","referral","","","",1,1,466,0 + "2021-12-31","Google","organic","","","",25,32,85,31 + "2021-12-31","Pinterest","referral","","","",22,24,88,22 + "2021-12-31","yahoo","organic","","","",3,3,1899,1 + "2022-01-01","","","","","",37,38,1137,35 + "2022-01-01","Bing","organic","","","",2,2,171,1 + "2022-01-01","DuckDuckGo","organic","","","",2,3,0,3 + "2022-01-01","Hacker News","referral","","","",1,1,0,1 + "2022-01-01","Google","referral","","","",1,1,0,1 + "2022-01-01","Google","organic","","","",21,23,115,19 + "2022-01-01","Pinterest","referral","","","",29,30,0,30 + "2022-01-01","yahoo","organic","","","",3,3,419,2 + "2022-01-06","","","","","",37,38,430,36 + "2022-01-06","Bing","organic","","","how lucky am I as UInt64",1,1,0,1 + "2022-01-06","Bing","organic","","","",3,3,10,2 + """ + }, + %{ + name: "imported_visitors.csv", + body: """ + "date","visitors","pageviews","bounces","visits","visit_duration" + "2011-12-25",5,50,2,7,8640 + "2011-12-26",3,4,2,3,43 + "2011-12-27",3,6,2,4,2313 + "2011-12-28",6,30,4,8,2264 + "2011-12-29",4,8,5,6,136 + "2011-12-30",1,1,1,1,0 + "2012-01-01",3,15,0,3,1593 + "2012-01-02",1,1,1,1,0 + "2012-01-03",2,2,2,2,0 + "2012-01-04",5,12,2,5,1127 + "2012-01-05",2,2,2,2,0 + "2012-01-06",2,2,2,2,0 + "2012-01-07",3,7,2,3,80 + "2012-01-08",1,1,1,1,0 + "2012-01-13",2,2,2,2,0 + "2012-01-14",1,1,1,1,0 + "2012-01-15",2,6,1,2,82 + "2012-01-16",1,1,1,1,0 + "2012-01-17",1,1,1,1,0 + "2012-01-25",1,11,0,1,91 + "2012-01-26",3,9,1,3,146 + "2012-01-29",1,1,1,1,0 + "2012-02-03",1,2,0,1,6 + "2012-02-07",1,1,1,1,0 + "2012-02-27",1,1,1,1,0 + "2012-03-02",1,3,0,1,70 + "2012-03-18",1,1,1,1,0 + "2012-03-19",1,2,2,2,0 + "2012-03-21",1,2,0,1,22 + "2012-03-22",1,2,0,1,48 + "2012-03-28",1,3,0,1,208 + "2012-03-29",4,18,4,8,154 + "2012-04-15",1,1,1,1,0 + "2012-04-16",1,1,1,1,0 + "2012-04-24",1,1,1,1,0 + "2012-05-25",1,1,1,1,0 + "2012-05-31",1,1,1,1,0 + "2012-06-12",2,2,2,2,0 + "2012-07-01",1,2,0,1,6 + "2012-07-02",1,1,1,1,0 + "2012-07-19",1,1,1,1,0 + "2012-07-22",1,13,0,1,344 + "2012-07-23",5,26,2,7,936 + "2012-07-24",4,69,2,7,12746 + "2012-07-25",10,118,7,15,7815 + "2012-07-26",5,20,4,8,2172 + "2012-07-27",15,55,10,16,6712 + "2012-07-28",3,18,1,3,931 + "2012-07-29",3,3,2,3,6 + "2012-07-30",4,5,2,4,246 + "2012-07-31",6,40,2,7,3146 + "2012-08-01",3,52,2,8,5509 + "2012-08-02",77,127,68,78,3426 + "2012-08-03",28,38,24,29,1808 + "2012-08-04",11,38,9,15,5482 + "2012-08-05",14,46,12,17,3928 + "2012-08-06",11,58,5,14,9140 + "2012-08-07",17,112,13,21,12435 + "2012-08-08",31,53,25,34,2871 + "2012-08-09",14,23,7,15,265 + "2012-08-10",5,8,3,5,468 + "2012-08-11",4,13,2,6,1618 + "2012-08-12",4,5,3,4,4 + "2012-08-13",3,3,3,3,0 + "2012-08-14",3,3,3,3,0 + "2012-08-15",3,3,2,3,127 + "2012-08-16",5,8,3,5,231 + "2012-08-17",16,31,11,18,36929 + "2012-08-18",2,20,2,3,2121 + "2012-08-19",15,63,9,16,6688 + "2012-08-20",8,29,3,9,1884 + "2012-08-21",16,61,11,20,8529 + "2012-08-22",142,199,127,148,7577 + "2012-08-23",69,95,57,71,1339 + "2012-08-24",22,139,13,30,15027 + "2012-08-25",12,76,8,15,8813 + "2012-08-26",19,66,17,24,7716 + "2012-08-27",18,26,15,19,1968 + "2012-08-28",17,80,12,22,9624 + "2012-08-29",49,76,38,50,4581 + "2012-08-30",16,33,13,20,2441 + "2012-08-31",6,22,3,6,4429 + "2012-09-01",22,75,13,26,10118 + "2012-09-02",114,192,98,119,15659 + "2012-09-03",38,46,28,40,1208 + "2012-09-04",39,56,33,41,1686 + "2012-09-05",32,50,29,36,1000 + "2012-09-06",51,112,45,58,12959 + "2012-09-07",34,42,23,34,563 + "2012-09-08",110,189,97,111,12067 + "2012-09-09",69,81,62,70,1259 + "2012-09-10",18,37,14,21,1151 + "2012-09-11",27,45,20,32,3352 + "2012-09-12",17,38,11,18,3608 + "2012-09-13",24,68,14,27,11067 + "2012-09-14",15,28,10,17,977 + "2012-09-15",10,11,8,10,78 + "2012-09-16",7,7,7,7,0 + "2012-09-17",47,89,39,51,9309 + "2012-09-18",19,48,12,22,4102 + """ + } + ] + + uploads = + for %{name: name, body: body} <- csvs do + key = "#{site.id}/#{name}" + ExAws.request!(ExAws.S3.put_object("imports", key, body, content_type: "text/csv")) + %{"filename" => name, "s3_path" => key} + end + + {:ok, job} = + CSVImporter.new_import( + site, + user, + # to satisfy the non null constraints on the table I'm providing "0" dates (according to ClickHouse) + start_date: ~D[1970-01-01], + end_date: ~D[1970-01-01], + uploads: uploads + ) + + job = Repo.reload!(job) + + assert :ok = Plausible.Workers.ImportAnalytics.perform(job) + + # on successfull import the start and end dates are updated + assert %SiteImport{ + start_date: ~D[2011-12-25], + end_date: ~D[2022-02-01], + source: :csv, + status: :completed + } = Repo.get_by!(SiteImport, site_id: site.id) + + assert 3406 == Plausible.Stats.Clickhouse.imported_pageview_count(site) + end + test "invalid CSV" end end From ab9820156d0a41a539008c5189f3fb083e3ba796 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:14:59 +0800 Subject: [PATCH 10/17] add failing test --- lib/plausible/imported/csv_importer.ex | 4 + test/plausible/imported/csv_importer_test.exs | 725 ++---------------- 2 files changed, 70 insertions(+), 659 deletions(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index b23c4f98e04a..b0bd88ec3540 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -77,6 +77,10 @@ defmodule Plausible.Imported.CSVImporter do start_date: Enum.min_by(ranges, & &1.first, Date).first, end_date: Enum.max_by(ranges, & &1.last, Date).last }} + rescue + # we are cancelling on any ArgumentError or ClickHouse errors + e in [ArgumentError, Ch.Error] -> + {:error, Exception.message(e)} end input_structures = %{ diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index 14136a9280e5..7cd06174ad8c 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -90,87 +90,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-31","Mobile App",1,1,85,0 "2021-12-31","Safari",37,45,1957,42 "2021-12-31","Samsung Internet",1,1,199,0 - "2022-01-01","'DuckDuckBot-Https",17,18,0,18 - "2022-01-01","Chrome",30,33,1149,29 - "2022-01-01","Edge",1,1,0,1 - "2022-01-01","Firefox",1,1,0,1 - "2022-01-01","Mobile App",1,1,0,1 - "2022-01-01","Mobile App",5,5,0,5 - "2022-01-01","Mobile App",2,2,0,2 - "2022-01-01","Opera",1,1,0,1 - "2022-01-01","Safari",36,38,693,33 - "2022-01-01","Samsung Internet",2,2,0,2 - "2022-01-02","Chrome",38,38,1411,34 - "2022-01-02","Firefox",2,2,0,2 - "2022-01-02","Mobile App",3,3,36,2 - "2022-01-02","Mobile App",9,9,0,9 - "2022-01-02","Mobile App",1,1,0,1 - "2022-01-02","Safari",50,57,1089,52 - "2022-01-03","Chrome",35,40,2067,34 - "2022-01-03","Edge",3,3,0,3 - "2022-01-03","Firefox",1,1,105,0 - "2022-01-03","Mobile App",3,3,124,2 - "2022-01-03","Mobile App",10,10,137,9 - "2022-01-03","Mobile App",1,1,0,1 - "2022-01-03","Safari",53,55,5786,45 - "2022-01-03","Samsung Internet",2,2,0,2 - "2022-01-04","Amazon Silk",1,1,0,1 - "2022-01-04","Chrome",22,23,250,21 - "2022-01-04","Edge",3,3,0,3 - "2022-01-04","Firefox",1,1,0,1 - "2022-01-04","Mobile App",1,1,0,1 - "2022-01-04","Mobile App",12,12,248,9 - "2022-01-04","Mobile App",1,1,0,1 - "2022-01-04","Mobile App",1,1,0,1 - "2022-01-04","Opera",1,1,0,1 - "2022-01-04","Safari",43,47,1806,43 - "2022-01-04","Samsung Internet",1,1,0,1 - "2022-01-05","Chrome",36,39,1133,34 - "2022-01-05","Edge",3,3,0,3 - "2022-01-05","Firefox",2,2,0,2 - "2022-01-05","Mobile App",1,1,147,0 - "2022-01-05","Mobile App",1,1,0,1 - "2022-01-05","Safari",38,38,1125,36 - "2022-01-05","Samsung Internet",1,1,0,1 - "2022-01-06","'DuckDuckBot-Https",18,18,0,18 - "2022-01-06","Chrome",36,37,1897,32 - "2022-01-06","Edge",2,2,10,1 - "2022-01-06","Firefox",1,1,0,1 - "2022-01-06","Internet Explorer",1,1,0,1 - "2022-01-06","Mobile App",1,1,0,1 - "2022-01-06","Mobile App",8,8,0,8 - "2022-01-06","Mobile App",2,2,0,2 - "2022-01-06","Safari",40,45,1206,42 - "2022-01-06","iubenda-radar",5,5,0,5 - "2022-01-07","Chrome",25,26,630,23 - "2022-01-07","Edge",1,1,0,1 - "2022-01-07","Firefox",1,1,0,1 - "2022-01-07","Safari",32,33,856,30 - "2022-01-07","Samsung Internet",3,3,0,3 - "2022-01-07","YaBrowser",1,1,0,1 - "2022-01-08","Chrome",33,33,229,31 - "2022-01-08","Edge",3,3,1700,2 - "2022-01-08","Firefox",1,1,94,0 - "2022-01-08","Mobile App",1,1,0,1 - "2022-01-08","Mobile App",4,4,10,3 - "2022-01-08","Mobile App",2,2,0,2 - "2022-01-08","Opera",1,1,0,1 - "2022-01-08","Safari",47,51,4070,45 - "2022-01-09","Amazon Silk",1,1,0,1 - "2022-01-09","Chrome",23,25,1012,20 - "2022-01-09","Edge",4,4,0,4 - "2022-01-09","Mobile App",3,4,16,3 - "2022-01-09","Mobile App",11,12,203,9 - "2022-01-09","Mobile App",1,1,0,1 - "2022-01-09","Safari",42,46,636,44 - "2022-01-09","Samsung Internet",2,2,0,2 - "2022-01-10","Amazon Silk",1,1,0,1 - "2022-01-10","Chrome",18,19,0,19 - "2022-01-10","Edge",1,1,0,1 - "2022-01-10","Firefox",2,2,0,2 - "2022-01-10","Mobile App",1,1,0,1 - "2022-01-10","Mobile App",4,4,0,4 - "2022-01-10","Safari",41,44,160,41 """ }, %{ @@ -189,94 +108,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2022-01-02","Desktop",28,28,86,26 "2022-01-02","Mobile",66,73,2450,65 "2022-01-02","Tablet",9,9,0,9 - "2022-01-03","Desktop",37,37,1855,30 - "2022-01-03","Mobile",64,68,6364,56 - "2022-01-03","Tablet",7,10,0,10 - "2022-01-04","Desktop",26,26,1330,22 - "2022-01-04","Mobile",58,63,954,59 - "2022-01-04","Tablet",3,3,20,2 - "2022-01-05","Desktop",30,32,1162,29 - "2022-01-05","Mobile",45,46,1084,43 - "2022-01-05","Tablet",7,7,159,5 - "2022-01-06","Desktop",53,53,114,51 - "2022-01-06","Mobile",55,61,2999,54 - "2022-01-06","Tablet",6,6,0,6 - "2022-01-07","Desktop",15,16,247,15 - "2022-01-07","Mobile",47,48,1239,43 - "2022-01-07","Tablet",1,1,0,1 - "2022-01-08","Desktop",27,28,2171,23 - "2022-01-08","Mobile",59,63,3932,57 - "2022-01-08","Tablet",5,5,0,5 - "2022-01-09","Desktop",31,33,1415,26 - "2022-01-09","Mobile",48,54,167,51 - "2022-01-09","Tablet",8,8,285,7 - "2022-01-10","Desktop",22,24,0,24 - "2022-01-10","Mobile",46,49,160,46 - "2022-01-10","Tablet",4,4,0,4 - "2022-01-11","Desktop",29,30,1835,23 - "2022-01-11","Mobile",39,40,185,35 - "2022-01-11","Tablet",10,10,0,10 - "2022-01-12","Desktop",16,16,1543,14 - "2022-01-12","Mobile",61,66,2113,57 - "2022-01-12","Tablet",7,7,18,6 - "2022-01-13","Desktop",34,35,3000,29 - "2022-01-13","Mobile",47,51,262,48 - "2022-01-13","Tablet",5,5,0,5 - "2022-01-14","Desktop",21,21,5,20 - "2022-01-14","Mobile",52,60,178,58 - "2022-01-14","Tablet",3,3,0,3 - "2022-01-15","Desktop",22,22,347,21 - "2022-01-15","Mobile",73,76,3633,72 - "2022-01-15","Tablet",4,4,231,3 - "2022-01-16","Desktop",30,34,829,30 - "2022-01-16","Mobile",64,65,2438,56 - "2022-01-16","Tablet",9,10,1449,9 - "2022-01-17","Desktop",41,42,550,39 - "2022-01-17","Mobile",59,65,2177,59 - "2022-01-17","Tablet",4,4,0,4 - "2022-01-18","Desktop",18,21,176,20 - "2022-01-18","Mobile",46,49,3004,42 - "2022-01-18","Tablet",4,4,117,3 - "2022-01-19","Desktop",15,15,0,15 - "2022-01-19","Mobile",49,53,3145,44 - "2022-01-19","Tablet",3,3,0,3 - "2022-01-20","Desktop",25,25,2734,20 - "2022-01-20","Mobile",41,45,1585,43 - "2022-01-20","Tablet",3,3,0,3 - "2022-01-21","Desktop",43,44,3041,39 - "2022-01-21","Mobile",47,52,2183,46 - "2022-01-21","Tablet",4,4,0,4 - "2022-01-22","Desktop",24,25,682,19 - "2022-01-22","Mobile",63,67,3209,55 - "2022-01-22","Tablet",3,3,591,2 - "2022-01-23","Desktop",20,20,594,18 - "2022-01-23","Mobile",64,70,2059,65 - "2022-01-23","Tablet",2,2,0,2 - "2022-01-24","Desktop",27,27,2004,25 - "2022-01-24","Mobile",58,62,659,56 - "2022-01-24","Tablet",5,5,0,5 - "2022-01-25","Desktop",32,37,280,34 - "2022-01-25","Mobile",34,36,467,30 - "2022-01-25","Tablet",4,4,0,4 - "2022-01-26","Desktop",14,15,0,15 - "2022-01-26","Mobile",57,61,4114,55 - "2022-01-26","Tablet",3,3,0,3 - "2022-01-27","Desktop",38,39,692,37 - "2022-01-27","Mobile",52,56,1237,49 - "2022-01-27","Tablet",2,2,0,2 - "2022-01-28","Desktop",18,20,161,18 - "2022-01-28","Mobile",56,62,0,62 - "2022-01-28","Tablet",2,3,0,3 - "2022-01-29","Desktop",32,32,946,29 - "2022-01-29","Mobile",66,69,2403,63 - "2022-01-29","Tablet",2,2,0,2 - "2022-01-30","Desktop",30,31,1189,28 - "2022-01-30","Mobile",56,59,2253,54 - "2022-01-30","Tablet",5,5,283,4 - "2022-01-31","Desktop",31,32,0,32 - "2022-01-31","Mobile",46,52,1515,46 - "2022-01-31","Tablet",2,2,382,1 - "2022-02-01","Desktop",26,26,211,23 """ }, %{ @@ -287,17 +118,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-30",1,1,0,1,"/15455127321321119046" "2021-12-30",1,1,43,0,"/10399835914295020763" "2021-12-30",1,1,0,1,"/9102354072466236765" - "2021-12-30",1,1,0,1,"/4457889102355683190" - "2021-12-30",1,1,0,1,"/12105301321223776356" - "2021-12-30",1,2,0,2,"/1526239929864936398" - "2021-12-30",3,3,0,3,"/12574817671425987843" - "2021-12-30",1,1,0,1,"/18226692671132987727" - "2021-12-30",1,1,0,1,"/13530392472417087202" - "2021-12-30",3,3,0,3,"/14629996869565295116" - "2021-12-30",2,2,0,2,"/16500758973580097263" - "2021-12-30",3,3,0,3,"/12300842990999856228" - "2021-12-30",2,2,75,1,"/17859765562822550514" - "2021-12-30",1,1,0,1,"/1250717791477281255" "2021-12-30",1,1,0,1,"/1586391735863371077" "2021-12-30",1,1,0,1,"/3457026921000639206" "2021-12-30",2,3,0,3,"/6077502147861556415" @@ -316,73 +136,11 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-31",1,1,45,0,"/11263893625781431659" "2021-12-31",1,1,0,1,"/16478773157730928089" "2021-12-31",1,1,0,1,"/1710995203264225236" - "2021-12-31",2,3,0,3,"/12574817671425987843" - "2021-12-31",1,2,0,2,"/4072002299714740082" - "2021-12-31",1,1,0,1,"/13544703054662457518" - "2021-12-31",1,1,0,1,"/3363270934034278688" - "2021-12-31",4,4,0,4,"/14629996869565295116" - "2021-12-31",2,2,284,0,"/5918158559063394909" - "2021-12-31",1,1,0,1,"/16500758973580097263" - "2021-12-31",1,1,0,1,"/15335700069940448722" - "2021-12-31",7,7,0,7,"/1250717791477281255" - "2021-12-31",3,3,0,3,"/3457026921000639206" - "2021-12-31",1,1,0,1,"/6077502147861556415" "2021-12-31",1,1,0,1,"/14280570555317344651" "2021-12-31",4,5,444,4,"/5284268072698982201" "2021-12-31",2,2,466,1,"/7478911940502018071" "2021-12-31",9,16,1455,15,"/13595620304963848161" "2021-12-31",25,25,88,23,"/9874837495456455794" - "2022-01-01",2,3,0,3,"/14776416252794997127" - "2022-01-01",1,1,0,1,"/18394065791342797546" - "2022-01-01",2,2,0,2,"/9102354072466236765" - "2022-01-01",2,2,0,2,"/2528868181315148597" - "2022-01-01",1,1,0,1,"/4457889102355683190" - "2022-01-01",1,1,0,1,"/12105301321223776356" - "2022-01-01",2,2,0,2,"/6318438003888425693" - "2022-01-01",2,2,0,2,"/5878724061840196349" - "2022-01-01",1,1,0,1,"/1526239929864936398" - "2022-01-01",2,2,0,2,"/7692634448754428624" - "2022-01-01",4,4,0,4,"/12574817671425987843" - "2022-01-01",1,1,0,1,"/14307781859600070983" - "2022-01-01",2,2,0,2,"/7110820102771013606" - "2022-01-01",3,3,0,3,"/14629996869565295116" - "2022-01-01",2,2,0,2,"/64779230489549655" - "2022-01-01",1,1,0,1,"/5551134200879446973" - "2022-01-01",1,1,25,0,"/11888590162960019765" - "2022-01-01",1,1,0,1,"/12567804068906971541" - "2022-01-01",1,1,0,1,"/14033373606203782378" - "2022-01-01",2,2,0,2,"/10341536794299767967" - "2022-01-01",1,1,0,1,"/12766075726240916242" - "2022-01-01",1,1,0,1,"/4362292817884281301" - "2022-01-01",1,2,0,2,"/1250717791477281255" - "2022-01-01",2,2,0,2,"/3457026921000639206" - "2022-01-01",1,1,0,1,"/6077502147861556415" - "2022-01-01",2,2,0,2,"/18342609412020394218" - "2022-01-01",1,1,0,1,"/12189381298883635575" - "2022-01-01",2,2,0,2,"/14280570555317344651" - "2022-01-01",5,6,435,4,"/5284268072698982201" - "2022-01-01",4,4,0,4,"/7478911940502018071" - "2022-01-01",1,1,0,1,"/6402607186523575652" - "2022-01-01",1,1,0,1,"/8587605562711759914" - "2022-01-01",1,1,0,1,"/9962503789684934900" - "2022-01-01",7,8,187,6,"/13595620304963848161" - "2022-01-01",1,1,0,1,"/17019199732013993436" - "2022-01-01",32,33,1195,29,"/9874837495456455794" - "2022-01-02",3,3,81,2,"/14776416252794997127" - "2022-01-02",3,6,33,5,"/9102354072466236765" - "2022-01-02",1,1,0,1,"/17912324809189526683" - "2022-01-02",1,1,0,1,"/2528868181315148597" - "2022-01-02",1,1,0,1,"/4457889102355683190" - "2022-01-02",1,1,0,1,"/4897404798407749335" - "2022-01-02",1,1,0,1,"/5290432771315151230" - "2022-01-02",1,1,0,1,"/3861893315241731546" - "2022-01-02",1,1,0,1,"/12571626520250599954" - "2022-01-02",11,11,832,10,"/12574817671425987843" - "2022-01-02",2,2,0,2,"/15791285625100101971" - "2022-01-02",2,2,0,2,"/16602440950175620202" - "2022-01-02",1,1,0,1,"/3363270934034278688" - "2022-01-02",6,6,122,5,"/14629996869565295116" - "2022-01-02",1,2,0,2,"/17952015439202551610" """ }, %{ @@ -395,19 +153,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-30",1,1,"/4457889102355683190" "2021-12-30",1,1,"/12105301321223776356" "2021-12-30",1,2,"/1526239929864936398" - "2021-12-30",3,3,"/12574817671425987843" - "2021-12-30",1,1,"/18226692671132987727" - "2021-12-30",1,1,"/13530392472417087202" - "2021-12-30",3,3,"/14629996869565295116" - "2021-12-30",2,2,"/16500758973580097263" - "2021-12-30",3,3,"/12300842990999856228" - "2021-12-30",2,2,"/17859765562822550514" - "2021-12-30",1,1,"/1250717791477281255" - "2021-12-30",1,1,"/1586391735863371077" - "2021-12-30",1,1,"/3457026921000639206" - "2021-12-30",2,3,"/6077502147861556415" - "2021-12-30",1,1,"/14280570555317344651" - "2021-12-30",3,3,"/5284268072698982201" "2021-12-30",1,1,"/7478911940502018071" "2021-12-30",1,1,"/6402607186523575652" "2021-12-30",2,2,"/9962503789684934900" @@ -421,74 +166,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-31",1,1,"/11263893625781431659" "2021-12-31",1,1,"/16478773157730928089" "2021-12-31",1,1,"/1710995203264225236" - "2021-12-31",2,3,"/12574817671425987843" - "2021-12-31",1,2,"/4072002299714740082" - "2021-12-31",1,1,"/13544703054662457518" - "2021-12-31",1,1,"/3363270934034278688" - "2021-12-31",4,4,"/14629996869565295116" - "2021-12-31",2,2,"/5918158559063394909" - "2021-12-31",1,1,"/16500758973580097263" - "2021-12-31",1,1,"/15335700069940448722" - "2021-12-31",7,7,"/1250717791477281255" - "2021-12-31",3,3,"/3457026921000639206" - "2021-12-31",1,1,"/6077502147861556415" - "2021-12-31",1,1,"/14280570555317344651" - "2021-12-31",4,5,"/5284268072698982201" - "2021-12-31",2,2,"/7478911940502018071" - "2021-12-31",9,16,"/13595620304963848161" - "2021-12-31",25,25,"/9874837495456455794" - "2022-01-01",2,3,"/14776416252794997127" - "2022-01-01",1,1,"/18394065791342797546" - "2022-01-01",2,2,"/9102354072466236765" - "2022-01-01",2,2,"/2528868181315148597" - "2022-01-01",1,1,"/4457889102355683190" - "2022-01-01",1,1,"/12105301321223776356" - "2022-01-01",2,2,"/6318438003888425693" - "2022-01-01",2,2,"/5878724061840196349" - "2022-01-01",1,1,"/1526239929864936398" - "2022-01-01",2,2,"/7692634448754428624" - "2022-01-01",4,4,"/12574817671425987843" - "2022-01-01",1,1,"/14307781859600070983" - "2022-01-01",2,2,"/7110820102771013606" - "2022-01-01",3,3,"/14629996869565295116" - "2022-01-01",2,2,"/64779230489549655" - "2022-01-01",1,1,"/5551134200879446973" - "2022-01-01",1,1,"/11888590162960019765" - "2022-01-01",1,1,"/12567804068906971541" - "2022-01-01",1,1,"/14033373606203782378" - "2022-01-01",2,2,"/10341536794299767967" - "2022-01-01",1,1,"/12766075726240916242" - "2022-01-01",1,1,"/4362292817884281301" - "2022-01-01",1,2,"/1250717791477281255" - "2022-01-01",2,2,"/3457026921000639206" - "2022-01-01",1,1,"/6077502147861556415" - "2022-01-01",2,2,"/18342609412020394218" - "2022-01-01",1,1,"/12189381298883635575" - "2022-01-01",2,2,"/14280570555317344651" - "2022-01-01",5,6,"/5284268072698982201" - "2022-01-01",4,4,"/7478911940502018071" - "2022-01-01",1,1,"/6402607186523575652" - "2022-01-01",1,1,"/8587605562711759914" - "2022-01-01",1,1,"/9962503789684934900" - "2022-01-01",7,8,"/13595620304963848161" - "2022-01-01",1,1,"/17019199732013993436" - "2022-01-01",32,33,"/9874837495456455794" - "2022-01-02",4,4,"/14776416252794997127" - "2022-01-02",3,5,"/9102354072466236765" - "2022-01-02",1,1,"/17912324809189526683" - "2022-01-02",1,1,"/2528868181315148597" - "2022-01-02",1,1,"/4457889102355683190" - "2022-01-02",1,1,"/4897404798407749335" - "2022-01-02",1,1,"/5290432771315151230" - "2022-01-02",1,1,"/3861893315241731546" - "2022-01-02",1,1,"/12571626520250599954" - "2022-01-02",10,10,"/12574817671425987843" - "2022-01-02",1,1,"/15302856869047719385" - "2022-01-02",2,2,"/15791285625100101971" - "2022-01-02",2,2,"/16602440950175620202" - "2022-01-02",1,1,"/3363270934034278688" - "2022-01-02",6,6,"/14629996869565295116" - "2022-01-02",1,2,"/17952015439202551610" """ }, %{ @@ -504,28 +181,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-30","BE","",2792196,1,1,0,1 "2021-12-30","BR","",0,1,1,0,1 "2021-12-30","CA","",0,1,1,0,1 - "2021-12-30","CA","",5907364,1,1,0,1 - "2021-12-30","CA","",5918118,1,1,0,1 - "2021-12-30","CA","",5946768,1,1,0,1 - "2021-12-30","CA","",6066513,2,2,0,2 - "2021-12-30","CA","",6122091,1,1,0,1 - "2021-12-30","CA","",6141256,1,1,0,1 - "2021-12-30","CA","",6167865,1,1,0,1 - "2021-12-30","CN","",1784658,1,1,0,1 - "2021-12-30","CN","",1796236,1,1,0,1 - "2021-12-30","DE","",2865716,1,1,0,1 - "2021-12-30","DE","",2874225,1,1,0,1 - "2021-12-30","DE","",2950159,1,1,0,1 - "2021-12-30","DO","",3492908,2,2,0,2 - "2021-12-30","FR","",2996944,1,1,0,1 - "2021-12-30","GB","",2641523,1,1,0,1 - "2021-12-30","GB","",2643179,1,1,0,1 - "2021-12-30","GB","",2643743,1,1,0,1 - "2021-12-30","GB","",2644688,1,1,0,1 - "2021-12-30","GB","",2646504,1,1,0,1 - "2021-12-30","GB","",2653822,1,1,0,1 - "2021-12-30","IT","",3173435,1,1,0,1 - "2021-12-30","NL","",2747373,2,2,75,1 "2021-12-30","PL","",0,1,1,0,1 "2021-12-30","PL","",756135,1,1,0,1 "2021-12-30","US","",0,1,1,0,1 @@ -534,57 +189,10 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-30","US","",0,1,1,0,1 "2021-12-30","US","",4063926,1,1,0,1 "2021-12-30","US","",4074013,1,3,0,3 - "2021-12-30","US","",4159077,1,1,0,1 - "2021-12-30","US","",4170688,1,1,0,1 - "2021-12-30","US","",4180183,1,1,0,1 - "2021-12-30","US","",4193699,1,1,0,1 - "2021-12-30","US","",4212992,1,1,0,1 - "2021-12-30","US","",4234436,1,1,0,1 - "2021-12-30","US","",4255836,1,1,0,1 - "2021-12-30","US","",4373238,1,1,0,1 - "2021-12-30","US","",4374798,1,1,0,1 - "2021-12-30","US","",4390705,1,1,0,1 - "2021-12-30","US","",4532846,1,1,0,1 - "2021-12-30","US","",4682665,1,1,0,1 - "2021-12-30","US","",4685987,1,1,0,1 - "2021-12-30","US","",4726311,1,1,0,1 - "2021-12-30","US","",4786619,1,1,0,1 - "2021-12-30","US","",4898015,1,1,0,1 - "2021-12-30","US","",4984247,1,1,0,1 - "2021-12-30","US","",5028537,2,2,0,2 - "2021-12-30","US","",5079991,1,1,0,1 "2021-12-30","US","",5089478,1,1,0,1 - "2021-12-30","US","",5118743,1,1,0,1 - "2021-12-30","US","",5178713,1,1,0,1 - "2021-12-30","US","",5337561,1,1,0,1 - "2021-12-30","US","",5368361,1,1,0,1 - "2021-12-30","US","",5393049,1,1,0,1 - "2021-12-30","US","",5809844,2,3,0,3 - "2021-12-30","US","",5814916,1,2,0,2 - "2021-12-30","ZA","",0,1,1,0,1 - "2021-12-30","ZA","",0,1,1,0,1 - "2021-12-30","ZA","",936374,1,1,0,1 - "2021-12-30","ZA","",967106,1,1,0,1 - "2021-12-30","ZA","",1105777,1,1,0,1 - "2021-12-31","AU","",2078025,1,1,0,1 "2021-12-31","AU","",2147714,3,3,0,3 "2021-12-31","AU","",2158177,2,2,0,2 "2021-12-31","CA","",0,1,1,0,1 - "2021-12-31","CA","",5907364,2,2,0,2 - "2021-12-31","CA","",5937615,1,1,0,1 - "2021-12-31","CA","",5990579,1,1,0,1 - "2021-12-31","CA","",6050610,1,1,0,1 - "2021-12-31","CA","",6087892,1,2,0,2 - "2021-12-31","CA","",6167865,3,3,0,3 - "2021-12-31","FR","",2988507,1,1,0,1 - "2021-12-31","GB","",0,1,1,0,1 - "2021-12-31","GB","",2640194,1,2,0,2 - "2021-12-31","GB","",2643743,2,2,0,2 - "2021-12-31","GB","",2644688,1,1,0,1 - "2021-12-31","GB","",2646274,1,1,0,1 - "2021-12-31","GB","",2654675,1,1,45,0 - "2021-12-31","GB","",2657562,1,1,0,1 - "2021-12-31","IE","",2964574,1,1,0,1 "2021-12-31","IT","",3176959,1,1,85,0 "2021-12-31","KR","",1835848,1,1,0,1 "2021-12-31","LV","",456172,1,1,0,1 @@ -615,92 +223,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2022-01-01","Mac",6,6,0,6 "2022-01-01","Windows",9,9,1117,7 "2022-01-01","iOS",38,40,693,35 - "2022-01-02","Android",21,21,1406,18 - "2022-01-02","Chrome OS",2,2,0,2 - "2022-01-02","Mac",18,18,86,16 - "2022-01-02","Windows",8,8,0,8 - "2022-01-02","iOS",54,61,1044,56 - "2022-01-03","Android",26,31,2009,26 - "2022-01-03","Chrome OS",1,1,58,0 - "2022-01-03","GNU/Linux",3,3,0,3 - "2022-01-03","Mac",25,25,1692,20 - "2022-01-03","Windows",8,8,105,7 - "2022-01-03","iOS",45,47,4355,40 - "2022-01-04","Android",21,22,250,20 - "2022-01-04","GNU/Linux",1,1,0,1 - "2022-01-04","Mac",18,18,1330,14 - "2022-01-04","Windows",7,7,0,7 - "2022-01-04","iOS",40,44,724,41 - "2022-01-05","Android",18,19,775,16 - "2022-01-05","Mac",12,12,1125,10 - "2022-01-05","Windows",18,20,37,19 - "2022-01-05","iOS",34,34,468,32 - "2022-01-06","",23,23,0,23 - "2022-01-06","Android",26,27,1793,23 - "2022-01-06","GNU/Linux",1,1,0,1 - "2022-01-06","Mac",16,16,0,16 - "2022-01-06","Windows",13,13,114,11 - "2022-01-06","iOS",35,40,1206,37 - "2022-01-07","Android",21,21,383,19 - "2022-01-07","Chrome OS",1,1,0,1 - "2022-01-07","Mac",6,6,0,6 - "2022-01-07","Windows",8,9,247,8 - "2022-01-07","iOS",27,28,856,25 - "2022-01-08","Android",26,26,163,25 - "2022-01-08","Chrome OS",1,1,0,1 - "2022-01-08","GNU/Linux",2,2,0,2 - "2022-01-08","Mac",17,18,405,15 - "2022-01-08","Windows",7,7,1766,5 - "2022-01-08","iOS",38,42,3769,37 - "2022-01-09","Android",15,16,151,14 - "2022-01-09","GNU/Linux",1,1,0,1 - "2022-01-09","Mac",19,20,554,16 - "2022-01-09","Windows",11,12,861,9 - "2022-01-09","iOS",41,46,301,44 - "2022-01-10","Android",11,13,0,13 - "2022-01-10","GNU/Linux",4,4,0,4 - "2022-01-10","Mac",13,15,0,15 - "2022-01-10","Windows",5,5,0,5 - "2022-01-10","iOS",39,40,160,37 - "2022-01-11","Android",19,19,68,16 - "2022-01-11","GNU/Linux",1,1,12,0 - "2022-01-11","Mac",14,14,700,10 - "2022-01-11","Windows",14,15,1123,13 - "2022-01-11","iOS",30,31,117,29 - "2022-01-12","Android",29,31,1735,26 - "2022-01-12","Chrome OS",1,1,1431,0 - "2022-01-12","GNU/Linux",1,1,0,1 - "2022-01-12","Mac",10,10,0,10 - "2022-01-12","Windows",4,4,112,3 - "2022-01-12","iOS",39,42,396,37 - "2022-01-13","Android",22,22,262,19 - "2022-01-13","Chrome OS",1,1,0,1 - "2022-01-13","Mac",18,19,2845,16 - "2022-01-13","Windows",15,15,155,12 - "2022-01-13","iOS",30,34,0,34 - "2022-01-14","Android",20,25,0,25 - "2022-01-14","Chrome OS",1,1,0,1 - "2022-01-14","GNU/Linux",1,1,0,1 - "2022-01-14","Mac",11,11,0,11 - "2022-01-14","Windows",8,8,5,7 - "2022-01-14","iOS",35,38,178,36 - "2022-01-15","Android",27,28,48,27 - "2022-01-15","Mac",15,15,347,14 - "2022-01-15","Windows",7,7,0,7 - "2022-01-15","iOS",50,52,3816,48 - "2022-01-16","Android",22,24,2292,20 - "2022-01-16","Chrome OS",1,1,0,1 - "2022-01-16","Mac",17,21,737,20 - "2022-01-16","Windows",12,12,92,9 - "2022-01-16","iOS",51,51,1595,45 - "2022-01-17","Android",25,29,1576,25 - "2022-01-17","GNU/Linux",2,2,0,2 - "2022-01-17","Mac",20,21,81,20 - "2022-01-17","Windows",19,19,469,17 - "2022-01-17","iOS",38,40,601,38 - "2022-01-18","Android",17,18,68,17 - "2022-01-18","Chrome OS",1,1,0,1 - "2022-01-18","GNU/Linux",3,3,0,3 """ }, %{ @@ -711,22 +233,6 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-30",1,1,1,0,"lucky.numbers.com","/14776416252794997127" "2021-12-30",6,6,6,0,"lucky.numbers.com","/14776416252794997127" "2021-12-30",1,1,1,0,"lucky.numbers.com","/9102354072466236765" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/4457889102355683190" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/12105301321223776356" - "2021-12-30",1,2,2,0,"lucky.numbers.com","/1526239929864936398" - "2021-12-30",3,3,3,0,"lucky.numbers.com","/12574817671425987843" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/18226692671132987727" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/13530392472417087202" - "2021-12-30",3,3,3,0,"lucky.numbers.com","/14629996869565295116" - "2021-12-30",2,2,2,0,"lucky.numbers.com","/16500758973580097263" - "2021-12-30",3,3,3,0,"lucky.numbers.com","/12300842990999856228" - "2021-12-30",2,3,2,75,"lucky.numbers.com","/17859765562822550514" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/1250717791477281255" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/1586391735863371077" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/3457026921000639206" - "2021-12-30",2,3,3,0,"lucky.numbers.com","/6077502147861556415" - "2021-12-30",1,1,1,0,"lucky.numbers.com","/14280570555317344651" - "2021-12-30",3,3,3,0,"lucky.numbers.com","/5284268072698982201" "2021-12-30",1,1,1,0,"lucky.numbers.com","/7478911940502018071" "2021-12-30",1,1,1,0,"lucky.numbers.com","/6402607186523575652" "2021-12-30",2,2,2,0,"lucky.numbers.com","/9962503789684934900" @@ -738,75 +244,7 @@ defmodule Plausible.Imported.CSVImporterTest do "2021-12-31",1,1,1,0,"lucky.numbers.com","/7445073500314667742" "2021-12-31",1,1,1,0,"lucky.numbers.com","/4897404798407749335" "2021-12-31",1,2,1,29,"lucky.numbers.com","/11263893625781431659" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/16478773157730928089" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/1710995203264225236" - "2021-12-31",2,3,3,0,"lucky.numbers.com","/12574817671425987843" - "2021-12-31",1,2,2,0,"lucky.numbers.com","/4072002299714740082" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/13544703054662457518" - "2021-12-31",1,1,0,16,"lucky.numbers.com","/11454211114661615085" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/3363270934034278688" - "2021-12-31",4,4,4,0,"lucky.numbers.com","/14629996869565295116" - "2021-12-31",1,1,0,60,"lucky.numbers.com","/11880955414137016346" - "2021-12-31",2,4,2,224,"lucky.numbers.com","/5918158559063394909" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/16500758973580097263" - "2021-12-31",1,1,0,72,"lucky.numbers.com","/13683424886884591498" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/15335700069940448722" - "2021-12-31",7,7,7,0,"lucky.numbers.com","/1250717791477281255" - "2021-12-31",3,3,3,0,"lucky.numbers.com","/3457026921000639206" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/6077502147861556415" - "2021-12-31",1,1,1,0,"lucky.numbers.com","/14280570555317344651" - "2021-12-31",4,6,5,444,"lucky.numbers.com","/5284268072698982201" - "2021-12-31",2,3,2,394,"lucky.numbers.com","/7478911940502018071" - "2021-12-31",9,17,16,1455,"lucky.numbers.com","/13595620304963848161" - "2021-12-31",25,27,25,88,"lucky.numbers.com","/9874837495456455794" - "2022-01-01",2,3,3,0,"lucky.numbers.com","/14776416252794997127" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/18394065791342797546" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/9102354072466236765" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/2528868181315148597" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/4457889102355683190" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/12105301321223776356" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/6318438003888425693" "2022-01-01",2,2,2,0,"lucky.numbers.com","/5878724061840196349" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/1526239929864936398" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/7692634448754428624" - "2022-01-01",4,4,4,0,"lucky.numbers.com","/12574817671425987843" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/14307781859600070983" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/7110820102771013606" - "2022-01-01",3,3,3,0,"lucky.numbers.com","/14629996869565295116" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/64779230489549655" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/5551134200879446973" - "2022-01-01",1,2,0,189,"lucky.numbers.com","/8000643558843134787" - "2022-01-01",1,2,1,25,"lucky.numbers.com","/11888590162960019765" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/12567804068906971541" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/14033373606203782378" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/10341536794299767967" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/12766075726240916242" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/4362292817884281301" - "2022-01-01",1,2,2,0,"lucky.numbers.com","/1250717791477281255" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/3457026921000639206" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/6077502147861556415" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/18342609412020394218" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/12189381298883635575" - "2022-01-01",2,2,2,0,"lucky.numbers.com","/14280570555317344651" - "2022-01-01",5,12,6,246,"lucky.numbers.com","/5284268072698982201" - "2022-01-01",4,4,4,0,"lucky.numbers.com","/7478911940502018071" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/6402607186523575652" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/8587605562711759914" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/9962503789684934900" - "2022-01-01",7,10,8,187,"lucky.numbers.com","/13595620304963848161" - "2022-01-01",1,1,1,0,"lucky.numbers.com","/17019199732013993436" - "2022-01-01",32,37,33,1195,"lucky.numbers.com","/9874837495456455794" - "2022-01-02",4,5,4,81,"lucky.numbers.com","/14776416252794997127" - "2022-01-02",3,6,5,33,"lucky.numbers.com","/9102354072466236765" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/17912324809189526683" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/2528868181315148597" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/4457889102355683190" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/4897404798407749335" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/5290432771315151230" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/3861893315241731546" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/12571626520250599954" - "2022-01-02",11,12,10,660,"lucky.numbers.com","/12574817671425987843" - "2022-01-02",1,1,1,0,"lucky.numbers.com","/15302856869047719385" """ }, %{ @@ -849,104 +287,15 @@ defmodule Plausible.Imported.CSVImporterTest do "2011-12-28",6,30,4,8,2264 "2011-12-29",4,8,5,6,136 "2011-12-30",1,1,1,1,0 - "2012-01-01",3,15,0,3,1593 - "2012-01-02",1,1,1,1,0 - "2012-01-03",2,2,2,2,0 - "2012-01-04",5,12,2,5,1127 - "2012-01-05",2,2,2,2,0 - "2012-01-06",2,2,2,2,0 - "2012-01-07",3,7,2,3,80 - "2012-01-08",1,1,1,1,0 - "2012-01-13",2,2,2,2,0 - "2012-01-14",1,1,1,1,0 - "2012-01-15",2,6,1,2,82 - "2012-01-16",1,1,1,1,0 - "2012-01-17",1,1,1,1,0 - "2012-01-25",1,11,0,1,91 - "2012-01-26",3,9,1,3,146 - "2012-01-29",1,1,1,1,0 - "2012-02-03",1,2,0,1,6 - "2012-02-07",1,1,1,1,0 - "2012-02-27",1,1,1,1,0 - "2012-03-02",1,3,0,1,70 - "2012-03-18",1,1,1,1,0 - "2012-03-19",1,2,2,2,0 - "2012-03-21",1,2,0,1,22 - "2012-03-22",1,2,0,1,48 - "2012-03-28",1,3,0,1,208 - "2012-03-29",4,18,4,8,154 - "2012-04-15",1,1,1,1,0 - "2012-04-16",1,1,1,1,0 - "2012-04-24",1,1,1,1,0 - "2012-05-25",1,1,1,1,0 - "2012-05-31",1,1,1,1,0 - "2012-06-12",2,2,2,2,0 - "2012-07-01",1,2,0,1,6 - "2012-07-02",1,1,1,1,0 - "2012-07-19",1,1,1,1,0 - "2012-07-22",1,13,0,1,344 - "2012-07-23",5,26,2,7,936 - "2012-07-24",4,69,2,7,12746 - "2012-07-25",10,118,7,15,7815 - "2012-07-26",5,20,4,8,2172 - "2012-07-27",15,55,10,16,6712 - "2012-07-28",3,18,1,3,931 - "2012-07-29",3,3,2,3,6 - "2012-07-30",4,5,2,4,246 - "2012-07-31",6,40,2,7,3146 - "2012-08-01",3,52,2,8,5509 - "2012-08-02",77,127,68,78,3426 - "2012-08-03",28,38,24,29,1808 - "2012-08-04",11,38,9,15,5482 - "2012-08-05",14,46,12,17,3928 - "2012-08-06",11,58,5,14,9140 - "2012-08-07",17,112,13,21,12435 - "2012-08-08",31,53,25,34,2871 - "2012-08-09",14,23,7,15,265 - "2012-08-10",5,8,3,5,468 - "2012-08-11",4,13,2,6,1618 - "2012-08-12",4,5,3,4,4 - "2012-08-13",3,3,3,3,0 - "2012-08-14",3,3,3,3,0 - "2012-08-15",3,3,2,3,127 - "2012-08-16",5,8,3,5,231 - "2012-08-17",16,31,11,18,36929 - "2012-08-18",2,20,2,3,2121 - "2012-08-19",15,63,9,16,6688 - "2012-08-20",8,29,3,9,1884 - "2012-08-21",16,61,11,20,8529 - "2012-08-22",142,199,127,148,7577 - "2012-08-23",69,95,57,71,1339 - "2012-08-24",22,139,13,30,15027 - "2012-08-25",12,76,8,15,8813 - "2012-08-26",19,66,17,24,7716 - "2012-08-27",18,26,15,19,1968 - "2012-08-28",17,80,12,22,9624 - "2012-08-29",49,76,38,50,4581 - "2012-08-30",16,33,13,20,2441 - "2012-08-31",6,22,3,6,4429 - "2012-09-01",22,75,13,26,10118 - "2012-09-02",114,192,98,119,15659 - "2012-09-03",38,46,28,40,1208 - "2012-09-04",39,56,33,41,1686 - "2012-09-05",32,50,29,36,1000 - "2012-09-06",51,112,45,58,12959 - "2012-09-07",34,42,23,34,563 - "2012-09-08",110,189,97,111,12067 - "2012-09-09",69,81,62,70,1259 - "2012-09-10",18,37,14,21,1151 - "2012-09-11",27,45,20,32,3352 - "2012-09-12",17,38,11,18,3608 - "2012-09-13",24,68,14,27,11067 - "2012-09-14",15,28,10,17,977 - "2012-09-15",10,11,8,10,78 - "2012-09-16",7,7,7,7,0 - "2012-09-17",47,89,39,51,9309 - "2012-09-18",19,48,12,22,4102 """ } ] + on_exit(fn -> + keys = Enum.map(csvs, fn csv -> "#{site.id}/#{csv.name}" end) + ExAws.request!(ExAws.S3.delete_all_objects("imports", keys)) + end) + uploads = for %{name: name, body: body} <- csvs do key = "#{site.id}/#{name}" @@ -971,14 +320,72 @@ defmodule Plausible.Imported.CSVImporterTest do # on successfull import the start and end dates are updated assert %SiteImport{ start_date: ~D[2011-12-25], - end_date: ~D[2022-02-01], + end_date: ~D[2022-01-06], source: :csv, status: :completed } = Repo.get_by!(SiteImport, site_id: site.id) - assert 3406 == Plausible.Stats.Clickhouse.imported_pageview_count(site) + assert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99 end - test "invalid CSV" + test "fails on invalid CSV", %{site: site, user: user} do + csvs = [ + %{ + name: "imported_browsers.csv", + body: """ + "date","browser","visitors","visits","visit_duration","bounces" + "2021-12-30","Amazon Silk",2,2,0,2 + "2021-12-30","Chrome",31,32,329,29 + "2021-12-30","Edge",3,3,0,3 + "2021-12-30","Firefox",1,1,0,1 + "2021-12-30","Internet Explorer",1,1,0,1 + "2021-12-30","Mobile App",2,2,0,2 + "2021-12-31","Mobile App",4,4,0,4 + """ + }, + %{ + name: "imported_devices.csv", + body: """ + "date","device","visitors","visit_duration","bounces" + "2021-12-30","Desktop",28,ehhhh.... + """ + } + ] + + on_exit(fn -> + keys = Enum.map(csvs, fn csv -> "#{site.id}/#{csv.name}" end) + ExAws.request!(ExAws.S3.delete_all_objects("imports", keys)) + end) + + uploads = + for %{name: name, body: body} <- csvs do + key = "#{site.id}/#{name}" + ExAws.request!(ExAws.S3.put_object("imports", key, body, content_type: "text/csv")) + %{"filename" => name, "s3_path" => key} + end + + {:ok, job} = + CSVImporter.new_import( + site, + user, + # to satisfy the non null constraints on the table I'm providing "0" dates (according to ClickHouse) + start_date: ~D[1970-01-01], + end_date: ~D[1970-01-01], + uploads: uploads + ) + + job = Repo.reload!(job) + + assert {:discard, message} = Plausible.Workers.ImportAnalytics.perform(job) + assert message =~ "CANNOT_PARSE_INPUT_ASSERTION_FAILED" + + # on successfull import the start and end dates are updated + assert %SiteImport{id: import_id, source: :csv, status: :failed} = + Repo.get_by!(SiteImport, site_id: site.id) + + # ensure no browser left behind + imported_browsers_q = from b in "imported_browsers", where: b.import_id == ^import_id + assert Plausible.ClickhouseRepo.aggregate(imported_browsers_q, :count) == 0 + end end end From 3393ec5ca723eac585d5f08c63d54a10e7c6e132 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:36:46 +0800 Subject: [PATCH 11/17] add config test --- config/runtime.exs | 2 +- lib/plausible/imported/csv_importer.ex | 2 +- test/plausible/config_test.exs | 85 ++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index e14dfeb1b3a1..909ed4f45f26 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -742,7 +742,7 @@ unless s3_disabled? do s3_missing_env = Enum.filter(s3_env, &is_nil(&1.value)) unless s3_missing_env == [] do - raise """ + raise ArgumentError, """ Missing S3 configuration. Please set #{s3_missing_env |> Enum.map(& &1.name) |> Enum.join(", ")} environment variable(s): #{s3_missing_env |> Enum.map(fn %{name: name, example: example} -> "\t#{name}=#{example}" end) |> Enum.join("\n")} diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index b0bd88ec3540..d5e3fd306a95 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -78,7 +78,7 @@ defmodule Plausible.Imported.CSVImporter do end_date: Enum.max_by(ranges, & &1.last, Date).last }} rescue - # we are cancelling on any ArgumentError or ClickHouse errors + # we are cancelling on any argument or ClickHouse errors e in [ArgumentError, Ch.Error] -> {:error, Exception.message(e)} end diff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exs index 3d1741182c34..d2dbcf2491ec 100644 --- a/test/plausible/config_test.exs +++ b/test/plausible/config_test.exs @@ -166,6 +166,91 @@ defmodule Plausible.ConfigTest do end end + describe "s3" do + test "has required env vars" do + env = [ + {"S3_ACCESS_KEY_ID", nil}, + {"S3_SECRET_ACCESS_KEY", nil}, + {"S3_REGION", nil}, + {"S3_ENDPOINT", nil}, + {"S3_IMPORTS_BUCKET", nil}, + {"S3_HOST_FOR_CLICKHOUSE", nil} + ] + + result = + try do + runtime_config(env) + rescue + e -> e + end + + assert %ArgumentError{} = result + + assert Exception.message(result) == """ + Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT, S3_IMPORTS_BUCKET environment variable(s): + + \tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE + \tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + \tS3_REGION=us-east-1 + \tS3_ENDPOINT=https://.r2.cloudflarestorage.com + \tS3_IMPORTS_BUCKET=my-imports-bucket + """ + end + + test "renders only missing env vars" do + env = [ + {"S3_ACCESS_KEY_ID", nil}, + {"S3_SECRET_ACCESS_KEY", nil}, + {"S3_REGION", "eu-north-1"}, + {"S3_ENDPOINT", nil}, + {"S3_IMPORTS_BUCKET", "imports"} + ] + + result = + try do + runtime_config(env) + rescue + e -> e + end + + assert %ArgumentError{} = result + + assert Exception.message(result) == """ + Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT environment variable(s): + + \tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE + \tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + \tS3_ENDPOINT=https://.r2.cloudflarestorage.com + """ + end + + test "works when everything is set" do + env = [ + {"S3_ACCESS_KEY_ID", "minioadmin"}, + {"S3_SECRET_ACCESS_KEY", "minioadmin"}, + {"S3_REGION", "us-east-1"}, + {"S3_ENDPOINT", "http://localhost:9000"}, + {"S3_IMPORTS_BUCKET", "imports"}, + {"S3_HOST_FOR_CLICKHOUSE", nil} + ] + + config = runtime_config(env) + + assert config[:ex_aws] == [ + http_client: Plausible.S3.Client, + access_key_id: "minioadmin", + secret_access_key: "minioadmin", + region: "us-east-1", + s3: [scheme: "http://", host: "localhost", port: 9000] + ] + + assert get_in(config, [:plausible, Plausible.S3]) == [ + imports_bucket: "imports", + host_for_clickhouse: nil + ] + end + end + defp runtime_config(env) do put_system_env_undo(env) Config.Reader.read!("config/runtime.exs", env: :prod) From 3db1ea5bcc6d8ada1dd2a5aa2163c91999f2c4d5 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Sat, 24 Feb 2024 17:30:38 +0800 Subject: [PATCH 12/17] add minio to Makefile --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 0cf03b0727cd..c2c7dc722a8d 100644 --- a/Makefile +++ b/Makefile @@ -36,3 +36,9 @@ postgres-prod: ## Start a container with the same version of postgres as the one postgres-stop: ## Stop and remove the postgres container docker stop plausible_db && docker rm plausible_db + +minio: ## Start a transient container with a recent version of minio (s3) + docker run -d --rm -p 6000:6000 -p 6001:6001 --name plausible_minio minio/minio server /data --address ":6000" --console-address ":6001" + +minio-stop: ## Stop and remove the minio container + docker stop plausible_minio From 3a171d5abf95afa85dd6eafd61cd1d66ce555722 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Sat, 24 Feb 2024 20:14:55 +0800 Subject: [PATCH 13/17] testcontainers --- .github/workflows/elixir.yml | 16 +----- config/.env.dev | 4 +- config/.env.test | 4 +- config/runtime.exs | 8 --- lib/plausible/imported/csv_importer.ex | 24 ++++---- lib/plausible/s3.ex | 21 ------- mix.exs | 3 +- mix.lock | 4 ++ test/plausible/config_test.exs | 26 +++------ test/plausible/imported/csv_importer_test.exs | 57 ++++++++++++------- 10 files changed, 66 insertions(+), 101 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 617dc62d858d..cc1273390362 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -74,21 +74,9 @@ jobs: - name: Check Credo Warnings run: mix credo diff --from-git-merge-base origin/master - name: Run tests - run: mix test --include slow --max-failures 1 --warnings-as-errors + run: mix test --include slow --include minio --max-failures 1 --warnings-as-errors - name: Run tests (small build) - run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors - - name: Start MinIO (for S3 tests) - run: docker run -d -p 9000:9000 minio/minio server /data - - name: Create s3://imports bucket - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://imports - env: - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - AWS_EC2_METADATA_DISABLED: true - - name: Run tests (S3) - run: mix test --only minio --max-failures 1 --warnings-as-errors - env: - S3_HOST_FOR_CLICKHOUSE: 172.17.0.1 + run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors - name: Check Dialyzer run: mix dialyzer env: diff --git a/config/.env.dev b/config/.env.dev index d3f39d92a44c..b401d0503324 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -25,6 +25,4 @@ S3_DISABLED=false S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin S3_REGION=us-east-1 -S3_ENDPOINT=http://localhost:9000 -S3_IMPORTS_BUCKET=imports -S3_HOST_FOR_CLICKHOUSE=172.17.0.1 +S3_ENDPOINT=http://localhost:6000 diff --git a/config/.env.test b/config/.env.test index 699a90bb70ad..56092d916c48 100644 --- a/config/.env.test +++ b/config/.env.test @@ -20,6 +20,4 @@ S3_DISABLED=false S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin S3_REGION=us-east-1 -S3_ENDPOINT=http://localhost:9000 -S3_IMPORTS_BUCKET=imports -S3_HOST_FOR_CLICKHOUSE=172.17.0.1 +S3_ENDPOINT=http://localhost:6000 diff --git a/config/runtime.exs b/config/runtime.exs index 909ed4f45f26..788db3134191 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -727,10 +727,6 @@ unless s3_disabled? do %{ name: "S3_ENDPOINT", example: "https://.r2.cloudflarestorage.com" - }, - %{ - name: "S3_IMPORTS_BUCKET", - example: "my-imports-bucket" } ] @@ -765,8 +761,4 @@ unless s3_disabled? do scheme: s3_scheme <> "://", host: s3_host, port: s3_port - - config :plausible, Plausible.S3, - imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET"), - host_for_clickhouse: get_var_from_path_or_env(config_dir, "S3_HOST_FOR_CLICKHOUSE") end diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index d5e3fd306a95..f45ef83b919c 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -33,14 +33,13 @@ defmodule Plausible.Imported.CSVImporter do ranges = Enum.map(uploads, fn upload -> - %{"filename" => filename, "s3_path" => s3_path} = upload + %{"filename" => filename, "s3_url" => s3_url} = upload ".csv" = Path.extname(filename) table = Path.rootname(filename) ensure_importable_table!(table) s3_structure = input_structure!(table) - s3_url = Plausible.S3.import_clickhouse_url(s3_path) statement = """ @@ -49,16 +48,17 @@ defmodule Plausible.Imported.CSVImporter do FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\ """ - params = %{ - "table" => table, - "site_id" => site_id, - "import_id" => import_id, - "s3_url" => s3_url, - "s3_access_key_id" => s3_access_key_id, - "s3_secret_access_key" => s3_secret_access_key, - "s3_format" => "CSVWithNames", - "s3_structure" => s3_structure - } + params = + %{ + "table" => table, + "site_id" => site_id, + "import_id" => import_id, + "s3_url" => s3_url, + "s3_access_key_id" => s3_access_key_id, + "s3_secret_access_key" => s3_secret_access_key, + "s3_format" => "CSVWithNames", + "s3_structure" => s3_structure + } Ch.query!(ch, statement, params, timeout: :infinity) diff --git a/lib/plausible/s3.ex b/lib/plausible/s3.ex index dfefc896f2a7..d8f8ad65f3a9 100644 --- a/lib/plausible/s3.ex +++ b/lib/plausible/s3.ex @@ -3,9 +3,6 @@ defmodule Plausible.S3 do Helper functions for S3 exports/imports. """ - defp config, do: Application.fetch_env!(:plausible, __MODULE__) - defp config(key), do: Keyword.fetch!(config(), key) - @doc """ Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3. """ @@ -15,22 +12,4 @@ defmodule Plausible.S3 do %{access_key_id: access_key_id, secret_access_key: secret_access_key} = ExAws.Config.new(:s3) %{access_key_id: access_key_id, secret_access_key: secret_access_key} end - - @doc """ - Returns S3 URL for an object to be used by ClickHouse during imports from S3. - - In the current implementation the bucket goes into the path component: - - ${S3_ENDPOINT}/${S3_IMPORTS_BUCKET}/${S3_PATH} - - https://s3.us-east-1.amazonaws.com/my-plausible-imports/1/imported_browsers.csv - - """ - @spec import_clickhouse_url(Path.t()) :: :uri_string.uri_string() - def import_clickhouse_url(s3_path) do - %{scheme: scheme, host: host} = config = ExAws.Config.new(:s3) - host = Keyword.get(config(), :host_for_clickhouse) || host - port = ExAws.S3.Utils.sanitized_port_component(config) - Path.join(["#{scheme}#{host}#{port}", config(:imports_bucket), s3_path]) - end end diff --git a/mix.exs b/mix.exs index 38f7967dcfd8..4097302747f1 100644 --- a/mix.exs +++ b/mix.exs @@ -138,7 +138,8 @@ defmodule Plausible.MixProject do {:ecto_network, "~> 1.5.0"}, {:ex_aws, "~> 2.5"}, {:ex_aws_s3, "~> 2.5"}, - {:sweet_xml, "~> 0.7.4"} + {:sweet_xml, "~> 0.7.4"}, + {:testcontainers, "~> 1.6", only: [:test, :small_test]} ] end diff --git a/mix.lock b/mix.lock index f10c50458123..d5222e4d587a 100644 --- a/mix.lock +++ b/mix.lock @@ -47,6 +47,7 @@ "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.15.1", "e92ba17c41e7405b7784e0e65f406b5f17cfe313e0e70de9befd653e12854822", [:mix], [{:ex_cldr, "~> 2.34", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "31df8bd37688340f8819bdd770eb17d659652078d34db632b85d4a32864d6a25"}, "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.32.3", "b631ff94c982ec518e46bf4736000a30a33d6b58facc085d5f240305f512ad4a", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 0.3 or ~> 1.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.37", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, ">= 2.14.2", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "7b626ff1e59a0ec9c3c5db5ce9ca91a6995e2ab56426b71f3cbf67181ea225f5"}, "ex_doc": {:hex, :ex_doc, "0.31.1", "8a2355ac42b1cc7b2379da9e40243f2670143721dd50748bf6c3b1184dae2089", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "3178c3a407c557d8343479e1ff117a96fd31bafe52a039079593fb0524ef61b0"}, + "ex_docker_engine_api": {:hex, :ex_docker_engine_api, "1.43.1", "1161e34b6bea5cef84d8fdc1d5d510fcb0c463941ce84c36f4a0f44a9096eb96", [:mix], [{:hackney, "~> 1.20", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:tesla, "~> 1.7", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "ec8fc499389aeef56ddca67e89e9e98098cff50587b56e8b4613279f382793b1"}, "ex_json_logger": {:hex, :ex_json_logger, "1.4.0", "ad1dcc1cfe6940ee1d9d489b20757c89769626ce34c4957548d6fbe155cd96f1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "7548a1ecba290746e06214d2b3d8783c76760c779a8903a8e44bfd23a7340444"}, "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, "ex_money": {:hex, :ex_money, "5.15.3", "ea070eb1eefd22258aa288921ba482f1fa5f870d229069dc3d12458b7b8bf66d", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr_numbers, "~> 2.31", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:gringotts, "~> 1.1", [hex: :gringotts, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "3671f1808c428b7c4688650d43dc1af0b64c0eea822429a28c55cef15fb4fdc1"}, @@ -140,12 +141,15 @@ "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, "telemetry_registry": {:hex, :telemetry_registry, "0.3.1", "14a3319a7d9027bdbff7ebcacf1a438f5f5c903057b93aee484cca26f05bdcba", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6d0ca77b691cf854ed074b459a93b87f4c7f5512f8f7743c635ca83da81f939e"}, + "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"}, + "testcontainers": {:hex, :testcontainers, "1.6.0", "14b3251f01ce0b1ada716130d371ba0b6cb1ce2904aa38bd58e5ff4194f4d88f", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.3", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:ex_docker_engine_api, "~> 1.43.1", [hex: :ex_docker_engine_api, repo: "hexpm", optional: false]}, {:uuid, "~> 1.1", [hex: :uuid, repo: "hexpm", optional: false]}], "hexpm", "3f812407f232954999a3a2e05b2802e1d8d1afba120533c42b32c7cc91d35daf"}, "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.21.0", "042ab2c0c860652bc5cf69c94e3a31f96676d14682e22ec7813bd173ceff1788", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6cee6cffc35a390840d48d463541d50746a7b0e421acaadb833cfc7961e490e7"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "ua_inspector": {:hex, :ua_inspector, "3.8.0", "c0b0d13200a9bd509225f15ea8cf275c0bec27390a21c355746ff8b8a88c3e4d", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "7c980bae82a4754075b933e0f383935a681e5a2628856ad3ecf6eb80d8139539"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "unsafe": {:hex, :unsafe, "1.0.2", "23c6be12f6c1605364801f4b47007c0c159497d0446ad378b5cf05f1855c0581", [:mix], [], "hexpm", "b485231683c3ab01a9cd44cb4a79f152c6f3bb87358439c6f68791b85c2df675"}, + "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, diff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exs index d2dbcf2491ec..29d5db749fee 100644 --- a/test/plausible/config_test.exs +++ b/test/plausible/config_test.exs @@ -172,9 +172,7 @@ defmodule Plausible.ConfigTest do {"S3_ACCESS_KEY_ID", nil}, {"S3_SECRET_ACCESS_KEY", nil}, {"S3_REGION", nil}, - {"S3_ENDPOINT", nil}, - {"S3_IMPORTS_BUCKET", nil}, - {"S3_HOST_FOR_CLICKHOUSE", nil} + {"S3_ENDPOINT", nil} ] result = @@ -187,23 +185,21 @@ defmodule Plausible.ConfigTest do assert %ArgumentError{} = result assert Exception.message(result) == """ - Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT, S3_IMPORTS_BUCKET environment variable(s): + Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT environment variable(s): \tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \tS3_REGION=us-east-1 \tS3_ENDPOINT=https://.r2.cloudflarestorage.com - \tS3_IMPORTS_BUCKET=my-imports-bucket """ end test "renders only missing env vars" do env = [ - {"S3_ACCESS_KEY_ID", nil}, + {"S3_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"}, {"S3_SECRET_ACCESS_KEY", nil}, {"S3_REGION", "eu-north-1"}, - {"S3_ENDPOINT", nil}, - {"S3_IMPORTS_BUCKET", "imports"} + {"S3_ENDPOINT", nil} ] result = @@ -216,9 +212,8 @@ defmodule Plausible.ConfigTest do assert %ArgumentError{} = result assert Exception.message(result) == """ - Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT environment variable(s): + Missing S3 configuration. Please set S3_SECRET_ACCESS_KEY, S3_ENDPOINT environment variable(s): - \tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \tS3_ENDPOINT=https://.r2.cloudflarestorage.com """ @@ -229,9 +224,7 @@ defmodule Plausible.ConfigTest do {"S3_ACCESS_KEY_ID", "minioadmin"}, {"S3_SECRET_ACCESS_KEY", "minioadmin"}, {"S3_REGION", "us-east-1"}, - {"S3_ENDPOINT", "http://localhost:9000"}, - {"S3_IMPORTS_BUCKET", "imports"}, - {"S3_HOST_FOR_CLICKHOUSE", nil} + {"S3_ENDPOINT", "http://localhost:6000"} ] config = runtime_config(env) @@ -241,12 +234,7 @@ defmodule Plausible.ConfigTest do access_key_id: "minioadmin", secret_access_key: "minioadmin", region: "us-east-1", - s3: [scheme: "http://", host: "localhost", port: 9000] - ] - - assert get_in(config, [:plausible, Plausible.S3]) == [ - imports_bucket: "imports", - host_for_clickhouse: nil + s3: [scheme: "http://", host: "localhost", port: 6000] ] end end diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index 7cd06174ad8c..d1ad57360ef6 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -1,17 +1,24 @@ defmodule Plausible.Imported.CSVImporterTest do use Plausible.DataCase, async: true - - doctest Plausible.Imported.CSVImporter, import: true + alias Plausible.Imported.{CSVImporter, SiteImport} + alias Testcontainers.MinioContainer + require SiteImport @moduletag :minio - # uses https://min.io - # docker run -d --rm -p 9000:9000 -p 9001:9001 --name minio minio/minio server /data --console-address ":9001" - # docker exec minio mc alias set local http://localhost:9000 minioadmin minioadmin - # docker exec minio mc mb local/imports + setup_all do + Testcontainers.start_link() - alias Plausible.Imported.{CSVImporter, SiteImport} - require SiteImport + {:ok, minio} = Testcontainers.start_container(MinioContainer.new()) + on_exit(fn -> :ok = Testcontainers.stop_container(minio.container_id) end) + connection_opts = MinioContainer.connection_opts(minio) + + bucket = "imports" + ExAws.request!(ExAws.S3.put_bucket(bucket, "us-east-1"), connection_opts) + on_exit(fn -> ExAws.request!(ExAws.S3.delete_bucket(bucket), connection_opts) end) + + {:ok, container: minio, bucket: bucket} + end describe "new_import/3 and parse_args/1" do setup [:create_user, :create_new_site] @@ -32,8 +39,11 @@ defmodule Plausible.Imported.CSVImporterTest do uploads = Enum.map(tables, fn table -> filename = "#{table}.csv" - s3_path = "#{site.id}/#{filename}" - %{"filename" => filename, "s3_path" => s3_path} + + %{ + "filename" => filename, + "s3_url" => "https://bucket-name.s3.eu-north-1.amazonaws.com/#{site.id}/#{filename}" + } end) assert {:ok, job} = @@ -65,7 +75,7 @@ defmodule Plausible.Imported.CSVImporterTest do describe "import_data/2" do setup [:create_user, :create_new_site] - test "imports tables from S3", %{site: site, user: user} do + test "imports tables from S3", %{site: site, user: user, bucket: bucket, container: minio} do csvs = [ %{ name: "imported_browsers.csv", @@ -291,16 +301,18 @@ defmodule Plausible.Imported.CSVImporterTest do } ] + connection_opts = MinioContainer.connection_opts(minio) + on_exit(fn -> keys = Enum.map(csvs, fn csv -> "#{site.id}/#{csv.name}" end) - ExAws.request!(ExAws.S3.delete_all_objects("imports", keys)) + ExAws.request!(ExAws.S3.delete_all_objects(bucket, keys), connection_opts) end) uploads = for %{name: name, body: body} <- csvs do key = "#{site.id}/#{name}" - ExAws.request!(ExAws.S3.put_object("imports", key, body, content_type: "text/csv")) - %{"filename" => name, "s3_path" => key} + ExAws.request!(ExAws.S3.put_object(bucket, key, body), connection_opts) + %{"filename" => name, "s3_url" => s3_url(minio, bucket, key)} end {:ok, job} = @@ -328,7 +340,7 @@ defmodule Plausible.Imported.CSVImporterTest do assert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99 end - test "fails on invalid CSV", %{site: site, user: user} do + test "fails on invalid CSV", %{site: site, user: user, bucket: bucket, container: minio} do csvs = [ %{ name: "imported_browsers.csv", @@ -352,23 +364,24 @@ defmodule Plausible.Imported.CSVImporterTest do } ] + connection_opts = MinioContainer.connection_opts(minio) + on_exit(fn -> keys = Enum.map(csvs, fn csv -> "#{site.id}/#{csv.name}" end) - ExAws.request!(ExAws.S3.delete_all_objects("imports", keys)) + ExAws.request!(ExAws.S3.delete_all_objects(bucket, keys), connection_opts) end) uploads = for %{name: name, body: body} <- csvs do key = "#{site.id}/#{name}" - ExAws.request!(ExAws.S3.put_object("imports", key, body, content_type: "text/csv")) - %{"filename" => name, "s3_path" => key} + ExAws.request!(ExAws.S3.put_object(bucket, key, body), connection_opts) + %{"filename" => name, "s3_url" => s3_url(minio, bucket, key)} end {:ok, job} = CSVImporter.new_import( site, user, - # to satisfy the non null constraints on the table I'm providing "0" dates (according to ClickHouse) start_date: ~D[1970-01-01], end_date: ~D[1970-01-01], uploads: uploads @@ -379,7 +392,6 @@ defmodule Plausible.Imported.CSVImporterTest do assert {:discard, message} = Plausible.Workers.ImportAnalytics.perform(job) assert message =~ "CANNOT_PARSE_INPUT_ASSERTION_FAILED" - # on successfull import the start and end dates are updated assert %SiteImport{id: import_id, source: :csv, status: :failed} = Repo.get_by!(SiteImport, site_id: site.id) @@ -388,4 +400,9 @@ defmodule Plausible.Imported.CSVImporterTest do assert Plausible.ClickhouseRepo.aggregate(imported_browsers_q, :count) == 0 end end + + defp s3_url(minio, bucket, key) do + port = minio |> MinioContainer.connection_opts() |> Keyword.fetch!(:port) + Path.join(["http://172.17.0.1:#{port}", bucket, key]) + end end From f912209d4cf1ffff75fc39e7b9976afb672c9b24 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Sun, 25 Feb 2024 00:10:27 +0800 Subject: [PATCH 14/17] remove extra whitespace --- .github/workflows/elixir.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index cc1273390362..467ed708be11 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -76,7 +76,7 @@ jobs: - name: Run tests run: mix test --include slow --include minio --max-failures 1 --warnings-as-errors - name: Run tests (small build) - run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors + run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors - name: Check Dialyzer run: mix dialyzer env: From 0c191cd3a1241bdbd0dce8867eee69427161551d Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Sun, 25 Feb 2024 00:29:00 +0800 Subject: [PATCH 15/17] explain the implementation a bit --- lib/plausible/imported/csv_importer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index f45ef83b919c..7e6a5aabab02 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -1,6 +1,6 @@ defmodule Plausible.Imported.CSVImporter do @moduledoc """ - CSV importer from S3. + CSV importer from S3 that uses ClickHouse [s3 table function.](https://clickhouse.com/docs/en/sql-reference/table-functions/s3) """ use Plausible.Imported.Importer From ab4f9c0351b234b1dcfefcbac6aa6dfe6e509992 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Mon, 26 Feb 2024 23:30:25 +0800 Subject: [PATCH 16/17] account for async deletes in tests --- test/plausible/imported/csv_importer_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index d1ad57360ef6..9cf4c098543e 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -397,7 +397,7 @@ defmodule Plausible.Imported.CSVImporterTest do # ensure no browser left behind imported_browsers_q = from b in "imported_browsers", where: b.import_id == ^import_id - assert Plausible.ClickhouseRepo.aggregate(imported_browsers_q, :count) == 0 + assert await_clickhouse_count(imported_browsers_q, 0) end end From 723b31fdc19c85d1001efa1dd292c8bbcd738629 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 27 Feb 2024 18:22:17 +0800 Subject: [PATCH 17/17] bounces is UInt32 Co-authored-by: Adrian Gruntkowski --- lib/plausible/imported/csv_importer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 7e6a5aabab02..d2d02e5426bb 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -89,7 +89,7 @@ defmodule Plausible.Imported.CSVImporter do "imported_devices" => "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "imported_entry_pages" => - "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt64", + "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32", "imported_exit_pages" => "date Date, exit_page String, visitors UInt64, exits UInt64", "imported_locations" => "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",