Merge pull request #182 from inaka/elbrujohalcon.182.correct_date__an…

…d_datetime_in_

Correct date  and datetime in the riak store

Makefile

@@ -2,16 +2,19 @@ PROJECT = sumo_db
 
 CONFIG ?= test/test.config
 
-DEPS = lager emysql emongo tirerl epgsql worker_pool riakc uuid
+DEPS = lager uuid emysql emongo tirerl epgsql worker_pool riakc iso8601
+SHELL_DEPS = sync
 
+dep_sync = git https://github.com/inaka/sync.git 0.1.3
 dep_lager = git https://github.com/basho/lager.git 2.1.1
-dep_emysql = git https://github.com/Eonblast/Emysql.git v0.4.1
+dep_emysql = git https://github.com/inaka/Emysql.git 0.4.2
 dep_emongo = git https://github.com/inaka/emongo.git v0.2.1
-dep_tirerl = git https://github.com/inaka/tirerl 0.1.7
+dep_tirerl = git https://github.com/inaka/tirerl 7ac7d57a24
 dep_epgsql = git https://github.com/epgsql/epgsql 2.0.0
-dep_worker_pool = git https://github.com/inaka/worker_pool.git 1.0.2
-dep_riakc = git https://github.com/inaka/riak-erlang-client.git 2.1.1-dialyzed
-dep_uuid = git git://github.com/okeuday/uuid.git v1.4.0
+dep_worker_pool = git https://github.com/inaka/worker_pool.git 1.0.3
+dep_riakc = git https://github.com/inaka/riak-erlang-client.git 2.1.1-R18
+dep_uuid = git https://github.com/okeuday/uuid.git 31f408f4ef
+dep_iso8601 = git https://github.com/zerotao/erlang_iso8601.git 0d14540
 
 TEST_DEPS = mixer
 dep_mixer = git git://github.com/inaka/mixer.git 0.1.2

rebar.config

@@ -19,15 +19,16 @@
   warn_untyped_record, debug_info
 ]}.
 {deps, [
-  {lager,       ".*", {git, "git://github.com/basho/lager.git",          "2.1.1"}},
-  {emysql,      ".*", {git, "git://github.com/Eonblast/Emysql.git",      "v0.4.1"}},
-  {emongo,      ".*", {git, "git://github.com/inaka/emongo.git",         "v0.2.1"}},
-  {tirerl,      ".*", {git, "git://github.com/inaka/tirerl",             "0.1.7"}},
-  {epgsql,      ".*", {git, "git://github.com/epgsql/epgsql",            "2.0.0"}},
-  {worker_pool, ".*", {git, "git://github.com/inaka/worker_pool.git",    "1.0.2"}},
-  {riakc,       ".*", {git, "git://github.com/inaka/riak-erlang-client", "2.1.1-dialyzed"}},
-  {uuid,        ".*", {git, "git://github.com/okeuday/uuid.git",         "v1.4.0"}},
-  {mixer,       ".*", {git, "git://github.com/inaka/mixer",              "0.1.2"}}
+  {lager,       ".*", {git, "https://github.com/basho/lager.git",          "2.1.1"}},
+  {emysql,      ".*", {git, "https://github.com/inaka/Emysql.git",         "0.4.2"}},
+  {emongo,      ".*", {git, "https://github.com/inaka/emongo.git",         "v0.2.1"}},
+  {tirerl,      ".*", {git, "https://github.com/inaka/tirerl",             "7ac7d57a24"}},
+  {epgsql,      ".*", {git, "https://github.com/epgsql/epgsql",            "2.0.0"}},
+  {worker_pool, ".*", {git, "https://github.com/inaka/worker_pool.git",    "1.0.3"}},
+  {riakc,       ".*", {git, "https://github.com/inaka/riak-erlang-client", "2.1.1-R18"}},
+  {uuid,        ".*", {git, "https://github.com/okeuday/uuid.git",         "v1.5.0"}},
+  {mixer,       ".*", {git, "https://github.com/inaka/mixer",              "0.1.2"}},
+  {iso8601,     ".*", {git, "https://github.com/kivra/erlang_iso8601.git", "1.1.2"}}
 ]}.
 {xref_warnings, true}.
 {xref_checks, [undefined_function_calls, undefined_functions, locals_not_used, deprecated_function_calls, deprecated_functions]}.

src/sumo_store_elasticsearch.erl

@@ -74,7 +74,7 @@ persist(Doc, #{index := Index, pool_name := PoolName} = State) ->
     IdField = sumo_internal:id_field_name(DocName),
     Id =  sumo_internal:get_field(IdField, Doc),
 
-    Fields = sumo_internal:doc_fields(Doc),
+    Fields = normalize_fields(sumo_internal:doc_fields(Doc)),
 
     Doc1 =
         case Id of
@@ -236,8 +236,23 @@ build_mapping(MappingType, Fields) ->
         fun
             (Field, Acc) ->
                 Name = sumo_internal:field_name(Field),
-                FieldType = sumo_internal:field_type(Field),
+                FieldType = normalize_type(sumo_internal:field_type(Field)),
                 maps:put(Name, #{type => FieldType}, Acc)
         end,
     Properties = lists:foldl(Fun, #{}, Fields),
     maps:from_list([{MappingType, #{properties => Properties}}]).
+
+normalize_type(date) -> binary;
+normalize_type(datetime) -> binary;
+normalize_type(Type) -> Type.
+
+normalize_fields(Doc) ->
+  FieldList = maps:to_list(Doc),
+  lists:foldl(
+    fun ({K, {_, _, _} = FieldValue}, AccDoc) ->
+          maps:put(K, iso8601:format({FieldValue, {0, 0, 0}}), AccDoc);
+        ({K, {{_, _, _}, {_, _, _}} = FieldValue}, AccDoc) ->
+          maps:put(K, iso8601:format(FieldValue), AccDoc);
+        ({_K, _FieldValue}, AccDoc) ->
+          AccDoc
+    end, Doc, FieldList).

src/sumo_store_mysql.erl

@@ -429,6 +429,10 @@ build_docs(DocName, #result_packet{rows = Rows, field_list = Fields}) ->
   [build_doc(sumo_internal:new_doc(DocName), FieldNames, Row) || Row <- Rows].
 
 build_doc(Doc, [], []) -> Doc;
+build_doc(Doc, [FieldName|FieldNames], [{date, Value}|Values]) ->
+  build_doc(sumo_internal:set_field(FieldName, Value, Doc), FieldNames, Values);
+build_doc(Doc, [FieldName|FieldNames], [{datetime, Value}|Values]) ->
+  build_doc(sumo_internal:set_field(FieldName, Value, Doc), FieldNames, Values);
 build_doc(Doc, [FieldName|FieldNames], [Value|Values]) ->
   build_doc(sumo_internal:set_field(FieldName, Value, Doc), FieldNames, Values).
 

src/sumo_store_riak.erl

@@ -113,7 +113,7 @@ init(Opts) ->
 ) -> sumo_store:result(sumo_internal:doc(), state()).
 persist(Doc,
         #state{conn = Conn, bucket = Bucket, put_opts = Opts} = State) ->
-  {Id, NewDoc} = new_doc(Doc, State),
+  {Id, NewDoc} = new_doc(sleep(Doc), State),
   case update_map(Conn, Bucket, Id, doc_to_rmap(NewDoc), Opts) of
     {error, Error} ->
       {error, Error, State};
@@ -223,6 +223,7 @@ find_by(DocName, Conditions, Limit, Offset, State) when is_list(Conditions) ->
 find_by(DocName, Conditions, Limit, Offset, State) ->
   find_by_query(DocName, Conditions, Limit, Offset, State).
 
+%% @private
 find_by_id_field(DocName, Key, State) ->
   #state{conn = Conn, bucket = Bucket, get_opts = Opts} = State,
   case fetch_map(Conn, Bucket, to_bin(Key), Opts) of
@@ -235,11 +236,13 @@ find_by_id_field(DocName, Key, State) ->
       {error, Error, State}
   end.
 
+%% @private
 find_by_query(DocName, Conditions, undefined, undefined, State) ->
   #state{conn = Conn, index = Index} = State,
   Query = build_query(Conditions),
   find_all_by_query(DocName, Conn, Index, Query, State);
 
+%% @private
 find_by_query(DocName, Conditions, Limit, Offset, State) ->
   #state{conn = Conn, index = Index} = State,
   Query = build_query(Conditions),
@@ -248,6 +251,7 @@ find_by_query(DocName, Conditions, Limit, Offset, State) ->
     {error, Error} -> {error, Error, State}
   end.
 
+%% @private
 find_all_by_query(DocName, Conn, Index, Query, State) ->
   FirstQuery =
     case search_docs_by(DocName, Conn, Index, Query, 0, 0) of
@@ -306,7 +310,7 @@ map_to_rmap(Map) ->
   sumo:schema_name(), riakc_map:crdt_map()
 ) -> sumo_internal:doc().
 rmap_to_doc(DocName, RMap) ->
-  sumo_internal:new_doc(DocName, rmap_to_map(RMap)).
+  wakeup(sumo_internal:new_doc(DocName, rmap_to_map(RMap))).
 
 -spec rmap_to_map(riakc_map:crdt_map()) -> map().
 rmap_to_map(RMap) ->
@@ -365,6 +369,67 @@ build_query(Conditions) ->
 %% Private API.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+%% @private
+sleep(Doc) ->
+  DTFields = datetime_fields(Doc),
+  Encode =
+    fun({FieldName, FieldType, FieldValue}, Acc) ->
+      case {FieldType, is_datetime(FieldValue)} of
+        {datetime, true} ->
+          sumo_internal:set_field(FieldName, iso8601:format(FieldValue), Acc);
+        {date, true} ->
+          DateTime = {FieldValue, {0, 0, 0}},
+          sumo_internal:set_field(FieldName, iso8601:format(DateTime), Acc);
+        _ ->
+          Acc
+      end
+    end,
+  lists:foldl(Encode, Doc, DTFields).
+
+%% @private
+wakeup(Doc) ->
+  DTFields = datetime_fields(Doc),
+  Decode =
+    fun({FieldName, FieldType, FieldValue}, Acc) ->
+      case FieldType of
+        datetime when FieldValue /= <<"undefined">> ->
+          sumo_internal:set_field(FieldName, iso8601:parse(FieldValue), Acc);
+        date when FieldValue /= <<"undefined">> ->
+          {Date, _} = iso8601:parse(FieldValue),
+          sumo_internal:set_field(FieldName, Date, Acc);
+        _ ->
+          Acc
+      end
+    end,
+  lists:foldl(Decode, Doc, DTFields).
+
+%% @private
+datetime_fields(Doc) ->
+  DocName = sumo_internal:doc_name(Doc),
+  Schema = sumo_internal:get_schema(DocName),
+  SchemaFields = sumo_internal:schema_fields(Schema),
+  Filter =
+    fun(Field, Acc) ->
+      FieldType = sumo_internal:field_type(Field),
+      case FieldType of
+        T when T =:= datetime; T =:= date ->
+          FieldName = sumo_internal:field_name(Field),
+          FieldValue = sumo_internal:get_field(FieldName, Doc),
+          [{FieldName, FieldType, FieldValue} | Acc];
+        _ ->
+          Acc
+      end
+    end,
+  lists:foldl(Filter, [], SchemaFields).
+
+%% @private
+is_datetime({{_, _, _} = Date, {_, _, _}}) ->
+  calendar:valid_date(Date);
+is_datetime({_, _, _} = Date) ->
+  calendar:valid_date(Date);
+is_datetime(_) ->
+  false.
+
 %% @private
 doc_id(Doc) ->
   DocName = sumo_internal:doc_name(Doc),
@@ -423,7 +488,7 @@ kv_to_doc(DocName, KV) ->
         NK = normalize_doc_fields(K),
         sumo_internal:set_field(to_atom(NK), V, Acc)
       end,
-  lists:foldl(F, sumo_internal:new_doc(DocName), KV).
+  wakeup(lists:foldl(F, sumo_internal:new_doc(DocName), KV)).
 
 %% @private
 normalize_doc_fields(Src) ->

test/sumo_basic_SUITE.erl

@@ -12,7 +12,8 @@
          find_all/1,
          find_by/1,
          delete_all/1,
-         delete/1
+         delete/1,
+         check_proper_dates/1
         ]).
 
 -define(EXCLUDED_FUNS,
@@ -70,6 +71,11 @@ delete_all(_Config) ->
 delete(_Config) ->
   run_all_stores(fun delete_module/1).
 
+check_proper_dates(_Config) ->
+  lists:foreach(
+    fun check_proper_dates_module/1,
+    sumo_test_utils:people_with_proper_dates()).
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% Internal functions
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -139,6 +145,20 @@ delete_module(Module) ->
 
   1 = length(All) - length(NewAll).
 
+check_proper_dates_module(Module) ->
+  [P0] = sumo:find_by(Module, [{name, <<"A">>}]),
+  P1 = sumo:find(Module, Module:id(P0)),
+  [P2 | _] = sumo:find_all(Module),
+
+  {Date, _} = calendar:universal_time(),
+
+  Date = Module:birthdate(P0),
+  {Date, {_, _, _}} = Module:created_at(P0),
+  Date = Module:birthdate(P1),
+  {Date, {_, _, _}} = Module:created_at(P1),
+  Date = Module:birthdate(P2),
+  {Date, {_, _, _}} = Module:created_at(P2).
+
 %%% Helper
 
 -spec run_all_stores(fun()) -> ok.

test/sumo_config_SUITE.erl

@@ -28,8 +28,10 @@
 
 -spec all() -> [atom()].
 all() ->
-  Exports = ?MODULE:module_info(exports),
-  [F || {F, _} <- Exports, not lists:member(F, ?EXCLUDED_FUNS)].
+  case lists:member(sumo_test_people_mysql, sumo_test_utils:all_people()) of
+    true -> [check_overrun_handler];
+    false -> []
+  end.
 
 -spec init_per_suite(config()) -> config().
 init_per_suite(Config) ->

test/sumo_test_people.erl

@@ -8,13 +8,15 @@
          sumo_sleep/1
         ]).
 
--export([new/2, new/3, new/4, name/1, id/1, age/1]).
+-export([new/2, new/3, new/4, name/1, id/1, age/1, birthdate/1, created_at/1]).
 
 -record(person, {id :: integer(),
                  name :: string(),
                  last_name :: string(),
                  age :: integer(),
-                 address :: string()}).
+                 address :: string(),
+                 birthdate  :: calendar:date(),
+                 created_at :: calendar:datetime()}).
 
 -type person() :: #person{}.
 
@@ -28,7 +30,9 @@ sumo_sleep(Person) ->
     name => Person#person.name,
     last_name => Person#person.last_name,
     age => Person#person.age,
-    address => Person#person.address}.
+    address => Person#person.address,
+    birthdate  => Person#person.birthdate,
+    created_at => Person#person.created_at}.
 
 -spec sumo_wakeup(sumo:doc()) -> person().
 sumo_wakeup(Person) ->
@@ -37,27 +41,27 @@ sumo_wakeup(Person) ->
     name = maps:get(name, Person),
     last_name = maps:get(last_name, Person),
     age = maps:get(age, Person),
-    address = maps:get(address, Person)
+    address = maps:get(address, Person),
+    birthdate  = maps:get(birthdate, Person),
+    created_at = maps:get(created_at, Person)
   }.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%% Exported
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-new(Name, LastName) ->
-  #person{name = Name,
-          last_name = LastName}.
+new(Name, LastName) -> new(Name, LastName, undefined).
 
-new(Name, LastName, Age) ->
-  #person{name = Name,
-          last_name = LastName,
-          age = Age}.
+new(Name, LastName, Age) -> new(Name, LastName, Age, undefined).
 
 new(Name, LastName, Age, Address) ->
+  Datetime = {Date, _} = calendar:universal_time(),
   #person{name = Name,
-          last_name = LastName,
-          age = Age,
-          address = Address}.
+          last_name   = LastName,
+          age         = Age,
+          address     = Address,
+          birthdate   = Date,
+          created_at  = Datetime}.
 
 name(Person) ->
   Person#person.name.
@@ -67,3 +71,9 @@ id(Person) ->
 
 age(Person) ->
   Person#person.age.
+
+birthdate(Person) ->
+  Person#person.birthdate.
+
+created_at(Person) ->
+  Person#person.created_at.

test/sumo_test_people_elasticsearch.erl

@@ -11,20 +11,28 @@
           new/4,
           name/1,
           id/1,
-          age/1
+          age/1,
+          birthdate/1,
+          created_at/1
          ]
         }
        ]).
 
 -export([sumo_schema/0]).
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% sumo_doc callbacks
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 -spec sumo_schema() -> sumo:schema().
 sumo_schema() ->
-    Fields =
-    [sumo:new_field(id,        string, [id, not_null, auto_increment]),
-     sumo:new_field(name,      string,  [{length, 255}, not_null]),
-     sumo:new_field(last_name, string,  [{length, 255}, not_null]),
-     sumo:new_field(age,       integer),
-     sumo:new_field(address,   string, [{length, 255}])
+  Fields =
+    [sumo:new_field(id,         string, [id]),
+     sumo:new_field(name,       string, [{length, 255}, not_null]),
+     sumo:new_field(last_name,  string, [{length, 255}, not_null]),
+     sumo:new_field(age,        integer),
+     sumo:new_field(address,    string, [{length, 255}]) %,
+     % sumo:new_field(birthdate,  date),
+     % sumo:new_field(created_at, datetime)
     ],
-    sumo:new_schema(?MODULE, Fields).
+  sumo:new_schema(?MODULE, Fields).