Avoid interrupting ra system start when pre init fails.

In practice this has proven too strict so now we will only log
errors as they occur, not stop the pre-init process from completing.

Also avoid asserting on a missing checkpoints directory as it isn't necessary
and the next server start will take care of that.

src/ra_lib.erl

@@ -529,7 +529,7 @@ merge_with_1(none, Result, _, _) ->
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
-lists_chink_test() ->
+lists_chunk_test() ->
     ?assertError(invalid_size, lists_chunk(0, [a])),
     ?assertMatch([], lists_chunk(2, [])),
     ?assertMatch([[a]], lists_chunk(2, [a])),

src/ra_log_pre_init.erl

@@ -46,9 +46,9 @@ init([System]) ->
                  ok -> ok
              catch _:Err ->
                        ?ERROR("pre_init failed in system ~s for UId ~ts with name ~ts"
-                              " This error may need manual intervention",
-                              [System, UId, Name]),
-                       throw({stop, {error, Err}})
+                              " This error may need manual intervention, Error ~p",
+                              [System, UId, Name, Err]),
+                       ok
              end
          end|| {Name, UId} <- Regd],
     {ok, #state{} , hibernate}.
@@ -95,7 +95,7 @@ pre_init(System, UId) ->
                                 {error, Err} ->
                                     ?ERROR("pre_init failed to read config file for UId '~ts', Err ~p",
                                            [UId, Err]),
-                                    exit({pre_init_failed, Err})
+                                    ok
                             end;
                         false ->
                             ?INFO("pre_init UId '~ts' is registered but no data directory was found",

src/ra_snapshot.erl

@@ -224,43 +224,47 @@ find_checkpoints(#?MODULE{uid = UId,
                           module = Module,
                           current = Current,
                           checkpoint_directory = CheckpointDir} = State) ->
-    true = ra_lib:is_dir(CheckpointDir),
-    CurrentIdx = case Current of
-                     undefined ->
-                         -1;
-                     {I, _} ->
-                         I
-                 end,
-    {ok, CPFiles0} = prim_file:list_dir(CheckpointDir),
-    CPFiles = lists:reverse(lists:sort(CPFiles0)),
-    Checkpoints =
-        lists:filtermap(
-          fun(File) ->
-                  CP = filename:join(CheckpointDir, File),
-                  case Module:validate(CP) of
-                      ok ->
-                          {ok, #{index := Idx, term := Term}} =
+    case ra_lib:is_dir(CheckpointDir) of
+        false ->
+            State;
+        true ->
+            CurrentIdx = case Current of
+                             undefined ->
+                                 -1;
+                             {I, _} ->
+                                 I
+                         end,
+            {ok, CPFiles0} = prim_file:list_dir(CheckpointDir),
+            CPFiles = lists:reverse(lists:sort(CPFiles0)),
+            Checkpoints =
+            lists:filtermap(
+              fun(File) ->
+                      CP = filename:join(CheckpointDir, File),
+                      case Module:validate(CP) of
+                          ok ->
+                              {ok, #{index := Idx, term := Term}} =
                               Module:read_meta(CP),
-                          case Idx > CurrentIdx of
-                              true ->
-                                  {true, {Idx, Term}};
-                              false ->
-                                  ?INFO("ra_snapshot: ~ts: removing "
-                                        "checkpoint ~s as was older than the "
-                                        "current snapshot.",
-                                        [UId, CP]),
-                                  delete(CheckpointDir, {Idx, Term}),
-                                  false
-                          end;
-                      Err ->
-                          ?INFO("ra_snapshot: ~ts: removing checkpoint ~s as "
-                                "did not validate. Err: ~w",
-                                [UId, CP, Err]),
-                          ra_lib:recursive_delete(CP),
-                          false
-                  end
-          end, CPFiles),
-    State#?MODULE{checkpoints = Checkpoints}.
+                              case Idx > CurrentIdx of
+                                  true ->
+                                      {true, {Idx, Term}};
+                                  false ->
+                                      ?INFO("ra_snapshot: ~ts: removing "
+                                            "checkpoint ~s as was older than the "
+                                            "current snapshot.",
+                                            [UId, CP]),
+                                      delete(CheckpointDir, {Idx, Term}),
+                                      false
+                              end;
+                          Err ->
+                              ?INFO("ra_snapshot: ~ts: removing checkpoint ~s as "
+                                    "did not validate. Err: ~w",
+                                    [UId, CP, Err]),
+                              ra_lib:recursive_delete(CP),
+                              false
+                      end
+              end, CPFiles),
+            State#?MODULE{checkpoints = Checkpoints}
+    end.
 
 -spec init_ets() -> ok.
 init_ets() ->

test/coordination_SUITE.erl

@@ -902,7 +902,8 @@ segment_writer_or_wal_crash_follower(Config) ->
                               LastIdxs =
                               [begin
                                    {ok, #{current_term := T,
-                                          log := #{last_index := L}}, _} =
+                                          log := #{last_index := L,
+                                                   cache_size := 0}}, _} =
                                    ra:member_overview(S),
                                    {T, L}
                                end || {_, _N} = S <- ServerIds],
@@ -934,8 +935,6 @@ segment_writer_or_wal_crash_follower(Config) ->
 
          %% assert stuff
          await_condition(AwaitReplicated, 100),
-         ?assertMatch({ok, #{log := #{cache_size := 0}}, _},
-                      ra:member_overview(Follower)),
          %% follower hasn't crashed
          ?assertEqual(FollowerPid, ct_rpc:call(FollowerNode, erlang, whereis,
                                                [FollowerName]))

test/ra_log_2_SUITE.erl

@@ -33,6 +33,8 @@ all_tests() ->
      recovery,
      recover_many,
      recovery_with_missing_directory,
+     recovery_with_missing_checkpoints_directory,
+     recovery_with_missing_config_file,
      wal_crash_recover,
      wal_down_read_availability,
      wal_down_append_throws,
@@ -653,6 +655,53 @@ recovery_with_missing_directory(Config) ->
 
     ok.
 
+recovery_with_missing_checkpoints_directory(Config) ->
+    %% checking that the ra system can be restarted even if the checkpoints
+    %% directory is missing, it will be created the next time the
+    %% log is initialised
+    logger:set_primary_config(level, debug),
+    UId = ?config(uid, Config),
+    Log0 = ra_log_init(Config),
+    ra_log:close(Log0),
+
+    ServerDataDir = ra_env:server_data_dir(default, UId),
+    CheckpointsDir = filename:join(ServerDataDir, "checkpoints"),
+    ok = ra_lib:recursive_delete(CheckpointsDir),
+    ?assertNot(filelib:is_dir(CheckpointsDir)),
+
+    application:stop(ra),
+    start_ra(Config),
+
+    Log5 = ra_log_init(Config),
+    ra_log:close(Log5),
+    ok = ra_lib:recursive_delete(ServerDataDir),
+    ?assertNot(filelib:is_dir(ServerDataDir)),
+
+    ok.
+
+recovery_with_missing_config_file(Config) ->
+    %% checking that the ra system can be restarted even when the config
+    %% file is missing
+    logger:set_primary_config(level, debug),
+    UId = ?config(uid, Config),
+    Log0 = ra_log_init(Config),
+    ra_log:close(Log0),
+
+    ServerDataDir = ra_env:server_data_dir(default, UId),
+    ConfigFile = filename:join(ServerDataDir, "config"),
+    file:delete(ConfigFile),
+    ?assertNot(filelib:is_file(ConfigFile)),
+
+    application:stop(ra),
+    start_ra(Config),
+
+    Log5 = ra_log_init(Config),
+    ra_log:close(Log5),
+    ok = ra_lib:recursive_delete(ServerDataDir),
+    ?assertNot(filelib:is_dir(ServerDataDir)),
+
+    ok.
+
 resend_write(Config) ->
     % logger:set_primary_config(level, debug),
     % simulate lost messages requiring the ra server to resend in flight