[server][Action] Reexecute action mechanism.

server/src/ActionManager.cc

@@ -549,6 +549,52 @@ void ActionManager::checkEvents(const EventInfoList &eventList)
 	}
 }
 
+void ActionManager::reExecuteUnfinishedAction(void)
+{
+	ThreadLocalDBCache cache;
+	DBTablesAction &dbAction = cache.getAction();
+	DBTablesMonitoring &dbMonitoring = cache.getMonitoring();
+
+	ActionLogList actionLogList;
+	const vector<int> targetStatuses{ACTLOG_STAT_QUEUING,
+	                                 ACTLOG_STAT_STARTED,
+	                                 ACTLOG_STAT_RESIDENT_QUEUING,
+	                                 ACTLOG_STAT_LAUNCHING_RESIDENT};
+
+	if (!dbAction.getTargetStatusesLogs(actionLogList, targetStatuses)) {
+		MLPL_INFO("Hatohol does not have unfinished action.\n");
+		return;
+	}
+
+	for (const auto &actionLog: actionLogList) {
+		EventInfoList eventList;
+		EventsQueryOption eventOption(USER_ID_SYSTEM);
+
+		eventOption.setTargetServerId(actionLog.serverId);
+		eventOption.setEventIds({actionLog.eventId});
+		dbMonitoring.getEventInfoList(eventList, eventOption);
+		if (eventList.empty())
+		        continue;
+		auto const &eventInfo = *eventList.cbegin();
+
+		ActionDefList actionList;
+		ActionsQueryOption actionOption(USER_ID_SYSTEM);
+		ActionIdList actionIdList{actionLog.actionId};
+		actionOption.setActionIdList(actionIdList);
+		dbAction.getActionList(actionList, actionOption);
+
+		if (actionList.empty())
+		        continue;
+		dbAction.updateLogStatusToAborted(actionLog.id);
+		runAction(*actionList.cbegin(), eventInfo, dbAction);
+		const string message =
+		   StringUtils::sprintf("Action log ID(%" FMT_GEN_ID "): "
+		                        "Update log status to aborted(7).\n",
+		                        actionLog.id);
+		MLPL_WARN("%s", message.c_str());
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Protected methods
 // ---------------------------------------------------------------------------

server/src/ActionManager.h

@@ -59,6 +59,7 @@ class ActionManager
 	ActionManager(void);
 	virtual ~ActionManager();
 	void checkEvents(const EventInfoList &eventList);
+	void reExecuteUnfinishedAction(void);
 
 protected:
 	/**

server/src/DBTablesAction.cc

@@ -823,6 +823,19 @@ void DBTablesAction::updateLogStatusToStart(const ActionLogIdType &logId)
 	getDBAgent().runTransaction(arg);
 }
 
+void DBTablesAction::updateLogStatusToAborted(const ActionLogIdType &logId)
+{
+	DBAgent::UpdateArg arg(tableProfileActionLogs);
+
+	const char *actionLogIdColumnName =
+	  COLUMN_DEF_ACTION_LOGS[IDX_ACTION_LOGS_ACTION_LOG_ID].columnName;
+	arg.condition = StringUtils::sprintf("%s=%" FMT_ACTION_LOG_ID,
+                                             actionLogIdColumnName, logId);
+	arg.add(IDX_ACTION_LOGS_STATUS, ACTLOG_STAT_ABORTED);
+
+	getDBAgent().update(arg);
+}
+
 bool DBTablesAction::getLog(ActionLog &actionLog, const ActionLogIdType &logId)
 {
 	const ColumnDef *def = COLUMN_DEF_ACTION_LOGS;
@@ -846,6 +859,25 @@ bool DBTablesAction::getLog(
 	return getLog(actionLog, condition);
 }
 
+bool DBTablesAction::getTargetStatusesLogs(
+  ActionLogList &actionLogList, const vector<int> &targetStatuses)
+{
+	SeparatorInjector commaInjector(",");
+	string statuses;
+	for (const auto &targetStatus: targetStatuses) {
+		commaInjector(statuses);
+		statuses += to_string(targetStatus);
+	}
+	const ColumnDef *def = COLUMN_DEF_ACTION_LOGS;
+	const char *idColNameStat = def[IDX_ACTION_LOGS_STATUS].columnName;
+	const char *idColNameEvtId = def[IDX_ACTION_LOGS_EVENT_ID].columnName;
+	string condition = StringUtils::sprintf(
+	                     "%s in (%s) AND %s!=''",
+	                     idColNameStat, statuses.c_str(), idColNameEvtId);
+
+	return getLogs(actionLogList, condition);
+}
+
 bool DBTablesAction::isIncidentSenderEnabled(void)
 {
 	ActionDefList actionDefList;
@@ -948,6 +980,78 @@ bool DBTablesAction::getLog(ActionLog &actionLog, const string &condition)
 	return true;
 }
 
+// TODO: Extract commonly used lines in getLog() above and reduce the similar lines. 
+bool DBTablesAction::getLogs(ActionLogList &actionLogList,
+                             const string &condition)
+{
+	DBAgent::SelectExArg arg(tableProfileActionLogs);
+	arg.condition = condition;
+	arg.add(IDX_ACTION_LOGS_ACTION_LOG_ID);
+	arg.add(IDX_ACTION_LOGS_ACTION_ID);
+	arg.add(IDX_ACTION_LOGS_STATUS);
+	arg.add(IDX_ACTION_LOGS_STARTER_ID);
+	arg.add(IDX_ACTION_LOGS_QUEUING_TIME);
+	arg.add(IDX_ACTION_LOGS_START_TIME);
+	arg.add(IDX_ACTION_LOGS_END_TIME);
+	arg.add(IDX_ACTION_LOGS_EXEC_FAILURE_CODE);
+	arg.add(IDX_ACTION_LOGS_EXIT_CODE);
+	arg.add(IDX_ACTION_LOGS_SERVER_ID);
+	arg.add(IDX_ACTION_LOGS_EVENT_ID);
+
+	getDBAgent().runTransaction(arg);
+
+	const auto &grpList = arg.dataTable->getItemGroupList();
+	if (grpList.empty())
+		return false;
+
+	for (const auto &itemGrp: grpList) {
+		ItemGroupStream itemGroupStream(itemGrp);
+		actionLogList.push_back(ActionLog());
+		ActionLog &actionLog = actionLogList.back();
+		actionLog.nullFlags = 0;
+
+		itemGroupStream >> actionLog.id;
+		itemGroupStream >> actionLog.actionId;
+		itemGroupStream >> actionLog.status;
+		itemGroupStream >> actionLog.starterId;
+
+		// queing time
+		if (itemGroupStream.getItem()->isNull())
+			actionLog.nullFlags |= ACTLOG_FLAG_QUEUING_TIME;
+		itemGroupStream >> actionLog.queuingTime;
+
+		// start time
+		if (itemGroupStream.getItem()->isNull())
+			actionLog.nullFlags |= ACTLOG_FLAG_START_TIME;
+		itemGroupStream >> actionLog.startTime;
+
+		// end time
+		if (itemGroupStream.getItem()->isNull())
+			actionLog.nullFlags |= ACTLOG_FLAG_END_TIME;
+		itemGroupStream >> actionLog.endTime;
+
+		// failure code
+		itemGroupStream >> actionLog.failureCode;
+
+		// exit code
+		if (itemGroupStream.getItem()->isNull())
+			actionLog.nullFlags |= ACTLOG_FLAG_EXIT_CODE;
+		itemGroupStream >> actionLog.exitCode;
+
+		// server id
+		if (itemGroupStream.getItem()->isNull())
+			actionLog.nullFlags |= ACTLOG_FLAG_EXIT_CODE;
+		itemGroupStream >> actionLog.serverId;
+
+		// event id
+		if (itemGroupStream.getItem()->isNull())
+			actionLog.nullFlags |= ACTLOG_FLAG_EXIT_CODE;
+		itemGroupStream >> actionLog.eventId;
+	}
+
+	return true;
+}
+
 HatoholError DBTablesAction::checkPrivilegeForAdd(
   const OperationPrivilege &privilege, const ActionDef &actionDef)
 {

server/src/DBTablesAction.h

@@ -173,9 +173,15 @@ struct ActionLog {
 	int      endTime;
 	int      failureCode;
 	int      exitCode;
+	ServerIdType serverId;
+	EventIdType  eventId;
 	uint32_t nullFlags;
 };
 
+typedef std::list<ActionLog>          ActionLogList;
+typedef ActionLogList::iterator       ActionLogListIterator;
+typedef ActionLogList::const_iterator ActionLogListConstIterator;
+
 struct ChildSigInfo {
 	pid_t pid;
 	int   code;
@@ -217,6 +223,7 @@ enum ActionLogStatus {
 	// Resident acitons for an actio ID are executed in series
 	// This status is used for waiting actions.
 	ACTLOG_STAT_RESIDENT_QUEUING,
+	ACTLOG_STAT_ABORTED,
 };
 
 enum ActionLogExecFailureCode {
@@ -342,6 +349,13 @@ class DBTablesAction : public DBTables {
 	 */
 	void updateLogStatusToStart(const ActionLogIdType &logId);
 
+	/**
+	 * Update the status in action log to ACTLOG_STAT_FAILED.
+	 *
+	 * @param logId A logID to be updated.
+	 */
+	void updateLogStatusToAborted(const ActionLogIdType &logId);
+
 	/**
 	 * Get the action log.
 	 * @param actionLog
@@ -365,6 +379,9 @@ class DBTablesAction : public DBTables {
 	bool getLog(ActionLog &actionLog, const ServerIdType &serverId,
 	            const EventIdType &eventId);
 
+	bool getTargetStatusesLogs(ActionLogList &actionLogList,
+	                           const std::vector<int> &targetStatuses);
+
 	/**
 	 * Check whether IncidentSender type action exists or not
 	 *
@@ -391,6 +408,7 @@ class DBTablesAction : public DBTables {
 	 * @return true if the log is found. Otherwise false.
 	 */
 	bool getLog(ActionLog &actionLog, const std::string &condition);
+	bool getLogs(ActionLogList &actionLogList, const std::string &condition);
 
 	HatoholError checkPrivilegeForAdd(
 	  const OperationPrivilege &privilege, const ActionDef &actionDef);

server/src/DBTablesMonitoring.cc

@@ -1011,6 +1011,7 @@ struct EventsQueryOption::Impl {
 	set<string> incidentStatuses;
 	vector<string> groupByColumns;
 	list<string> hostnameList;
+	list<EventIdType> eventIds;
 
 	Impl()
 	: limitOfUnifiedId(NO_LIMIT),
@@ -1022,7 +1023,8 @@ struct EventsQueryOption::Impl {
 	  triggerId(ALL_TRIGGERS),
 	  beginTime({0, 0}),
 	  endTime({0, 0}),
-	  hostnameList({})
+	  hostnameList({}),
+	  eventIds({})
 	{
 	}
 };
@@ -1137,6 +1139,11 @@ string EventsQueryOption::getCondition(void) const
 			     makeHostnameListCondition(m_impl->hostnameList));
 	}
 
+	if (!m_impl->eventIds.empty()) {
+		addCondition(condition,
+		             makeEventIdListCondition(m_impl->eventIds));
+	}
+
 	string typeCondition;
 	for (const auto &type: m_impl->eventTypes) {
 		addCondition(
@@ -1368,6 +1375,11 @@ const std::set<EventType> &EventsQueryOption::getEventTypes(void) const
 	return m_impl->eventTypes;
 }
 
+void EventsQueryOption::setEventIds(const list<EventIdType> &eventIds)
+{
+	m_impl->eventIds = eventIds;
+}
+
 void EventsQueryOption::setTriggerSeverities(
   const set<TriggerSeverityType> &severities)
 {
@@ -1417,6 +1429,22 @@ string EventsQueryOption::makeHostnameListCondition(
 	return condition;
 }
 
+string EventsQueryOption::makeEventIdListCondition(
+  const list<EventIdType> &eventIds) const
+{
+	string condition;
+	const ColumnDef &colId = COLUMN_DEF_EVENTS[IDX_EVENTS_ID];
+	SeparatorInjector commaInjector(",");
+	DBTermCStringProvider rhs(*getDBTermCodec());
+	condition = StringUtils::sprintf("%s in (", colId.columnName);
+	for (const auto &eventId : eventIds) {
+		commaInjector(condition);
+		condition += StringUtils::sprintf("%s", rhs(eventId.c_str()));
+	}
+	condition += ")";
+	return condition;
+}
+
 //
 // TriggersQueryOption
 //

server/src/DBTablesMonitoring.h

@@ -74,6 +74,7 @@ class EventsQueryOption : public HostResourceQueryOption {
 
 	void setEventTypes(const std::set<EventType> &types);
 	const std::set<EventType> &getEventTypes(void) const;
+	void setEventIds(const std::list<EventIdType> &eventIds);
 	void setTriggerSeverities(const std::set<TriggerSeverityType> &severities);
 	const std::set<TriggerSeverityType> &getTriggerSeverities(void) const;
 	void setTriggerStatuses(const std::set<TriggerStatusType> &statuses);
@@ -85,6 +86,9 @@ class EventsQueryOption : public HostResourceQueryOption {
 	std::string makeHostnameListCondition(
 	  const std::list<std::string> &hostnameList) const;
 
+	std::string makeEventIdListCondition(
+	  const std::list<EventIdType> &eventIds) const;
+
 private:
 	struct Impl;
 	std::unique_ptr<Impl> m_impl;

server/src/main.cc

@@ -46,6 +46,7 @@ using namespace mlpl;
 #include "ConfigManager.h"
 #include "ThreadLocalDBCache.h"
 #include "ChildProcessManager.h"
+#include "ActionManager.h"
 
 static string pidFilePath;
 static int pipefd[2];
@@ -244,6 +245,11 @@ int mainRoutine(int argc, char *argv[])
 
 	// setup configuration database
 	ThreadLocalDBCache cache;
+
+	// Re execute unfinished action
+	ActionManager actionManager;
+	actionManager.reExecuteUnfinishedAction();
+
 	// start REST server
 	// 'rest' is on a stack. The destructor of it will be automatically
 	// called at the end of this function.