From a8069b9bedacbe819840da4a50133fe86065a24c Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 Jan 2020 14:01:21 -0500 Subject: [PATCH] Improve data warehouse export logging (#1223) --- .../DataWarehouse/Export/BatchProcessor.php | 129 +++++++++++++----- classes/DataWarehouse/Export/FileManager.php | 50 ++++--- .../Export/FileWriter/FileWriterFactory.php | 14 +- classes/DataWarehouse/Export/QueryHandler.php | 88 +++++++++++- .../WarehouseExportControllerProvider.php | 93 ++++++++++--- .../warehouse-export/GET-requests.schema.json | 11 ++ tests/component/lib/Export/ExportDBTest.php | 1 + 7 files changed, 309 insertions(+), 77 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 8a401631d2..b16c37a660 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -16,6 +16,15 @@ class BatchProcessor extends Loggable { + // Constants used in log messages. + const LOG_MODULE_KEY = 'module'; + const LOG_MODULE = 'data-warehouse-export'; + const LOG_MESSAGE_KEY = 'message'; + const LOG_EVENT_KEY = 'event'; + const LOG_REQUEST_ID_KEY = 'batch_export_request.id'; + const LOG_USER_ID_KEY = 'Users.id'; + const LOG_STACKTRACE_KEY = 'stacktrace'; + /** * Database handle for moddb. * @var \CCR\DB\iDatabase @@ -54,9 +63,9 @@ public function __construct(Log $logger = null) // Must set properties that are used in `setLogger` before calling the // parent constructor. $this->fileManager = new FileManager($logger); + $this->queryHandler = new QueryHandler($logger); parent::__construct($logger); $this->dbh = DB::factory('database'); - $this->queryHandler = new QueryHandler(); $this->realmManager = new RealmManager(); } @@ -71,6 +80,7 @@ public function setLogger(Log $logger = null) { parent::setLogger($logger); $this->fileManager->setLogger($logger); + $this->queryHandler->setLogger($logger); return $this; } @@ -106,7 +116,10 @@ public function processRequests() */ private function processSubmittedRequests() { - $this->logger->info('Processing submitted requests'); + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Processing submitted requests' + ]); foreach ($this->queryHandler->listSubmittedRecords() as $request) { $this->processSubmittedRequest($request); } @@ -119,18 +132,24 @@ private function processSubmittedRequests() */ private function processSubmittedRequest(array $request) { + $requestId = $request['id']; + $userId = $request['user_id']; + $this->logger->info([ - 'message' => 'Processing request', - 'batch_export_request.id' => $request['id'] + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Processing request', + self::LOG_USER_ID_KEY => $userId, + self::LOG_REQUEST_ID_KEY => $requestId ]); - $user = XDUser::getUserByID($request['user_id']); + $user = XDUser::getUserByID($userId); if ($user === null) { $this->logger->err([ - 'message' => 'User not found', - 'Users.id' => $request['user_id'], - 'batch_export_request.id' => $request['id'] + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'User not found', + self::LOG_USER_ID_KEY => $userId, + self::LOG_REQUEST_ID_KEY => $requestId ]); return; } @@ -138,35 +157,45 @@ private function processSubmittedRequest(array $request) try { $this->dbh->beginTransaction(); if (!$this->dryRun) { - $this->queryHandler->submittedToAvailable($request['id']); + $this->queryHandler->submittedToAvailable($requestId); } $dataSet = $this->getDataSet($request, $user); $format = $this->dryRun ? 'null' : $request['export_file_format']; $dataFile = $this->fileManager->writeDataSetToFile($dataSet, $format); if (!$this->dryRun) { - $this->fileManager->createZipFile($dataFile, $request); + $zipFile = $this->fileManager->createZipFile($dataFile, $request); } + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_EVENT_KEY => 'CREATED_EXPORT_FILE', + self::LOG_MESSAGE_KEY => 'Created data warehouse export zip file', + self::LOG_USER_ID_KEY => $userId, + self::LOG_REQUEST_ID_KEY => $requestId, + 'file_path' => $zipFile + ]); + // Delete file that was added to zip archive. if (!$this->dryRun && !unlink($dataFile)) { - $this->logger->err(sprintf( - 'Failed to delete temporary data file "%s"', - $dataFile - )); + $this->logger->err([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => sprintf('Failed to delete temporary data file "%s"', $dataFile) + ]); } // Query for same record to get expiration date. - $request = $this->queryHandler->getRequestRecord($request['id']); + $request = $this->queryHandler->getRequestRecord($requestId); $this->sendExportSuccessEmail($user, $request); $this->dbh->commit(); } catch (Exception $e) { $this->dbh->rollback(); $this->logger->err([ - 'message' => 'Failed to export data: ' . $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Failed to export data: ' . $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() ]); if (!$this->dryRun) { - $this->queryHandler->submittedToFailed($request['id']); + $this->queryHandler->submittedToFailed($requestId); } $this->sendExportFailureEmail($user, $request, $e); } @@ -180,7 +209,10 @@ private function processSubmittedRequest(array $request) */ private function processExpiringRequests() { - $this->logger->info('Processing expiring requests'); + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Processing expiring requests' + ]); foreach ($this->queryHandler->listExpiringRecords() as $request) { $this->processExpiringRequest($request); } @@ -194,12 +226,16 @@ private function processExpiringRequests() private function processExpiringRequest(array $request) { $this->logger->info([ - 'message' => 'Expiring request', - 'batch_export_request.id' => $request['id'] + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Expiring request', + self::LOG_REQUEST_ID_KEY => $request['id'] ]); if ($this->dryRun) { - $this->logger->notice('dry run: Not expiring export file'); + $this->logger->notice([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'dry run: Not expiring export file' + ]); return; } @@ -211,8 +247,9 @@ private function processExpiringRequest(array $request) } catch (Exception $e) { $this->dbh->rollback(); $this->logger->err([ - 'message' => 'Failed to expire record: ' . $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Failed to expire record: ' . $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() ]); } } @@ -225,7 +262,10 @@ private function processExpiringRequest(array $request) */ private function processDeletedRequests() { - $this->logger->info('Processing deleted requests'); + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Processing deleted requests' + ]); $this->fileManager->removeDeletedRequests( array_map( function ($request) { @@ -247,12 +287,13 @@ function ($request) { private function getDataSet(array $request, XDUser $user) { $this->logger->info([ - 'message' => 'Querying data', - 'Users.id' => $user->getUserID(), + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Querying data', + self::LOG_USER_ID_KEY => $user->getUserID(), 'user_email' => $user->getEmailAddress(), 'user_first_name' => $user->getFirstName(), 'user_last_name' => $user->getLastName(), - 'batch_export_request.id' => $request['id'], + self::LOG_REQUEST_ID_KEY => $request['id'], 'realm' => $request['realm'], 'start_date' => $request['start_date'], 'end_date' => $request['end_date'] @@ -260,7 +301,10 @@ private function getDataSet(array $request, XDUser $user) try { $className = $this->realmManager->getRawDataQueryClass($request['realm']); - $this->logger->debug(sprintf('Instantiating query class "%s"', $className)); + $this->logger->debug([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => sprintf('Instantiating query class "%s"', $className) + ]); $query = new $className( [ 'start_date' => $request['start_date'], @@ -272,8 +316,9 @@ private function getDataSet(array $request, XDUser $user) return $dataSet; } catch (Exception $e) { $this->logger->err([ - 'message' => $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() ]); throw new Exception('Failed to create batch export query', 0, $e); } @@ -288,11 +333,18 @@ private function getDataSet(array $request, XDUser $user) private function sendExportSuccessEmail(XDUser $user, array $request) { if ($this->dryRun) { - $this->logger->notice('dry run: Not sending success email'); + $this->logger->notice([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'dry run: Not sending success email' + ]); return; } - $this->logger->info('Sending success email'); + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_EVENT_KEY => 'SENDING_SUCCESS_EMAIL', + self::LOG_MESSAGE_KEY => 'Sending success email' + ]); // Remove time from expiration date time. list($expirationDate) = explode(' ', $request['export_expires_datetime']); @@ -332,13 +384,18 @@ private function sendExportFailureEmail( Exception $e ) { if ($this->dryRun) { - $this->logger->notice('dry run: Not sending failure email'); + $this->logger->notice([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'dry run: Not sending failure email' + ]); return; } $this->logger->info([ - 'message' => 'Sending failure email', - 'batch_export_request.id' => $request['id'] + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_EVENT_KEY => 'SENDING_FAILURE_EMAIL', + self::LOG_MESSAGE_KEY => 'Sending failure email', + self::LOG_REQUEST_ID_KEY => $request['id'] ]); $message = $e->getMessage(); diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index 12691066c0..6912ac9ac6 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -15,6 +15,14 @@ */ class FileManager extends Loggable { + // Constants used in log messages. + const LOG_MODULE_KEY = 'module'; + const LOG_MODULE = 'data-warehouse-export'; + const LOG_MESSAGE_KEY = 'message'; + const LOG_STACKTRACE_KEY = 'stacktrace'; + const LOG_REQUEST_ID_KEY = 'batch_export_request.id'; + const LOG_ZIP_FILE_KEY = 'zip_file'; + /** * Data warehouse batch export directory path. * @@ -50,8 +58,9 @@ public function __construct(Log $logger = null) ); } catch (Exception $e) { $this->logger->err([ - 'message' => $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() ]); throw new Exception('Export directory is not configured', 0, $e); } @@ -149,7 +158,8 @@ public function getZipFileName(array $request) public function writeDataSetToFile(BatchDataset $dataSet, $format) { $this->logger->info([ - 'message' => 'Writing data to file', + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Writing data to file', 'format' => $format ]); @@ -170,7 +180,8 @@ public function writeDataSetToFile(BatchDataset $dataSet, $format) ); $this->logger->debug([ - 'message' => 'Created file writer', + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Created file writer', 'file_writer' => $fileWriter ]); @@ -185,8 +196,9 @@ public function writeDataSetToFile(BatchDataset $dataSet, $format) return $dataFile; } catch (Exception $e) { $this->logger->err([ - 'message' => $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() ]); throw new Exception('Failed to write data set to file', 0, $e); } @@ -205,10 +217,11 @@ public function createZipFile($dataFile, array $request) $zipFile = $this->getExportDataFilePath($request['id']); $this->logger->info([ - 'message' => 'Creating zip file', - 'batch_export_request.id' => $request['id'], + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Creating zip file', + self::LOG_REQUEST_ID_KEY => $request['id'], 'data_file' => $dataFile, - 'zip_file' => $zipFile + self::LOG_ZIP_FILE_KEY => $zipFile ]); try { @@ -245,8 +258,9 @@ public function createZipFile($dataFile, array $request) return $zipFile; } catch (Exception $e) { $this->logger->err([ - 'message' => $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() ]); throw new Exception('Failed to create zip file', 0, $e); } @@ -263,9 +277,10 @@ public function removeExportFile($id) $zipFile = $this->getExportDataFilePath($id); $this->logger->info([ - 'message' => 'Removing export file', - 'batch_export_request.id' => $id, - 'zip_file' => $zipFile + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Removing export file', + self::LOG_REQUEST_ID_KEY => $id, + self::LOG_ZIP_FILE_KEY => $zipFile ]); if (!unlink($zipFile)) { @@ -285,9 +300,10 @@ public function removeDeletedRequests(array $deletedRequestIds) $exportFile = $this->getExportDataFilePath($id); if (is_file($exportFile)) { $this->logger->info([ - 'message' => 'Removing export file', - 'batch_export_request.id' => $id, - 'zip_file' => $exportFile + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Removing export file', + self::LOG_REQUEST_ID_KEY => $id, + self::LOG_ZIP_FILE_KEY => $exportFile ]); if (!unlink($exportFile)) { throw new Exception(sprintf( diff --git a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php index ed54fd1392..93ed1892ae 100644 --- a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php +++ b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php @@ -9,6 +9,13 @@ */ class FileWriterFactory extends Loggable { + // Constants used in log messages. + const LOG_MODULE_KEY = 'module'; + const LOG_MODULE = 'data-warehouse-export'; + const LOG_MESSAGE_KEY = 'message'; + const LOG_FORMAT_KEY = 'format'; + const LOG_FILE_KEY = 'file'; + /** * Create a file writer for the given format and file. * @@ -19,9 +26,10 @@ class FileWriterFactory extends Loggable public function createFileWriter($format, $file) { $this->logger->debug([ - 'message' => 'Creating new file writer', - 'format' => $format, - 'file' => $file + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Creating new file writer', + self::LOG_FORMAT_KEY => $format, + self::LOG_FILE_KEY => $file ]); switch (strtolower($format)) { diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index b7d89aa0a3..f7cc1bceb7 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -29,9 +29,25 @@ use Exception; use CCR\DB; +use CCR\Loggable; +use Log; -class QueryHandler +class QueryHandler extends Loggable { + // Constants used in log messages. + const LOG_MODULE_KEY = 'module'; + const LOG_MODULE = 'data-warehouse-export'; + const LOG_MESSAGE_KEY = 'message'; + const LOG_EVENT_KEY = 'event'; + const LOG_TABLE_KEY = 'table'; + const LOG_TABLE = 'moddb.batch_export_requests'; + const LOG_ID_KEY = 'id'; + const LOG_USER_ID_KEY = 'user_id'; + const LOG_STACKTRACE_KEY = 'stacktrace'; + const LOG_REALM_KEY = 'realm'; + const LOG_START_DATE_KEY = 'start_date'; + const LOG_END_DATE_KEY = 'end_date'; + /** * Database handle. * @var \CCR\DB\iDatabase @@ -68,8 +84,9 @@ class QueryHandler */ private $whereDeleted = "WHERE is_deleted = 1 "; - public function __construct() + public function __construct(Log $logger = null) { + parent::__construct($logger); $this->dbh = DB::factory('database'); } @@ -121,11 +138,28 @@ function ($request) use ($realm, $startDate, $endDate, $format) { 'export_file_format' => $format ); + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Creating data warehouse export record', + self::LOG_EVENT_KEY => 'INSERT', + self::LOG_TABLE_KEY => self::LOG_TABLE, + self::LOG_USER_ID_KEY => $userId, + self::LOG_REALM_KEY => $realm, + self::LOG_START_DATE_KEY => $startDate, + self::LOG_END_DATE_KEY => $endDate, + 'format' => $format + ]); + $id = $this->dbh->insert($sql, $params); $this->dbh->commit(); return $id; } catch (Exception $e) { $this->dbh->rollBack(); + $this->logger->err([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Record creation failed: ' . $e->getMessage(), + self::LOG_STACKTRACE_KEY => $e->getTraceAsString() + ]); throw $e; } } @@ -169,6 +203,13 @@ public function submittedToFailed($id) SET export_succeeded = 0 " . $this->whereSubmitted . "AND id = :id"; + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Transitioning data warehouse export record to failed state', + self::LOG_EVENT_KEY => 'UPDATE_STATE_TO_FAILED', + self::LOG_TABLE_KEY => self::LOG_TABLE, + self::LOG_ID_KEY => $id + ]); return $this->dbh->execute($sql, array('id' => $id)); } @@ -201,6 +242,14 @@ public function submittedToAvailable($id) 'id' => $id ); + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Transitioning data warehouse export record to available state', + self::LOG_EVENT_KEY => 'UPDATE_STATE_TO_AVAILABLE', + self::LOG_TABLE_KEY => self::LOG_TABLE, + self::LOG_ID_KEY => $id + ]); + return $this->dbh->execute($sql, $params); } @@ -214,6 +263,13 @@ public function availableToExpired($id) { $sql = "UPDATE batch_export_requests SET export_expired = 1 " . $this->whereAvailable . 'AND id = :id'; + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Transitioning data warehouse export record to expired state', + self::LOG_EVENT_KEY => 'UPDATE_STATE_TO_EXPIRED', + self::LOG_TABLE_KEY => self::LOG_TABLE, + self::LOG_ID_KEY => $id + ]); return $this->dbh->execute($sql, array('id' => $id)); } @@ -332,6 +388,7 @@ public function listUserRequestsByState($userId) export_created_datetime, export_file_format, requested_datetime, + downloaded_datetime, "; $fromTable = "FROM batch_export_requests "; $userClause = "AND user_id = :user_id "; @@ -356,6 +413,33 @@ public function listUserRequestsByState($userId) public function deleteRequest($id, $userId) { $sql = "UPDATE batch_export_requests SET is_deleted = 1 WHERE id = :request_id AND user_id = :user_id"; + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Deleting data warehouse export record', + self::LOG_EVENT_KEY => 'UPDATE_STATE_TO_DELETED', + self::LOG_TABLE_KEY => self::LOG_TABLE, + self::LOG_ID_KEY => $id, + self::LOG_USER_ID_KEY => $userId + ]); return $this->dbh->execute($sql, array('request_id' => $id, 'user_id' => $userId)); } + + /** + * Update the downloaded datetime for a record. + * + * @param integer $id Export request primary key. + * @return integer Count of updated rows--should be 1 if successful. + */ + public function updateDownloadedDatetime($id) + { + $sql = 'UPDATE batch_export_requests SET downloaded_datetime = NOW() WHERE id = :request_id'; + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Updating data warehouse export record downloaded datetime', + self::LOG_EVENT_KEY => 'UPDATE_DOWNLOADED_DATETIME', + self::LOG_TABLE_KEY => self::LOG_TABLE, + self::LOG_ID_KEY => $id + ]); + return $this->dbh->execute($sql, ['request_id' => $id]); + } } diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 106973bb7d..50724b41db 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -3,6 +3,7 @@ namespace Rest\Controllers; use CCR\DB; +use CCR\Log; use DataWarehouse\Export\FileManager; use DataWarehouse\Export\QueryHandler; use DataWarehouse\Export\RealmManager; @@ -17,6 +18,20 @@ class WarehouseExportControllerProvider extends BaseControllerProvider { + // Constants used in log messages. + const LOG_MODULE_KEY = 'module'; + const LOG_MODULE = 'data-warehouse-export'; + const LOG_MESSAGE_KEY = 'message'; + const LOG_EVENT_KEY = 'event'; + const LOG_ID_KEY = 'id'; + const LOG_USER_ID_KEY = 'user_id'; + + // Constants used in JSON responses. + const JSON_SUCCESS_KEY = 'success'; + const JSON_MESSAGE_KEY = 'message'; + const JSON_DATA_KEY = 'data'; + const JSON_TOTAL_KEY = 'total'; + /** * @var DataWarehouse\Export\QueryHandler */ @@ -27,11 +42,24 @@ class WarehouseExportControllerProvider extends BaseControllerProvider */ private $realmManager; + /** + * @var \CCR\Log + */ + private $logger; + public function __construct(array $params = []) { parent::__construct($params); + $this->logger = Log::factory( + 'data-warehouse-export-rest', + [ + 'console' => false, + 'file' => false, + 'mail' => false + ] + ); $this->realmManager = new RealmManager(); - $this->queryHandler = new QueryHandler(); + $this->queryHandler = new QueryHandler($this->logger); } /** @@ -85,9 +113,9 @@ function ($realm) { return $app->json( [ - 'success' => true, - 'data' => array_values($realms), - 'total' => count($realms) + self::JSON_SUCCESS_KEY => true, + self::JSON_DATA_KEY => array_values($realms), + self::JSON_TOTAL_KEY => count($realms) ] ); } @@ -106,9 +134,9 @@ public function getRequests(Request $request, Application $app) $results = $this->queryHandler->listUserRequestsByState($user->getUserId()); return $app->json( [ - 'success' => true, - 'data' => $results, - 'total' => count($results) + self::JSON_SUCCESS_KEY => true, + self::JSON_DATA_KEY => $results, + self::JSON_TOTAL_KEY => count($results) ] ); } @@ -178,10 +206,10 @@ function ($realm) { } return $app->json([ - 'success' => true, - 'message' => 'Created export request', - 'data' => [['id' => $id]], - 'total' => 1 + self::JSON_SUCCESS_KEY => true, + self::JSON_MESSAGE_KEY => 'Created export request', + self::JSON_DATA_KEY => [['id' => $id]], + self::JSON_TOTAL_KEY => 1 ]); } @@ -231,6 +259,18 @@ function ($request) use ($id) { throw new AccessDeniedHttpException('Exported data is not readable'); } + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Sending data warehouse export file', + self::LOG_EVENT_KEY => 'DOWNLOAD', + self::LOG_ID_KEY => $id, + self::LOG_USER_ID_KEY => $user->getUserId() + ]); + + if ($request['downloaded_datetime'] === null) { + $this->queryHandler->updateDownloadedDatetime($request['id']); + } + return $app->sendFile( $file, 200, @@ -263,11 +303,19 @@ public function deleteRequest(Request $request, Application $app, $id) throw new NotFoundHttpException('Export request not found'); } + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Deleted data warehouse export request', + self::LOG_EVENT_KEY => 'DELETE_BY_USER', + self::LOG_ID_KEY => $id, + self::LOG_USER_ID_KEY => $user->getUserId() + ]); + return $app->json([ - 'success' => true, - 'message' => 'Deleted export request', - 'data' => [['id' => $id]], - 'total' => 1 + self::JSON_SUCCESS_KEY => true, + self::JSON_MESSAGE_KEY => 'Deleted export request', + self::JSON_DATA_KEY => [['id' => $id]], + self::JSON_TOTAL_KEY => 1 ]); } @@ -319,6 +367,13 @@ public function deleteRequests(Request $request, Application $app) if ($count === 0) { throw new NotFoundHttpException('Export request not found'); } + $this->logger->info([ + self::LOG_MODULE_KEY => self::LOG_MODULE, + self::LOG_MESSAGE_KEY => 'Deleted data warehouse export request', + self::LOG_EVENT_KEY => 'DELETE_BY_USER', + self::LOG_ID_KEY => $id, + self::LOG_USER_ID_KEY => $user->getUserId() + ]); } $dbh->commit(); @@ -331,15 +386,15 @@ public function deleteRequests(Request $request, Application $app) } return $app->json([ - 'success' => true, - 'message' => 'Deleted export requests', - 'data' => array_map( + self::JSON_SUCCESS_KEY => true, + self::JSON_MESSAGE_KEY => 'Deleted export requests', + self::JSON_DATA_KEY => array_map( function ($id) { return ['id' => $id]; }, $requestIds ), - 'total' => count($requestIds) + self::JSON_TOTAL_KEY => count($requestIds) ]); } } diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json index 219dfaf2ac..fd7eea9a37 100644 --- a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json +++ b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json @@ -88,6 +88,17 @@ "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$" }, + "downloaded_datetime": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$" + } + ] + }, "state": { "type": "string", "enum": [ diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 775e87e3dc..a6ef81c3cc 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -454,6 +454,7 @@ public function testUserRecordReportStates() 'export_created_datetime', 'export_file_format', 'requested_datetime', + 'downloaded_datetime', 'state' );