From e81df7b8d6a685a45b3afe4ac553256a5be8eaaa Mon Sep 17 00:00:00 2001 From: Adam Halfar Date: Thu, 22 Aug 2024 16:47:13 +0200 Subject: [PATCH 1/4] Add deflate and inflate of panel content, changed gc chance from 1% to 10%, added 10% chance to vacuum when sqlite is used, added pragma commands for optimalization when using sqlite --- src/Model/Entity/Panel.php | 29 ++++++++++++++++++++++++++- src/Model/Table/RequestsTable.php | 33 +++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/Model/Entity/Panel.php b/src/Model/Entity/Panel.php index cae68512..a50c760e 100644 --- a/src/Model/Entity/Panel.php +++ b/src/Model/Entity/Panel.php @@ -36,7 +36,7 @@ class Panel extends Entity protected array $_hidden = ['content']; /** - * Read the stream contents. + * Read the stream contents or inflate deflated data. * * Over certain sizes PDO will return file handles. * For backwards compatibility and consistency we smooth over that difference here. @@ -46,10 +46,37 @@ class Panel extends Entity */ protected function _getContent(mixed $content): string { + if (is_string($content)) { + $contentInflated = @gzinflate($content); + if ($contentInflated) { + return $contentInflated; + } + + return $content; + } + if (is_resource($content)) { return (string)stream_get_contents($content); } + return ''; + } + + /** + * Deflate the string data before saving it into database + * + * @param mixed $content Content + * @return mixed + */ + protected function _setContent(mixed $content): mixed + { + if (is_string($content)) { + $contentDeflated = gzdeflate($content, 9); + if ($contentDeflated) { + $content = $contentDeflated; + } + } + return $content; } } diff --git a/src/Model/Table/RequestsTable.php b/src/Model/Table/RequestsTable.php index 9b31f10f..a2a165e4 100644 --- a/src/Model/Table/RequestsTable.php +++ b/src/Model/Table/RequestsTable.php @@ -86,7 +86,17 @@ public function findRecent(SelectQuery $query): SelectQuery */ protected function shouldGc(): bool { - return rand(1, 100) === 100; + return rand(1, 10) === 10; + } + + /** + * Check if garbage collection vacuum should be run + * + * @return bool + */ + protected function shouldGcVacuum(): bool + { + return rand(1, 10) === 10; } /** @@ -131,7 +141,26 @@ public function gc(): void $conn = $this->getConnection(); if ($conn->getDriver() instanceof Sqlite) { - $conn->execute('VACUUM;'); + $conn->execute(' + PRAGMA auto_vacuum = FULL; + PRAGMA journal_mode = OFF; + PRAGMA synchronous = OFF; + PRAGMA foreign_keys = OFF; + PRAGMA temp_store = MEMORY; + PRAGMA automatic_index = OFF; + '); + + if (!$this->shouldGcVacuum()) { + return; + } + + try { + $conn->execute('VACUUM;'); + } catch (PDOException) { + if (is_file(TMP . 'debug_kit.sqlite')) { + unlink(TMP . 'debug_kit.sqlite'); + } + } } } catch (PDOException $e) { Log::warning('Unable to garbage collect requests table. This is probably due to concurrent requests.'); From e238bcb412019433f8c550557d338a3fafe77d29 Mon Sep 17 00:00:00 2001 From: Adam Halfar Date: Thu, 22 Aug 2024 16:55:10 +0200 Subject: [PATCH 2/4] Fix tests --- src/Model/Entity/Panel.php | 6 ++++-- src/Model/Table/RequestsTable.php | 29 ++++++++++++----------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/Model/Entity/Panel.php b/src/Model/Entity/Panel.php index a50c760e..716eda46 100644 --- a/src/Model/Entity/Panel.php +++ b/src/Model/Entity/Panel.php @@ -47,8 +47,10 @@ class Panel extends Entity protected function _getContent(mixed $content): string { if (is_string($content)) { + // phpcs:disable $contentInflated = @gzinflate($content); - if ($contentInflated) { + // phpcs:enable + if ($contentInflated !== false) { return $contentInflated; } @@ -72,7 +74,7 @@ protected function _setContent(mixed $content): mixed { if (is_string($content)) { $contentDeflated = gzdeflate($content, 9); - if ($contentDeflated) { + if ($contentDeflated !== false) { $content = $contentDeflated; } } diff --git a/src/Model/Table/RequestsTable.php b/src/Model/Table/RequestsTable.php index a2a165e4..e6f238f8 100644 --- a/src/Model/Table/RequestsTable.php +++ b/src/Model/Table/RequestsTable.php @@ -1,4 +1,5 @@ hasMany('DebugKit.Panels', [ 'sort' => ['Panels.title' => 'ASC'], ]); @@ -62,8 +62,7 @@ public function initialize(array $config): void * * @return string */ - public static function defaultConnectionName(): string - { + public static function defaultConnectionName(): string { return 'debug_kit'; } @@ -73,8 +72,7 @@ public static function defaultConnectionName(): string * @param \Cake\ORM\Query\SelectQuery $query The query * @return \Cake\ORM\Query\SelectQuery The query. */ - public function findRecent(SelectQuery $query): SelectQuery - { + public function findRecent(SelectQuery $query): SelectQuery { return $query->orderBy(['Requests.requested_at' => 'DESC']) ->limit(10); } @@ -84,8 +82,7 @@ public function findRecent(SelectQuery $query): SelectQuery * * @return bool */ - protected function shouldGc(): bool - { + protected function shouldGc(): bool { return rand(1, 10) === 10; } @@ -94,8 +91,7 @@ protected function shouldGc(): bool * * @return bool */ - protected function shouldGcVacuum(): bool - { + protected function shouldGcVacuum(): bool { return rand(1, 10) === 10; } @@ -108,8 +104,7 @@ protected function shouldGcVacuum(): bool * * @return void */ - public function gc(): void - { + public function gc(): void { if (!$this->shouldGc()) { return; } @@ -157,9 +152,9 @@ public function gc(): void try { $conn->execute('VACUUM;'); } catch (PDOException) { - if (is_file(TMP . 'debug_kit.sqlite')) { - unlink(TMP . 'debug_kit.sqlite'); - } + // phpcs:disable + @unlink(TMP . 'debug_kit.sqlite'); + // phpcs:enable } } } catch (PDOException $e) { From a3fc2ca22baa71364ef8c394062a4a5d7ccbc30f Mon Sep 17 00:00:00 2001 From: Adam Halfar Date: Thu, 22 Aug 2024 17:04:53 +0200 Subject: [PATCH 3/4] Revert unwanted formatting, add checks if functions exists, add inflate for content loaded from resource --- src/Model/Entity/Panel.php | 18 ++++++++---------- src/Model/Table/RequestsTable.php | 22 ++++++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Model/Entity/Panel.php b/src/Model/Entity/Panel.php index 716eda46..d8db7aaf 100644 --- a/src/Model/Entity/Panel.php +++ b/src/Model/Entity/Panel.php @@ -46,22 +46,20 @@ class Panel extends Entity */ protected function _getContent(mixed $content): string { - if (is_string($content)) { + if (is_resource($content)) { + $content = (string)stream_get_contents($content); + } + + if (is_string($content) && function_exists('gzinflate')) { // phpcs:disable $contentInflated = @gzinflate($content); // phpcs:enable if ($contentInflated !== false) { return $contentInflated; } - - return $content; } - if (is_resource($content)) { - return (string)stream_get_contents($content); - } - - return ''; + return $content; } /** @@ -72,9 +70,9 @@ protected function _getContent(mixed $content): string */ protected function _setContent(mixed $content): mixed { - if (is_string($content)) { + if (is_string($content) && function_exists('gzdeflate')) { $contentDeflated = gzdeflate($content, 9); - if ($contentDeflated !== false) { + if ($contentDeflated) { $content = $contentDeflated; } } diff --git a/src/Model/Table/RequestsTable.php b/src/Model/Table/RequestsTable.php index e6f238f8..9c726ab4 100644 --- a/src/Model/Table/RequestsTable.php +++ b/src/Model/Table/RequestsTable.php @@ -1,5 +1,4 @@ hasMany('DebugKit.Panels', [ 'sort' => ['Panels.title' => 'ASC'], ]); @@ -62,7 +63,8 @@ public function initialize(array $config): void { * * @return string */ - public static function defaultConnectionName(): string { + public static function defaultConnectionName(): string + { return 'debug_kit'; } @@ -72,7 +74,8 @@ public static function defaultConnectionName(): string { * @param \Cake\ORM\Query\SelectQuery $query The query * @return \Cake\ORM\Query\SelectQuery The query. */ - public function findRecent(SelectQuery $query): SelectQuery { + public function findRecent(SelectQuery $query): SelectQuery + { return $query->orderBy(['Requests.requested_at' => 'DESC']) ->limit(10); } @@ -82,7 +85,8 @@ public function findRecent(SelectQuery $query): SelectQuery { * * @return bool */ - protected function shouldGc(): bool { + protected function shouldGc(): bool + { return rand(1, 10) === 10; } @@ -91,7 +95,8 @@ protected function shouldGc(): bool { * * @return bool */ - protected function shouldGcVacuum(): bool { + protected function shouldGcVacuum(): bool + { return rand(1, 10) === 10; } @@ -104,7 +109,8 @@ protected function shouldGcVacuum(): bool { * * @return void */ - public function gc(): void { + public function gc(): void + { if (!$this->shouldGc()) { return; } From 2548003a6119de7baef8a428e650ac93d46688bf Mon Sep 17 00:00:00 2001 From: Adam Halfar Date: Thu, 22 Aug 2024 17:06:43 +0200 Subject: [PATCH 4/4] Add missing strict check --- src/Model/Entity/Panel.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Entity/Panel.php b/src/Model/Entity/Panel.php index d8db7aaf..49f3bf02 100644 --- a/src/Model/Entity/Panel.php +++ b/src/Model/Entity/Panel.php @@ -72,7 +72,7 @@ protected function _setContent(mixed $content): mixed { if (is_string($content) && function_exists('gzdeflate')) { $contentDeflated = gzdeflate($content, 9); - if ($contentDeflated) { + if ($contentDeflated !== false) { $content = $contentDeflated; } }