From afd675bf01ef7bbad9ba576b1c789059c9b004e4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 31 Jan 2025 12:00:15 +0900 Subject: [PATCH 1/2] Implement Count Queries --- src/Database/Adapter.php | 13 +- src/Database/Adapter/MariaDB.php | 46 +++++-- src/Database/Adapter/Mongo.php | 2 +- src/Database/Adapter/Postgres.php | 48 +++++-- src/Database/Adapter/SQL.php | 12 +- src/Database/Query.php | 21 +++ src/Database/Validator/Queries.php | 15 +++ src/Database/Validator/Queries/Documents.php | 2 + src/Database/Validator/Query/Base.php | 2 + src/Database/Validator/Query/Count.php | 76 +++++++++++ tests/e2e/Adapter/Base.php | 130 ++++++++++++++++++- tests/unit/QueryTest.php | 8 +- tests/unit/Validator/Query/CountTest.php | 58 +++++++++ 13 files changed, 397 insertions(+), 36 deletions(-) create mode 100644 src/Database/Validator/Query/Count.php create mode 100644 tests/unit/Validator/Query/CountTest.php diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index b16ecdd23..525ada17f 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -901,11 +901,11 @@ abstract public function getSupportForCastIndexArray(): bool; abstract public function getSupportForUpserts(): bool; /** - * Are sum queries supported? + * Are aggregate queries supported? * * @return bool */ - abstract public function getSupportForSum(): bool; + abstract public function getSupportForAggregateQueries(): bool; /** * Get current attribute count from collection document @@ -996,17 +996,18 @@ protected function getAttributeSelections(array $queries): array } /** - * Get all sum attributes from queries + * Get all attributes from an aggregate query * - * @param Query[] $queries + * @param array $queries + * @param string $queryType * @return array> */ - protected function getAttributeSums(array $queries): array + protected function getAttributeSums(array $queries, string $queryType): array { $selections = []; foreach ($queries as $query) { - if ($query->getMethod() === Query::TYPE_SUM) { + if ($query->getMethod() === $queryType) { $selections[] = $query->getValues(); } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index e665a968b..c3a6a9f98 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2139,7 +2139,6 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $sqlLimit .= \is_null($offset) ? '' : ' OFFSET :offset'; $selections = $this->getAttributeSelections($queries); - $sumSelections = $this->getAttributeSums($queries); $sql = " SELECT {$this->getAttributeProjection($selections, 'table_main')} @@ -2149,8 +2148,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, {$sqlLimit} "; - if (!empty($sumSelections)) { - $sql = "SELECT {$this->getSumQueries($sumSelections)} FROM ({$sql}) table_sum;"; + $aggregateQueries = array_filter($queries, fn ($query) => in_array($query->getMethod(), Query::AGGREGATE_TYPES)); + + if (!empty($aggregateQueries)) { + $sql = $this->handleAggregateQueries($sql, $aggregateQueries); } else { $sql .= ';'; } @@ -2645,16 +2646,6 @@ public function getSupportForUpserts(): bool return true; } - /** - * Are sum queries supported? - * - * @return bool - */ - public function getSupportForSum(): bool - { - return true; - } - /** * Set max execution time * @param int $milliseconds @@ -2779,6 +2770,35 @@ public function getSchemaAttributes(string $collection): array } } + /** + * Handle Aggregate Queries + * + * @param string $sql + * @param array $aggregateQueries + * @return string + */ + public function handleAggregateQueries(string $sql, array $aggregateQueries): string + { + // There should not be multiple types of aggregate queries + // Multiple of the same type should be allowed + $aggregateType = $aggregateQueries[0]->getMethod(); + + switch ($aggregateType) { + case Query::TYPE_COUNT: + $countSelections = []; + foreach ($aggregateQueries as $query) { + $attribute = $query->getAttribute(); + $countSelections[] = "COUNT(`{$attribute}`) as `{$attribute}`"; + } + return "SELECT " . implode(', ', $countSelections) . " FROM ({$sql}) table_count;"; + case Query::TYPE_SUM: + $selections = $this->getAttributeSums($aggregateQueries, $aggregateType); + return "SELECT {$this->getSumQueries($selections)} FROM ({$sql}) table_sum;"; + default: + throw new DatabaseException('Unknown aggregate type: ' . $aggregateType); + } + } + public function getSupportForSchemaAttributes(): bool { return true; diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 7a1f97b8d..a1baa3c3f 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1820,7 +1820,7 @@ public function getSupportForUpserts(): bool return false; } - public function getSupportForSum(): bool + public function getSupportForAggregateQueries(): bool { return false; } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c71de02fc..b4c0de5a3 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1917,7 +1917,6 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $sqlLimit = \is_null($limit) ? '' : 'LIMIT :limit'; $sqlLimit .= \is_null($offset) ? '' : ' OFFSET :offset'; $selections = $this->getAttributeSelections($queries); - $sumSelections = $this->getAttributeSums($queries); $sql = " SELECT {$this->getAttributeProjection($selections, 'table_main')} @@ -1927,8 +1926,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, {$sqlLimit} "; - if (!empty($sumSelections)) { - $sql = "SELECT {$this->getSumQueries($sumSelections)} FROM ({$sql}) table_sum;"; + $aggregateQueries = array_filter($queries, fn ($query) => in_array($query->getMethod(), Query::AGGREGATE_TYPES)); + + if (!empty($aggregateQueries)) { + $sql = $this->handleAggregateQueries($sql, $aggregateQueries); } else { $sql .= ';'; } @@ -2480,16 +2481,6 @@ public function getSupportForUpserts(): bool return false; } - /** - * Are sum queries supported? - * - * @return bool - */ - public function getSupportForSum(): bool - { - return true; - } - /** * @return string */ @@ -2530,6 +2521,37 @@ protected function processException(PDOException $e): \Exception return $e; } + /** + * Handle Aggregate Queries + * + * @param string $sql + * @param array $aggregateQueries + * @return string + */ + public function handleAggregateQueries(string $sql, array $aggregateQueries): string + { + if (empty($aggregateQueries)) { + return $sql; + } + + $aggregateType = $aggregateQueries[0]->getMethod(); + + switch ($aggregateType) { + case Query::TYPE_COUNT: + $countSelections = []; + foreach ($aggregateQueries as $query) { + $attribute = $query->getAttribute(); + $countSelections[] = "COUNT(\"{$attribute}\") as \"{$attribute}\""; + } + return "SELECT " . implode(', ', $countSelections) . " FROM ({$sql}) table_count;"; + case Query::TYPE_SUM: + $selections = $this->getAttributeSums($aggregateQueries, $aggregateType); + return "SELECT {$this->getSumQueries($selections)} FROM ({$sql}) table_sum;"; + default: + throw new DatabaseException('Unknown aggregate type: ' . $aggregateType); + } + } + /** * @return string */ diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 97056abab..a3a6df11e 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -402,6 +402,16 @@ public function getSupportForGetConnectionId(): bool return true; } + /** + * Are aggregate queries supported? + * + * @return bool + */ + public function getSupportForAggregateQueries(): bool + { + return true; + } + /** * Get current attribute count from collection document * @@ -1134,7 +1144,7 @@ public function getSQLConditions(array $queries = [], string $separator = 'AND') $conditions = []; foreach ($queries as $query) { - if ($query->getMethod() === Query::TYPE_SELECT || $query->getMethod() === Query::TYPE_SUM) { + if ($query->getMethod() === Query::TYPE_SELECT || in_array($query->getMethod(), Query::AGGREGATE_TYPES)) { continue; } diff --git a/src/Database/Query.php b/src/Database/Query.php index 006586a16..422950303 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -25,6 +25,9 @@ class Query public const TYPE_SELECT = 'select'; public const TYPE_SUM = 'sum'; + // Aggregate methods + public const TYPE_COUNT = 'count'; + // Order methods public const TYPE_ORDER_DESC = 'orderDesc'; public const TYPE_ORDER_ASC = 'orderAsc'; @@ -63,6 +66,7 @@ class Query self::TYPE_CURSOR_BEFORE, self::TYPE_AND, self::TYPE_OR, + self::TYPE_COUNT, ]; protected const LOGICAL_TYPES = [ @@ -70,6 +74,11 @@ class Query self::TYPE_OR, ]; + public const AGGREGATE_TYPES = [ + self::TYPE_COUNT, + self::TYPE_SUM, + ]; + protected string $method = ''; protected string $attribute = ''; protected bool $onArray = false; @@ -217,6 +226,7 @@ public static function isMethod(string $value): bool self::TYPE_OR, self::TYPE_AND, self::TYPE_SUM, + self::TYPE_COUNT, self::TYPE_SELECT => true, default => false, }; @@ -571,6 +581,17 @@ public static function endsWith(string $attribute, string $value): self return new self(self::TYPE_ENDS_WITH, $attribute, [$value]); } + /** + * Helper method to create Query with sum method + * + * @param string $attribute + * @return Query + */ + public static function count(string $attribute): self + { + return new self(self::TYPE_COUNT, $attribute); + } + /** * @param array $queries * @return Query diff --git a/src/Database/Validator/Queries.php b/src/Database/Validator/Queries.php index 2373e7e9d..391866054 100644 --- a/src/Database/Validator/Queries.php +++ b/src/Database/Validator/Queries.php @@ -83,6 +83,20 @@ public function isValid($value): bool } } + $methodsUsed = array_map( + fn (Query|string $query) => $query instanceof Query ? $query->getMethod() : $query, + $value + ); + + $methodsUsed = array_unique($methodsUsed); + $aggregateMethods = array_intersect($methodsUsed, Query::AGGREGATE_TYPES); + + // Check if multiple aggregate methods are used + if (count($aggregateMethods) > 1) { + $this->message = 'Invalid query: Multiple types of aggregate methods are not supported'; + return false; + } + if ($query->isNested()) { if (!self::isValid($query->getValues())) { return false; @@ -95,6 +109,7 @@ public function isValid($value): bool Query::TYPE_SUM => Base::METHOD_TYPE_SUM, Query::TYPE_LIMIT => Base::METHOD_TYPE_LIMIT, Query::TYPE_OFFSET => Base::METHOD_TYPE_OFFSET, + Query::TYPE_COUNT => Base::METHOD_TYPE_COUNT, Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE => Base::METHOD_TYPE_CURSOR, Query::TYPE_ORDER_ASC, diff --git a/src/Database/Validator/Queries/Documents.php b/src/Database/Validator/Queries/Documents.php index e16ce8428..266871cae 100644 --- a/src/Database/Validator/Queries/Documents.php +++ b/src/Database/Validator/Queries/Documents.php @@ -6,6 +6,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\IndexedQueries; +use Utopia\Database\Validator\Query\Count; use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\Query\Filter; use Utopia\Database\Validator\Query\Limit; @@ -68,6 +69,7 @@ public function __construct( new Order($attributes), new Select($attributes), new Sum($attributes), + new Count($attributes), ]; parent::__construct($attributes, $indexes, $validators); diff --git a/src/Database/Validator/Query/Base.php b/src/Database/Validator/Query/Base.php index ef419e4cc..e41fd22ed 100644 --- a/src/Database/Validator/Query/Base.php +++ b/src/Database/Validator/Query/Base.php @@ -13,6 +13,8 @@ abstract class Base extends Validator public const METHOD_TYPE_FILTER = 'filter'; public const METHOD_TYPE_SELECT = 'select'; public const METHOD_TYPE_SUM = 'sum'; + public const METHOD_TYPE_COUNT = 'count'; + protected string $message = 'Invalid query'; diff --git a/src/Database/Validator/Query/Count.php b/src/Database/Validator/Query/Count.php new file mode 100644 index 000000000..91f106b43 --- /dev/null +++ b/src/Database/Validator/Query/Count.php @@ -0,0 +1,76 @@ + + */ + protected array $schema = []; + + /** + * List of internal attributes + * + * @var array + */ + protected const INTERNAL_ATTRIBUTES = [ + '$id', + '$internalId', + '$createdAt', + '$updatedAt', + '$permissions', + '$collection', + ]; + + /** + * @param array $attributes + */ + public function __construct(array $attributes = []) + { + foreach ($attributes as $attribute) { + $this->schema[$attribute->getAttribute('key', $attribute->getAttribute('$id'))] = $attribute->getArrayCopy(); + } + } + + /** + * Is valid. + * + * Returns true if method is TYPE_SUM selections are valid + * + * Otherwise, returns false + * + * @param Query $value + * @return bool + */ + public function isValid($value): bool + { + if (!$value instanceof Query) { + return false; + } + + if ($value->getMethod() !== Query::TYPE_COUNT) { + return false; + } + + $internalKeys = \array_map( + fn ($attr) => $attr['$id'], + Database::INTERNAL_ATTRIBUTES + ); + + if (!isset($this->schema[$value->getAttribute()])) { + $this->message = 'Attribute not found in schema: ' . $value->getAttribute(); + return false; + } + return true; + } + + public function getMethodType(): string + { + return self::METHOD_TYPE_COUNT; + } +} diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 91da329ff..bde69229a 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17525,7 +17525,7 @@ public function testUpdateDocumentsRelationships(): void public function testSumQueries(): void { - if (!static::getDatabase()->getAdapter()->getSupportForSum()) { + if (!static::getDatabase()->getAdapter()->getSupportForAggregateQueries()) { $this->expectNotToPerformAssertions(); return; } @@ -17679,6 +17679,134 @@ public function testSumQueries(): void $database->deleteCollection('testSumQueries'); } + + public function testCountQueries(): void + { + if (!static::getDatabase()->getAdapter()->getSupportForAggregateQueries()) { + $this->expectNotToPerformAssertions(); + return; + } + + $database = static::getDatabase(); + + $database->createCollection('testCountQueries', attributes: [ + new Document([ + '$id' => ID::custom('integer'), + 'type' => Database::VAR_INTEGER, + 'size' => 64, + 'required' => false, + ]), + new Document([ + '$id' => ID::custom('string'), + 'type' => Database::VAR_STRING, + 'size' => 64, + 'required' => false, + ]), + ], permissions: [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ], documentSecurity: false); + + $database->createDocuments('testCountQueries', [ + new Document([ + '$id' => ID::unique(), + 'integer' => 10, + 'string' => 'test1', + ]), + new Document([ + '$id' => ID::unique(), + 'integer' => 20, + 'string' => 'test2', + ]), + new Document([ + '$id' => ID::unique(), + 'integer' => 30, + 'string' => null, + ]), + new Document([ + '$id' => ID::unique(), + 'integer' => null, + 'string' => null, + ]), + ]); + + $this->assertEquals(4, $database->count('testCountQueries')); + + $documents = $database->find('testCountQueries', [ + Query::count('integer'), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(3, $documents[0]->getAttribute('integer')); + + $documents = $database->find('testCountQueries', [ + Query::count('string'), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(2, $documents[0]->getAttribute('string')); + + // Expect Fail: Count of non-existent attribute + try { + $database->find('testCountQueries', [ + Query::count('nonExistent'), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Attribute not found in schema: nonExistent', $e->getMessage()); + } + + // Test mixing count with other queries + $documents = $database->find('testCountQueries', [ + Query::count('integer'), + Query::limit(1), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(1, $documents[0]->getAttribute('integer')); + + $documents = $database->find('testCountQueries', [ + Query::count('integer'), + Query::greaterThan('integer', 10), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(2, $documents[0]->getAttribute('integer')); + + $documents = $database->find('testCountQueries', [ + Query::count('integer'), + Query::lessThan('integer', 11), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(1, $documents[0]->getAttribute('integer')); + + // Test multiple count queries + $documents = $database->find('testCountQueries', [ + Query::count('integer'), + Query::count('string'), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(3, $documents[0]->getAttribute('integer')); + $this->assertEquals(2, $documents[0]->getAttribute('string')); + + // Expect Fail: Mixing aggregate queries with other aggregate queries + try { + $database->find('testCountQueries', [ + Query::count('integer'), + Query::sum(['string']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Multiple types of aggregate methods are not supported', $e->getMessage()); + } + + $database->deleteCollection('testCountQueries'); + } + public function testNestedQueryValidation(): void { $this->getDatabase()->createCollection(__FUNCTION__, [ diff --git a/tests/unit/QueryTest.php b/tests/unit/QueryTest.php index 3875c24ff..6ca961cd1 100644 --- a/tests/unit/QueryTest.php +++ b/tests/unit/QueryTest.php @@ -203,6 +203,11 @@ public function testParse(): void $this->assertEquals(null, $query->getAttribute()); $this->assertEquals(['title', 'director'], $query->getValues()); + $query = Query::parse(Query::count('actors')->toString()); + $this->assertEquals('count', $query->getMethod()); + $this->assertEquals('actors', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + $json = Query::or([ Query::equal('actors', ['Brad Pitt']), Query::equal('actors', ['Johnny Depp']) @@ -270,6 +275,7 @@ public function testIsMethod(): void $this->assertTrue(Query::isMethod('sum')); $this->assertTrue(Query::isMethod('or')); $this->assertTrue(Query::isMethod('and')); + $this->assertTrue(Query::isMethod('count')); $this->assertTrue(Query::isMethod(Query::TYPE_EQUAL)); $this->assertTrue(Query::isMethod(Query::TYPE_NOT_EQUAL)); @@ -292,7 +298,7 @@ public function testIsMethod(): void $this->assertTrue(Query::isMethod(QUERY::TYPE_SUM)); $this->assertTrue(Query::isMethod(QUERY::TYPE_OR)); $this->assertTrue(Query::isMethod(QUERY::TYPE_AND)); - + $this->assertTrue(Query::isMethod(QUERY::TYPE_COUNT)); $this->assertFalse(Query::isMethod('invalid')); $this->assertFalse(Query::isMethod('lte ')); } diff --git a/tests/unit/Validator/Query/CountTest.php b/tests/unit/Validator/Query/CountTest.php new file mode 100644 index 000000000..26cb0bc71 --- /dev/null +++ b/tests/unit/Validator/Query/CountTest.php @@ -0,0 +1,58 @@ +validator = new Count( + attributes: [ + new Document([ + '$id' => 'value', + 'key' => 'value', + 'type' => Database::VAR_INTEGER, + 'array' => false, + ]), + new Document([ + '$id' => 'valueFloat', + 'key' => 'valueFloat', + 'type' => Database::VAR_FLOAT, + 'array' => false, + ]), + new Document([ + '$id' => 'valueStr', + 'key' => 'valueStr', + 'type' => Database::VAR_STRING, + 'array' => false, + ]), + ], + ); + } + + public function testValueSuccess(): void + { + $this->assertTrue($this->validator->isValid(Query::count('value'))); + $this->assertTrue($this->validator->isValid(Query::count('valueFloat'))); + } + + public function testValueFailure(): void + { + $this->assertFalse($this->validator->isValid(Query::limit(1))); + $this->assertEquals('Invalid query', $this->validator->getDescription()); + $this->assertFalse($this->validator->isValid(Query::count('valueStr'))); + } +} From 8d9f0ca880936371423ec7c009c4bb877630e043 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 3 Feb 2025 10:55:12 +0900 Subject: [PATCH 2/2] Fix unit tests --- tests/unit/Validator/Query/CountTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/Validator/Query/CountTest.php b/tests/unit/Validator/Query/CountTest.php index 26cb0bc71..fe0467181 100644 --- a/tests/unit/Validator/Query/CountTest.php +++ b/tests/unit/Validator/Query/CountTest.php @@ -53,6 +53,6 @@ public function testValueFailure(): void { $this->assertFalse($this->validator->isValid(Query::limit(1))); $this->assertEquals('Invalid query', $this->validator->getDescription()); - $this->assertFalse($this->validator->isValid(Query::count('valueStr'))); + $this->assertFalse($this->validator->isValid(Query::count('notInSchema'))); } }