From 4d25336d8717af0c2d91511771f8c32dcec5b0ad Mon Sep 17 00:00:00 2001 From: Sobuno Date: Wed, 1 Jan 2025 04:17:55 +0100 Subject: [PATCH] More WIP --- app/Support/Search/OperatorQuerySearch.php | 13 +- app/Support/Search/QueryParser2.php | 136 +++++++++++ app/Support/Search/QueryParserInterface.php | 23 +- ...ractQueryParserInterfaceParseQueryTest.php | 226 ++++++++++++++---- .../Search/QueryParserParseQueryTest.php | 4 +- 5 files changed, 341 insertions(+), 61 deletions(-) create mode 100644 app/Support/Search/QueryParser2.php diff --git a/app/Support/Search/OperatorQuerySearch.php b/app/Support/Search/OperatorQuerySearch.php index ca989bd911..596aedd7d0 100644 --- a/app/Support/Search/OperatorQuerySearch.php +++ b/app/Support/Search/OperatorQuerySearch.php @@ -166,9 +166,15 @@ class OperatorQuerySearch implements SearchInterface switch (true) { case $node instanceof Word: - $allWords = (string) $node->getValue(); - app('log')->debug(sprintf('Add words "%s" to search string', $allWords)); - $this->words[] = $allWords; + $word = (string) $node->getValue(); + if($node->isProhibited()) { + app('log')->debug(sprintf('Exclude word "%s" from search string', $word)); + $this->prohibitedWords[] = $word; + } else { + app('log')->debug(sprintf('Add word "%s" to search string', $word)); + $this->words[] = $word; + } + break; case $node instanceof Field: @@ -176,6 +182,7 @@ class OperatorQuerySearch implements SearchInterface break; case $node instanceof Subquery: + //TODO: Handle Subquery prohibition, i.e. flip all prohibition flags inside the subquery foreach ($node->getNodes() as $subNode) { $this->handleSearchNode($subNode); } diff --git a/app/Support/Search/QueryParser2.php b/app/Support/Search/QueryParser2.php new file mode 100644 index 0000000000..76e2a816e4 --- /dev/null +++ b/app/Support/Search/QueryParser2.php @@ -0,0 +1,136 @@ +query = $query; + $this->position = 0; + return $this->parseQuery(); + } + + private function parseQuery(): array + { + $nodes = []; + $token = $this->buildNextNode(); + + while ($token !== null) { + $nodes[] = $token; + $token = $this->buildNextNode(); + } + + return $nodes; + } + + private function buildNextNode(): ?Node + { + $tokenUnderConstruction = ''; + $inQuotes = false; + $fieldName = ''; + $prohibited = false; + + while ($this->position < strlen($this->query)) { + $char = $this->query[$this->position]; + + // If we're in a quoted string, we treat all characters except another quote as ordinary characters + if ($inQuotes) { + if($char !== '"') { + $tokenUnderConstruction .= $char; + $this->position++; + continue; + } else { + $this->position++; + return $this->createNode($tokenUnderConstruction, $fieldName, $prohibited); + } + } + + switch ($char) { + case '-': + if ($tokenUnderConstruction === '') { + // A minus sign at the beginning of a token indicates prohibition + $prohibited = true; + } else { + // In any other location, it's just a normal character + $tokenUnderConstruction .= $char; + } + break; + + case '"': + if ($tokenUnderConstruction === '') { + // A quote sign at the beginning of a token indicates the start of a quoted string + $inQuotes = true; + } else { + // In any other location, it's just a normal character + $tokenUnderConstruction .= $char; + } + break; + + case '(': + if ($tokenUnderConstruction === '') { + // A left parentheses at the beginning of a token indicates the start of a subquery + $this->position++; + return new Subquery($this->parseQuery(), $prohibited); + } else { + // In any other location, it's just a normal character + $tokenUnderConstruction .= $char; + } + break; + + case ')': + if ($tokenUnderConstruction !== '') { + $this->position++; + return $this->createNode($tokenUnderConstruction, $fieldName, $prohibited); + } + $this->position++; + return null; + + + case ':': + if ($tokenUnderConstruction !== '') { + // If we meet a colon with a left-hand side string, we know we're in a field and are about to set up the value + $fieldName = $tokenUnderConstruction; + $tokenUnderConstruction = ''; + } else { + // In any other location, it's just a normal character + $tokenUnderConstruction .= $char; + } + break; + + case ' ': + // A space indicates the end of a token construction if non-empty, otherwise it's just ignored + if ($tokenUnderConstruction !== '') { + $this->position++; + return $this->createNode($tokenUnderConstruction, $fieldName, $prohibited); + } + break; + + default: + $tokenUnderConstruction .= $char; + } + + $this->position++; + } + + return $fieldName !== '' || $tokenUnderConstruction !== '' + ? $this->createNode($tokenUnderConstruction, $fieldName, $prohibited) + : null; + } + + private function createNode(string $token, string $fieldName, bool $prohibited): Node + { + if (strlen($fieldName) > 0) { + return new Field(trim($fieldName), trim($token), $prohibited); + } + return new Word(trim($token), $prohibited); + } +} diff --git a/app/Support/Search/QueryParserInterface.php b/app/Support/Search/QueryParserInterface.php index d56494ce70..93f88d1e75 100644 --- a/app/Support/Search/QueryParserInterface.php +++ b/app/Support/Search/QueryParserInterface.php @@ -27,10 +27,12 @@ abstract class Node class Word extends Node { private string $value; + private bool $prohibited; - public function __construct(string $value) + public function __construct(string $value, bool $prohibited = false) { $this->value = $value; + $this->prohibited = $prohibited; } public function getValue(): string @@ -38,9 +40,14 @@ class Word extends Node return $this->value; } + public function isProhibited(): bool + { + return $this->prohibited; + } + public function __toString(): string { - return $this->value; + return ($this->prohibited ? '-' : '') . $this->value; } } @@ -89,12 +96,15 @@ class Subquery extends Node /** @var Node[] */ private array $nodes; + private bool $prohibited; + /** * @param Node[] $nodes */ - public function __construct(array $nodes) + public function __construct(array $nodes, bool $prohibited = false) { $this->nodes = $nodes; + $this->prohibited = $prohibited; } /** @@ -105,8 +115,13 @@ class Subquery extends Node return $this->nodes; } + public function isProhibited(): bool + { + return $this->prohibited; + } + public function __toString(): string { - return '(' . implode(' ', array_map(fn($node) => (string)$node, $this->nodes)) . ')'; + return ($this->prohibited ? '-' : '') . '(' . implode(' ', array_map(fn($node) => (string)$node, $this->nodes)) . ')'; } } diff --git a/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php b/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php index 8c4758d2e1..5b27e4caeb 100644 --- a/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php +++ b/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php @@ -65,35 +65,6 @@ abstract class AbstractQueryParserInterfaceParseQueryTest extends TestCase $this->assertEquals('100', $result[0]->getValue()); } - /*public function testGivenNestedSubqueryWhenParsingQueryThenReturnsSubqueryNode(): void - { - $result = $this->createParser()->parse('(amount:100 (description_contains:"test payment" -has_attachments:true))'); - - $this->assertIsArray($result); - $this->assertCount(1, $result); - $this->assertInstanceOf(Subquery::class, $result[0]); - - $nodes = $result[0]->getNodes(); - $this->assertCount(2, $nodes); - - $this->assertInstanceOf(Field::class, $nodes[0]); - $this->assertEquals('amount', $nodes[0]->getOperator()); - $this->assertEquals('100', $nodes[0]->getValue()); - - $this->assertInstanceOf(Subquery::class, $nodes[1]); - $subNodes = $nodes[1]->getNodes(); - $this->assertCount(2, $subNodes); - - $this->assertInstanceOf(Field::class, $subNodes[0]); - $this->assertEquals('description_contains', $subNodes[0]->getOperator()); - $this->assertEquals('test payment', $subNodes[0]->getValue()); - - $this->assertInstanceOf(Field::class, $subNodes[1]); - $this->assertTrue($subNodes[1]->isProhibited()); - $this->assertEquals('has_attachments', $subNodes[1]->getOperator()); - $this->assertEquals('true', $subNodes[1]->getValue()); - }*/ - public function testGivenSimpleWordWhenParsingQueryThenReturnsWordNode(): void { $result = $this->createParser()->parse('groceries'); @@ -166,14 +137,23 @@ abstract class AbstractQueryParserInterfaceParseQueryTest extends TestCase $this->assertEquals('true', $result[0]->getValue()); } - /*public function testGivenIncompleteFieldOperatorWhenParsingQueryThenHandlesGracefully(): void + public function testGivenFieldOperatorWithBlankValueWhenParsingQueryThenReturnsCorrectNodes(): void { $result = $this->createParser()->parse('amount:'); $this->assertInstanceOf(Field::class, $result[0]); $this->assertEquals('amount', $result[0]->getOperator()); $this->assertEquals('', $result[0]->getValue()); - }*/ + } + + public function testGivenFieldOperatorWithEmptyQuotedStringWhenParsingQueryThenReturnsCorrectNodes(): void + { + $result = $this->createParser()->parse('amount:""'); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('amount', $result[0]->getOperator()); + $this->assertEquals('', $result[0]->getValue()); + } public function testGivenUnterminatedQuoteWhenParsingQueryThenHandlesGracefully(): void { @@ -184,30 +164,172 @@ abstract class AbstractQueryParserInterfaceParseQueryTest extends TestCase $this->assertEquals('unterminated', $result[0]->getValue()); } - public function testGivenWordFollowedBySubqueryWithoutSpaceWhenParsingQueryThenReturnsCorrectNodes(): void -{ - $result = $this->createParser()->parse('groceries(amount:100 description_contains:"test")'); + public function testGivenWordFollowedBySubqueryWhenParsingQueryThenReturnsCorrectNodes(): void + { + $result = $this->createParser()->parse('groceries (amount:100 description_contains:"test")'); - $this->assertIsArray($result); - $this->assertCount(2, $result); + $this->assertIsArray($result); + $this->assertCount(2, $result); - // Test the word node - $this->assertInstanceOf(Word::class, $result[0]); - $this->assertEquals('groceries', $result[0]->getValue()); + // Test the word node + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('groceries', $result[0]->getValue()); - // Test the subquery node - $this->assertInstanceOf(Subquery::class, $result[1]); - $nodes = $result[1]->getNodes(); - $this->assertCount(2, $nodes); + // Test the subquery node + $this->assertInstanceOf(Subquery::class, $result[1]); + $nodes = $result[1]->getNodes(); + $this->assertCount(2, $nodes); - // Test first field in subquery - $this->assertInstanceOf(Field::class, $nodes[0]); - $this->assertEquals('amount', $nodes[0]->getOperator()); - $this->assertEquals('100', $nodes[0]->getValue()); + // Test first field in subquery + $this->assertInstanceOf(Field::class, $nodes[0]); + $this->assertEquals('amount', $nodes[0]->getOperator()); + $this->assertEquals('100', $nodes[0]->getValue()); - // Test second field in subquery - $this->assertInstanceOf(Field::class, $nodes[1]); - $this->assertEquals('description_contains', $nodes[1]->getOperator()); - $this->assertEquals('test', $nodes[1]->getValue()); -} + // Test second field in subquery + $this->assertInstanceOf(Field::class, $nodes[1]); + $this->assertEquals('description_contains', $nodes[1]->getOperator()); + $this->assertEquals('test', $nodes[1]->getValue()); + } + + public function testGivenMultipleFieldsWithQuotedValuesWhenParsingQueryThenReturnsFieldNodes(): void + { + $result = $this->createParser()->parse('description:"shopping at market" notes:"paid in cash" category:"groceries and food"'); + + $this->assertIsArray($result); + $this->assertCount(3, $result); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('description', $result[0]->getOperator()); + $this->assertEquals('shopping at market', $result[0]->getValue()); + + $this->assertInstanceOf(Field::class, $result[1]); + $this->assertEquals('notes', $result[1]->getOperator()); + $this->assertEquals('paid in cash', $result[1]->getValue()); + + $this->assertInstanceOf(Field::class, $result[2]); + $this->assertEquals('category', $result[2]->getOperator()); + $this->assertEquals('groceries and food', $result[2]->getValue()); + } + + public function testGivenSubqueryAfterFieldValueWhenParsingQueryThenReturnsCorrectNodes(): void + { + $result = $this->createParser()->parse('amount:100 (description:"market" category:food)'); + + $this->assertIsArray($result); + $this->assertCount(2, $result); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('amount', $result[0]->getOperator()); + $this->assertEquals('100', $result[0]->getValue()); + + $this->assertInstanceOf(Subquery::class, $result[1]); + $nodes = $result[1]->getNodes(); + $this->assertCount(2, $nodes); + + $this->assertInstanceOf(Field::class, $nodes[0]); + $this->assertEquals('description', $nodes[0]->getOperator()); + $this->assertEquals('market', $nodes[0]->getValue()); + + $this->assertInstanceOf(Field::class, $nodes[1]); + $this->assertEquals('category', $nodes[1]->getOperator()); + $this->assertEquals('food', $nodes[1]->getValue()); + } + + public function testGivenMultipleFieldsWithQuotedValuesWithoutSpacesWhenParsingQueryThenReturnsFieldNodes(): void + { + $result = $this->createParser()->parse('description:"shopping market"category:"groceries"notes:"cash payment"'); + + $this->assertIsArray($result); + $this->assertCount(3, $result); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('description', $result[0]->getOperator()); + $this->assertEquals('shopping market', $result[0]->getValue()); + + $this->assertInstanceOf(Field::class, $result[1]); + $this->assertEquals('category', $result[1]->getOperator()); + $this->assertEquals('groceries', $result[1]->getValue()); + + $this->assertInstanceOf(Field::class, $result[2]); + $this->assertEquals('notes', $result[2]->getOperator()); + $this->assertEquals('cash payment', $result[2]->getValue()); + } + + public function testGivenStringWithSingleQuoteInMiddleWhenParsingQueryThenReturnsWordNode(): void + { + $result = $this->createParser()->parse('stringWithSingle"InMiddle'); + + $this->assertIsArray($result); + $this->assertCount(1, $result); + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('stringWithSingle"InMiddle', $result[0]->getValue()); + } + + public function testGivenWordStartingWithColonWhenParsingQueryThenReturnsWordNode(): void + { + $result = $this->createParser()->parse(':startingWithColon'); + + $this->assertIsArray($result); + $this->assertCount(1, $result); + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals(':startingWithColon', $result[0]->getValue()); + } + + public function testGivenComplexNestedSubqueriesWhenParsingQueryThenReturnsCorrectNodes(): void + { + $result = $this->createParser()->parse('shopping (amount:50 market (-category:food word description:"test phrase" (has_notes:true)))'); + + $this->assertIsArray($result); + $this->assertCount(2, $result); + + // Test the first word node + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('shopping', $result[0]->getValue()); + + // Test first level subquery + $this->assertInstanceOf(Subquery::class, $result[1]); + /** @var Subquery $firstLevelSubquery */ + $firstLevelSubquery = $result[1]; + $level1Nodes = $firstLevelSubquery->getNodes(); + $this->assertCount(3, $level1Nodes); + + // Test field in first level + $this->assertInstanceOf(Field::class, $level1Nodes[0]); + $this->assertEquals('amount', $level1Nodes[0]->getOperator()); + $this->assertEquals('50', $level1Nodes[0]->getValue()); + + // Test word in first level + $this->assertInstanceOf(Word::class, $level1Nodes[1]); + $this->assertEquals('market', $level1Nodes[1]->getValue()); + + // Test second level subquery + $this->assertInstanceOf(Subquery::class, $level1Nodes[2]); + $level2Nodes = $level1Nodes[2]->getNodes(); + $this->assertCount(4, $level2Nodes); + + // Test prohibited field in second level + $this->assertInstanceOf(Field::class, $level2Nodes[0]); + $this->assertTrue($level2Nodes[0]->isProhibited()); + $this->assertEquals('category', $level2Nodes[0]->getOperator()); + $this->assertEquals('food', $level2Nodes[0]->getValue()); + + // Test word in second level + $this->assertInstanceOf(Word::class, $level2Nodes[1]); + $this->assertEquals('word', $level2Nodes[1]->getValue()); + + // Test field with quoted value in second level + $this->assertInstanceOf(Field::class, $level2Nodes[2]); + $this->assertEquals('description', $level2Nodes[2]->getOperator()); + $this->assertEquals('test phrase', $level2Nodes[2]->getValue()); + + // Test third level subquery + $this->assertInstanceOf(Subquery::class, $level2Nodes[3]); + $level3Nodes = $level2Nodes[3]->getNodes(); + $this->assertCount(1, $level3Nodes); + + // Test field in third level + $this->assertInstanceOf(Field::class, $level3Nodes[0]); + $this->assertEquals('has_notes', $level3Nodes[0]->getOperator()); + $this->assertEquals('true', $level3Nodes[0]->getValue()); + } } diff --git a/tests/unit/Support/Search/QueryParserParseQueryTest.php b/tests/unit/Support/Search/QueryParserParseQueryTest.php index a5c4e85963..f05301f29c 100644 --- a/tests/unit/Support/Search/QueryParserParseQueryTest.php +++ b/tests/unit/Support/Search/QueryParserParseQueryTest.php @@ -2,7 +2,7 @@ namespace Tests\unit\Support\Search; -use FireflyIII\Support\Search\QueryParser; +use FireflyIII\Support\Search\QueryParser2; use FireflyIII\Support\Search\QueryParserInterface; use Tests\unit\Support\Search\AbstractQueryParserInterfaceParseQueryTest; @@ -20,6 +20,6 @@ final class QueryParserParseQueryTest extends AbstractQueryParserInterfaceParseQ { protected function createParser(): QueryParserInterface { - return new QueryParser(); + return new QueryParser2(); } }