Update README.md

README.md

@@ -52,15 +52,15 @@ main layers:
 Basic usage:
 ------------
 
-```
+```php
 <?php
 
 use ...
 
 $platform = Platform::get(Platform::MYSQL, '8.0'); // version defaults to x.x.99 when no patch number is given
-$settings = new ParserSettings($platform);
+$config = new ParserConfig($platform);
 $session = new Session($platform);
-$parser = new Parser($settings, $session);
+$parser = new Parser($config, $session);
 
 // returns a Generator. will not parse anything if you don't iterate over it
 $commands = $parser->parse('SELECT foo FROM ...');

composer.json

@@ -16,7 +16,7 @@
     },
     "require-dev": {
         "dogma/dogma-dev": "0.1.29",
-        "phpstan/phpstan": "1.9.15",
+        "phpstan/phpstan": "1.12.3",
         "phpstan/phpstan-strict-rules": "^1.0",
         "amphp/parallel-functions": "1.1.0",
         "rector/rector": "0.14.8",
@@ -26,7 +26,7 @@
         "classmap": ["sources"]
     },
     "autoload-dev": {
-        "classmap": ["tests", "build"]
+        "classmap": ["tests", "build", "vendor/dogma/dogma-debug"]
     },
     "minimum-stability": "dev",
     "prefer-stable": true,
@@ -90,7 +90,10 @@
         "tests:mysql": "php tests/Mysql/test.php",
         "tests:mysql-s": "php tests/Mysql/test.php --single",
 
-        "phpstan:run": "php vendor/phpstan/phpstan/phpstan analyse -c build/phpstan/phpstan.neon --memory-limit=512M",
+        "phpstan:run": [
+            "Composer\\Config::disableProcessTimeout",
+            "php vendor/phpstan/phpstan/phpstan analyse -vvv -c build/phpstan/phpstan.neon --memory-limit=512M"
+        ],
         "phpstan:all": [
             "php84 vendor/phpstan/phpstan/phpstan analyse -c build/phpstan/phpstan.neon --memory-limit=512M",
             "php83 vendor/phpstan/phpstan/phpstan analyse -c build/phpstan/phpstan.neon --memory-limit=512M",

doc/token-types.md

@@ -2,44 +2,85 @@
 Token type hierarchy:
 ---------------------
 
-TokenType constants:
-
-- **WHITESPACE**
-- **COMMENT**
-    - **BLOCK_COMMENT** - `/* ... * /`
-        - **OPTIONAL_COMMENT** - `/*! ... * /`
-        - **HINT_COMMENT** - `/*+ ... * /`
-    - **DOUBLE_HYPHEN_COMMENT** - `-- ...`
-    - **DOUBLE_SLASH_COMMENT** - `// ...`
-    - **HASH_COMMENT** - `# ...`
-- **NAME**
-    - **UNQUOTED_NAME** - `table1` etc.
-        - **KEYWORD** - `datetime` etc.
-            - **RESERVED** - `SELECT` etc.
-                - **OPERATOR** - `AND`, `OR` etc.
-    - **DOUBLE_QUOTED_STRING** - `"table1"` (standard, MySQL in ANSI_STRINGS mode)
-    - **BACKTICK_QUOTED_STRING** - `` `table1` `` (MySQL, PostgreSQL, Sqlite)
-    - **SQUARE_BRACKETED_STRING** - `[table1]` (MSSQL, SqLite)
-    - **AT_VARIABLE** - `@var`, `@@global`, `@'192.168.0.1'` (also includes host names)
-        - **SINGLE_QUOTED_STRING** - `@'var'`
-        - **DOUBLE_QUOTED_STRING** - `@"var"`
-        - **BACKTICK_QUOTED_STRING** - `` @`var` ``
-- **VALUE**
-    - **STRING**
-        - **SINGLE_QUOTED_STRING** - `'string'` (standard)
-        - **DOUBLE_QUOTED_STRING** - `"string"` (MySQL in default mode)
-        * **DOLLAR_QUOTED_STRING** - `$foo$table1$foo$` (PostgreSQL)
-    - **NUMBER**
-        - **INT**
-            - **UINT**
-    - **BINARY_LITERAL**
-    - **HEXADECIMAL_LITERAL**
-    - **UUID** - e.g. `3E11FA47-71CA-11E1-9E33-C80AA9429562`
-- **SYMBOL** - `(`, `)`, `[`, `]`, `{`, `}`, `.`, `,`, `;`
-    - **OPERATOR** - `+`, `||` etc.
-- **PLACEHOLDER** - placeholder for a parameter
-    - **QUESTION_MARK_PLACEHOLDER** - `?` (SQL, Doctrine, Laravel)
-    - **NUMBERED_QUESTION_MARK_PLACEHOLDER** - `?123` (Doctrine)
-    - **DOUBLE_COLON_PLACEHOLDER** - `:foo` (Doctrine, Laravel)
-- **DELIMITER** - default `;`
-- **DELIMITER_DEFINITION**
+Token type and additional token info are packed into 31 bit int
+
+### TokenType constants (unique):
+
+- **1K WHITESPACE**
+- COMMENT
+    - **2K LINE_COMMENT**
+        - DOUBLE_HYPHEN_COMMENT - `-- ...`
+        - DOUBLE_SLASH_COMMENT - `// ...`
+        - HASH_COMMENT - `# ...`
+    - **4K BLOCK_COMMENT** - `/* ... * /`
+        - OPTIONAL_COMMENT - `/*! ... * /`
+        - HINT_COMMENT - `/*+ ... * /`
+- NAME
+    - **32K UNQUOTED_NAME** - `table1` etc.
+        - **16K KEYWORD** - `datetime` etc.
+            - **8K RESERVED** - `SELECT` etc.
+                - **512 OPERATOR** - `AND`, `OR` etc.
+    - **64K QUOTED_NAME**
+        - DOUBLE_QUOTED_NAME - `"table1"` (standard, MySQL in ANSI_STRINGS mode)
+        - BACKTICK_QUOTED_NAME - `` `table1` `` (MySQL, PostgreSQL, Sqlite)
+        - SQUARE_BRACKETED_NAME - `[table1]` (MSSQL, SqLite)
+    - **128K AT_VARIABLE** - `@var`, `@@global`, `@'192.168.0.1'` (also includes host names)
+        - SINGLE_QUOTED_AT_VAR - `@'var'`
+        - DOUBLE_QUOTED_AT_VAR - `@"var"`
+        - BACKTICK_QUOTED_AT_VAR - `` @`var` ``
+- VALUE
+    - **1M NUMBER**
+        - **512K INT**
+            - **256K UINT**
+    - **2M STRING**
+        - SINGLE_QUOTED_STRING - `'string'` (standard)
+        - DOUBLE_QUOTED_STRING - `"string"` (MySQL in default mode)
+        - DOLLAR_QUOTED_STRING - `$$table1$$` (PostgreSQL)
+    - **4M BIT_STRING**
+        - BINARY_LITERAL
+        - OCTAL_LITERAL (PostgreSQL)
+        - HEXADECIMAL_LITERAL
+    - **8M UUID** - e.g. `3E11FA47-71CA-11E1-9E33-C80AA9429562`
+- **16M SYMBOL** - `(`, `)`, `[`, `]`, `{`, `}`, `.`, `,`, `;`
+    - **512 OPERATOR** - `+`, `||` etc.
+    - OPTIMIZER_HINT_START - `/*+`
+    - OPTIMIZER_HINT_END - `*/`
+    - N/A **CHARSET_INTRODUCER** - `N`
+    - N/A DOLLAR_QUOTE - `$foo$` (PostgreSQL)
+- **32M PLACEHOLDER** - placeholder for a parameter
+    - QUESTION_MARK_PLACEHOLDER - `?` (SQL, Doctrine, Laravel)
+    - NUMBERED_QUESTION_MARK_PLACEHOLDER - `?123` (Doctrine)
+    - DOUBLE_COLON_PLACEHOLDER - `:foo` (Doctrine, Laravel)
+- **64M DELIMITER** - default `;`
+- **128M DELIMITER_DEFINITION**
+- **256M END**
+- **1G INVALID**
+
+values 512 and 512M are free fow now
+
+### Token info (non unique):
+
+Comment type:
+-  **1** DOUBLE_HYPHEN_COMMENT
+-  **2** DOUBLE_SLASH_COMMENT
+-  **4** HASH_COMMENT
+-  **8** OPTIONAL_COMMENT
+- **16** OPTIMIZER_HINT_COMMENT
+
+Quoting:
+-  **1** SINGLE_QUOTED `'`
+-  **2** DOUBLE_QUOTED `"`
+-  **4** BACKTICK_QUOTED `` ` ``
+-  **8** SQUARE_BRACKETED `[]`
+- **16** DOUBLE_DOLLAR_QUOTED `$$`
+- **32** DOLLAR_TAG_QUOTED `$foo$string value$foo$`
+- **64** (reserved)
+
+Base:
+-  **1** (reserved for single quoted literals)
+-  **2** (reserved for double quoted literals)
+-  **4** binary
+-  **8** octal (PostgreSQL)
+- **16** hexadecimal
+- **32** (base 32?)
+- **64** (base 64?)

sources/Formatter/Formatter.php

@@ -38,6 +38,7 @@
 use function preg_match;
 use function str_replace;
 use function strpos;
+use function strtoupper;
 
 class Formatter
 {
@@ -124,12 +125,13 @@ public function formatName(string $name): string
         $sqlMode = $this->session->getMode();
         $quote = $sqlMode->containsAny(SqlMode::ANSI_QUOTES) ? '"' : '`';
         $name = str_replace($quote, $quote . $quote, $name);
+        $upper = strtoupper($name);
 
         $needsQuoting = $this->quoteAllNames
+            || isset($this->platform->reserved[$upper])
             || strpos($name, $quote) !== false // contains quote
             || preg_match('~[\pL_]~u', $name) === 0 // does not contain letters
-            || preg_match('~[\pC\pM\pS\pZ\p{Pd}\p{Pe}\p{Pf}\p{Pi}\p{Po}\p{Ps}]~u', ltrim($name, '@')) !== 0 // contains control, mark, symbols, whitespace, punctuation except _
-            || $this->platform->isReserved($name);
+            || preg_match('~[\pC\pM\pS\pZ\p{Pd}\p{Pe}\p{Pf}\p{Pi}\p{Po}\p{Ps}]~u', ltrim($name, '@')) !== 0; // contains control, mark, symbols, whitespace, punctuation except _
 
         if ($needsQuoting && !$sqlMode->containsAny(SqlMode::NO_BACKSLASH_ESCAPES)) {
             $name = str_replace($this->escapeKeys, $this->escapeValues, $name);

sources/Parser/Dal/ReplicationCommandsParser.php

@@ -13,9 +13,12 @@
 use LogicException;
 use SqlFtw\Parser\ExpressionParser;
 use SqlFtw\Parser\InvalidValueException;
+use SqlFtw\Parser\InvalidVersionException;
 use SqlFtw\Parser\ParserException;
 use SqlFtw\Parser\TokenList;
 use SqlFtw\Parser\TokenType;
+use SqlFtw\Platform\Features\Feature;
+use SqlFtw\Platform\Platform;
 use SqlFtw\Sql\Dal\Replication\ChangeMasterToCommand;
 use SqlFtw\Sql\Dal\Replication\ChangeReplicationFilterCommand;
 use SqlFtw\Sql\Dal\Replication\ChangeReplicationSourceToCommand;
@@ -55,10 +58,13 @@
 class ReplicationCommandsParser
 {
 
+    private Platform $platform;
+
     private ExpressionParser $expressionParser;
 
-    public function __construct(ExpressionParser $expressionParser)
+    public function __construct(Platform $platform, ExpressionParser $expressionParser)
     {
+        $this->platform = $platform;
         $this->expressionParser = $expressionParser;
     }
 
@@ -511,7 +517,9 @@ public function parseStartGroupReplication(TokenList $tokenList): StartGroupRepl
         $keywords = [Keyword::USER, Keyword::PASSWORD, Keyword::DEFAULT_AUTH];
         $keyword = $tokenList->getAnyKeyword(...$keywords);
         while ($keyword !== null) {
-            $tokenList->check('group replication credentials', 80021);
+            if (!isset($this->platform->features[Feature::GROUP_REPLICATION_CREDENTIALS])) {
+                throw new InvalidVersionException(Feature::GROUP_REPLICATION_CREDENTIALS, $this->platform, $tokenList);
+            }
             $tokenList->passSymbol('=');
             if ($keyword === Keyword::USER) {
                 $user = $tokenList->expectString();

sources/Parser/Dal/UserCommandsParser.php

@@ -9,8 +9,10 @@
 
 namespace SqlFtw\Parser\Dal;
 
+use LogicException;
 use SqlFtw\Parser\ParserException;
 use SqlFtw\Parser\TokenList;
+use SqlFtw\Platform\Features\Feature;
 use SqlFtw\Platform\Platform;
 use SqlFtw\Sql\Dal\User\AddAuthFactor;
 use SqlFtw\Sql\Dal\User\AlterAuthOption;
@@ -55,7 +57,6 @@
 use SqlFtw\Sql\Dal\User\UserPrivilege;
 use SqlFtw\Sql\Dal\User\UserPrivilegeResource;
 use SqlFtw\Sql\Dal\User\UserPrivilegeResourceType;
-use SqlFtw\Sql\Dal\User\UserPrivilegeType;
 use SqlFtw\Sql\Dal\User\UserResourceOption;
 use SqlFtw\Sql\Dal\User\UserResourceOptionType;
 use SqlFtw\Sql\Dal\User\UserTlsOption;
@@ -74,6 +75,13 @@
 class UserCommandsParser
 {
 
+    private Platform $platform;
+
+    public function __construct(Platform $platform)
+    {
+        $this->platform = $platform;
+    }
+
     private const RESOURCE_PRIVILEGES = [
         UserPrivilegeResourceType::TABLE => [
             StaticUserPrivilege::ALL,
@@ -368,7 +376,7 @@ private function parseAuthOptionParts(TokenList $tokenList, bool $currentUser =
         if (!$currentUser && $authPlugin !== null && $tokenList->hasKeyword(Keyword::AS)) {
             $as = $tokenList->expectStringValue();
         } elseif ($tokenList->hasKeyword(Keyword::BY)) {
-            if ($tokenList->using(Platform::MYSQL, null, 50799) && $tokenList->hasKeyword(Keyword::PASSWORD)) {
+            if (isset($this->platform->features[Feature::DEPRECATED_IDENTIFIED_BY_PASSWORD]) && $tokenList->hasKeyword(Keyword::PASSWORD)) {
                 $oldHashedPassword = true;
                 $password = $tokenList->expectStringValue();
             } elseif ($tokenList->hasKeywords(Keyword::RANDOM, Keyword::PASSWORD)) {
@@ -755,8 +763,9 @@ private function parsePrivilegesList(TokenList $tokenList, bool $ifExists = fals
                     }
                 }
             }
-            /** @var UserPrivilegeType $type */
-            $type = $type;
+            if ($type === null) {
+                throw new LogicException('Type cannot be null here.');
+            }
 
             $columns = null;
             if ($tokenList->hasSymbol('(')) {
@@ -976,9 +985,9 @@ public function parseSetPassword(TokenList $tokenList): SetPasswordCommand
 
         $passwordFunction = $password = $replace = null;
         if ($tokenList->hasOperator(Operator::EQUAL)) {
-            $passwordFunction = $tokenList->using(null, 50700)
-                ? $tokenList->getAnyKeyword(Keyword::PASSWORD)
-                : $tokenList->getAnyKeyword(Keyword::PASSWORD, Keyword::OLD_PASSWORD);
+            $passwordFunction = isset($this->platform->functions[BuiltInFunction::OLD_PASSWORD])
+                ? $tokenList->getAnyKeyword(Keyword::PASSWORD, Keyword::OLD_PASSWORD)
+                : $tokenList->getAnyKeyword(Keyword::PASSWORD);
             if ($passwordFunction !== null) {
                 $tokenList->expectSymbol('(');
             }

sources/Parser/Ddl/IndexCommandsParser.php

@@ -11,8 +11,10 @@
 
 use Dogma\Re;
 use SqlFtw\Parser\ExpressionParser;
+use SqlFtw\Parser\InvalidVersionException;
 use SqlFtw\Parser\ParserException;
 use SqlFtw\Parser\TokenList;
+use SqlFtw\Platform\Features\Feature;
 use SqlFtw\Platform\Platform;
 use SqlFtw\Sql\Ddl\Index\CreateIndexCommand;
 use SqlFtw\Sql\Ddl\Index\DropIndexCommand;
@@ -28,8 +30,8 @@
 use SqlFtw\Sql\Order;
 use SqlFtw\Sql\SqlMode;
 use function count;
+use function strcasecmp;
 use function strlen;
-use function strtoupper;
 
 class IndexCommandsParser
 {
@@ -124,7 +126,7 @@ public function parseIndexDefinition(TokenList $tokenList, bool $inTable = false
             $name = $tokenList->expectName(EntityType::INDEX);
         }
 
-        if ($name !== null && strtoupper($name) === Keyword::PRIMARY) {
+        if ($name !== null && strcasecmp($name, Keyword::PRIMARY) === 0) {
             throw new ParserException('Invalid index name.', $tokenList);
         }
 
@@ -169,7 +171,7 @@ public function parseIndexDefinition(TokenList $tokenList, bool $inTable = false
                 $withParser = $tokenList->expectName(EntityType::INDEX_PARSER);
             } elseif ($keyword === Keyword::COMMENT) {
                 $commentString = $tokenList->expectString();
-                $limit = $this->platform->getMaxLengths()[EntityType::INDEX_COMMENT];
+                $limit = $this->platform->maxLengths[EntityType::INDEX_COMMENT];
                 if (strlen($commentString) > $limit && $tokenList->getSession()->getMode()->containsAny(SqlMode::STRICT_ALL_TABLES)) {
                     throw new ParserException("Index comment length exceeds limit of {$limit} bytes.", $tokenList);
                 }
@@ -185,11 +187,15 @@ public function parseIndexDefinition(TokenList $tokenList, bool $inTable = false
             } elseif ($keyword === Keyword::INVISIBLE) {
                 $visible = false;
             } elseif ($keyword === Keyword::ENGINE_ATTRIBUTE) {
-                $tokenList->check(Keyword::ENGINE_ATTRIBUTE, 80021);
+                if (!isset($this->platform->features[Feature::ENGINE_ATTRIBUTE])) {
+                    throw new InvalidVersionException(Feature::ENGINE_ATTRIBUTE, $this->platform, $tokenList);
+                }
                 $tokenList->passSymbol('=');
                 $engineAttribute = $tokenList->expectString();
             } elseif ($keyword === Keyword::SECONDARY_ENGINE_ATTRIBUTE) {
-                $tokenList->check(Keyword::SECONDARY_ENGINE_ATTRIBUTE, 80021);
+                if (!isset($this->platform->features[Feature::SECONDARY_ENGINE_ATTRIBUTE])) {
+                    throw new InvalidVersionException(Feature::SECONDARY_ENGINE_ATTRIBUTE, $this->platform, $tokenList);
+                }
                 $tokenList->passSymbol('=');
                 $secondaryEngineAttribute = $tokenList->expectString();
             }
@@ -219,7 +225,9 @@ private function parseIndexParts(TokenList $tokenList): array
         $parts = [];
         do {
             if ($tokenList->hasSymbol('(')) {
-                $tokenList->check('functional indexes', 80013);
+                if (!isset($this->platform->features[Feature::FUNCTIONAL_INDEXES])) {
+                    throw new InvalidVersionException(Feature::FUNCTIONAL_INDEXES, $this->platform, $tokenList);
+                }
                 $expression = $this->expressionParser->parseExpression($tokenList);
                 $tokenList->expectSymbol(')');