From 1afd75a3505ab1ee6e09d3325c976d307ccf815b Mon Sep 17 00:00:00 2001 From: Joby Elliott Date: Fri, 17 Aug 2018 11:32:10 -0600 Subject: [PATCH] initial commit --- .gitignore | 6 + .travis.yml | 29 ++ LICENSE.md | 7 + README.md | 82 ++++++ benchmark/results.txt | 23 ++ benchmark/run.php | 143 ++++++++++ composer.json | 41 +++ phpunit.xml | 15 + src/DSO.php | 119 ++++++++ src/DSOFactoryInterface.php | 23 ++ src/DSOInterface.php | 27 ++ src/DriverFactory.php | 25 ++ src/Drivers/AbstractDriver.php | 120 ++++++++ src/Drivers/DSODriverInterface.php | 19 ++ src/Drivers/MySQLDriver.php | 88 ++++++ src/Drivers/PostgreSQLDriver.php | 87 ++++++ src/Factory.php | 268 ++++++++++++++++++ src/LegacyDrivers/AbstractLegacyDriver.php | 183 ++++++++++++ src/LegacyDrivers/MySQL56Driver.php | 35 +++ src/LegacyDrivers/README.md | 59 ++++ src/LegacyDrivers/SQLiteDriver.php | 94 ++++++ src/LegacyDrivers/destructr_json_extract.sql | 29 ++ src/Search.php | 59 ++++ tests/DSOTest.php | 48 ++++ tests/Drivers/AbstractDriverTest.php | 218 ++++++++++++++ .../AbstractDriverIntegrationTest.php | 180 ++++++++++++ .../MySQLDriverIntegrationTest.php | 16 ++ tests/Drivers/MySQLDriverTest.php | 15 + tests/FactoryTest.php | 112 ++++++++ tests/HarnessDriver.php | 65 +++++ .../SQLiteDriverIntegrationTest.php | 23 ++ tests/LegacyDrivers/MySQL56DriverTest.php | 23 ++ tests/LegacyDrivers/SQLiteDriverTest.php | 21 ++ 33 files changed, 2302 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 benchmark/results.txt create mode 100644 benchmark/run.php create mode 100644 composer.json create mode 100644 phpunit.xml create mode 100644 src/DSO.php create mode 100644 src/DSOFactoryInterface.php create mode 100644 src/DSOInterface.php create mode 100644 src/DriverFactory.php create mode 100644 src/Drivers/AbstractDriver.php create mode 100644 src/Drivers/DSODriverInterface.php create mode 100644 src/Drivers/MySQLDriver.php create mode 100644 src/Drivers/PostgreSQLDriver.php create mode 100644 src/Factory.php create mode 100644 src/LegacyDrivers/AbstractLegacyDriver.php create mode 100644 src/LegacyDrivers/MySQL56Driver.php create mode 100644 src/LegacyDrivers/README.md create mode 100644 src/LegacyDrivers/SQLiteDriver.php create mode 100644 src/LegacyDrivers/destructr_json_extract.sql create mode 100644 src/Search.php create mode 100644 tests/DSOTest.php create mode 100644 tests/Drivers/AbstractDriverTest.php create mode 100644 tests/Drivers/IntegrationTests/AbstractDriverIntegrationTest.php create mode 100644 tests/Drivers/IntegrationTests/MySQLDriverIntegrationTest.php create mode 100644 tests/Drivers/MySQLDriverTest.php create mode 100644 tests/FactoryTest.php create mode 100644 tests/HarnessDriver.php create mode 100644 tests/LegacyDrivers/IntegrationTests/SQLiteDriverIntegrationTest.php create mode 100644 tests/LegacyDrivers/MySQL56DriverTest.php create mode 100644 tests/LegacyDrivers/SQLiteDriverTest.php diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..88e1243 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +vendor +composer.lock +.phpunit.result.cache +test.php +test.sqlite +.DS_Store diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..51513b3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +language: php + +php: + - 7.1 + - 7.2 + +install: composer install + +dist: trusty + +sudo: required + +services: + - mysql + +addons: + apt: + sources: + - mysql-5.7-trusty + packages: + - mysql-server + - mysql-client + +before_install: + - sudo mysql_upgrade + - sudo service mysql restart + +before_script: + - mysql -e 'create database phpunit;' diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..59c8c4d --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,7 @@ +Copyright (c) 2018 Joby Elliott + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9581cf5 --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +# Destructr + +Destructr is a specialized ORM that allows a seamless mix of structured, relational data with unstructured JSON data. + +## Getting started + +The purpose of Destructr is to allow many "types" of objects to be stored in a single table. +Every Destructr Data Object (DSO) simply contains an array of data that will be saved into the database as JSON. +Array access is also flattened, using dots as delimiters, so rather than reading `$dso["foo"]["bar"]` you access that data via `$dso["foo.bar"]`. +This is for two reasons. +It sidesteps the issue of updating nested array values by reference, and it creates an unambiguous way of locating a node of the unstructured data with a string, which mirrors how we can write SQL queries to reference them. + +If this sounds like an insanely slow idea, that's because it is. +Luckily MySQL and MariaDB have mechanisms we can take advantage of to make generated columns from any part of the unstructured data, so that pieces of it can be pulled out into their own virtual columns for indexing and faster searching/sorting. + +### Database driver and factory + +In order to read/write objects from a database table, you'll need to configure a Driver and Factory class. + +```php +// DriverFactory::factory() has the same arguments as PDO::__construct +// You can also construct a driver directly, from a class in Drivers, +// but for common databases DriverFactory::factory should pick the right class +$driver = \Digraph\DriverFactory::factory( + 'mysql:host=127.0.0.1', + 'username', + 'password' +); +// Driver is then used to construct a Factory +$factory = new \Digraph\Destructr\Factory( + $driver, //driver is used to manage connection and generate queries + 'dso_objects' //all of a Factory's data is stored in a single table +); +``` + +### Creating a new record + +Next, you can use the factory to create a new record object. + +```php +// by default all objects are the DSO class, but factories can be made to use +// other classes depending on what data objects are instantiated with +$obj = $factory->create(); + +// returns boolean indicating whether insertion succeeded +// insert() must be called before update() will work +$obj->insert(); + +// set a new value and call update() to save changes into database. update() +// will return true without doing anything if no changes have been made. +$obj['foo.bar'] = 'some value'; +$obj->update(); + +// deleting an object will by default just set dso.deleted to the current time +// objects with a non-null dso.deleted are excluded from queries by default +// delete() calls update() inside it, so its effect is immediate +$obj->delete(); + +// objects that were deleted via default delete() are recoverable via undelete() +// undelete() also calls update() for you +$obj->undelete(); + +// objects can be actually removed from the table by calling delete(true) +$obj->delete(true); +``` + +## Requirements + +This system relies **heavily** on the JSON features of the underlying database. +This means it cannot possibly run without a database that supports JSON features. +Exact requirements are in flux during development, but basically if a database doesn't have JSON functions it's probably impossible for Destructr to ever work with it. + +In practice this means Destructr will **never** be able to run on less than the following versions of the following popular databases: + +* MySQL >=5.7 +* MariaDB >=10.2 +* PostgreSQL >=9.3 +* SQL Server >=2016 + +Theoretically Destructr is also an excellent fit for NoSQL databases. +If I ever find myself needing it there's a good chance it's possible to write drivers for running it on something like MongoDB as well. +It might even be kind of easy. diff --git a/benchmark/results.txt b/benchmark/results.txt new file mode 100644 index 0000000..07f78d7 --- /dev/null +++ b/benchmark/results.txt @@ -0,0 +1,23 @@ +Date: 2018-08-10T03:16:29+01:00 +Machine: PAULZE +Ops per: 1000 +Each result is the average number of milliseconds it took to complete each operation. Lower is better. + +Digraph\Destructr\Drivers\MySQLDriver +insert: 30.19ms +update: 31.07ms +search vcol: 147.44ms +search json: 154.34ms + +Digraph\Destructr\LegacyDrivers\MySQL56Driver +insert: 32.89ms +update: 25.11ms +search vcol: 174.05ms +search json: 192.56ms + +Digraph\Destructr\LegacyDrivers\SQLiteDriver +insert: 140.28ms +update: 145.12ms +search vcol: 157.92ms +search json: 165.01ms + diff --git a/benchmark/run.php b/benchmark/run.php new file mode 100644 index 0000000..6eaf699 --- /dev/null +++ b/benchmark/run.php @@ -0,0 +1,143 @@ + $config) { + $driver = new $class(@$config['dsn'], @$config['username'], @$config['password'], @$config['options']); + $factory = new Factory($driver, $config['table']); + $factory->createTable(); + benchmark_empty($factory); + $out[] = $class; + $out[] = benchmark_insert($factory); + $out[] = benchmark_update($factory); + $out[] = benchmark_search_vcol($factory); + $out[] = benchmark_search_json($factory); + $out[]= ''; +} +$out[] = ''; + +file_put_contents(__DIR__.'/results.txt', implode(PHP_EOL, $out)); + +/** + * The classes and connection settings for benchmarking + */ +function drivers_list() +{ + @unlink(__DIR__.'/test.sqlite'); + $out = []; + $out[MySQLDriver::class] = [ + 'table' => 'benchmark57', + 'dsn' => 'mysql:host=127.0.0.1;dbname=phpunit', + 'username' => 'travis' + ]; + $out[MySQL56Driver::class] = [ + 'table' => 'benchmark56', + 'dsn' => 'mysql:host=127.0.0.1;dbname=phpunit', + 'username' => 'travis' + ]; + $out[SQLiteDriver::class] = [ + 'table' => 'benchmark', + 'dsn' => 'sqlite:'.__DIR__.'/test.sqlite' + ]; + return $out; +} + +/** + * Empties a table before beginning + */ +function benchmark_empty(&$factory) +{ + global $dsos; + $dsos = []; + foreach ($factory->search()->execute([], null) as $o) { + $o->delete(true); + } +} + +/** + * Benchmark insert operations + */ +function benchmark_insert(&$factory) +{ + global $dsos; + $start = microtime(true); + for ($i=0; $i < OPS_PER; $i++) { + $dsos[$i] = $factory->create( + [ + 'dso.id'=>'benchmark-'.$i, + 'dso.type'=>'benchmark-'.($i%2?'odd':'even'), + 'benchmark.mod'=>($i%2?'odd':'even') + ] + ); + $dsos[$i]->insert(); + } + $end = microtime(true); + $per = round(($end-$start)*100000/OPS_PER)/100; + return 'insert: '.$per.'ms'; +} + +/** + * Benchmark update operations + */ +function benchmark_update(&$factory) +{ + global $dsos; + $start = microtime(true); + for ($i=0; $i < OPS_PER; $i++) { + $dsos[$i]['benchmark.int'] = $i; + $dsos[$i]['benchmark.string'] = 'benchmark-'.$i; + $dsos[$i]->update(); + } + $end = microtime(true); + $per = round(($end-$start)*100000/OPS_PER)/100; + return 'update: '.$per.'ms'; +} + +/** + * Benchmark searching on a vcol + */ +function benchmark_search_vcol(&$factory) +{ + $start = microtime(true); + for ($i=0; $i < OPS_PER; $i++) { + $search = $factory->search(); + $search->where('${dso.type} = :type'); + $search->execute([':type'=>'benchmark-odd']); + } + $end = microtime(true); + $per = round(($end-$start)*100000/OPS_PER)/100; + return 'search vcol: '.$per.'ms'; +} + +/** + * Benchmark searching on a JSON value + */ +function benchmark_search_json(&$factory) +{ + $start = microtime(true); + for ($i=0; $i < OPS_PER; $i++) { + $search = $factory->search(); + $search->where('${benchmark.mod} = :type'); + $search->execute([':type'=>'even']); + } + $end = microtime(true); + $per = round(($end-$start)*100000/OPS_PER)/100; + return 'search json: '.$per.'ms'; +} diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..366dde4 --- /dev/null +++ b/composer.json @@ -0,0 +1,41 @@ +{ + "name": "digraphcms/destructr", + "description": "A library for storing a mix of structured and unstructured data in relational databases", + "type": "library", + "license": "MIT", + "minimum-stability": "dev", + "prefer-stable": true, + "require": { + "php": ">=7.1", + "digraphcms/utilities": "^0.6", + "mofodojodino/profanity-filter": "^1.3" + }, + "require-dev": { + "phpunit/phpunit": "^7", + "phpunit/dbunit": "^4.0" + }, + "scripts": { + "test": [ + "phpunit" + ], + "test-local": [ + "phpunit --testsuite Local" + ], + "test-db": [ + "phpunit --testsuite DB" + ], + "test-legacydb": [ + "phpunit --testsuite LegacyDB" + ] + }, + "autoload": { + "psr-4": { + "Digraph\\Destructr\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Digraph\\Destructr\\": "tests/" + } + } +} diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000..65c3167 --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,15 @@ + + + + tests + tests/Drivers + tests/LegacyDrivers + + + tests/Drivers + + + tests/LegacyDrivers + + + diff --git a/src/DSO.php b/src/DSO.php new file mode 100644 index 0000000..8d731ad --- /dev/null +++ b/src/DSO.php @@ -0,0 +1,119 @@ +resetChanges(); + parent::__construct($data); + $this->factory($factory); + $this->resetChanges(); + } + + public function hook_create() + { + //does nothing + } + public function hook_update() + { + //does nothing + } + + public function delete(bool $permanent = false) : bool + { + return $this->factory->delete($this, $permanent); + } + + public function undelete() : bool + { + return $this->factory->undelete($this); + } + + public function insert() : bool + { + return $this->factory()->insert($this); + } + + public function update() : bool + { + return $this->factory()->update($this); + } + + public function resetChanges() + { + $this->changes = new FlatArray(); + $this->removals = new FlatArray(); + } + + public function changes() : array + { + return $this->changes->get(); + } + + public function removals() : array + { + return $this->removals->get(); + } + + public function set(string $name = null, $value, $force=false) + { + $name = strtolower($name); + if ($this->get($name) == $value) { + return; + } + if (is_array($value)) { + //check for what's being removed + if (is_array($this->get($name))) { + foreach ($this->get($name) as $k => $v) { + if (!isset($value[$k])) { + if ($name) { + $k = $name.'.'.$k; + } + $this->unset($k); + } + } + } + //recursively set individual values so we can track them + foreach ($value as $k => $v) { + if ($name) { + $k = $name.'.'.$k; + } + $this->set($k, $v, $force); + } + } else { + $this->changes->set($name, $value); + unset($this->removals[$name]); + parent::set($name, $value); + } + } + + public function unset(?string $name) + { + if (isset($this[$name])) { + $this->removals->set($name, $this->get($name)); + unset($this->changes[$name]); + parent::unset($name); + } + } + + public function factory(DSOFactoryInterface &$factory = null) : ?DSOFactoryInterface + { + if ($factory) { + $this->factory = $factory; + } + return $this->factory; + } +} diff --git a/src/DSOFactoryInterface.php b/src/DSOFactoryInterface.php new file mode 100644 index 0000000..90d2545 --- /dev/null +++ b/src/DSOFactoryInterface.php @@ -0,0 +1,23 @@ + */ +namespace Digraph\Destructr; + +interface DSOFactoryInterface +{ + public function __construct(Drivers\DSODriverInterface &$driver, string $table); + + public function class(array $data) : ?string; + + public function createTable() : bool; + public function create(array $data = array()) : DSOInterface; + public function read(string $value, string $field = 'dso.id', $deleted = false) : ?DSOInterface; + public function insert(DSOInterface &$dso) : bool; + public function update(DSOInterface &$dso) : bool; + public function delete(DSOInterface &$dso, bool $permanent = false) : bool; + + public function search() : Search; + public function executeSearch(Search $search, array $params = array(), $deleted = false) : array; +} diff --git a/src/DSOInterface.php b/src/DSOInterface.php new file mode 100644 index 0000000..373eeee --- /dev/null +++ b/src/DSOInterface.php @@ -0,0 +1,27 @@ + Drivers\MySQLDriver::class, + 'mariadb' => Drivers\MySQLDriver::class, + 'pgsql' => Driver\MySQLDriver::class + ]; + + public static function factory(string $dsn, string $username=null, string $password=null, array $options=null, string $type = null) : ?Drivers\DSODriverInterface + { + if (!$type) { + $type = array_shift(explode(':', $dsn, 2)); + } + $type = strtolower($type); + if ($class = @static::$map[$type]) { + return new $class($dsn, $username, $password, $options); + } else { + return null; + } + } +} diff --git a/src/Drivers/AbstractDriver.php b/src/Drivers/AbstractDriver.php new file mode 100644 index 0000000..68922db --- /dev/null +++ b/src/Drivers/AbstractDriver.php @@ -0,0 +1,120 @@ +pdo = new \PDO($dsn, $username, $password, $options)) { + throw new \Exception("Error creating PDO connection"); + } + } + + protected function expandPaths($value) + { + if ($value === null) { + return null; + } + $value = preg_replace_callback( + '/\$\{([^\}\\\]+)\}/', + function ($matches) { + return $this->expandPath($matches[1]); + }, + $value + ); + return $value; + } + + public function errorInfo() + { + return $this->pdo->errorInfo(); + } + + public function createTable(string $table, array $virtualColumns) : bool + { + $sql = $this->sql_ddl([ + 'table'=>$table, + 'virtualColumns'=>$virtualColumns + ]); + return $this->pdo->exec($sql) !== false; + } + + public function update(string $table, DSOInterface $dso) : bool + { + if (!$dso->changes() && !$dso->removals()) { + return true; + } + $s = $this->getStatement( + 'setJSON', + ['table'=>$table] + ); + return $s->execute([ + ':dso_id' => $dso['dso.id'], + ':data' => json_encode($dso->get()) + ]); + } + + public function delete(string $table, DSOInterface $dso) : bool + { + $s = $this->getStatement( + 'delete', + ['table'=>$table] + ); + return $s->execute([ + ':dso_id' => $dso['dso.id'] + ]); + } + + public function select(string $table, Search $search, array $params) + { + $s = $this->getStatement( + 'select', + ['table'=>$table,'search'=>$search] + ); + if (!$s->execute($params)) { + return []; + } + return $s->fetchAll(\PDO::FETCH_ASSOC); + } + + public function insert(string $table, DSOInterface $dso) : bool + { + return $this->getStatement( + 'insert', + ['table'=>$table] + )->execute( + [':data'=>json_encode($dso->get())] + ); + } + + protected function getStatement(string $type, $args=array()) : \PDOStatement + { + $fn = 'sql_'.$type; + if (!method_exists($this, $fn)) { + throw new \Exception("Error getting SQL statement, driver doesn't have a method named $fn"); + } + $sql = $this->$fn($args); + $stmt = $this->pdo->prepare($sql); + if (!$stmt) { + $this->lastPreparationErrorOn = $sql; + throw new \Exception("Error preparing statement: ".implode(': ', $this->pdo->errorInfo()), 1); + } + return $stmt; + //TODO: turn this on someday and see if caching statements helps in the real world + // $sql = $this->$fn($args); + // $id = md5($sql); + // if (!isset($this->prepared[$id])) { + // $this->prepared[$id] = $this->pdo->prepare($sql); + // } + // return @$this->prepared[$id]; + } +} diff --git a/src/Drivers/DSODriverInterface.php b/src/Drivers/DSODriverInterface.php new file mode 100644 index 0000000..59cdadf --- /dev/null +++ b/src/Drivers/DSODriverInterface.php @@ -0,0 +1,19 @@ += 5.7 + * * MariaDB >= 10.2 + */ +class MySQLDriver extends AbstractDriver +{ + /** + * Within the search we expand strings like ${dso.id} into JSON queries. + * Note that the Search will have already had these strings expanded into + * column names if there are virtual columns configured for them. That + * happens in the Factory before it gets here. + */ + protected function sql_select($args) + { + //extract query parts from Search and expand paths + $where = $this->expandPaths($args['search']->where()); + $order = $this->expandPaths($args['search']->order()); + $limit = $args['search']->limit(); + $offset = $args['search']->offset(); + //select from + $out = ["SELECT * FROM `{$args['table']}`"]; + //where statement + if ($where !== null) { + $out[] = "WHERE ".$where; + } + //order statement + if ($order !== null) { + $out[] = "ORDER BY ".$order; + } + //limit + if ($limit !== null) { + $out[] = "LIMIT ".$limit; + } + //offset + if ($offset !== null) { + $out[] = "OFFSET ".$offset; + } + //return + return implode(PHP_EOL, $out).';'; + } + + protected function sql_ddl($args=array()) + { + $out = []; + $out[] = "CREATE TABLE `{$args['table']}` ("; + $lines = []; + $lines[] = "`json_data` JSON DEFAULT NULL"; + foreach ($args['virtualColumns'] as $path => $col) { + $lines[] = "`{$col['name']}` {$col['type']} GENERATED ALWAYS AS (".$this->expandPath($path).") VIRTUAL"; + } + foreach ($args['virtualColumns'] as $path => $col) { + if (@$col['unique'] && $as = @$col['index']) { + $lines[] = "UNIQUE KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as"; + } elseif ($as = @$col['index']) { + $lines[] = "KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as"; + } + } + $out[] = implode(','.PHP_EOL, $lines); + $out[] = ") ENGINE=InnoDB DEFAULT CHARSET=utf8;"; + return implode(PHP_EOL, $out); + } + + protected function expandPath(string $path) : string + { + return "JSON_UNQUOTE(JSON_EXTRACT(`json_data`,'$.{$path}'))"; + } + + protected function sql_setJSON($args) + { + return 'UPDATE `'.$args['table'].'` SET `json_data` = :data WHERE `dso_id` = :dso_id;'; + } + + protected function sql_insert($args) + { + return "INSERT INTO `{$args['table']}` (`json_data`) VALUES (:data);"; + } + + protected function sql_delete($args) + { + return 'DELETE FROM `'.$args['table'].'` WHERE `dso_id` = :dso_id;'; + } +} diff --git a/src/Drivers/PostgreSQLDriver.php b/src/Drivers/PostgreSQLDriver.php new file mode 100644 index 0000000..1c30dc6 --- /dev/null +++ b/src/Drivers/PostgreSQLDriver.php @@ -0,0 +1,87 @@ +=9.3 + * + * Eventually, anyway. At the moment it's untested and probably doesn't work. + */ +class PostgreSQLDriver extends AbstractDriver +{ + /** + * Within the search we expand strings like ${dso.id} into JSON queries. + * Note that the Search will have already had these strings expanded into + * column names if there are virtual columns configured for them. That + * happens in the Factory before it gets here. + */ + protected function sql_select($args) + { + //extract query parts from Search and expand paths + $where = $this->expandPaths($args['search']->where()); + $order = $this->expandPaths($args['search']->order()); + $limit = $args['search']->limit(); + $offset = $args['search']->offset(); + //select from + $out = ["SELECT * FROM `{$args['table']}`"]; + //where statement + if ($where !== null) { + $out[] = "WHERE ".$where; + } + //order statement + if ($order !== null) { + $out[] = "ORDER BY ".$order; + } + //limit + if ($limit !== null) { + $out[] = "LIMIT ".$limit; + } + //offset + if ($offset !== null) { + $out[] = "OFFSET ".$offset; + } + //return + return implode(PHP_EOL, $out).';'; + } + + protected function sql_ddl($args=array()) + { + $out = []; + $out[] = "CREATE TABLE `{$args['table']}` ("; + $lines = []; + $lines[] = "`json_data` JSON DEFAULT NULL"; + foreach ($args['virtualColumns'] as $path => $col) { + $lines[] = "`{$col['name']}` {$col['type']} GENERATED ALWAYS AS (".$this->expandPath($path).") VIRTUAL"; + } + foreach ($args['virtualColumns'] as $path => $col) { + if (@$col['unique'] && $as = @$col['index']) { + $lines[] = "UNIQUE KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as"; + } elseif ($as = @$col['index']) { + $lines[] = "KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as"; + } + } + $out[] = implode(','.PHP_EOL, $lines); + $out[] = ") ENGINE=InnoDB DEFAULT CHARSET=utf8;"; + return implode(PHP_EOL, $out); + } + + protected function expandPath(string $path) : string + { + return "JSON_UNQUOTE(JSON_EXTRACT(`json_data`,'$.{$path}'))"; + } + + protected function sql_setJSON($args) + { + return 'UPDATE `'.$args['table'].'` SET `json_data` = :data WHERE `dso_id` = :dso_id;'; + } + + protected function sql_insert($args) + { + return "INSERT INTO `{$args['table']}` (`json_data`) VALUES (:data);"; + } + + protected function sql_delete($args) + { + return 'DELETE FROM `'.$args['table'].'` WHERE `dso_id` = :dso_id;'; + } +} diff --git a/src/Factory.php b/src/Factory.php new file mode 100644 index 0000000..8ecf914 --- /dev/null +++ b/src/Factory.php @@ -0,0 +1,268 @@ + [ + 'name'=>'dso_id', + 'type'=>'VARCHAR(16)', + 'index' => 'BTREE', + 'unique' => true + ], + 'dso.type' => [ + 'name'=>'dso_type', + 'type'=>'VARCHAR(30)', + 'index'=>'BTREE' + ], + 'dso.deleted' => [ + 'name'=>'dso_deleted', + 'type'=>'BIGINT', + 'index'=>'BTREE' + ] + ]; + /** + * This cannot be modified by extending classes, it's used by legacy drivers + */ + const CORE_VIRTUAL_COLUMNS = [ + 'dso.id' => [ + 'name'=>'dso_id', + 'type'=>'VARCHAR(16)', + 'index' => 'BTREE', + 'unique' => true + ], + 'dso.type' => [ + 'name'=>'dso_type', + 'type'=>'VARCHAR(30)', + 'index'=>'BTREE' + ], + 'dso.deleted' => [ + 'name'=>'dso_deleted', + 'type'=>'BIGINT', + 'index'=>'BTREE' + ] + ]; + + public function __construct(Drivers\DSODriverInterface &$driver, string $table) + { + $this->driver = $driver; + $this->table = $table; + } + + protected function hook_create(DSOInterface &$dso) + { + if (!$dso->get('dso.id')) { + $dso->set('dso.id', static::generate_id(static::ID_CHARS, static::ID_LENGTH), true); + } + if (!$dso->get('dso.created.date')) { + $dso->set('dso.created.date', time()); + } + if (!$dso->get('dso.created.user')) { + $dso->set('dso.created.user', ['ip'=>@$_SERVER['REMOTE_ADDR']]); + } + } + + protected function hook_update(DSOInterface &$dso) + { + $dso->set('dso.modified.date', time()); + $dso->set('dso.modified.user', ['ip'=>@$_SERVER['REMOTE_ADDR']]); + } + + public function class(array $data) : ?string + { + return null; + } + + public function delete(DSOInterface &$dso, bool $permanent = false) : bool + { + if ($permanent) { + return $this->driver->delete($this->table, $dso); + } + $dso['dso.deleted'] = time(); + return $this->update($dso); + } + + public function undelete(DSOInterface &$dso) : bool + { + unset($dso['dso.deleted']); + return $this->update($dso); + } + + public function create(array $data = array()) : DSOInterface + { + if (!($class = $this->class($data))) { + $class = DSO::class; + } + $dso = new $class($data, $this); + $this->hook_create($dso); + $dso->hook_create(); + $dso->resetChanges(); + return $dso; + } + + public function createTable() : bool + { + return $this->driver->createTable( + $this->table, + ($this->driver::EXTENSIBLE_VIRTUAL_COLUMNS?$this->virtualColumns:$this::CORE_VIRTUAL_COLUMNS) + ); + } + + protected function virtualColumnName($path) : ?string + { + if ($this->driver::EXTENSIBLE_VIRTUAL_COLUMNS) { + $vcols = $this->virtualColumns; + } else { + $vcols = static::CORE_VIRTUAL_COLUMNS; + } + return @$vcols[$path]['name']; + } + + public function update(DSOInterface &$dso) : bool + { + if (!$dso->changes() && !$dso->removals()) { + return true; + } + $this->hook_update($dso); + $dso->hook_update(); + $out = $this->driver->update($this->table, $dso); + $dso->resetChanges(); + return $out; + } + + public function search() : Search + { + return new Search($this); + } + + protected function makeObjectFromRow($arr) + { + $data = json_decode($arr['json_data'], true); + return $this->create($data); + } + + protected function makeObjectsFromRows($arr) + { + foreach ($arr as $key => $value) { + $arr[$key] = $this->makeObjectFromRow($value); + } + return $arr; + } + + public function executeSearch(Search $search, array $params = array(), $deleted = false) : array + { + //add deletion clause and expand column names + $search = $this->preprocessSearch($search, $deleted); + //run select + $r = $this->driver->select( + $this->table, + $search, + $params + ); + return $this->makeObjectsFromRows($r); + } + + public function read(string $value, string $field = 'dso.id', $deleted = false) : ?DSOInterface + { + $search = $this->search(); + $search->where('${'.$field.'} = :value'); + if ($results = $search->execute([':value'=>$value], $deleted)) { + return array_shift($results); + } + return null; + } + + public function insert(DSOInterface &$dso) : bool + { + $this->hook_update($dso); + $dso->hook_update(); + $dso->resetChanges(); + return $this->driver->insert($this->table, $dso); + } + + protected function preprocessSearch($input, $deleted) + { + //clone search so we're not accidentally messing with a reference + $search = new Search($this); + $search->where($input->where()); + $search->order($input->order()); + /* add deletion awareness to where clause */ + if ($deleted !== null) { + $where = $search->where(); + if ($deleted === true) { + $added = '${dso.deleted} is not null'; + } else { + $added = '${dso.deleted} is null'; + } + if ($where) { + $where = '('.$where.') AND '.$added; + } else { + $where = $added; + } + $search->where($where); + } + /* expand virtual column names */ + foreach (['where','order'] as $clause) { + if ($value = $search->$clause()) { + $value = preg_replace_callback( + '/\$\{([^\}\\\]+)\}/', + function ($matches) { + /* depends on whether a virtual column is expected for this value */ + if ($vcol = $this->virtualColumnName($matches[1])) { + return "`$vcol`"; + } + return $matches[0]; + }, + $value + ); + $search->$clause($value); + } + } + /* return search */ + return $search; + } + + protected static function generate_id($chars, $length) : string + { + $check = new Check(); + do { + $id = ''; + while (strlen($id) < $length) { + $id .= substr( + $chars, + rand(0, strlen($chars)-1), + 1 + ); + } + } while ($check->hasProfanity($id)); + return $id; + } +} diff --git a/src/LegacyDrivers/AbstractLegacyDriver.php b/src/LegacyDrivers/AbstractLegacyDriver.php new file mode 100644 index 0000000..5847258 --- /dev/null +++ b/src/LegacyDrivers/AbstractLegacyDriver.php @@ -0,0 +1,183 @@ + $row) { + $new = new FlatArray(); + foreach (json_decode($row['json_data'], true) as $key => $value) { + $new->set(str_replace('|', '.', $key), $value); + } + $results[$rkey]['json_data'] = json_encode($new->get()); + } + return $results; + } + + protected function sql_select($args) + { + //extract query parts from Search and expand paths + $where = $this->expandPaths($args['search']->where()); + $order = $this->expandPaths($args['search']->order()); + $limit = $args['search']->limit(); + $offset = $args['search']->offset(); + //select from + $out = ["SELECT * FROM `{$args['table']}`"]; + //where statement + if ($where !== null) { + $out[] = "WHERE ".$where; + } + //order statement + if ($order !== null) { + $out[] = "ORDER BY ".$order; + } + //limit + if ($limit !== null) { + $out[] = "LIMIT ".$limit; + } + //offset + if ($offset !== null) { + $out[] = "OFFSET ".$offset; + } + //return + return implode(PHP_EOL, $out).';'; + } + + protected function sql_ddl($args=array()) + { + $out = []; + $out[] = "CREATE TABLE `{$args['table']}` ("; + $lines = []; + $lines[] = "`json_data` TEXT DEFAULT NULL"; + foreach ($args['virtualColumns'] as $path => $col) { + $lines[] = "`{$col['name']}` {$col['type']}"; + } + foreach ($args['virtualColumns'] as $path => $col) { + if (@$col['unique'] && $as = @$col['index']) { + $lines[] = "UNIQUE KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as"; + } elseif ($as = @$col['index']) { + $lines[] = "KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as"; + } + } + $out[] = implode(','.PHP_EOL, $lines); + $out[] = ") ENGINE=InnoDB DEFAULT CHARSET=utf8;"; + return implode(PHP_EOL, $out); + } + + public function update(string $table, DSOInterface $dso) : bool + { + if (!$dso->changes() && !$dso->removals()) { + return true; + } + $s = $this->getStatement( + 'setJSON', + ['table'=>$table] + ); + $params = $this->legacyParams($dso); + $out = $s->execute($params); + return $out; + } + + protected function sql_setJSON($args) + { + $out = []; + $out[] = 'UPDATE `'.$args['table'].'`'; + $out[] = 'SET'; + foreach (Factory::CORE_VIRTUAL_COLUMNS as $v) { + $out[] = '`'.$v['name'].'` = :'.$v['name'].','; + } + $out[] = '`json_data` = :data'; + $out[] = 'WHERE `dso_id` = :dso_id'; + return implode(PHP_EOL, $out).';'; + } + + public function insert(string $table, DSOInterface $dso) : bool + { + $s = $this->getStatement( + 'insert', + ['table'=>$table] + ); + $params = $this->legacyParams($dso); + return $s->execute($params); + } + + protected function legacyParams(DSOInterface $dso) + { + $params = [':data' => $this->json_encode($dso->get())]; + foreach (Factory::CORE_VIRTUAL_COLUMNS as $vk => $vv) { + $params[':'.$vv['name']] = $dso->get($vk); + } + return $params; + } + + protected function sql_insert($args) + { + $out = []; + $out[] = 'INSERT INTO `'.$args['table'].'`'; + $out[] = '(`json_data`,`dso_id`,`dso_type`,`dso_deleted`)'; + $out[] = 'VALUES (:data, :dso_id, :dso_type, :dso_deleted)'; + return implode(PHP_EOL, $out).';'; + } + + public function delete(string $table, DSOInterface $dso) : bool + { + $s = $this->getStatement( + 'delete', + ['table'=>$table] + ); + $out = $s->execute([ + ':dso_id' => $dso['dso.id'] + ]); + // if (!$out) { + // var_dump($s->errorInfo()); + // } + return $out; + } + + protected function sql_delete($args) + { + return 'DELETE FROM `'.$args['table'].'` WHERE `dso_id` = :dso_id;'; + } + + public function json_encode($a, array &$b = null, string $prefix = '') + { + if ($b === null) { + $b = []; + $this->json_encode($a, $b, ''); + return json_encode($b); + } else { + if (is_array($a)) { + foreach ($a as $ak => $av) { + if ($prefix == '') { + $nprefix = $ak; + } else { + $nprefix = $prefix.'|'.$ak; + } + $this->json_encode($av, $b, $nprefix); + } + } else { + $b[$prefix] = $a; + } + } + } +} diff --git a/src/LegacyDrivers/MySQL56Driver.php b/src/LegacyDrivers/MySQL56Driver.php new file mode 100644 index 0000000..9d6e7e2 --- /dev/null +++ b/src/LegacyDrivers/MySQL56Driver.php @@ -0,0 +1,35 @@ +createLegacyUDF(); + return parent::createTable($table, $virtualColumns); + } + + public function createLegacyUDF() + { + $drop = $this->pdo->exec('DROP FUNCTION IF EXISTS `destructr_json_extract`;'); + $create = $this->pdo->exec(file_get_contents(__DIR__.'/destructr_json_extract.sql')); + } + + protected function expandPath(string $path) : string + { + $path = str_replace('.', '|', $path); + return "destructr_json_extract(`json_data`,'$.{$path}')"; + } +} diff --git a/src/LegacyDrivers/README.md b/src/LegacyDrivers/README.md new file mode 100644 index 0000000..dd450de --- /dev/null +++ b/src/LegacyDrivers/README.md @@ -0,0 +1,59 @@ +# Destructr legacy drivers + +This folder holds what are called "legacy drivers." +These drivers attempt to extend Destructr support to as many databases as possible, +even in some cases at the expense of functionality. + +All these drivers are designed to support the following features: +* Inserting objects (and disallowing duplicates at the *SQL* level) +* Updating existing objects +* Deleting objects +* Searching by exact text matches + +The things many of this sort of driver will *not* ever support: +* Performance optimization via virtual columns. Many of them will have their + performance lightly optimized for common system queries, just by hard-coding + virtual columns for `dso.id`, `dso.deleted`, and `dso.type` +* Complex queries, like joins, REGEX, or LIKE queries. + +## Current state of legacy drivers + +### SQLite 3 + +**\Digraph\Destructr\LegacyDrivers\SQLiteDriver** + +**Overall support level: Highly functional, a touch slow** + +Via LegacyDrivers\SQLiteDriver Destructr now has surprisingly good support for +SQLite 3. It accomplishes this by using SQLite's user-defined functions feature +to offload JSON parsing to PHP. This means all the JSON parsing is actually up +to spec, storage doesn't need to be flattened like the other legacy drivers, +and tables are dramatically more easily forward-compatible. + +If you're considering using legacy MySQL, you should really seriously consider +just using SQLite instead. In benchmarks with 1000 records SQLite's performance +is actually BETTER than MySQL 5.6 for everything but insert/update operations. +In some cases it appears to even be *significantly* faster while also having the +distinct advantage of not using any goofy home-rolled JSON extraction funcitons. + +So unless you have a good reason to use MySQL 5.6, you're probably best off +using SQLite if you don't have access to a fully supported database version. + +### MySQL 5.6 + +**\Digraph\Destructr\LegacyDrivers\MySQL56Driver** + +**Overall support level: Decent performance, highly suspect accuracy** + +LegacyDrivers\MySQL56Driver provides bare-minimum support for MySQL < 5.7. +This driver now passes the basic tests and basic integration tests, but hasn't +been verified in the slightest beyond that. + +It flattens unstructured JSON and uses a highly dodgy user-defined function to +extract values from it. There are absolutely edge cases that will extract the +wrong data. That said, outside of those edge cases it should actually work +fairly well. All the sorting and filtering is happening in SQL, and things +should mostly be fairly predictable. + +This driver should be your last resort. I cannot emphasize enough that this +thing is extremely kludgey and should not be trusted. diff --git a/src/LegacyDrivers/SQLiteDriver.php b/src/LegacyDrivers/SQLiteDriver.php new file mode 100644 index 0000000..08f008b --- /dev/null +++ b/src/LegacyDrivers/SQLiteDriver.php @@ -0,0 +1,94 @@ +pdo->sqliteCreateFunction( + 'DESTRUCTR_JSON_EXTRACT', + '\\Digraph\\Destructr\\LegacyDrivers\\SQLiteDriver::JSON_EXTRACT', + 2 + ); + } + + public static function JSON_EXTRACT($json, $path) + { + $path = substr($path, 2); + $path = explode('.', $path); + $arr = json_decode($json, true); + $out = &$arr; + while ($key = array_shift($path)) { + if (isset($out[$key])) { + $out = &$out[$key]; + } else { + return null; + } + } + return $out; + } + + public function createTable(string $table, array $virtualColumns) : bool + { + $sql = $this->sql_ddl([ + 'table'=>$table, + ]); + $out = $this->pdo->exec($sql) !== false; + foreach (Factory::CORE_VIRTUAL_COLUMNS as $key => $vcol) { + $idxResult = true; + if (@$vcol['unique']) { + $idxResult = $this->pdo->exec('CREATE UNIQUE INDEX '.$table.'_'.$vcol['name'].'_idx on `'.$table.'`(`'.$vcol['name'].'`)') !== false; + } elseif (@$vcol['index']) { + $idxResult = $this->pdo->exec('CREATE INDEX '.$table.'_'.$vcol['name'].'_idx on `'.$table.'`(`'.$vcol['name'].'`)') !== false; + } + if (!$idxResult) { + $out = false; + } + } + return $out; + } + + protected function sql_ddl($args=array()) + { + $out = []; + $out[] = "CREATE TABLE `{$args['table']}` ("; + $lines = []; + $lines[] = "`json_data` TEXT DEFAULT NULL"; + foreach (Factory::CORE_VIRTUAL_COLUMNS as $path => $col) { + $lines[] = "`{$col['name']}` {$col['type']}"; + } + $out[] = implode(','.PHP_EOL, $lines); + $out[] = ");"; + return implode(PHP_EOL, $out); + } + + protected function expandPath(string $path) : string + { + return "DESTRUCTR_JSON_EXTRACT(`json_data`,'$.{$path}')"; + } + + public function json_encode($a, ?array &$b = null, string $prefix = '') + { + return json_encode($a); + } +} diff --git a/src/LegacyDrivers/destructr_json_extract.sql b/src/LegacyDrivers/destructr_json_extract.sql new file mode 100644 index 0000000..9b0597b --- /dev/null +++ b/src/LegacyDrivers/destructr_json_extract.sql @@ -0,0 +1,29 @@ +CREATE FUNCTION `destructr_json_extract`( +details TEXT, +required_field VARCHAR (255) +) RETURNS TEXT CHARSET utf8 +BEGIN + DECLARE search_term TEXT; + SET details = SUBSTRING_INDEX(details, "{", -1); + SET details = SUBSTRING_INDEX(details, "}", 1); + SET search_term = CONCAT('"', SUBSTRING_INDEX(required_field,'$.', - 1), '"'); + IF INSTR(details, search_term) > 0 THEN + RETURN TRIM( + BOTH '"' FROM SUBSTRING_INDEX( + SUBSTRING_INDEX( + SUBSTRING_INDEX( + details, + search_term, + - 1 + ), + ',"', + 1 + ), + ':', + -1 + ) + ); + ELSE + RETURN NULL; + END IF; +END; diff --git a/src/Search.php b/src/Search.php new file mode 100644 index 0000000..6e16f6e --- /dev/null +++ b/src/Search.php @@ -0,0 +1,59 @@ +factory = $factory; + } + + public function execute(array $params = array(), $deleted = false) + { + return $this->factory->executeSearch($this, $params, $deleted); + } + + public function where(string $set = null) : ?string + { + return $this->valueFunction('where', $set); + } + + public function order(string $set = null) : ?string + { + return $this->valueFunction('order', $set); + } + + public function limit(int $set = null) : ?int + { + return $this->valueFunction('limit', $set); + } + + public function offset(int $set = null) : ?int + { + return $this->valueFunction('offset', $set); + } + + public function serialize() + { + return json_encode( + [$this->where(),$this->order(),$this->limit(),$this->offset()] + ); + } + + public function unserialize($string) + { + list($where, $order, $limit, $offset) = json_decode($string, true); + $this->where($where); + $this->order($order); + $this->limit($limit); + $this->offset($offset); + } +} diff --git a/tests/DSOTest.php b/tests/DSOTest.php new file mode 100644 index 0000000..6538b08 --- /dev/null +++ b/tests/DSOTest.php @@ -0,0 +1,48 @@ + 'b', + 'c' => 'd', + 'e' => [1,2,3] + ]); + $this->assertEquals([], $dso->changes()); + $this->assertEquals([], $dso->removals()); + //not actually a change, shouldn't trigger + $dso['a'] = 'b'; + $this->assertEquals([], $dso->changes()); + $this->assertEquals([], $dso->removals()); + //not actually a change, shouldn't trigger + $dso['e'] = [1,2,3]; + $this->assertEquals([], $dso->changes()); + $this->assertEquals([], $dso->removals()); + //changing a should trigger changes but not removals + $dso['a'] = 'B'; + $this->assertEquals(['a'=>'B'], $dso->changes()); + $this->assertEquals([], $dso->removals()); + //removing c should trigger removals but not change changes + unset($dso['c']); + $this->assertEquals(['a'=>'B'], $dso->changes()); + $this->assertEquals(['c'=>'d'], $dso->removals()); + //setting c back should remove it from removals, but add it to changes + $dso['c'] = 'd'; + $this->assertEquals(['a'=>'B','c'=>'d'], $dso->changes()); + $this->assertEquals([], $dso->removals()); + //unsetting c again should remove it from changes, but add it back to removals + unset($dso['c']); + $this->assertEquals(['a'=>'B'], $dso->changes()); + $this->assertEquals(['c'=>'d'], $dso->removals()); + //resetting changes + $dso->resetChanges(); + $this->assertEquals([], $dso->changes()); + $this->assertEquals([], $dso->removals()); + } +} diff --git a/tests/Drivers/AbstractDriverTest.php b/tests/Drivers/AbstractDriverTest.php new file mode 100644 index 0000000..4f23d9d --- /dev/null +++ b/tests/Drivers/AbstractDriverTest.php @@ -0,0 +1,218 @@ + [ + 'name'=>'dso_id', + 'type'=>'VARCHAR(16)', + 'index' => 'BTREE', + 'unique' => true + ], + 'dso.type' => [ + 'name'=>'dso_type', + 'type'=>'VARCHAR(30)', + 'index'=>'BTREE' + ], + 'dso.deleted' => [ + 'name'=>'dso_deleted', + 'type'=>'BIGINT', + 'index'=>'BTREE' + ] + ]; + + public function testCreateTable() + { + $driver = $this->createDriver(); + $res = $driver->createTable('testCreateTable', $this->virtualColumns); + $this->assertTrue($res); + $this->assertEquals(0, $this->getConnection()->getRowCount('testCreateTable')); + $this->assertFalse($driver->createTable('testCreateTable', $this->virtualColumns)); + } + + public function testInsert() + { + $driver = $this->createDriver(); + $driver->createTable('testInsert', $this->virtualColumns); + //test inserting an object + $o = new DSO(['dso.id'=>'first-inserted']); + $this->assertTrue($driver->insert('testInsert', $o)); + $this->assertEquals(1, $this->getConnection()->getRowCount('testInsert')); + //test inserting a second object + $o = new DSO(['dso.id'=>'second-inserted']); + $this->assertTrue($driver->insert('testInsert', $o)); + $this->assertEquals(2, $this->getConnection()->getRowCount('testInsert')); + //test inserting a second object with an existing id, it shouldn't work + $o = new DSO(['dso.id'=>'first-inserted']); + $this->assertFalse($driver->insert('testInsert', $o)); + $this->assertEquals(2, $this->getConnection()->getRowCount('testInsert')); + } + + public function testSelect() + { + $driver = $this->createDriver(); + $driver->createTable('testSelect', $this->virtualColumns); + //set up dummy data + $this->setup_testSelect(); + //empty search + $search = new Search(); + $results = $driver->select('testSelect', $search, []); + $this->assertSame(4, count($results)); + //sorting by json value sort + $search = new Search(); + $search->order('${sort} asc'); + $results = $driver->select('testSelect', $search, []); + $this->assertSame(4, count($results)); + $results = array_map( + function ($a) { + return json_decode($a['json_data'], true); + }, + $results + ); + $this->assertSame('item-a-1', $results[0]['dso']['id']); + $this->assertSame('item-b-1', $results[1]['dso']['id']); + $this->assertSame('item-a-2', $results[2]['dso']['id']); + $this->assertSame('item-b-2', $results[3]['dso']['id']); + // search with no results, searching by virtual column + $search = new Search(); + $search->where('`dso_type` = :param'); + $results = $driver->select('testSelect', $search, [':param'=>'type-none']); + $this->assertSame(0, count($results)); + // search with no results, searching by json field + $search = new Search(); + $search->where('${foo} = :param'); + $results = $driver->select('testSelect', $search, [':param'=>'nonexistent foo value']); + $this->assertSame(0, count($results)); + } + + public function testDelete() + { + $driver = $this->createDriver(); + $driver->createTable('testDelete', $this->virtualColumns); + //set up dummy data + $this->setup_testDelete(); + //try deleting an item + $dso = new DSO(['dso.id'=>'item-a-1']); + $driver->delete('testDelete', $dso); + $this->assertEquals(3, $this->getConnection()->getRowCount('testDelete')); + //try deleting an item at the other end of the table + $dso = new DSO(['dso.id'=>'item-b-2']); + $driver->delete('testDelete', $dso); + $this->assertEquals(2, $this->getConnection()->getRowCount('testDelete')); + } + + protected function setup_testDelete() + { + $driver = $this->createDriver(); + $driver->insert('testDelete', new DSO([ + 'dso'=>['id'=>'item-a-1','type'=>'type-a'], + 'foo'=>'bar', + 'sort'=>'a' + ])); + $driver->insert('testDelete', new DSO([ + 'dso'=>['id'=>'item-a-2','type'=>'type-a'], + 'foo'=>'baz', + 'sort'=>'c' + ])); + $driver->insert('testDelete', new DSO([ + 'dso'=>['id'=>'item-b-1','type'=>'type-b'], + 'foo'=>'buz', + 'sort'=>'b' + ])); + $driver->insert('testDelete', new DSO([ + 'dso'=>['id'=>'item-b-2','type'=>'type-b','deleted'=>100], + 'foo'=>'quz', + 'sort'=>'d' + ])); + } + + protected function setup_testSelect() + { + $driver = $this->createDriver(); + $driver->insert('testSelect', new DSO([ + 'dso'=>['id'=>'item-a-1','type'=>'type-a'], + 'foo'=>'bar', + 'sort'=>'a' + ])); + $driver->insert('testSelect', new DSO([ + 'dso'=>['id'=>'item-a-2','type'=>'type-a'], + 'foo'=>'baz', + 'sort'=>'c' + ])); + $driver->insert('testSelect', new DSO([ + 'dso'=>['id'=>'item-b-1','type'=>'type-b'], + 'foo'=>'buz', + 'sort'=>'b' + ])); + $driver->insert('testSelect', new DSO([ + 'dso'=>['id'=>'item-b-2','type'=>'type-b','deleted'=>100], + 'foo'=>'quz', + 'sort'=>'d' + ])); + } + + /** + * Creates a Driver from class constants, so extending classes can test + * different databases. + */ + public function createDriver() + { + $class = static::DRIVER_CLASS; + return new $class( + static::DRIVER_DSN, + static::DRIVER_USERNAME, + static::DRIVER_PASSWORD, + static::DRIVER_OPTIONS + ); + } + + public static function setUpBeforeClass() + { + $pdo = static::createPDO(); + $pdo->exec('DROP TABLE testCreateTable'); + $pdo->exec('DROP TABLE testInsert'); + $pdo->exec('DROP TABLE testSelect'); + $pdo->exec('DROP TABLE testDelete'); + } + + protected static function createPDO() + { + return new \PDO( + static::DRIVER_DSN, + static::DRIVER_USERNAME, + static::DRIVER_PASSWORD, + static::DRIVER_OPTIONS + ); + } + + public function getConnection() + { + return $this->createDefaultDBConnection($this->createPDO(), 'phpunit'); + } + + public function getDataSet() + { + return new \PHPUnit\DbUnit\DataSet\DefaultDataSet(); + } +} diff --git a/tests/Drivers/IntegrationTests/AbstractDriverIntegrationTest.php b/tests/Drivers/IntegrationTests/AbstractDriverIntegrationTest.php new file mode 100644 index 0000000..bfa034e --- /dev/null +++ b/tests/Drivers/IntegrationTests/AbstractDriverIntegrationTest.php @@ -0,0 +1,180 @@ +exec('DROP TABLE '.static::TEST_TABLE); + } + + public function testCreateTable() + { + $factory = $this->createFactory(); + //should work the first time + $this->assertTrue($factory->createTable()); + //but not the second time, because it already exists + $this->assertFalse($factory->createTable()); + //table should exist and have zero rows + $this->assertEquals(0, $this->getConnection()->getRowCount(static::TEST_TABLE)); + } + + public function testInsert() + { + $startRowCount = $this->getConnection()->getRowCount(static::TEST_TABLE); + $factory = $this->createFactory(); + //inserting a freshly created object should return true + $o = $factory->create(['dso.id'=>'object-one']); + $this->assertTrue($o->insert()); + //inserting it a second time should not + $this->assertFalse($o->insert()); + //there should now be one more row + $this->assertEquals($startRowCount+1, $this->getConnection()->getRowCount(static::TEST_TABLE)); + } + + public function testReadAndUpdate() + { + $startRowCount = $this->getConnection()->getRowCount(static::TEST_TABLE); + $factory = $this->createFactory(); + //insert some new objects + $a1 = $factory->create(['foo'=>'bar']); + $a1->insert(); + $b1 = $factory->create(['foo.bar'=>'baz']); + $b1->insert(); + //read objects back out + $a2 = $factory->read($a1['dso.id']); + $b2 = $factory->read($b1['dso.id']); + //objects should be the same + $this->assertEquals($a1->get(), $a2->get()); + $this->assertEquals($b1->get(), $b2->get()); + //alter things in the objects and update them + $a2['foo'] = 'baz'; + $b2['foo.bar'] = 'bar'; + $a2->update(); + $b2->update(); + //read objects back out a third time + $a3 = $factory->read($a1['dso.id']); + $b3 = $factory->read($b1['dso.id']); + //objects should be the same + $this->assertEquals($a2->get(), $a3->get()); + $this->assertEquals($b2->get(), $b3->get()); + //they should not be the same as the originals from the beginning + $this->assertNotEquals($a1->get(), $a3->get()); + $this->assertNotEquals($b1->get(), $b3->get()); + //there should now be two more rows + $this->assertEquals($startRowCount+2, $this->getConnection()->getRowCount(static::TEST_TABLE)); + } + + public function testDelete() + { + $startRowCount = $this->getConnection()->getRowCount(static::TEST_TABLE); + $factory = $this->createFactory(); + //insert some new objects + $a1 = $factory->create(['testDelete'=>'undelete me']); + $a1->insert(); + $b1 = $factory->create(['testDelete'=>'should be permanently deleted']); + $b1->insert(); + //there should now be two more rows + $this->assertEquals($startRowCount+2, $this->getConnection()->getRowCount(static::TEST_TABLE)); + //delete one permanently and the other not, both shoudl take effect immediately + $a1->delete(); + $b1->delete(true); + //there should now be only one more row + $this->assertEquals($startRowCount+1, $this->getConnection()->getRowCount(static::TEST_TABLE)); + //a should be possible to read a back out with the right flags + $this->assertNull($factory->read($a1['dso.id'])); + $this->assertNotNull($factory->read($a1['dso.id'], 'dso.id', true)); + $this->assertNotNull($factory->read($a1['dso.id'], 'dso.id', null)); + //undelete a, should have update() inside it + $a1->undelete(); + //it should be possible to read a back out with different flags + $this->assertNotNull($factory->read($a1['dso.id'])); + $this->assertNull($factory->read($a1['dso.id'], 'dso.id', true)); + $this->assertNotNull($factory->read($a1['dso.id'], 'dso.id', null)); + } + + public function testSearch() + { + $startRowCount = $this->getConnection()->getRowCount(static::TEST_TABLE); + $factory = $this->createFactory(); + //insert some dummy data + $factory->create([ + 'testSearch' => 'a', + 'a' => '1', + 'b' => '2' + ])->insert(); + $factory->create([ + 'testSearch' => 'b', + 'a' => '2', + 'b' => '1' + ])->insert(); + $factory->create([ + 'testSearch' => 'c', + 'a' => '3', + 'b' => '4' + ])->insert(); + $factory->create([ + 'testSearch' => 'a', + 'a' => '4', + 'b' => '3' + ])->insert(); + //there should now be four more rows + $this->assertEquals($startRowCount+4, $this->getConnection()->getRowCount(static::TEST_TABLE)); + //TODO: test some searches + } + + /** + * Creates a Driver from class constants, so extending classes can test + * different databases. + */ + public function createDriver() + { + $class = static::DRIVER_CLASS; + return new $class( + static::DRIVER_DSN, + static::DRIVER_USERNAME, + static::DRIVER_PASSWORD, + static::DRIVER_OPTIONS + ); + } + + public function createFactory() + { + $driver = $this->createDriver(); + return new Factory( + $driver, + static::TEST_TABLE + ); + } + + protected static function createPDO() + { + return new \PDO( + static::DRIVER_DSN, + static::DRIVER_USERNAME, + static::DRIVER_PASSWORD, + static::DRIVER_OPTIONS + ); + } + + public function getConnection() + { + return $this->createDefaultDBConnection($this->createPDO(), 'phpunit'); + } + + public function getDataSet() + { + return new \PHPUnit\DbUnit\DataSet\DefaultDataSet(); + } +} diff --git a/tests/Drivers/IntegrationTests/MySQLDriverIntegrationTest.php b/tests/Drivers/IntegrationTests/MySQLDriverIntegrationTest.php new file mode 100644 index 0000000..ac138ac --- /dev/null +++ b/tests/Drivers/IntegrationTests/MySQLDriverIntegrationTest.php @@ -0,0 +1,16 @@ +search(); + $s->where('foo = bar'); + + //default execute, should be adding dso_deleted is null to where + $s->execute(['p'=>'q']); + $this->assertEquals('table_name', $d->last_select['table']); + $this->assertEquals('(foo = bar) AND `dso_deleted` is null', $d->last_select['search']->where()); + $this->assertEquals(['p'=>'q'], $d->last_select['params']); + + //execute with deleted=true, should be adding dso_deleted is not null to where + $s->execute(['p'=>'q'], true); + $this->assertEquals('table_name', $d->last_select['table']); + $this->assertEquals('(foo = bar) AND `dso_deleted` is not null', $d->last_select['search']->where()); + $this->assertEquals(['p'=>'q'], $d->last_select['params']); + + //execute with deleted=null, shouldn't touch where + $s->execute(['p'=>'q'], null); + $this->assertEquals('table_name', $d->last_select['table']); + $this->assertEquals('foo = bar', $d->last_select['search']->where()); + $this->assertEquals(['p'=>'q'], $d->last_select['params']); + } + + public function testInsert() + { + $d = new HarnessDriver('testInsert'); + $f = new Factory($d, 'table_name'); + //creating a new object + $o = $f->create(); + $o->insert(); + $this->assertEquals('table_name', $d->last_insert['table']); + $this->assertEquals($o->get('dso.id'), $d->last_insert['dso']->get('dso.id')); + + //creating a second object to verify + $o = $f->create(); + $o->insert(); + $this->assertEquals('table_name', $d->last_insert['table']); + $this->assertEquals($o->get('dso.id'), $d->last_insert['dso']->get('dso.id')); + } + + public function testUpdate() + { + $d = new HarnessDriver('testUpdate'); + $f = new Factory($d, 'table_name'); + + //creatingtwo new objects + $o1 = $f->create(['dso.id'=>'object1id']); + $o2 = $f->create(['dso.id'=>'object2id']); + //initially, updating shouldn't do anything because there are no changes + $o1->update(); + $this->assertNull($d->last_update); + //after making changes updating should do something + $o1['foo'] = 'bar'; + $o1->update(); + $this->assertEquals('table_name', $d->last_update['table']); + $this->assertEquals($o1->get('dso.id'), $d->last_update['dso']->get('dso.id')); + //updating second object to verify + $o2['foo'] = 'bar'; + $o2->update(); + $this->assertEquals('table_name', $d->last_update['table']); + $this->assertEquals($o2->get('dso.id'), $d->last_update['dso']->get('dso.id')); + //calling update on first object shouldn't do anything, because it hasn't changed + $o1->update(); + $this->assertEquals('table_name', $d->last_update['table']); + $this->assertEquals($o2->get('dso.id'), $d->last_update['dso']->get('dso.id')); + //after making changes updating should do something + $o1['foo'] = 'baz'; + $o1->update(); + $this->assertEquals('table_name', $d->last_update['table']); + $this->assertEquals($o1->get('dso.id'), $d->last_update['dso']->get('dso.id')); + } + + public function testDelete() + { + $d = new HarnessDriver('testInsert'); + $f = new Factory($d, 'table_name'); + + //non-permanent delete shouldn't do anything with deletion, but should call update + $o = $f->create(); + $o->delete(); + $this->assertNull($d->last_delete); + $this->assertEquals('table_name', $d->last_update['table']); + $this->assertEquals($o->get('dso.id'), $d->last_update['dso']->get('dso.id')); + //undelete should also not engage delete, but should call update + $d->last_update = null; + $o->undelete(); + $this->assertNull($d->last_delete); + $this->assertEquals('table_name', $d->last_update['table']); + $this->assertEquals($o->get('dso.id'), $d->last_update['dso']->get('dso.id')); + + //non-permanent delete shouldn't do anything with update, but should call delete + $d->last_update = null; + $o = $f->create(); + $o->delete(true); + $this->assertNull($d->last_update); + $this->assertEquals('table_name', $d->last_delete['table']); + $this->assertEquals($o->get('dso.id'), $d->last_delete['dso']->get('dso.id')); + } +} diff --git a/tests/HarnessDriver.php b/tests/HarnessDriver.php new file mode 100644 index 0000000..0c99502 --- /dev/null +++ b/tests/HarnessDriver.php @@ -0,0 +1,65 @@ +last_select = [ + 'table' => $table, + 'search' => $search, + 'params' => $params + ]; + return []; + } + + public function insert(string $table, DSOInterface $dso) : bool + { + $this->dsn = 'inserting'; + $this->last_insert = [ + 'table' => $table, + 'dso' => $dso + ]; + return true; + } + + public function update(string $table, DSOInterface $dso) : bool + { + $this->last_update = [ + 'table' => $table, + 'dso' => $dso + ]; + return true; + } + + public function delete(string $table, DSOInterface $dso) : bool + { + $this->last_delete = [ + 'table' => $table, + 'dso' => $dso + ]; + return true; + } + + public function errorInfo() + { + return []; + } +} diff --git a/tests/LegacyDrivers/IntegrationTests/SQLiteDriverIntegrationTest.php b/tests/LegacyDrivers/IntegrationTests/SQLiteDriverIntegrationTest.php new file mode 100644 index 0000000..0911078 --- /dev/null +++ b/tests/LegacyDrivers/IntegrationTests/SQLiteDriverIntegrationTest.php @@ -0,0 +1,23 @@ +createLegacyUDF(); + return $class; + } +} diff --git a/tests/LegacyDrivers/SQLiteDriverTest.php b/tests/LegacyDrivers/SQLiteDriverTest.php new file mode 100644 index 0000000..708f05d --- /dev/null +++ b/tests/LegacyDrivers/SQLiteDriverTest.php @@ -0,0 +1,21 @@ +