Schema management (#1)

* saves schemas to database, uses those schemas - still need to finish methods for update/add/remove operations on columns, for updating schemas

* fix for schema table creation

* possible test fixes

* possibly working, dropped some non-integration tests that didn't work with schema changes

* seems working, but still needs schema updating

* mysql's schema updates seem to be working

* mariadb driver tests

* change to how saving schema for createTable works

* very basic tests for schema management

* test that schemas are updated in schema table

.travis.yml

@@ -6,6 +6,8 @@ php:
   - 7.4
 before_install:
   - mysql -e 'CREATE DATABASE test'
+  - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=destructrtest -e MYSQL_USER=destructrtest -e MYSQL_PASSWORD=destructrtest -e MYSQL_ROOT_PASSWORD=verysecret mariadb:10.2 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB
+  - sleep 15
 install:
   - composer install
 script: composer test

composer.json

@@ -21,6 +21,9 @@
         "test-mysql": [
             "phpunit --testsuite MySQL"
         ],
+        "test-mariadb": [
+            "phpunit --testsuite MariaDB"
+        ],
         "test-sqlite": [
             "phpunit --testsuite SQLite"
         ]

examples/example_factory.php

@@ -4,30 +4,26 @@ use Destructr\Factory;
 
 class ExampleFactory extends Factory {
     /**
-     * Virtual columns are only supported by modern SQL servers. Most of the
+     * Example factory with a different schema, to index on random_data, but not
-     * legacy drivers will only use the ones defined in CORE_VIRTUAL_COLUMNS,
+     * by dso_type.
-     * but that should be handled automatically.
+     * 
+     * Also uses a different column name for dso.id
      */
-    protected $virtualColumns = [
+    protected $schema = [
         'dso.id' => [
-            'name'=>'dso_id',
+            'name'=>'dso_id_other_name',
             'type'=>'VARCHAR(16)',
             'index' => 'BTREE',
             'unique' => true,
             'primary' => true
         ],
-        'dso.type' => [
-            'name'=>'dso_type',
-            'type'=>'VARCHAR(30)',
-            'index'=>'BTREE'
-        ],
         'dso.deleted' => [
             'name'=>'dso_deleted',
-            'type'=>'BIGINT',
+            'type'=>'INT',
             'index'=>'BTREE'
         ],
-        'example.indexed' => [
+        'random_data' => [
-            'name'=>'example_indexed',
+            'name'=>'random_data',
             'type'=>'VARCHAR(100)',
             'index'=>'BTREE'
         ]

examples/mariadb.php

@@ -15,17 +15,19 @@ $driver = \Destructr\DriverFactory::factoryFromPDO(
 
 /*
 Creates a factory using the table 'example_table', and creates
-the necessary table. Note that createTable() can safely be called
+the necessary table. Note that prepareEnvironment() can safely be called
 multiple times.
 */
-$factory = new \Destructr\Factory($driver, 'example_table');
+include __DIR__ . '/example_factory.php';
-$factory->createTable();
+$factory = new ExampleFactory($driver, 'example_table');
+$factory->prepareEnvironment();
+$factory->updateEnvironment();
 
 /*
-The following can be uncommented to insert 1,000 dummy records
+The following can be uncommented to insert dummy records
 into the given table.
 */
-// for($i = 0; $i < 1000; $i++) {
+// for($i = 0; $i < 10; $i++) {
 //     $obj = $factory->create(
 //         [
 //             'dso.type'=>'foobar',

examples/mysql.php

@@ -0,0 +1,52 @@
+<?php
+
+use Destructr\Factory;
+
+include __DIR__ . '/../vendor/autoload.php';
+
+/*
+Constructing MariaDB drivers should be done using factoryFromPDO,
+so that they use the MariaDB driver instead of the MySQL driver.
+*/
+$driver = \Destructr\DriverFactory::factory(
+    'mysql:server=localhost;port=3306;dbname=destructr',
+    'root'
+);
+
+/*
+Creates a factory using the table 'example_table', and creates
+the necessary table. Note that prepareEnvironment() can safely be called
+multiple times.
+*/
+include __DIR__ . '/example_factory.php';
+$factory = new Factory($driver, 'example_table');
+$factory->prepareEnvironment();
+$factory->updateEnvironment();
+
+/*
+The following can be uncommented to insert dummy records
+into the given table.
+*/
+// for($i = 0; $i < 100; $i++) {
+//     $obj = $factory->create(
+//         [
+//             'dso.type'=>'foobar',
+//             'random_data' => md5(rand())
+//         ]
+//     );
+//     $obj->insert();
+// }
+
+/*
+Search by random data field
+*/
+// $search = $factory->search();
+// $search->where('${random_data} = :q');
+// $result = $search->execute(['q'=>'rw7nivub9bhhh3t4']);
+
+/*
+Search by dso.id, which is much faster because it's indexed
+*/
+// $search = $factory->search();
+// $search->where('${dso.id} = :q');
+// $result = $search->execute(['q'=>'rw7nivub9bhhh3t4']);

examples/sqlite.php

@@ -1,29 +1,34 @@
 <?php
+
+use Destructr\Factory;
+
 include __DIR__ . '/../vendor/autoload.php';
 
 /*
 SQLite drivers can be created by the default factory.
 A charset of UTF8 should be specified, to avoid character encoding
 issues.
-*/
+ */
 $driver = \Destructr\DriverFactory::factory(
-    'sqlite:'.__DIR__.'/example.sqlite'
+    'sqlite:' . __DIR__ . '/example.sqlite'
 );
 
 /*
 Creates a factory using the table 'example_table', and creates
-the necessary table. Note that createTable() can safely be called
+the necessary table. Note that prepareEnvironment() can safely be called
 multiple times.
-*/
+ */
-$factory = new \Destructr\Factory($driver, 'example_table');
+include __DIR__ . '/example_factory.php';
-$factory->createTable();
+$factory = new Factory($driver, 'example_table');
+$factory->prepareEnvironment();
+$factory->updateEnvironment();
 
 /*
-The following can be uncommented to insert 1,000 dummy records
+The following can be uncommented to insert dummy records
 into the given table.
-*/
+ */
 // ini_set('max_execution_time','0');
-// for($i = 0; $i < 1000; $i++) {
+// for($i = 0; $i < 10; $i++) {
 //     $obj = $factory->create(
 //         [
 //             'dso.type'=>'foobar',
@@ -35,18 +40,19 @@ into the given table.
 
 /*
 Search by random data field
-*/
+ */
 $search = $factory->search();
 $search->where('${random_data} LIKE :q');
 $result = $search->execute(['q'=>'%ab%']);
 foreach($result as $r) {
+    var_dump($r->get());
     $r['random_data_2'] = md5(rand());
     $r->update();
 }
 
 /*
 Search by dso.id, which is much faster because it's indexed
-*/
+ */
 // $search = $factory->search();
 // $search->where('${dso.id} = :q');
 // $result = $search->execute(['q'=>'rw7nivub9bhhh3t4']);

phpunit.xml

@@ -3,6 +3,9 @@
     <testsuite name="MySQL">
       <directory>tests/Drivers/MySQL</directory>
     </testsuite>
+    <testsuite name="MariaDB">
+      <directory>tests/Drivers/MariaDB</directory>
+    </testsuite>
     <testsuite name="SQLite">
       <directory>tests/Drivers/SQLite</directory>
     </testsuite>

src/DSO.php

@@ -5,7 +5,7 @@ namespace Destructr;
 use \Flatrr\FlatArray;
 
 /**
- * Interface for DeStructure Objects (DSOs). These are the class that is
+ * Interface for DeStructured Objects (DSOs). These are the class that is
  * actually used for storing and retrieving partially-structured data from the
  * database.
  */
@@ -15,7 +15,7 @@ class DSO extends FlatArray implements DSOInterface
     protected $changes;
     protected $removals;
 
-    public function __construct(array $data = null, DSOFactoryInterface $factory = null)
+    public function __construct(array $data = null, Factory $factory = null)
     {
         $this->resetChanges();
         parent::__construct($data);
@@ -32,22 +32,22 @@ class DSO extends FlatArray implements DSOInterface
         //does nothing
     }
 
-    public function delete(bool $permanent = false) : bool
+    public function delete(bool $permanent = false): bool
     {
         return $this->factory->delete($this, $permanent);
     }
 
-    public function undelete() : bool
+    public function undelete(): bool
     {
         return $this->factory->undelete($this);
     }
 
-    public function insert() : bool
+    public function insert(): bool
     {
         return $this->factory()->insert($this);
     }
 
-    public function update(bool $sneaky = false) : bool
+    public function update(bool $sneaky = false): bool
     {
         return $this->factory()->update($this);
     }
@@ -58,17 +58,17 @@ class DSO extends FlatArray implements DSOInterface
         $this->removals = new FlatArray();
     }
 
-    public function changes() : array
+    public function changes(): array
     {
         return $this->changes->get();
     }
 
-    public function removals() : array
+    public function removals(): array
     {
         return $this->removals->get();
     }
 
-    public function set(string $name = null, $value, $force=false)
+    public function set(?string $name, $value, $force = false)
     {
         $name = strtolower($name);
         if ($this->get($name) === $value) {
@@ -80,7 +80,7 @@ class DSO extends FlatArray implements DSOInterface
                 foreach ($this->get($name) as $k => $v) {
                     if (!isset($value[$k])) {
                         if ($name) {
-                            $k = $name.'.'.$k;
+                            $k = $name . '.' . $k;
                         }
                         $this->unset($k);
                     }
@@ -91,7 +91,7 @@ class DSO extends FlatArray implements DSOInterface
             //recursively set individual values so we can track them
             foreach ($value as $k => $v) {
                 if ($name) {
-                    $k = $name.'.'.$k;
+                    $k = $name . '.' . $k;
                 }
                 $this->set($k, $v, $force);
             }
@@ -102,8 +102,7 @@ class DSO extends FlatArray implements DSOInterface
         }
     }
 
-    public function unset(?string $name)
+    function unset(?string $name) {
-    {
         if (isset($this[$name])) {
             $this->removals->set($name, $this->get($name));
             unset($this->changes[$name]);
@@ -111,7 +110,7 @@ class DSO extends FlatArray implements DSOInterface
         }
     }
 
-    public function factory(DSOFactoryInterface $factory = null) : ?DSOFactoryInterface
+    public function factory(Factory $factory = null): ?Factory
     {
         if ($factory) {
             $this->factory = $factory;

src/DSOFactoryInterface.php

@@ -1,23 +0,0 @@
-<?php
-/* Destructr | https://github.com/jobyone/destructr | MIT License */
-namespace Destructr;
-
-interface DSOFactoryInterface
-{
-    public function __construct(Drivers\DSODriverInterface $driver, string $table);
-
-    public function class(array $data) : ?string;
-    public function virtualColumns(): array;
-
-    public function createTable() : bool;
-    public function create(array $data = array()) : DSOInterface;
-    public function read(string $value, string $field = 'dso.id', $deleted = false) : ?DSOInterface;
-    public function insert(DSOInterface $dso) : bool;
-    public function update(DSOInterface $dso, bool $sneaky = false) : bool;
-    public function delete(DSOInterface $dso, bool $permanent = false) : bool;
-    public function quote(string $str) : string;
-
-    public function search() : Search;
-    public function executeSearch(Search $search, array $params = array(), $deleted = false) : array;
-    public function executeCount(Search $search, array $params = array(), $deleted = false) : ?int;
-}

src/DSOInterface.php

@@ -11,17 +11,17 @@ use Flatrr\FlatArrayInterface;
  */
 interface DSOInterface extends FlatArrayInterface
 {
-    public function __construct(array $data = null, DSOFactoryInterface $factory = null);
+    public function __construct(array $data = null, Factory $factory = null);
-    public function factory(DSOFactoryInterface $factory = null) : ?DSOFactoryInterface;
+    public function factory(Factory $factory = null): ?Factory;
 
-    public function set(string $name = null, $value, $force=false);
+    public function set(?string $name, $value, $force = false);
 
     public function resetChanges();
-    public function changes() : array;
+    public function changes(): array;
-    public function removals() : array;
+    public function removals(): array;
 
-    public function insert() : bool;
+    public function insert(): bool;
-    public function update(bool $sneaky = false) : bool;
+    public function update(bool $sneaky = false): bool;
-    public function delete(bool $permanent = false) : bool;
+    public function delete(bool $permanent = false): bool;
-    public function undelete() : bool;
+    public function undelete(): bool;
 }

src/DriverFactory.php

@@ -10,7 +10,7 @@ class DriverFactory
         'sqlite' => Drivers\SQLiteDriver::class,
     ];
 
-    public static function factory(string $dsn, string $username = null, string $password = null, array $options = null, string $type = null): ?Drivers\DSODriverInterface
+    public static function factory(string $dsn, string $username = null, string $password = null, array $options = null, string $type = null): ?Drivers\AbstractDriver
     {
         if (!$type) {
             $type = @array_shift(explode(':', $dsn, 2));
@@ -23,7 +23,7 @@ class DriverFactory
         }
     }
 
-    public static function factoryFromPDO(\PDO $pdo, string $type = null): ?Drivers\DSODriverInterface
+    public static function factoryFromPDO(\PDO $pdo, string $type = null): ?Drivers\AbstractDriver
     {
         if (!$type) {
             $type = $pdo->getAttribute(\PDO::ATTR_DRIVER_NAME);

src/Drivers/AbstractDriver.php

@@ -6,139 +6,16 @@ use Destructr\DSOInterface;
 use Destructr\Search;
 use PDO;
 
-abstract class AbstractDriver implements DSODriverInterface
+abstract class AbstractDriver
 {
-    public $lastPreparationErrorOn;
+    abstract public function errorInfo();
-    public $pdo;
+    abstract public function update(string $table, DSOInterface $dso): bool;
-
+    abstract public function delete(string $table, DSOInterface $dso): bool;
-    abstract protected function sql_select(array $args): string;
+    abstract public function count(string $table, Search $search, array $params): int;
-    abstract protected function sql_count(array $args): string;
+    abstract public function select(string $table, Search $search, array $params);
-    abstract protected function sql_ddl(array $args = []): string;
+    abstract public function insert(string $table, DSOInterface $dso): bool;
-    abstract protected function expandPath(string $path): string;
+    abstract public function prepareEnvironment(string $table, array $schem): bool;
-    abstract protected function sql_setJSON(array $args): string;
+    abstract public function beginTransaction(): bool;
-    abstract protected function sql_insert(array $args): string;
+    abstract public function commit(): bool;
-    abstract protected function sql_delete(array $args): string;
+    abstract public function rollBack(): bool;
-
-    public function __construct(string $dsn = null, string $username = null, string $password = null, array $options = null)
-    {
-        if ($dsn) {
-            if (!($pdo = new \PDO($dsn, $username, $password, $options))) {
-                throw new \Exception("Error creating PDO connection");
-            }
-            $this->pdo($pdo);
-        }
-    }
-
-    public function pdo(\PDO $pdo = null): ?\PDO
-    {
-        if ($pdo) {
-            $pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, false);
-            $this->pdo = $pdo;
-        }
-        return $this->pdo;
-    }
-
-    protected function expandPaths($value)
-    {
-        if ($value === null) {
-            return null;
-        }
-        $value = preg_replace_callback(
-            '/\$\{([^\}\\\]+)\}/',
-            function ($matches) {
-                return $this->expandPath($matches[1]);
-            },
-            $value
-        );
-        return $value;
-    }
-
-    public function errorInfo()
-    {
-        return $this->pdo->errorInfo();
-    }
-
-    public function createTable(string $table, array $virtualColumns): bool
-    {
-        $sql = $this->sql_ddl([
-            'table' => $table,
-            'virtualColumns' => $virtualColumns,
-        ]);
-        return $this->pdo->exec($sql) !== false;
-    }
-
-    public function update(string $table, DSOInterface $dso): bool
-    {
-        if (!$dso->changes() && !$dso->removals()) {
-            return true;
-        }
-        $s = $this->getStatement(
-            'setJSON',
-            ['table' => $table]
-        );
-        return $s->execute([
-            ':dso_id' => $dso['dso.id'],
-            ':data' => json_encode($dso->get()),
-        ]);
-    }
-
-    public function delete(string $table, DSOInterface $dso): bool
-    {
-        $s = $this->getStatement(
-            'delete',
-            ['table' => $table]
-        );
-        return $s->execute([
-            ':dso_id' => $dso['dso.id'],
-        ]);
-    }
-
-    public function count(string $table, Search $search, array $params)
-    {
-        $s = $this->getStatement(
-            'count',
-            ['table' => $table, 'search' => $search]
-        );
-        if (!$s->execute($params)) {
-            return null;
-        }
-        return intval($s->fetchAll(\PDO::FETCH_COLUMN)[0]);
-    }
-
-    public function select(string $table, Search $search, array $params)
-    {
-        $s = $this->getStatement(
-            'select',
-            ['table' => $table, 'search' => $search]
-        );
-        if (!$s->execute($params)) {
-            return [];
-        }
-        return @$s->fetchAll(\PDO::FETCH_ASSOC);
-    }
-
-    public function insert(string $table, DSOInterface $dso): bool
-    {
-        return $this->getStatement(
-            'insert',
-            ['table' => $table]
-        )->execute(
-            [':data' => json_encode($dso->get())]
-        );
-    }
-
-    protected function getStatement(string $type, $args = array()): \PDOStatement
-    {
-        $fn = 'sql_' . $type;
-        if (!method_exists($this, $fn)) {
-            throw new \Exception("Error getting SQL statement, driver doesn't have a method named $fn");
-        }
-        $sql = $this->$fn($args);
-        $stmt = $this->pdo->prepare($sql);
-        if (!$stmt) {
-            $this->lastPreparationErrorOn = $sql;
-            throw new \Exception("Error preparing statement: " . implode(': ', $this->pdo->errorInfo()), 1);
-        }
-        return $stmt;
-    }
 }

src/Drivers/AbstractSQLDriver.php

@@ -0,0 +1,353 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+namespace Destructr\Drivers;
+
+use Destructr\DSOInterface;
+use Destructr\Search;
+use PDO;
+
+abstract class AbstractSQLDriver extends AbstractDriver
+{
+    public $lastPreparationErrorOn;
+    public $pdo;
+    protected $schemas = [];
+
+    abstract protected function sql_ddl(array $args = []): string;
+    abstract protected function expandPath(string $path): string;
+    abstract protected function sql_set_json(array $args): string;
+    abstract protected function sql_insert(array $args): string;
+    abstract protected function sql_create_schema_table(): string;
+    abstract protected function sql_table_exists(string $table): string;
+    abstract protected function buildIndexes(string $table, array $schema): bool;
+    abstract protected function addColumns($table, $schema): bool;
+    abstract protected function removeColumns($table, $schema): bool;
+    abstract protected function rebuildSchema($table, $schema): bool;
+
+    public function __construct(string $dsn = null, string $username = null, string $password = null, array $options = null)
+    {
+        if ($dsn) {
+            if (!($pdo = new \PDO($dsn, $username, $password, $options))) {
+                throw new \Exception("Error creating PDO connection");
+            }
+            $this->pdo($pdo);
+        }
+    }
+
+    public function tableExists(string $table): bool
+    {
+        $stmt = $this->pdo()->prepare($this->sql_table_exists($table));
+        if ($stmt && $stmt->execute() !== false) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public function createSchemaTable()
+    {
+        $this->pdo->exec($this->sql_create_schema_table());
+        return $this->tableExists('destructr_schema');
+    }
+
+    public function pdo(\PDO $pdo = null): ?\PDO
+    {
+        if ($pdo) {
+            $pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, false);
+            $this->pdo = $pdo;
+        }
+        return $this->pdo;
+    }
+
+    public function beginTransaction(): bool
+    {
+        return $this->pdo->beginTransaction();
+    }
+
+    public function commit(): bool
+    {
+        return $this->pdo->commit();
+    }
+
+    public function rollBack(): bool
+    {
+        return $this->pdo->rollBack();
+    }
+
+    protected function expandPaths($value)
+    {
+        if ($value === null) {
+            return null;
+        }
+        $value = preg_replace_callback(
+            '/\$\{([^\}\\\]+)\}/',
+            function ($matches) {
+                return $this->expandPath($matches[1]);
+            },
+            $value
+        );
+        return $value;
+    }
+
+    public function errorInfo()
+    {
+        return $this->pdo->errorInfo();
+    }
+
+    public function prepareEnvironment(string $table, array $schema): bool
+    {
+        $this->beginTransaction();
+        if ($this->createSchemaTable() && $this->createTable($table, $schema)) {
+            $this->commit();
+            return true;
+        } else {
+            $this->rollBack();
+            return false;
+        }
+    }
+
+    public function updateEnvironment(string $table, array $schema): bool
+    {
+        $this->beginTransaction();
+        if ($this->updateTable($table, $schema)) {
+            $this->commit();
+            return true;
+        } else {
+            $this->rollBack();
+            return false;
+        }
+    }
+
+    protected function updateTable($table, $schema): bool
+    {
+        $current = $this->getSchema($table);
+        $new = $schema;
+        if (!$current || $schema == $current) {
+            return true;
+        }
+        //do nothing with totally unchanged columns
+        foreach ($current as $c_id => $c) {
+            foreach ($schema as $n_id => $n) {
+                if ($n == $c && $n_id == $c_id) {
+                    unset($current[$c_id]);
+                    unset($new[$n_id]);
+                }
+            }
+        }
+        $removed = $current;
+        $added = $new;
+        //apply changes
+        $out = [
+            'removeColumns' => $this->removeColumns($table, $removed),
+            'addColumns' => $this->addColumns($table, $added),
+            'rebuildSchema' => $this->rebuildSchema($table, $schema),
+            'buildIndexes' => $this->buildIndexes($table, $schema),
+            'saveSchema' => $this->saveSchema($table, $schema),
+        ];
+        foreach ($out as $k => $v) {
+            if (!$v) {
+                user_error("An error occurred during updateTable for $table. The error happened during $k.", E_USER_WARNING);
+            }
+        }
+        return !!array_filter($out);
+    }
+
+    public function createTable(string $table, array $schema): bool
+    {
+        // check if table exists, if it doesn't, save into schema table
+        $saveSchema = !$this->tableExists($table);
+        // create table from scratch
+        $sql = $this->sql_ddl([
+            'table' => $table,
+            'schema' => $schema,
+        ]);
+        $out = $this->pdo->exec($sql) !== false;
+        if ($out) {
+            $this->buildIndexes($table, $schema);
+            if ($saveSchema) {
+                $this->saveSchema($table, $schema);
+            }
+        }
+        return $out;
+    }
+
+    public function getSchema(string $table): ?array
+    {
+        if (!isset($this->schemas[$table])) {
+            $s = $this->getStatement(
+                'get_schema',
+                ['table' => $table]
+            );
+            if (!$s->execute(['table' => $table])) {
+                $this->schemas[$table] = null;
+            } else {
+                if ($row = $s->fetch(\PDO::FETCH_ASSOC)) {
+                    $this->schemas[$table] = @json_decode($row['schema_schema'], true);
+                } else {
+                    $this->schemas[$table] = null;
+                }
+            }
+        }
+        return @$this->schemas[$table];
+    }
+
+    public function saveSchema(string $table, array $schema): bool
+    {
+        $out = $this->pdo->exec(
+            $this->sql_save_schema($table, $schema)
+        ) !== false;
+        unset($this->schemas[$table]);
+        return $out;
+    }
+
+    protected function sql_save_schema(string $table, array $schema)
+    {
+        $time = time();
+        $table = $this->pdo->quote($table);
+        $schema = $this->pdo->quote(json_encode($schema));
+        return <<<EOT
+INSERT INTO `destructr_schema`
+(schema_time,schema_table,schema_schema)
+VALUES ($time,$table,$schema);
+EOT;
+    }
+
+    protected function sql_get_schema(array $args)
+    {
+        return <<<EOT
+SELECT * FROM `destructr_schema`
+WHERE `schema_table` = :table
+ORDER BY `schema_time` desc
+LIMIT 1
+EOT;
+    }
+
+    public function update(string $table, DSOInterface $dso): bool
+    {
+        if (!$dso->changes() && !$dso->removals()) {
+            return true;
+        }
+        $s = $this->getStatement(
+            'set_json',
+            ['table' => $table]
+        );
+        return $s->execute([
+            ':dso_id' => $dso['dso.id'],
+            ':data' => json_encode($dso->get()),
+        ]);
+    }
+
+    public function delete(string $table, DSOInterface $dso): bool
+    {
+        $s = $this->getStatement(
+            'delete',
+            ['table' => $table]
+        );
+        return $s->execute([
+            ':dso_id' => $dso['dso.id'],
+        ]);
+    }
+
+    public function count(string $table, Search $search, array $params): int
+    {
+        $s = $this->getStatement(
+            'count',
+            ['table' => $table, 'search' => $search]
+        );
+        if (!$s->execute($params)) {
+            return null;
+        }
+        return intval($s->fetchAll(\PDO::FETCH_COLUMN)[0]);
+    }
+
+    public function select(string $table, Search $search, array $params)
+    {
+        $s = $this->getStatement(
+            'select',
+            ['table' => $table, 'search' => $search]
+        );
+        if (!$s->execute($params)) {
+            return [];
+        }
+        return @$s->fetchAll(\PDO::FETCH_ASSOC);
+    }
+
+    public function insert(string $table, DSOInterface $dso): bool
+    {
+        return $this->getStatement(
+            'insert',
+            ['table' => $table]
+        )->execute(
+            [':data' => json_encode($dso->get())]
+        );
+    }
+
+    protected function getStatement(string $type, $args = array()): \PDOStatement
+    {
+        $fn = 'sql_' . $type;
+        if (!method_exists($this, $fn)) {
+            throw new \Exception("Error getting SQL statement, driver doesn't have a method named $fn");
+        }
+        $sql = $this->$fn($args);
+        $stmt = $this->pdo->prepare($sql);
+        if (!$stmt) {
+            $this->lastPreparationErrorOn = $sql;
+            throw new \Exception("Error preparing statement: " . implode(': ', $this->pdo->errorInfo()), 1);
+        }
+        return $stmt;
+    }
+
+    /**
+     * Within the search we expand strings like ${dso.id} into JSON queries.
+     * Note that the Search will have already had these strings expanded into
+     * column names if there are virtual columns configured for them. That
+     * happens in the Factory before it gets here.
+     */
+    protected function sql_select(array $args): string
+    {
+        //extract query parts from Search and expand paths
+        $where = $this->expandPaths($args['search']->where());
+        $order = $this->expandPaths($args['search']->order());
+        $limit = $args['search']->limit();
+        $offset = $args['search']->offset();
+        //select from
+        $out = ["SELECT * FROM `{$args['table']}`"];
+        //where statement
+        if ($where !== null) {
+            $out[] = "WHERE " . $where;
+        }
+        //order statement
+        if ($order !== null) {
+            $out[] = "ORDER BY " . $order;
+        }
+        //limit
+        if ($limit !== null) {
+            $out[] = "LIMIT " . $limit;
+        }
+        //offset
+        if ($offset !== null) {
+            $out[] = "OFFSET " . $offset;
+        }
+        //return
+        return implode(PHP_EOL, $out) . ';';
+    }
+
+    protected function sql_count(array $args): string
+    {
+        //extract query parts from Search and expand paths
+        $where = $this->expandPaths($args['search']->where());
+        //select from
+        $out = ["SELECT count(dso_id) FROM `{$args['table']}`"];
+        //where statement
+        if ($where !== null) {
+            $out[] = "WHERE " . $where;
+        }
+        //return
+        return implode(PHP_EOL, $out) . ';';
+    }
+
+    protected function sql_delete(array $args): string
+    {
+        return 'DELETE FROM `' . $args['table'] . '` WHERE `dso_id` = :dso_id;';
+    }
+
+}

src/Drivers/DSODriverInterface.php

@@ -1,20 +0,0 @@
-<?php
-/* Destructr | https://github.com/jobyone/destructr | MIT License */
-namespace Destructr\Drivers;
-
-use Destructr\DSOInterface;
-use Destructr\Search;
-
-interface DSODriverInterface
-{
-    public function __construct(string $dsn = null, string $username = null, string $password = null, array $options = null);
-    public function pdo(\PDO $pdo = null): ?\PDO;
-
-    public function createTable(string $table, array $virtualColumns): bool;
-    public function select(string $table, Search $search, array $params);
-    public function insert(string $table, DSOInterface $dso): bool;
-    public function update(string $table, DSOInterface $dso): bool;
-    public function delete(string $table, DSOInterface $dso): bool;
-
-    public function errorInfo();
-}

src/Drivers/MariaDBDriver.php

@@ -13,26 +13,49 @@ class MariaDBDriver extends MySQLDriver
         $out[] = "CREATE TABLE IF NOT EXISTS `{$args['table']}` (";
         $lines = [];
         $lines[] = "`json_data` JSON DEFAULT NULL";
-        foreach ($args['virtualColumns'] as $path => $col) {
+        foreach ($args['schema'] as $path => $col) {
-            $line = "`{$col['name']}` {$col['type']} GENERATED ALWAYS AS (" . $this->expandPath($path) . ")";
+            $line = "`{$col['name']}` {$col['type']} GENERATED ALWAYS AS (" . $this->expandPath($path) . ") VIRTUAL";
-            if (@$col['primary']) {
-                $line .= ' PERSISTENT';
-            } else {
-                $line .= ' VIRTUAL';
-            }
             $lines[] = $line;
         }
-        foreach ($args['virtualColumns'] as $path => $col) {
-            if (@$col['primary']) {
-                $lines[] = "UNIQUE KEY (`{$col['name']}`)";
-            } elseif (@$col['unique'] && $as = @$col['index']) {
-                $lines[] = "UNIQUE KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as";
-            } elseif ($as = @$col['index']) {
-                $lines[] = "KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as";
-            }
-        }
         $out[] = implode(',' . PHP_EOL, $lines);
         $out[] = ") ENGINE=InnoDB DEFAULT CHARSET=utf8;";
-        return implode(PHP_EOL, $out);
+        $out = implode(PHP_EOL, $out);
+        return $out;
+    }
+
+    protected function buildIndexes(string $table, array $schema): bool
+    {
+        foreach ($schema as $path => $col) {
+            if (@$col['primary']) {
+                $this->pdo->exec(
+                    "CREATE UNIQUE INDEX `{$table}_{$col['name']}_idx` ON {$table} (`{$col['name']}`) USING BTREE"
+                );
+            } elseif (@$col['unique'] && $as = @$col['index']) {
+                $this->pdo->exec(
+                    "CREATE UNIQUE INDEX `{$table}_{$col['name']}_idx` ON {$table} (`{$col['name']}`) USING $as"
+                );
+            } elseif ($as = @$col['index']) {
+                $this->pdo->exec(
+                    "CREATE INDEX `{$table}_{$col['name']}_idx` ON {$table} (`{$col['name']}`) USING $as"
+                );
+            }
+        }
+        return true;
+    }
+
+    protected function addColumns($table, $schema): bool
+    {
+        $out = true;
+        foreach ($schema as $path => $col) {
+            $line = "ALTER TABLE `{$table}` ADD COLUMN `${col['name']}` {$col['type']} GENERATED ALWAYS AS (" . $this->expandPath($path) . ")";
+            if (@$col['primary']) {
+                $line .= ' PERSISTENT;';
+            } else {
+                $line .= ' VIRTUAL;';
+            }
+            $out = $out &&
+            $this->pdo->exec($line) !== false;
+        }
+        return $out;
     }
 }

src/Drivers/MySQLDriver.php

@@ -5,64 +5,15 @@ namespace Destructr\Drivers;
 /**
  * What this driver supports: MySQL >= 5.7.8
  */
-class MySQLDriver extends AbstractDriver
+class MySQLDriver extends AbstractSQLDriver
 {
-    /**
-     * Within the search we expand strings like ${dso.id} into JSON queries.
-     * Note that the Search will have already had these strings expanded into
-     * column names if there are virtual columns configured for them. That
-     * happens in the Factory before it gets here.
-     */
-    protected function sql_select(array $args): string
-    {
-        //extract query parts from Search and expand paths
-        $where = $this->expandPaths($args['search']->where());
-        $order = $this->expandPaths($args['search']->order());
-        $limit = $args['search']->limit();
-        $offset = $args['search']->offset();
-        //select from
-        $out = ["SELECT * FROM `{$args['table']}`"];
-        //where statement
-        if ($where !== null) {
-            $out[] = "WHERE " . $where;
-        }
-        //order statement
-        if ($order !== null) {
-            $out[] = "ORDER BY " . $order;
-        }
-        //limit
-        if ($limit !== null) {
-            $out[] = "LIMIT " . $limit;
-        }
-        //offset
-        if ($offset !== null) {
-            $out[] = "OFFSET " . $offset;
-        }
-        //return
-        return implode(PHP_EOL, $out) . ';';
-    }
-
-    protected function sql_count(array $args): string
-    {
-        //extract query parts from Search and expand paths
-        $where = $this->expandPaths($args['search']->where());
-        //select from
-        $out = ["SELECT count(dso_id) FROM `{$args['table']}`"];
-        //where statement
-        if ($where !== null) {
-            $out[] = "WHERE " . $where;
-        }
-        //return
-        return implode(PHP_EOL, $out) . ';';
-    }
-
     protected function sql_ddl(array $args = []): string
     {
         $out = [];
         $out[] = "CREATE TABLE IF NOT EXISTS `{$args['table']}` (";
         $lines = [];
         $lines[] = "`json_data` JSON DEFAULT NULL";
-        foreach ($args['virtualColumns'] as $path => $col) {
+        foreach ($args['schema'] as $path => $col) {
             $line = "`{$col['name']}` {$col['type']} GENERATED ALWAYS AS (" . $this->expandPath($path) . ")";
             if (@$col['primary']) {
                 $line .= ' STORED';
@@ -71,18 +22,30 @@ class MySQLDriver extends AbstractDriver
             }
             $lines[] = $line;
         }
-        foreach ($args['virtualColumns'] as $path => $col) {
-            if (@$col['primary']) {
-                $lines[] = "PRIMARY KEY (`{$col['name']}`)";
-            } elseif (@$col['unique'] && $as = @$col['index']) {
-                $lines[] = "UNIQUE KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as";
-            } elseif ($as = @$col['index']) {
-                $lines[] = "KEY `{$args['table']}_{$col['name']}_idx` (`{$col['name']}`) USING $as";
-            }
-        }
         $out[] = implode(',' . PHP_EOL, $lines);
         $out[] = ") ENGINE=InnoDB DEFAULT CHARSET=utf8;";
-        return implode(PHP_EOL, $out);
+        $out = implode(PHP_EOL, $out);
+        return $out;
+    }
+
+    protected function buildIndexes(string $table, array $schema): bool
+    {
+        foreach ($schema as $path => $col) {
+            if (@$col['primary']) {
+                $this->pdo->exec(
+                    "CREATE UNIQUE INDEX `{$table}_{$col['name']}_idx` ON {$table} (`{$col['name']}`) USING BTREE"
+                );
+            } elseif (@$col['unique'] && $as = @$col['index']) {
+                $this->pdo->exec(
+                    "CREATE UNIQUE INDEX `{$table}_{$col['name']}_idx` ON {$table} (`{$col['name']}`) USING $as"
+                );
+            } elseif ($as = @$col['index']) {
+                $this->pdo->exec(
+                    "CREATE INDEX `{$table}_{$col['name']}_idx` ON {$table} (`{$col['name']}`) USING $as"
+                );
+            }
+        }
+        return true;
     }
 
     protected function expandPath(string $path): string
@@ -90,7 +53,7 @@ class MySQLDriver extends AbstractDriver
         return "JSON_UNQUOTE(JSON_EXTRACT(`json_data`,'$.{$path}'))";
     }
 
-    protected function sql_setJSON(array $args): string
+    protected function sql_set_json(array $args): string
     {
         return 'UPDATE `' . $args['table'] . '` SET `json_data` = :data WHERE `dso_id` = :dso_id;';
     }
@@ -100,8 +63,53 @@ class MySQLDriver extends AbstractDriver
         return "INSERT INTO `{$args['table']}` (`json_data`) VALUES (:data);";
     }
 
-    protected function sql_delete(array $args): string
+    protected function addColumns($table, $schema): bool
     {
-        return 'DELETE FROM `' . $args['table'] . '` WHERE `dso_id` = :dso_id;';
+        $out = true;
+        foreach ($schema as $path => $col) {
+            $line = "ALTER TABLE `{$table}` ADD COLUMN `${col['name']}` {$col['type']} GENERATED ALWAYS AS (" . $this->expandPath($path) . ")";
+            if (@$col['primary']) {
+                $line .= ' STORED;';
+            } else {
+                $line .= ' VIRTUAL;';
+            }
+            $out = $out &&
+            $this->pdo->exec($line) !== false;
+        }
+        return $out;
+    }
+
+    protected function removeColumns($table, $schema): bool
+    {
+        $out = true;
+        foreach ($schema as $path => $col) {
+            $out = $out &&
+            $this->pdo->exec("ALTER TABLE `{$table}` DROP COLUMN `${col['name']}`;") !== false;
+        }
+        return $out;
+    }
+
+    protected function rebuildSchema($table, $schema): bool
+    {
+        //this does nothing in databases that can generate columns themselves
+        return true;
+    }
+
+    protected function sql_create_schema_table(): string
+    {
+        return <<<EOT
+CREATE TABLE `destructr_schema` (
+    `schema_time` bigint NOT NULL,
+    `schema_table` varchar(100) NOT NULL,
+    `schema_schema` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL CHECK (json_valid(`schema_schema`)),
+    PRIMARY KEY (`schema_time`,`schema_table`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+EOT;
+    }
+
+    protected function sql_table_exists(string $table): string
+    {
+        $table = preg_replace('/[^a-zA-Z0-9\-_]/', '', $table);
+        return 'SELECT 1 FROM ' . $table . ' LIMIT 1';
     }
 }

src/Drivers/SQLiteDriver.php

@@ -3,9 +3,6 @@
 namespace Destructr\Drivers;
 
 use Destructr\DSOInterface;
-use Destructr\Factory;
-use Destructr\Search;
-use Flatrr\FlatArray;
 
 /**
  * What this driver supports: Any version of SQLite3 in PHP environments that allow
@@ -17,64 +14,8 @@ use Flatrr\FlatArray;
  * your updating through Destructr, but is something to be cognizent of if your
  * data is being updated outside Destructr.
  */
-class SQLiteDriver extends AbstractDriver
+class SQLiteDriver extends AbstractSQLDriver
 {
-    public function select(string $table, Search $search, array $params)
-    {
-        $results = parent::select($table, $search, $params);
-        foreach ($results as $rkey => $row) {
-            $new = new FlatArray();
-            foreach (json_decode($row['json_data'], true) as $key => $value) {
-                $new->set(str_replace('|', '.', $key), $value);
-            }
-            $results[$rkey]['json_data'] = json_encode($new->get());
-        }
-        return $results;
-    }
-
-    protected function sql_count(array $args): string
-    {
-        //extract query parts from Search and expand paths
-        $where = $this->expandPaths($args['search']->where());
-        //select from
-        $out = ["SELECT count(dso_id) FROM `{$args['table']}`"];
-        //where statement
-        if ($where !== null) {
-            $out[] = "WHERE " . $where;
-        }
-        //return
-        return implode(PHP_EOL, $out) . ';';
-    }
-
-    protected function sql_select(array $args): string
-    {
-        //extract query parts from Search and expand paths
-        $where = $this->expandPaths($args['search']->where());
-        $order = $this->expandPaths($args['search']->order());
-        $limit = $args['search']->limit();
-        $offset = $args['search']->offset();
-        //select from
-        $out = ["SELECT * FROM `{$args['table']}`"];
-        //where statement
-        if ($where !== null) {
-            $out[] = "WHERE " . $where;
-        }
-        //order statement
-        if ($order !== null) {
-            $out[] = "ORDER BY " . $order;
-        }
-        //limit
-        if ($limit !== null) {
-            $out[] = "LIMIT " . $limit;
-        }
-        //offset
-        if ($offset !== null) {
-            $out[] = "OFFSET " . $offset;
-        }
-        //return
-        return implode(PHP_EOL, $out) . ';';
-    }
-
     public function update(string $table, DSOInterface $dso): bool
     {
         if (!$dso->changes() && !$dso->removals()) {
@@ -82,7 +23,7 @@ class SQLiteDriver extends AbstractDriver
         }
         $columns = $this->dso_columns($dso);
         $s = $this->getStatement(
-            'setJSON',
+            'set_json',
             [
                 'table' => $table,
                 'columns' => $columns,
@@ -104,6 +45,72 @@ class SQLiteDriver extends AbstractDriver
         return $s->execute($columns);
     }
 
+    protected function updateTable($table, $schema): bool
+    {
+        $current = $this->getSchema($table);
+        if (!$current || $schema == $current) {
+            return true;
+        }
+        //create new table
+        $table_tmp = "{$table}_tmp_" . md5(rand());
+        $sql = $this->sql_ddl([
+            'table' => $table_tmp,
+            'schema' => $schema,
+        ]);
+        if ($this->pdo->exec($sql) === false) {
+            return false;
+        }
+        //copy data into it
+        $sql = ["INSERT INTO $table_tmp"];
+        $cols = ["json_data"];
+        $srcs = ["json_data"];
+        foreach ($schema as $path => $col) {
+            $cols[] = $col['name'];
+            $srcs[] = $this->expandPath($path);
+        }
+        $sql[] = '(' . implode(',', $cols) . ')';
+        $sql[] = 'SELECT';
+        $sql[] = implode(',', $srcs);
+        $sql[] = "FROM $table";
+        $sql = implode(PHP_EOL, $sql);
+        if ($this->pdo->exec($sql) === false) {
+            return false;
+        }
+        //remove old table, rename new table to old table
+        if ($this->pdo->exec("DROP TABLE $table") === false) {
+            return false;
+        }
+        if ($this->pdo->exec("ALTER TABLE $table_tmp RENAME TO $table") === false) {
+            return false;
+        }
+        //set up indexes
+        if (!$this->buildIndexes($table, $schema)) {
+            return false;
+        }
+        //save schema
+        $this->saveSchema($table, $schema);
+        //return result
+        return true;
+    }
+
+    protected function addColumns($table, $schema): bool
+    {
+        //does nothing
+        return true;
+    }
+
+    protected function removeColumns($table, $schema): bool
+    {
+        //does nothing
+        return true;
+    }
+
+    protected function rebuildSchema($table, $schema): bool
+    {
+        //does nothing
+        return true;
+    }
+
     protected function sql_insert(array $args): string
     {
         $out = [];
@@ -120,18 +127,18 @@ class SQLiteDriver extends AbstractDriver
         return $out;
     }
 
-    protected function sql_setJSON(array $args): string
+    protected function sql_set_json(array $args): string
     {
         $names = array_map(
             function ($e) {
-                return '`'.preg_replace('/^:/', '', $e).'` = '.$e;
+                return '`' . preg_replace('/^:/', '', $e) . '` = ' . $e;
             },
             array_keys($args['columns'])
         );
         $out = [];
         $out[] = 'UPDATE `' . $args['table'] . '`';
         $out[] = 'SET';
-        $out[] = implode(','.PHP_EOL,$names);
+        $out[] = implode(',' . PHP_EOL, $names);
         $out[] = 'WHERE `dso_id` = :dso_id';
         $out = implode(PHP_EOL, $out) . ';';
         return $out;
@@ -148,11 +155,6 @@ class SQLiteDriver extends AbstractDriver
         ]);
     }
 
-    protected function sql_delete(array $args): string
-    {
-        return 'DELETE FROM `' . $args['table'] . '` WHERE `dso_id` = :dso_id;';
-    }
-
     /**
      * Used to extract a list of column/parameter names for a given DSO, based
      * on the current values.
@@ -162,8 +164,8 @@ class SQLiteDriver extends AbstractDriver
      */
     protected function dso_columns(DSOInterface $dso)
     {
-        $columns = [':json_data' => $this->json_encode($dso->get())];
+        $columns = [':json_data' => json_encode($dso->get())];
-        foreach ($dso->factory()->virtualColumns() as $vk => $vv) {
+        foreach ($this->getSchema($dso->factory()->table()) ?? [] as $vk => $vv) {
             $columns[':' . $vv['name']] = $dso->get($vk);
         }
         return $columns;
@@ -206,27 +208,21 @@ class SQLiteDriver extends AbstractDriver
         return @"$out";
     }
 
-    public function createTable(string $table, array $virtualColumns): bool
+    protected function buildIndexes(string $table, array $schema): bool
     {
-        $sql = $this->sql_ddl([
+        $result = true;
-            'table' => $table,
+        foreach ($schema as $key => $vcol) {
-            'virtualColumns' => $virtualColumns,
-        ]);
-        $out = $this->pdo->exec($sql) !== false;
-        foreach ($virtualColumns as $key => $vcol) {
-            $idxResult = true;
             if (@$vcol['primary']) {
                 //sqlite automatically creates this index
             } elseif (@$vcol['unique']) {
-                $idxResult = $this->pdo->exec('CREATE UNIQUE INDEX ' . $table . '_' . $vcol['name'] . '_idx on `' . $table . '`(`' . $vcol['name'] . '`)') !== false;
+                $result = $result &&
+                $this->pdo->exec('CREATE UNIQUE INDEX ' . $table . '_' . $vcol['name'] . '_idx on `' . $table . '`(`' . $vcol['name'] . '`)') !== false;
             } elseif (@$vcol['index']) {
-                $idxResult = $this->pdo->exec('CREATE INDEX ' . $table . '_' . $vcol['name'] . '_idx on `' . $table . '`(`' . $vcol['name'] . '`)') !== false;
+                $idxResult = $result &&
-            }
+                $this->pdo->exec('CREATE INDEX ' . $table . '_' . $vcol['name'] . '_idx on `' . $table . '`(`' . $vcol['name'] . '`)') !== false;
-            if (!$idxResult) {
-                $out = false;
             }
         }
-        return $out;
+        return $result;
     }
 
     protected function sql_ddl(array $args = []): string
@@ -235,7 +231,7 @@ class SQLiteDriver extends AbstractDriver
         $out[] = "CREATE TABLE IF NOT EXISTS `{$args['table']}` (";
         $lines = [];
         $lines[] = "`json_data` TEXT DEFAULT NULL";
-        foreach ($args['virtualColumns'] as $path => $col) {
+        foreach ($args['schema'] as $path => $col) {
             $line = "`{$col['name']}` {$col['type']}";
             if (@$col['primary']) {
                 $line .= ' PRIMARY KEY';
@@ -253,8 +249,20 @@ class SQLiteDriver extends AbstractDriver
         return "DESTRUCTR_JSON_EXTRACT(`json_data`,'$.{$path}')";
     }
 
-    public function json_encode($a, ?array &$b = null, string $prefix = '')
+    protected function sql_create_schema_table(): string
     {
-        return json_encode($a);
+        return <<<EOT
+CREATE TABLE IF NOT EXISTS `destructr_schema`(
+    schema_time BIGINT NOT NULL,
+    schema_table VARCHAR(100) NOT NULL,
+    schema_schema TEXT NOT NULL
+);
+EOT;
+    }
+
+    protected function sql_table_exists(string $table): string
+    {
+        $table = preg_replace('/[^a-zA-Z0-9\-_]/', '', $table);
+        return 'SELECT 1 FROM ' . $table . ' LIMIT 1';
     }
 }

src/Factory.php

@@ -2,6 +2,7 @@
 /* Destructr | https://github.com/jobyone/destructr | MIT License */
 namespace Destructr;
 
+use Destructr\Drivers\AbstractDriver;
 use mofodojodino\ProfanityFilter\Check;
 
 /**
@@ -19,25 +20,30 @@ use mofodojodino\ProfanityFilter\Check;
  *  * Executing Searches (which largely consists of passing them to the Driver)
  *  * Inspecting unstructured data straight from the database and figuring out what class to make it (defaults to just DSO)
  */
-class Factory implements DSOFactoryInterface
+class Factory
 {
     const ID_CHARS = 'abcdefghijkmnorstuvwxyz0123456789';
     const ID_LENGTH = 16;
 
-    protected $driver;
-    protected $table;
     /**
-     * Virtual columns are only supported by modern SQL servers. Most of the
+     * @var Drivers\AbstractDriver
-     * legacy drivers will only use the ones defined in CORE_VIRTUAL_COLUMNS,
-     * but that should be handled automatically.
      */
-    protected $virtualColumns = [
+    protected $driver;
+    /**
+     * @var string
+     */
+    protected $table;
+
+    /**
+     * Virtual columns that should be created for sorting/indexing in the SQL server
+     */
+    protected $schema = [
         'dso.id' => [
-            'name' => 'dso_id',
+            'name' => 'dso_id', //column name to be used
-            'type' => 'VARCHAR(16)',
+            'type' => 'VARCHAR(16)', //column type
-            'index' => 'BTREE',
+            'index' => 'BTREE', //whether/how to index
-            'unique' => true,
+            'unique' => true, //whether column should be unique
-            'primary' => true,
+            'primary' => true, //whether column should be the primary key
         ],
         'dso.type' => [
             'name' => 'dso_type',
@@ -51,12 +57,33 @@ class Factory implements DSOFactoryInterface
         ],
     ];
 
-    public function __construct(Drivers\DSODriverInterface $driver, string $table)
+    public function __construct(Drivers\AbstractDriver $driver, string $table)
     {
         $this->driver = $driver;
         $this->table = $table;
     }
 
+    public function table(): string
+    {
+        return $this->table;
+    }
+
+    public function driver(): AbstractDriver
+    {
+        return $this->driver;
+    }
+
+    public function tableExists(): bool
+    {
+        return $this->driver->tableExists($this->table);
+    }
+
+    public function createSchemaTable(): bool
+    {
+        $this->driver->createSchemaTable('destructr_schema');
+        return $this->driver->tableExists('destructr_schema');
+    }
+
     public function quote(string $str): string
     {
         return $this->driver->pdo()->quote($str);
@@ -122,22 +149,30 @@ class Factory implements DSOFactoryInterface
         return $dso;
     }
 
-    public function createTable(): bool
+    public function prepareEnvironment(): bool
     {
-        return $this->driver->createTable(
+        return $this->driver->prepareEnvironment(
             $this->table,
-            $this->virtualColumns
+            $this->schema
         );
     }
 
-    public function virtualColumns(): array
+    public function updateEnvironment(): bool
     {
-        return $this->virtualColumns;
+        return $this->driver->updateEnvironment(
+            $this->table,
+            $this->schema
+        );
+    }
+
+    public function schema(): array
+    {
+        return $this->driver->getSchema($this->table) ?? $this->schema;
     }
 
     protected function virtualColumnName($path): ?string
     {
-        return @$this->virtualColumns[$path]['name'];
+        return @$this->schema()[$path]['name'];
     }
 
     public function update(DSOInterface $dso, bool $sneaky = false): bool

src/Search.php

@@ -2,8 +2,6 @@
 /* Destructr | https://github.com/jobyone/destructr | MIT License */
 namespace Destructr;
 
-use Destructr\DSOFactoryInterface;
-
 class Search implements \Serializable
 {
     protected $factory;
@@ -12,7 +10,7 @@ class Search implements \Serializable
     protected $limit;
     protected $offset;
 
-    public function __construct(DSOFactoryInterface $factory = null)
+    public function __construct(Factory $factory = null)
     {
         $this->factory = $factory;
     }

tests/Drivers/AbstractSQLDriverIntegrationTest.php

@@ -7,9 +7,10 @@ use Destructr\Factory;
 use PHPUnit\DbUnit\TestCaseTrait;
 use PHPUnit\Framework\TestCase;
 
-abstract class AbstractDriverIntegrationTest extends TestCase
+abstract class AbstractSQLDriverIntegrationTest extends TestCase
 {
     use TestCaseTrait;
+    const TEST_TABLE = 'integrationtest';
 
     public static function setUpBeforeClass()
     {
@@ -17,10 +18,10 @@ abstract class AbstractDriverIntegrationTest extends TestCase
         $pdo->exec('DROP TABLE ' . static::TEST_TABLE);
     }
 
-    public function testCreateTable()
+    public function testPrepareEnvironment()
     {
         $factory = $this->createFactory();
-        $factory->createTable();
+        $factory->prepareEnvironment();
         //table should exist and have zero rows
         $this->assertEquals(0, $this->getConnection()->getRowCount(static::TEST_TABLE));
     }

tests/Drivers/AbstractSQLDriverSchemaChangeTest.php

@@ -0,0 +1,134 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers;
+
+use PDO;
+use PHPUnit\DbUnit\TestCaseTrait;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * This class tests a Driver's ability to correctly change schemas.
+ */
+abstract class AbstractSQLDriverSchemaChangeTest extends TestCase
+{
+    use TestCaseTrait;
+    const TEST_TABLE = 'schematest';
+
+    public function testSchemaChanges()
+    {
+        // set up using schema A
+        $factory = $this->createFactoryA();
+        $factory->prepareEnvironment();
+        $factory->updateEnvironment();
+        // verify schema in database
+        $this->assertEquals(
+            $factory->schema,
+            $factory->driver()->getSchema('schematest')
+        );
+        // add some content
+        $new = $factory->create([
+            'dso.id' => 'dso1',
+            'test.a' => 'value a1',
+            'test.b' => 'value b1',
+            'test.c' => 'value c1',
+        ]);
+        $new->insert();
+        $new = $factory->create([
+            'dso.id' => 'dso2',
+            'test.a' => 'value a2',
+            'test.b' => 'value b2',
+            'test.c' => 'value c2',
+        ]);
+        $new->insert();
+        $new = $factory->create([
+            'dso.id' => 'dso3',
+            'test.a' => 'value a3',
+            'test.b' => 'value b3',
+            'test.c' => 'value c3',
+        ]);
+        $new->insert();
+        // verify data in table matches
+        $pdo = $this->createPDO();
+        $this->assertEquals(3, $this->getConnection()->getRowCount('schematest'));
+        for ($i = 1; $i <= 3; $i++) {
+            $row = $pdo->query('select dso_id, test_a, test_b from schematest where dso_id = "dso' . $i . '"')->fetch(PDO::FETCH_ASSOC);
+            $this->assertEquals(['dso_id' => "dso$i", 'test_a' => "value a$i", 'test_b' => "value b$i"], $row);
+        }
+        // change to schema B
+        sleep(1); //a table can't have its schema updated faster than once per second
+        $factory = $this->createFactoryB();
+        $factory->prepareEnvironment();
+        $factory->updateEnvironment();
+        // verify schema in database
+        $this->assertEquals(
+            $factory->schema,
+            $factory->driver()->getSchema('schematest')
+        );
+        // verify data in table matches
+        $pdo = $this->createPDO();
+        $this->assertEquals(3, $this->getConnection()->getRowCount('schematest'));
+        for ($i = 1; $i <= 3; $i++) {
+            $row = $pdo->query('select dso_id, test_a_2, test_c from schematest where dso_id = "dso' . $i . '"')->fetch(PDO::FETCH_ASSOC);
+            $this->assertEquals(['dso_id' => "dso$i", 'test_a_2' => "value a$i", 'test_c' => "value c$i"], $row);
+        }
+    }
+
+    protected static function createFactoryA()
+    {
+        $driver = static::createDriver();
+        $factory = new FactorySchemaA(
+            $driver,
+            static::TEST_TABLE
+        );
+        return $factory;
+    }
+
+    protected static function createFactoryB()
+    {
+        $driver = static::createDriver();
+        $factory = new FactorySchemaB(
+            $driver,
+            static::TEST_TABLE
+        );
+        return $factory;
+    }
+
+    protected static function createDriver()
+    {
+        $class = static::DRIVER_CLASS;
+        return new $class(
+            static::DRIVER_DSN,
+            static::DRIVER_USERNAME,
+            static::DRIVER_PASSWORD,
+            static::DRIVER_OPTIONS
+        );
+    }
+
+    protected static function createPDO()
+    {
+        return new \PDO(
+            static::DRIVER_DSN,
+            static::DRIVER_USERNAME,
+            static::DRIVER_PASSWORD,
+            static::DRIVER_OPTIONS
+        );
+    }
+
+    public function getConnection()
+    {
+        return $this->createDefaultDBConnection($this->createPDO(), 'phpunit');
+    }
+
+    public function getDataSet()
+    {
+        return new \PHPUnit\DbUnit\DataSet\DefaultDataSet();
+    }
+
+    public static function setUpBeforeClass()
+    {
+        $pdo = static::createPDO();
+        $pdo->exec('DROP TABLE schematest');
+        $pdo->exec('DROP TABLE destructr_schema');
+    }
+}

tests/Drivers/AbstractSQLDriverTest.php

@@ -13,18 +13,18 @@ use PHPUnit\Framework\TestCase;
  * This class tests a factory in isolation. In the name of simplicity it's a bit
  * simplistic, because it doesn't get the help of the Factory.
  *
- * There is also a class called AbstractDriverIntegrationTest that tests drivers
+ * There is also a class called AbstractSQLDriverIntegrationTest that tests drivers
  * through a Factory. The results of that are harder to interpret, but more
  * properly and thoroughly test the Drivers in a real environment.
  */
-abstract class AbstractDriverTest extends TestCase
+abstract class AbstractSQLDriverTest extends TestCase
 {
     use TestCaseTrait;
 
     /*
     In actual practice, these would come from a Factory
      */
-    protected $virtualColumns = [
+    protected $schema = [
         'dso.id' => [
             'name' => 'dso_id',
             'type' => 'VARCHAR(16)',
@@ -43,17 +43,22 @@ abstract class AbstractDriverTest extends TestCase
         ],
     ];
 
-    public function testCreateTable()
+    public function testPrepareEnvironment()
     {
         $driver = $this->createDriver();
-        $driver->createTable('testCreateTable', $this->virtualColumns);
+        $this->assertFalse($driver->tableExists('testPrepareEnvironment'));
-        $this->assertEquals(0, $this->getConnection()->getRowCount('testCreateTable'));
+        $this->assertFalse($driver->tableExists('destructr_schema'));
+        $driver->prepareEnvironment('testPrepareEnvironment', $this->schema);
+        $this->assertTrue($driver->tableExists('destructr_schema'));
+        $this->assertTrue($driver->tableExists('testPrepareEnvironment'));
+        $this->assertEquals(1, $this->getConnection()->getRowCount('destructr_schema'));
+        $this->assertEquals(0, $this->getConnection()->getRowCount('testPrepareEnvironment'));
     }
 
     public function testInsert()
     {
         $driver = $this->createDriver();
-        $driver->createTable('testInsert', $this->virtualColumns);
+        $driver->prepareEnvironment('testInsert', $this->schema);
         //test inserting an object
         $o = new DSO(['dso.id' => 'first-inserted'],new Factory($driver,'no_table'));
         $this->assertTrue($driver->insert('testInsert', $o));
@@ -62,16 +67,12 @@ abstract class AbstractDriverTest extends TestCase
         $o = new DSO(['dso.id' => 'second-inserted'],new Factory($driver,'no_table'));
         $this->assertTrue($driver->insert('testInsert', $o));
         $this->assertEquals(2, $this->getConnection()->getRowCount('testInsert'));
-        //test inserting a second object with an existing id, it shouldn't work
-        $o = new DSO(['dso.id' => 'first-inserted'],new Factory($driver,'no_table'));
-        $this->assertFalse($driver->insert('testInsert', $o));
-        $this->assertEquals(2, $this->getConnection()->getRowCount('testInsert'));
     }
 
     public function testSelect()
     {
         $driver = $this->createDriver();
-        $driver->createTable('testSelect', $this->virtualColumns);
+        $driver->prepareEnvironment('testSelect', $this->schema);
         //set up dummy data
         $this->setup_testSelect();
         //empty search
@@ -105,47 +106,6 @@ abstract class AbstractDriverTest extends TestCase
         $this->assertSame(0, count($results));
     }
 
-    public function testDelete()
-    {
-        $driver = $this->createDriver();
-        $driver->createTable('testDelete', $this->virtualColumns);
-        //set up dummy data
-        $this->setup_testDelete();
-        //try deleting an item
-        $dso = new DSO(['dso.id' => 'item-a-1'],new Factory($driver,'no_table'));
-        $driver->delete('testDelete', $dso);
-        $this->assertEquals(3, $this->getConnection()->getRowCount('testDelete'));
-        //try deleting an item at the other end of the table
-        $dso = new DSO(['dso.id' => 'item-b-2'],new Factory($driver,'no_table'));
-        $driver->delete('testDelete', $dso);
-        $this->assertEquals(2, $this->getConnection()->getRowCount('testDelete'));
-    }
-
-    protected function setup_testDelete()
-    {
-        $driver = $this->createDriver();
-        $driver->insert('testDelete', new DSO([
-            'dso' => ['id' => 'item-a-1', 'type' => 'type-a'],
-            'foo' => 'bar',
-            'sort' => 'a',
-        ],new Factory($driver,'no_table')));
-        $driver->insert('testDelete', new DSO([
-            'dso' => ['id' => 'item-a-2', 'type' => 'type-a'],
-            'foo' => 'baz',
-            'sort' => 'c',
-        ],new Factory($driver,'no_table')));
-        $driver->insert('testDelete', new DSO([
-            'dso' => ['id' => 'item-b-1', 'type' => 'type-b'],
-            'foo' => 'buz',
-            'sort' => 'b',
-        ],new Factory($driver,'no_table')));
-        $driver->insert('testDelete', new DSO([
-            'dso' => ['id' => 'item-b-2', 'type' => 'type-b', 'deleted' => 100],
-            'foo' => 'quz',
-            'sort' => 'd',
-        ],new Factory($driver,'no_table')));
-    }
-
     protected function setup_testSelect()
     {
         $driver = $this->createDriver();
@@ -189,10 +149,10 @@ abstract class AbstractDriverTest extends TestCase
     public static function setUpBeforeClass()
     {
         $pdo = static::createPDO();
-        $pdo->exec('DROP TABLE testCreateTable');
+        $pdo->exec('DROP TABLE testPrepareEnvironment');
         $pdo->exec('DROP TABLE testInsert');
         $pdo->exec('DROP TABLE testSelect');
-        $pdo->exec('DROP TABLE testDelete');
+        $pdo->exec('DROP TABLE destructr_schema');
     }
 
     protected static function createPDO()

tests/Drivers/FactorySchemaA.php

@@ -0,0 +1,30 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers;
+
+use Destructr\Factory;
+
+class FactorySchemaA extends Factory
+{
+    public $schema = [
+        'dso.id' => [
+            'name' => 'dso_id', //column name to be used
+            'type' => 'VARCHAR(16)', //column type
+            'index' => 'BTREE', //whether/how to index
+            'unique' => true, //whether column should be unique
+            'primary' => true, //whether column should be the primary key
+        ],
+        'test.a' => [
+            'name' => 'test_a',
+            'type' => 'VARCHAR(100)',
+            'index' => 'BTREE',
+        ]
+        ,
+        'test.b' => [
+            'name' => 'test_b',
+            'type' => 'VARCHAR(100)',
+            'index' => 'BTREE',
+        ],
+    ];
+}

tests/Drivers/FactorySchemaB.php

@@ -0,0 +1,30 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers;
+
+use Destructr\Factory;
+
+class FactorySchemaB extends Factory
+{
+    public $schema = [
+        'dso.id' => [
+            'name' => 'dso_id', //column name to be used
+            'type' => 'VARCHAR(16)', //column type
+            'index' => 'BTREE', //whether/how to index
+            'unique' => true, //whether column should be unique
+            'primary' => true, //whether column should be the primary key
+        ],
+        'test.a' => [
+            'name' => 'test_a_2',
+            'type' => 'VARCHAR(100)',
+            'index' => 'BTREE',
+        ]
+        ,
+        'test.c' => [
+            'name' => 'test_c',
+            'type' => 'VARCHAR(100)',
+            'index' => 'BTREE',
+        ],
+    ];
+}

tests/Drivers/MariaDB/MariaDBDriverIntegrationTest.php

@@ -0,0 +1,16 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers\MariaDB;
+
+use Destructr\Drivers\AbstractSQLDriverIntegrationTest;
+use Destructr\Drivers\MariaDBDriver;
+
+class MariaDBDriverIntegrationTest extends AbstractSQLDriverIntegrationTest
+{
+    const DRIVER_CLASS = MariaDBDriver::class;
+    const DRIVER_DSN = 'mysql:host=127.0.0.1;port=3307;dbname=destructrtest';
+    const DRIVER_USERNAME = 'destructrtest';
+    const DRIVER_PASSWORD = 'destructrtest';
+    const DRIVER_OPTIONS = null;
+}

tests/Drivers/MariaDB/MariaDBDriverSchemaChangeTest.php

@@ -0,0 +1,16 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers\MariaDB;
+
+use Destructr\Drivers\AbstractSQLDriverSchemaChangeTest;
+use Destructr\Drivers\MariaDBDriver;
+
+class MariaDBDriverSchemaChangeTest extends AbstractSQLDriverSchemaChangeTest
+{
+    const DRIVER_CLASS = MariaDBDriver::class;
+    const DRIVER_DSN = 'mysql:host=127.0.0.1;port=3307;dbname=destructrtest';
+    const DRIVER_USERNAME = 'destructrtest';
+    const DRIVER_PASSWORD = 'destructrtest';
+    const DRIVER_OPTIONS = null;
+}

tests/Drivers/MariaDB/MariaDBDriverTest.php

@@ -0,0 +1,16 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers\MariaDB;
+
+use Destructr\Drivers\AbstractSQLDriverTest;
+use Destructr\Drivers\MariaDBDriver;
+
+class MariaDBDriverTest extends AbstractSQLDriverTest
+{
+    const DRIVER_CLASS = MariaDBDriver::class;
+    const DRIVER_DSN = 'mysql:host=127.0.0.1;port=3307;dbname=destructrtest';
+    const DRIVER_USERNAME = 'destructrtest';
+    const DRIVER_PASSWORD = 'destructrtest';
+    const DRIVER_OPTIONS = null;
+}

tests/Drivers/MySQL/MySQLDriverIntegrationTest.php

@@ -3,15 +3,14 @@
 declare (strict_types = 1);
 namespace Destructr\Drivers\MySQL;
 
-use Destructr\Drivers\AbstractDriverIntegrationTest;
+use Destructr\Drivers\AbstractSQLDriverIntegrationTest;
 use Destructr\Drivers\MySQLDriver;
 
-class MySQLDriverIntegrationTest extends AbstractDriverIntegrationTest
+class MySQLDriverIntegrationTest extends AbstractSQLDriverIntegrationTest
 {
     const DRIVER_CLASS = MySQLDriver::class;
     const DRIVER_DSN = 'mysql:host=127.0.0.1;dbname=test';
     const DRIVER_USERNAME = 'root';
-    const DRIVER_PASSWORD = '';
+    const DRIVER_PASSWORD = null;
     const DRIVER_OPTIONS = null;
-    const TEST_TABLE = 'integrationtest';
 }

tests/Drivers/MySQL/MySQLDriverSchemaChangeTest.php

@@ -0,0 +1,16 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers\MySQL;
+
+use Destructr\Drivers\AbstractSQLDriverSchemaChangeTest;
+use Destructr\Drivers\MySQLDriver;
+
+class MySQLDriverSchemaChangeTest extends AbstractSQLDriverSchemaChangeTest
+{
+    const DRIVER_CLASS = MySQLDriver::class;
+    const DRIVER_DSN = 'mysql:host=127.0.0.1;dbname=test';
+    const DRIVER_USERNAME = 'root';
+    const DRIVER_PASSWORD = null;
+    const DRIVER_OPTIONS = null;
+}

tests/Drivers/MySQL/MySQLDriverTest.php

@@ -3,14 +3,14 @@
 declare (strict_types = 1);
 namespace Destructr\Drivers\MySQL;
 
-use Destructr\Drivers\AbstractDriverTest;
+use Destructr\Drivers\AbstractSQLDriverTest;
 use Destructr\Drivers\MySQLDriver;
 
-class MySQLDriverTest extends AbstractDriverTest
+class MySQLDriverTest extends AbstractSQLDriverTest
 {
     const DRIVER_CLASS = MySQLDriver::class;
     const DRIVER_DSN = 'mysql:host=127.0.0.1;dbname=test';
     const DRIVER_USERNAME = 'root';
-    const DRIVER_PASSWORD = '';
+    const DRIVER_PASSWORD = null;
     const DRIVER_OPTIONS = null;
 }

tests/Drivers/SQLite/SQLiteDriverIntegrationTest.php

@@ -3,15 +3,19 @@
 declare (strict_types = 1);
 namespace Destructr\Drivers\SQLite;
 
-use Destructr\Drivers\AbstractDriverIntegrationTest;
+use Destructr\Drivers\AbstractSQLDriverIntegrationTest;
 use Destructr\Drivers\SQLiteDriver;
 
-class SQLiteDriverIntegrationTest extends AbstractDriverIntegrationTest
+class SQLiteDriverIntegrationTest extends AbstractSQLDriverIntegrationTest
 {
     const DRIVER_CLASS = SQLiteDriver::class;
     const DRIVER_DSN = 'sqlite:'.__DIR__.'/integration.test.sqlite';
     const DRIVER_USERNAME = 'root';
     const DRIVER_PASSWORD = '';
     const DRIVER_OPTIONS = null;
-    const TEST_TABLE = 'integrationtest';
+
+    public static function setUpBeforeClass()
+    {
+        @unlink(__DIR__.'/integration.test.sqlite');
+    }
 }

tests/Drivers/SQLite/SQLiteDriverSchemaChangeTest.php

@@ -0,0 +1,21 @@
+<?php
+/* Destructr | https://github.com/jobyone/destructr | MIT License */
+declare (strict_types = 1);
+namespace Destructr\Drivers\SQLite;
+
+use Destructr\Drivers\AbstractSQLDriverSchemaChangeTest;
+use Destructr\Drivers\SQLiteDriver;
+
+class SQLiteDriverSchemaChangeTest extends AbstractSQLDriverSchemaChangeTest
+{
+    const DRIVER_CLASS = SQLiteDriver::class;
+    const DRIVER_DSN = 'sqlite:'.__DIR__.'/schema.test.sqlite';
+    const DRIVER_USERNAME = 'root';
+    const DRIVER_PASSWORD = '';
+    const DRIVER_OPTIONS = null;
+
+    public static function setUpBeforeClass()
+    {
+        @unlink(__DIR__.'/schema.test.sqlite');
+    }
+}

tests/Drivers/SQLite/SQLiteDriverTest.php

@@ -3,14 +3,19 @@
 declare (strict_types = 1);
 namespace Destructr\Drivers\SQLite;
 
-use Destructr\Drivers\AbstractDriverTest;
+use Destructr\Drivers\AbstractSQLDriverTest;
 use Destructr\Drivers\SQLiteDriver;
 
-class SQLiteDriverTest extends AbstractDriverTest
+class SQLiteDriverTest extends AbstractSQLDriverTest
 {
     const DRIVER_CLASS = SQLiteDriver::class;
     const DRIVER_DSN = 'sqlite:'.__DIR__.'/driver.test.sqlite';
     const DRIVER_USERNAME = 'root';
     const DRIVER_PASSWORD = '';
     const DRIVER_OPTIONS = null;
+
+    public static function setUpBeforeClass()
+    {
+        @unlink(__DIR__.'/driver.test.sqlite');
+    }
 }

tests/HarnessDriver.php

@@ -2,7 +2,7 @@
 /* Destructr | https://github.com/jobyone/destructr | MIT License */
 namespace Destructr;
 
-class HarnessDriver implements Drivers\DSODriverInterface
+class HarnessDriver extends Drivers\AbstractDriver
 {
     const EXTENSIBLE_VIRTUAL_COLUMNS = true;
     public $last_select;
@@ -18,7 +18,7 @@ class HarnessDriver implements Drivers\DSODriverInterface
         return null;
     }
 
-    public function createTable(string $table, array $virtualColumns) : bool
+    public function prepareEnvironment(string $table, array $schema) : bool
     {
         //TODO: add tests for this too
         return false;