Merge branch 'editing', split darkmode to own file

app/Exceptions/Handler.php

@@ -6,6 +6,7 @@ use Exception;
 use Illuminate\Auth\AuthenticationException;
 use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
 use Illuminate\Validation\ValidationException;
+use MightyPork\Exceptions\Exceptions\SimpleValidationException;
 
 class Handler extends ExceptionHandler
 {
@@ -15,7 +16,7 @@ class Handler extends ExceptionHandler
      * @var array
      */
     protected $dontReport = [
-        //
+        SimpleValidationException::class,
     ];
 
     /**

app/Exceptions/ViewException.php

@@ -1,14 +0,0 @@
-<?php
-
-namespace FlowBox\Exceptions;
-
-class ViewException extends FBRuntimeException
-{
-	public $captured;
-
-	public function __construct(\Exception $cause, $captured)
-	{
-		$this->captured = $captured;
-		parent::__construct($cause);
-	}
-}

app/Http/Controllers/Controller.php

@@ -3,6 +3,7 @@
 namespace App\Http\Controllers;
 
 use Illuminate\Foundation\Bus\DispatchesJobs;
+use Illuminate\Http\JsonResponse;
 use Illuminate\Http\Request;
 use Illuminate\Routing\Controller as BaseController;
 use Illuminate\Foundation\Validation\ValidatesRequests;
@@ -55,6 +56,11 @@ class Controller extends BaseController
         'mastodon', // mastodon fetching previews
     ];
 
+    protected function jsonResponse($data, $code=200)
+    {
+        return new JsonResponse($data, $code);
+    }
+
     // Hacks to allow recursive nesting of validations in string and array format
 
     public function makeValidator($data, $rules, $messages = array(), $customAttributes = array())

app/Http/Controllers/TableController.php

@@ -5,6 +5,7 @@ namespace App\Http\Controllers;
 use App\Models\Revision;
 use App\Models\Table;
 use App\Models\User;
+use App\Tables\Changeset;
 use App\Tables\Column;
 use App\Tables\ColumnNumerator;
 use App\Tables\CStructArrayExporter;
@@ -31,7 +32,7 @@ class TableController extends Controller
             ->where('name', $table)->first();
         if ($tableModel === null) abort(404, "No such table.");
 
-        // make it possible to show other revisions
+        // option to show other revisions
         if ($input->has('rev')) {
             $rev = (int)$input->rev;
             $revision = $tableModel->revisions()->orderBy('created_at')->skip($rev - 1)->first();
@@ -206,39 +207,13 @@ class TableController extends Controller
         }
 
         // --- DATA ---
-        $dataCsvLines = array_map('str_getcsv', explode("\n", $input->data));
+        $dataCsvLines = Utils::csvToArray($input->data);
 
         // Preparing data to insert into the Rows table
         $rowsToInsert = null;
         $rowNumerator = null;
         try {
-            $rowsToInsert = collect($dataCsvLines)->map(function ($row) use ($columns) {
-                if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
-                if (count($row) != count($columns)) {
-                    throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
-                }
-
-                $data = [];
-
-                foreach ($row as $i => $val) {
-                    $col = $columns[$i];
-
-                    if (strlen($val) > 1000) {
-                        // try to stop people inserting unstructured crap / malformed CSV
-                        throw new NotApplicableException("Value for column {$col->name} too long.");
-                    }
-                    $data[$col->id] = $col->cast($val);
-                }
-
-                // return in a format prepared for filling eloquent
-                return ['data' => $data];
-            })->filter()->all(); // remove nulls, to array
-
-            // attach _id labels to all rows
-            $rowNumerator = new RowNumerator(count($dataCsvLines));
-            foreach ($rowsToInsert as &$item) {
-                $item['data']['_id'] = $rowNumerator->next();
-            }
+            $rowsToInsert = Changeset::csvToRowsArray($columns, $dataCsvLines, true)->all();
         } catch (\Exception $e) {
             return $this->backWithErrors(['data' => $e->getMessage()]);
         }

app/Http/Controllers/TableEditController.php

@@ -0,0 +1,174 @@
+<?php
+
+
+namespace App\Http\Controllers;
+
+use App\Models\Table;
+use App\Models\User;
+use App\Tables\Changeset;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Input;
+use MightyPork\Exceptions\SimpleValidationException;
+use MightyPork\Utils\Utils;
+
+class TableEditController extends Controller
+{
+    /**
+     * Initialize the session-stored changeset, if not set yet
+     *
+     * @param Table $table
+     * @return Changeset
+     */
+    private function getChangeset(Table $table)
+    {
+        if (Input::has('reset')) {
+            session()->forget($table->draftSessionKey);
+        }
+
+        /** @var Changeset $changeset */
+        return session()->remember($table->draftSessionKey, function () use ($table) {
+            $changeset = new Changeset();
+            $changeset->table = $table;
+            $changeset->revision = $table->revision;
+            return $changeset;
+        });
+    }
+
+    private function storeChangeset(Changeset $chs)
+    {
+        session()->put($chs->table->draftSessionKey, $chs);
+    }
+
+    /**
+     * Discard draft and redirect to table view
+     */
+    public function discard(User $user, string $table)
+    {
+        /** @var Table $tableModel */
+        $tableModel = $user->tables()->where('name', $table)->first();
+        if ($tableModel === null) abort(404, "No such table.");
+
+        session()->forget($tableModel->draftSessionKey);
+
+        return redirect($tableModel->viewRoute);
+    }
+
+    public function draft(User $user, string $table, $tab = null)
+    {
+        /** @var Table $tableModel */
+        $tableModel = $user->tables()->where('name', $table)->first();
+        if ($tableModel === null) abort(404, "No such table.");
+
+        if ($tab == null) $tab = 'edit-rows';
+        $tabs = ['edit-rows', 'add-rows', 'manage-columns', 'review'];
+        if (!in_array($tab, $tabs)) abort(404, "No such tab: $tab");
+
+        $changeset = $this->getChangeset($tableModel);
+
+        if (Input::has('dump')) {
+            dd($changeset);
+        }
+
+        return $this->{camel_case($tab)}($changeset);
+    }
+
+    /** @noinspection PhpUnusedPrivateMethodInspection */
+    private function editRows(Changeset $changeset)
+    {
+        $revision = $changeset->revision;
+        $columns = $changeset->fetchAndTransformColumns();
+        $rows = $revision->rowsData($columns, true, false)->paginate(25, []);
+
+        return view('table.propose.edit-rows', [
+            'changeset' => $changeset,
+            'table' => $changeset->table,
+            'columns' => collect($columns),
+            'rows' => $rows,
+        ]);
+    }
+
+    /** @noinspection PhpUnusedPrivateMethodInspection */
+    private function addRows(Changeset $changeset)
+    {
+        $columns = $changeset->fetchAndTransformColumns();
+        $rows = $changeset->getAddedRows(25);
+
+        return view('table.propose.add-rows', [
+            'changeset' => $changeset,
+            'table' => $changeset->table,
+            'columns' => collect($columns),
+            'rows' => $rows,
+        ]);
+    }
+
+    public function draftUpdate(Request $request, User $user, string $table)
+    {
+        /** @var Table $tableModel */
+        $tableModel = $user->tables()->where('name', $table)->first();
+        if ($tableModel === null) abort(404, "No such table.");
+
+        $changeset = $this->getChangeset($tableModel);
+
+        $input = objBag($request->all(), false);
+
+        try {
+            $code = 200;
+            switch ($input->action) {
+                case 'row.update':
+                    $data = (object)$input->data;
+                    $changeset->rowUpdate($data);
+
+                    $resp = $changeset->fetchAndTransformRow($data->_id);
+                    break;
+
+                case 'row.remove':
+                    $isNew = $changeset->isNewRow($input->id);
+                    $changeset->rowRemove($input->id);
+                    $resp = $isNew ? null : $changeset->fetchAndTransformRow($input->id);
+                    break;
+
+                case 'row.restore':
+                    $changeset->rowRestore($input->id);
+                    $resp = $changeset->fetchAndTransformRow($input->id);
+                    break;
+
+                case 'rows.add':
+                    $changeset->addBlankRows($input->count);
+
+                    // rows.add is sent via a form
+                    if ($input->has('redirect')) {
+                        return redirect($input->redirect);
+                    } else {
+                        $resp = null;
+                    }
+                    break;
+
+                case 'rows.add-csv':
+                    try {
+                        $changeset->addFilledRows(Utils::csvToArray($input->data));
+                    } catch (\Exception $e) {
+                        return $this->backWithErrors(['data' => $e->getMessage()]);
+                    }
+
+                    // rows.add-csv is sent via a form
+                    if ($input->has('redirect')) {
+                        return redirect($input->redirect);
+                    } else {
+                        $resp = null;
+                    }
+                    break;
+
+                default:
+                    $resp = "Bad Action";
+                    $code = 400;
+                    break;
+            }
+        } catch (SimpleValidationException $e) {
+            return $this->jsonResponse(['errors' => $e->getMessageBag()->getMessages()], 400);
+        }
+
+        $this->storeChangeset($changeset);
+
+        return $this->jsonResponse($resp, $code);
+    }
+}

app/Models/BaseModel.php

@@ -26,7 +26,9 @@ class BaseModel extends Model
     public function getRelationValue($key)
     {
         if ($this->exists && !method_exists($this, $key)) {
-            throw new \LogicException("No attribute or relation ".var_export($key, true));
+            if (!isset($this->original[$key])) {
+                throw new \LogicException("No attribute or relation " . var_export($key, true));
+            }
         }
 
         return parent::getRelationValue($key);

app/Models/Proposal.php

@@ -3,6 +3,8 @@
 namespace App\Models;
 
 use App\Models\Concerns\Reportable;
+use App\Tables\Changeset;
+use MightyPork\Exceptions\NotApplicableException;
 
 /**
  * Change proposal
@@ -14,7 +16,7 @@ use App\Models\Concerns\Reportable;
  * @property int $revision_id
  * @property int $author_id
  * @property string $note
- * @property object $changes - JSONB
+ * @property Proposal $changes - JSONB
  * @property-read User $author
  * @property-read Table $table
  * @property-read Revision $revision
@@ -43,4 +45,60 @@ class Proposal extends BaseModel
     {
         return $this->belongsTo(Table::class);
     }
+
+    public function getChangesAttribute($value)
+    {
+        $changeset = Changeset::fromObject(fromJSON($value));
+        $changeset->revision = $this->revision;
+        $changeset->table = $this->table;
+        $changeset->note = $this->note;
+
+        return $changeset;
+    }
+
+    public function setChangesAttribute($value)
+    {
+        if ($value instanceof Changeset) {
+            $this->attributes['changes'] = toJSON($value->toObject());
+        } else {
+            throw new NotApplicableException("Only a Changeset may be set to Proposal->changes");
+        }
+    }
+
+    /**
+     * Create a new Proposal instance wrapping this changeset,
+     * owned by the currently logged in User. The created instance
+     * is NOT saved yet.
+     *
+     * @param Changeset $changeset - changeset to hydrate the proposal with
+     * @return Proposal
+     */
+    public static function fromChangeset(Changeset $changeset)
+    {
+        if (!$changeset->hasAnyChanges()) {
+            throw new NotApplicableException('No changes to propose.');
+        }
+
+        if ($changeset->note == null) {
+            throw new NotApplicableException('Proposal note must be filled.');
+        }
+
+        if ($changeset->table == null || !$changeset->table instanceof Table) {
+            throw new NotApplicableException('Table not assigned to Changeset');
+        }
+
+        if ($changeset->revision == null || !$changeset->revision instanceof Revision) {
+            throw new NotApplicableException('Revision not assigned to Changeset');
+        }
+
+        return new Proposal([
+            // relations
+            'table_id' => $changeset->table->getKey(),
+            'revision_id' => $changeset->revision->getKey(),
+            'author_id' => \Auth::user()->getKey(),
+            // the proposal info
+            'note' => $changeset->note,
+            'changes' => $changeset->toObject(),
+        ]);
+    }
 }

app/Models/Revision.php

@@ -40,12 +40,12 @@ class Revision extends BaseModel
      * @param Column[] $columns
      * @return \Illuminate\Database\Query\Builder|static
      */
-    public function rowsData($columns, $withId=true)
+    public function rowsData($columns, $withId=true, $named=true)
     {
         $selects = $withId ? ["data->>'_id' as _id"] : [];
 
         foreach ($columns as $col) {
-            $selects[] = "data->>'$col->id' as $col->name";
+            $selects[] = "data->>'$col->id' as " . ($named ? $col->name : $col->id);
         }
 
         return $this->rows()->select([])->selectRaw(implode(', ', $selects));

app/Models/Row.php

@@ -6,7 +6,7 @@ namespace App\Models;
  * Row in a data table
  *
  * @property int $id
- * @property string $data - JSONB, always containing _id
+ * @property object|\RowData $data - JSONB, always containing _id
  */
 class Row extends BaseModel
 {

app/Models/Table.php

@@ -22,7 +22,12 @@ use Illuminate\Database\Eloquent\Collection;
  * @property string $origin
  * @property int $visits
  * @property-read string $viewRoute
+ * @property-read string $draftRoute
  * @property-read string $settingsRoute
+ * @property-read string $deleteRoute
+ * @property-read string $draftSessionKey
+ * @property-read string $draftDiscardRoute
+ * @property-read string $draftUpdateRoute
  * @property-read User $owner
  * @property-read Table $parentTable
  * @property-read Table[]|Collection $forks
@@ -123,21 +128,36 @@ class Table extends BaseModel
 
     public function __get($name)
     {
-        if ($name == 'viewRoute') {
-            return route('table.view', ['user' => $this->cachedOwner()->name, 'table' => $this->name]);
+        if (ends_with($name, 'Route')) {
+            $arg = [
+                'user' => $this->cachedOwner()->name,
+                'table' => $this->name
+            ];
+
+            switch ($name) {
+                case 'viewRoute':     return route('table.view', $arg);
+                case 'settingsRoute': return route('table.conf', $arg);
+                case 'draftRoute':    return route('table.draft', $arg);
+                case 'deleteRoute':   return route('table.delete', $arg);
+                case 'draftDiscardRoute':    return route('table.draft-discard', $arg);
+                case 'draftUpdateRoute':    return route('table.draft-update', $arg);
+            }
         }
 
-        if ($name == 'settingsRoute') {
-            return route('table.conf', ['user' => $this->cachedOwner()->name, 'table' => $this->name]);
-        }
-
-        if ($name == 'deleteRoute') {
-            return route('table.delete', ['user' => $this->cachedOwner()->name, 'table' => $this->name]);
-        }
+        if ($name == 'draftSessionKey') return "proposal.{$this->id}";
 
         return parent::__get($name);
     }
 
+    public function getDraftRoute($tab=null)
+    {
+        return route('table.draft', [
+            'user' => $this->cachedOwner()->name,
+            'table' => $this->name,
+            'tab' => $tab,
+        ]);
+    }
+
     public function scopeForList(Builder $query)
     {
         return $query->with('revision:id,row_count')->with('owner:id,name,title')

app/Providers/AppServiceProvider.php

@@ -29,8 +29,17 @@ class AppServiceProvider extends ServiceProvider
         });
 
         \Blade::directive('tooltip', function($arg) {
+            $arg = trim($arg);
+            $placement = '';
+            if (starts_with($arg, ['top,', 'bottom,', 'left,', 'right,'])) {
+                list($placement, $arg) = explode(',', $arg);
+                $arg = trim($arg);
+            }
             $arge = e($arg);
-            return 'aria-label="' . $arge . '" title="' . $arge . '"';
+
+            $html = '';
+            if ($placement) $html .= 'data-placement="' . $placement . '" ';
+            return $html . 'data-toggle="tooltip" aria-label="' . $arge . '" title="' . $arge . '"';
         });
 
         \Blade::directive('sr', function($arg) {

app/Tables/BaseNumerator.php

@@ -0,0 +1,76 @@
+<?php
+
+
+namespace App\Tables;
+
+
+abstract class BaseNumerator
+{
+    /** @var int */
+    protected $next;
+
+    /** @var int */
+    protected $last;
+
+    /**
+     * BaseNumerator constructor.
+     *
+     * @param int|int[] $first - first index, or [first, last]
+     * @param int|null $last - last index, or null
+     */
+    public function __construct($first, $last = null)
+    {
+        if (is_array($first) && $last === null) {
+            list($first, $last) = $first;
+        }
+
+        $this->next = $first;
+        $this->last = $last;
+    }
+
+    /**
+     * Get next key, incrementing the internal state
+     *
+     * @return string
+     */
+    public function next()
+    {
+        if (!$this->hasMore())
+            throw new \OutOfBoundsException("Column numerator has run out of allocated GCID slots");
+
+        $key = $this->getKey($this->next);
+        $this->next++;
+        return $key;
+    }
+
+    /**
+     * Convert numeric index to a key
+     *
+     * @param int $index
+     * @return mixed
+     */
+    protected function getKey($index)
+    {
+        return $index; // simple default
+    }
+
+    /**
+     * @return bool - true iof there are more keys available
+     */
+    protected function hasMore()
+    {
+        return $this->next <= $this->last;
+    }
+
+    /**
+     * Generate all keys
+     *
+     * @return \Generator
+     */
+    public function generate()
+    {
+        while ($this->hasMore()) {
+            yield $this->next();
+        }
+    }
+}

app/Tables/Changeset.php

@@ -0,0 +1,467 @@
+<?php
+
+
+namespace App\Tables;
+
+use App\Models\Revision;
+use App\Models\Row;
+use App\Models\Table;
+use Illuminate\Pagination\Paginator;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Collection;
+use MightyPork\Exceptions\NotApplicableException;
+use MightyPork\Exceptions\NotExistException;
+use ReflectionClass;
+
+/**
+ * Object representing a set of table modifications
+ */
+class Changeset
+{
+    use SerializesModels;
+
+    const object_excluded_properties = [
+        'revision',
+        'table',
+        'note',
+    ];
+
+    /**
+     * @var Revision - base revision this changeset belongs to
+     */
+    public $revision;
+
+    /**
+     * @var Table - table this changeset belongs to
+     */
+    public $table;
+
+    /**
+     * @var string - user's note attached to this changeset (future proposal)
+     */
+    public $note = '';
+
+    /**
+     * Rows whose content changed, identified by _id.
+     * Only changed values are to be filled. Columns are identified by GCIDs
+     *
+     * Key'd by _id
+     *
+     * @var array|null - [_id -> [_id:…, cid:…, cid:…], …]
+     */
+    public $rowUpdates = [];
+
+    /**
+     * New rows in the full Row::data format, including GRIDs.
+     * Values are identified by GCIDs from previously defined, or new columns.
+     *
+     * @var array|null - [_id -> [_id:…, cid:…, cid:…], …]
+     */
+    public $newRows = [];
+
+    /**
+     * Rows to be removed
+     *
+     * @var int[]|null - GRIDs
+     */
+    public $removedRows = [];
+
+    /**
+     * Values changed in column specifications, such as name, title, etc.
+     * This does not affect the table rows in any way.
+     *
+     * Key'd by id
+     *
+     * @var array[] - column specification objects, with GCIDs, key'd by CID
+     */
+    public $columnUpdates = [];
+
+    /**
+     * New columns in the full format, including GCIDs
+     *
+     * @var array|null - [id -> [id:…, …], …]
+     */
+    public $newColumns = [];
+
+    /**
+     * When reordering columns, here is the column IDs array
+     * in the new order. Columns meanwhile removed from the table
+     * or added to it are to be ignored or appended at the end.
+     *
+     * This shall not be filled if merely editing or adding columns,
+     * unless the order is explicitly adjusted by the user.
+     *
+     * @var string[]|null - GCIDs
+     */
+    public $columnOrder = [];
+
+    /**
+     * Columns to be removed
+     *
+     * The data associated to those columns may or may not be removed from the Rows.
+     * It does not matter, other than in regard to the table size.
+     *
+     * @var int[]|null - GCIDs
+     */
+    public $removedColumns = [];
+
+    private function walkProps()
+    {
+        $properties = (new ReflectionClass($this))->getProperties();
+
+        foreach ($properties as $property) {
+            if (in_array($property->name, self::object_excluded_properties)) {
+                continue;
+            }
+
+            yield $property->name;
+        }
+    }
+
+    /**
+     * Reconstruct from a object, such as one found in Proposal.
+     * Note that the fromProposal() method should be used when the
+     * proposal is available, as it populates additional fields.
+     *
+     * @param \stdClass $changes
+     * @return Changeset
+     */
+    public static function fromObject($changes)
+    {
+        $changeset = new Changeset();
+
+        foreach ($changeset->walkProps() as $prop) {
+            if (isset($changes->$prop)) {
+                $changeset->$prop = $changes->$prop;
+            }
+        }
+
+        return $changeset;
+    }
+
+    /**
+     * Serialize to an object format that may be stored in a Proposal.
+     *
+     * @return \stdClass
+     */
+    public function toObject()
+    {
+        $object = new \stdClass();
+
+        foreach ($this->walkProps() as $prop) {
+            $object->$prop = $this->$prop;
+        }
+
+        return $object;
+    }
+
+    /**
+     * Check if there is any change in this changeset
+     *
+     * @return bool - any found
+     */
+    public function hasAnyChanges()
+    {
+        foreach ($this->walkProps() as $prop) {
+            if (!empty($this->$prop)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Decorate / transform a single row for the editor view
+     *
+     * @param object|\DecoratedRow $row - row, must be key'd by column ids
+     * @param bool $decorate - to add extra underscored info for the editor
+     * @return \DecoratedRow|object|null - null if not decorating and the row was removed
+     */
+    public function transformRow($row, $decorate)
+    {
+        if ($row instanceof Row) $row = (object)$row->getAttributes();
+
+        if ($decorate) {
+            $row->_remove = false;
+            $row->_changed = [];
+            $row->_orig = [];
+        }
+
+        if ($this->isNewRow($row->_id)) {
+            if ($decorate) {
+                $row->_new = true;
+            }
+            return $row;
+        }
+
+        // Removed rows - return as null
+        if (in_array($row->_id, $this->removedRows)) {
+            if ($decorate) {
+                $row->_remove = true;
+            } else {
+                return null;
+            }
+        }
+
+        // if marked for removal, hide changes
+        if (!$row->_remove) {
+            // Changed values
+            if (isset($this->rowUpdates[$row->_id])) {
+                $newVals = $this->rowUpdates[$row->_id];
+
+                if ($decorate) {
+                    $row->_changed = array_keys($newVals);
+                    $row->_orig = array_only((array)$row, $row->_changed);
+                }
+
+                $row = (object)array_merge((array)$row, $newVals);
+            }
+        }
+
+        // Drop deleted columns
+        if (!$decorate) {
+            foreach ($this->removedColumns as $colId) {
+                unset($row->$colId);
+            }
+        }
+
+        unset($row->_row_pivot);
+
+        return $row;
+    }
+
+    /**
+     * Decorate / transform columns (loaded from the source revision)
+     *
+     * @return Column[]
+     */
+    public function fetchAndTransformColumns()
+    {
+        /** @var Column[] - loaded and transformed columns, cached from previous call to transformColumns() */
+        static $cachedColumns = [];
+
+        if ($cachedColumns) return $cachedColumns;
+        $columns = Column::columnsFromJson($this->revision->columns);
+
+        // Modify columns
+        foreach ($columns as $column) {
+            if (isset($this->columnUpdates[$column->id])) {
+                $column->modifyByChangeset($this->columnUpdates[$column->id]);
+            }
+
+            if (in_array($column->id, $this->removedColumns)) {
+                $column->markForRemoval();
+            }
+        }
+
+        // Append new columns
+        foreach ($this->newColumns as $newColumn) {
+            $columns[] = $c = new Column($newColumn);
+            $c->markAsNew();
+        }
+
+        // Reorder
+        $colsById = collect($columns)->keyBy('id')->all();
+        $newOrder = [];
+        foreach ($this->columnOrder as $id) {
+            $newOrder[] = $colsById[$id];
+        }
+
+        $leftover_keys = array_diff(array_keys($colsById), $this->columnOrder);
+        foreach ($leftover_keys as $id) {
+            $newOrder[] = $colsById[$id];
+        }
+
+        return $cachedColumns = $newOrder;
+    }
+
+    public function rowRemove(int $id)
+    {
+        if ($this->isNewRow($id)) {
+            unset($this->newRows[$id]);
+        }
+        else {
+            $this->removedRows[] = $id;
+        }
+    }
+
+    public function rowRestore(int $id)
+    {
+        $this->removedRows = array_diff($this->removedRows, [$id]);
+    }
+
+    public function isNewRow(int $id)
+    {
+        return isset($this->newRows[$id]);
+    }
+
+    public function fetchAndTransformRow(int $id)
+    {
+        $r = $this->fetchRow($id);
+        $transformed = $this->transformRow($r, true);
+        return $transformed;
+    }
+
+    public function fetchColumns()
+    {
+        return Column::columnsFromJson($this->revision->columns);
+    }
+
+    /**
+     * Fetch an existing row from DB, or a new row.
+     *
+     * @param $id
+     * @return object
+     */
+    public function fetchRow(int $id)
+    {
+        if ($this->isNewRow($id)) {
+            return (object)$this->newRows[$id];
+        }
+
+        $r = $this->revision->rowsData($this->fetchColumns(), true, false)
+            ->whereRaw("data->>'_id' = ?", $id)->first();
+
+        if (!$r) throw new NotExistException("No such row _id = $id in this revision.");
+
+        // remove junk
+        unset($r->pivot_revision_id);
+        unset($r->pivot_row_id);
+
+        return (object)$r->getAttributes();
+    }
+
+    /**
+     * Apply a row update, adding the row to the list of changes, or removing it
+     * if all differences were undone.
+     *
+     * @param array|object $newVals - values of the new row
+     */
+    public function rowUpdate($newVals)
+    {
+        $newVals = (object)$newVals;
+
+        $_id = $newVals->_id;
+        $origRow = $this->fetchRow($_id);
+
+        /** @var Column[]|Collection $cols */
+        $cols = collect($this->fetchAndTransformColumns())->keyBy('id');
+
+        $updateObj = [];
+
+        foreach ($newVals as $colId => $value) {
+            if (starts_with($colId, '_')) continue; // internals
+
+            $col = $cols[$colId];
+            $value = $col->cast($value);
+            $origValue = $col->cast(isset($origRow->$colId) ? $origRow->$colId : null);
+
+            if ($value !== $origValue) {
+                $updateObj[$colId] = $value;
+            }
+        }
+
+        if ($this->isNewRow($_id)) {
+            $this->newRows[$_id] = array_merge($this->newRows[$_id], $updateObj);
+        }
+        else {
+            if (!empty($updateObj)) {
+                $this->rowUpdates[$_id] = $updateObj;
+            } else {
+                // remove possible old update record for this row, if nothing changes now
+                unset($this->rowUpdates[$_id]);
+            }
+        }
+    }
+
+    /**
+     * Get a page of added rows for display in the editor
+     *
+     * @param int $perPage
+     * @return \Illuminate\Pagination\LengthAwarePaginator|Collection|array
+     */
+    public function getAddedRows($perPage = 25)
+    {
+        return collection_paginate($this->newRows, $perPage);
+    }
+
+    /**
+     * @param Column[] $columns
+     * @param array $csvArray
+     * @param bool $forTableInsert
+     * @return \array[][]|Collection
+     */
+    public static function csvToRowsArray($columns, $csvArray, $forTableInsert)
+    {
+        /** @var Collection|array[][] $rows */
+        $rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) {
+            if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
+            if (count($row) != count($columns)) {
+                throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
+            }
+
+            $data = [];
+
+            foreach ($row as $i => $val) {
+                $col = $columns[$i];
+
+                if (strlen($val) > 1000) {
+                    // try to stop people inserting unstructured crap / malformed CSV
+                    throw new NotApplicableException("Value for column {$col->name} too long.");
+                }
+                $data[$col->id] = $col->cast($val);
+            }
+
+            if ($forTableInsert) {
+                return ['data' => $data];
+            } else {
+                return $data;
+            }
+        })->filter();
+
+
+        $rowNumerator = new RowNumerator(count($csvArray));
+
+        if ($forTableInsert) {
+            return $rows->map(function ($row) use (&$rowNumerator) {
+                $row['data']['_id'] = $rowNumerator->next();
+                return $row;
+            });
+        }
+        else {
+            return $rows->map(function ($row) use (&$rowNumerator) {
+                $row['_id'] = $rowNumerator->next();
+                return $row;
+            });
+        }
+    }
+
+    public function addBlankRows(int $count)
+    {
+        $numerator = new RowNumerator($count);
+
+        $columns = $this->fetchAndTransformColumns();
+        $template = [];
+        foreach ($columns as $column) {
+            $template[$column->id] = $column->cast(null);
+        }
+
+        foreach ($numerator->generate() as $_id) {
+            $row = $template;
+            $row['_id'] = $_id;
+            $this->newRows[$_id] = $row;
+        }
+    }
+
+    public function addFilledRows($csvArray)
+    {
+        /** @var Column[] $columns */
+        $columns = array_values($this->fetchAndTransformColumns());
+
+        $rows = self::csvToRowsArray($columns, $csvArray, false)
+            ->keyBy('_id');
+
+        $this->newRows = array_merge($this->newRows, $rows->all());
+    }
+}

app/Tables/Column.php

@@ -1,32 +1,105 @@
 <?php
 
-
 namespace App\Tables;
 
-
+use Illuminate\Contracts\Support\Arrayable;
 use JsonSerializable;
+use MightyPork\Exceptions\SimpleValidationException;
 use MightyPork\Exceptions\NotApplicableException;
 use MightyPork\Utils\Utils;
 
 /**
  * Helper class representing one column in a data table.
  *
- * @property-read string $id
- * @property-read string $type
- * @property-read string $name
- * @property-read string $title
+ * @property-read bool $id_modified
+ * @property-read bool $type_modified
+ * @property-read bool $name_modified
+ * @property-read bool $title_modified
+ *
+ * @property-read string $id_orig
+ * @property-read string $type_orig
+ * @property-read string $name_orig
+ * @property-read string $title_orig
  */
-class Column implements JsonSerializable
+class Column implements JsonSerializable, Arrayable
 {
     const colTypes = [
         'int', 'bool', 'float', 'string'
     ];
 
-    private $id;
-    private $type;
-    private $name;
-    private $title;
+    // marked public to make keyBy() work
 
+    public $id;
+    public $type;
+    public $name;
+    public $title;
+
+    /** @var bool - column is marked to be deleted by a changeset */
+    public $toRemove = false;
+
+    /** @var bool - column is new in this changeset */
+    public $isNew = false;
+
+    /** @var array - original attrib values if edited in a changeset */
+    private $orig_attribs = [];
+    /** @var array - list of attrib names that are modified by a changeset */
+    private $modified_attribs = [];
+
+    /**
+     * Mark for removal (used in editing GUI)
+     */
+    public function markForRemoval()
+    {
+        $this->toRemove = true;
+    }
+
+    /**
+     * Mark this column as new
+     */
+    public function markAsNew()
+    {
+        $this->isNew = true;
+    }
+
+    /**
+     * Modify by a changeset
+     *
+     * @param array $columnObject
+     */
+    public function modifyByChangeset(array $columnObject)
+    {
+        foreach ((array)$columnObject as $key => $value) {
+            if ($value != $this->$key) {
+                $this->modified_attribs[] = $key;
+                $this->orig_attribs[] = $this->$key;
+                $this->$key = $value;
+            }
+        }
+    }
+
+    public function __get($name)
+    {
+        if (ends_with($name, '_modified')) {
+            $basename = str_replace('_modified', '', $name);
+            if (property_exists($this, $basename)) {
+                return in_array($basename, $this->modified_attribs);
+            }
+        }
+
+        if (ends_with($name, '_orig')) {
+            $basename = str_replace('_orig', '', $name);
+            if (property_exists($this, $basename)) {
+                return $this->orig_attribs[$basename];
+            }
+        }
+
+        throw new NotApplicableException("No such column property: $name");
+    }
+
+    /**
+     * @param $columns
+     * @return Column[]
+     */
     public static function columnsFromJson($columns)
     {
         if (is_string($columns)) {
@@ -56,7 +129,7 @@ class Column implements JsonSerializable
     {
         $b = objBag($obj);
 
-        if (!$b->has('name')) throw new NotApplicableException('Missing name in column');
+        if (!$b->has('name') || $b->name == '') throw new NotApplicableException('Missing name in column');
         if (!$b->has('type')) throw new NotApplicableException('Missing type in column');
 
         if ($b->name[0] == '_') { // global row ID
@@ -73,15 +146,6 @@ class Column implements JsonSerializable
         $this->title = $b->title ?: $b->name;
     }
 
-    public function __get($name)
-    {
-        if (property_exists($this, $name)) {
-            return $this->$name;
-        }
-
-        throw new NotApplicableException("No such column property: $name");
-    }
-
     /**
      * @return array with keys {name, title, type}
      */
@@ -105,20 +169,24 @@ class Column implements JsonSerializable
     {
         switch ($this->type) {
             case 'int':
+                if (is_null($value)) return 0;
                 if (is_int($value)) return $value;
                 if (is_float($value)) return round($value);
                 if (is_numeric($value)) return intval($value);
-                throw new NotApplicableException("Could not convert value \"$value\" to int!");
+                throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to int!");
 
             case 'float':
+                if (is_null($value)) return 0;
                 if (is_int($value) || is_float($value)) return (float)$value;
                 if (is_numeric($value)) return floatval($value);
-                throw new NotApplicableException("Could not convert value \"$value\" to float!");
+                throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to float!");
 
             case 'bool':
+                if (is_null($value)) return false;
                 return Utils::parseBool($value);
 
             case 'string':
+                if (is_null($value)) return "";
                 return "$value";
 
             default:

app/Tables/ColumnNumerator.php

@@ -17,36 +17,23 @@ use MightyPork\Utils\Utils;
  * Thanks to this uniqueness, it could even be possible to compare or merge forks
  * of the same table.
  */
-class ColumnNumerator
+class ColumnNumerator extends BaseNumerator
 {
     const ALPHABET = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
                      'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'];
 
-    /** @var int */
-    private $next;
-    /** @var int */
-    private $last;
-
     /**
      * Create numerator for the given number of columns
+     *
+     * @param int $capacity - how many
      */
     public function __construct($capacity)
     {
-        list($this->next, $this->last) = Row::allocateColIDs($capacity);
+        parent::__construct(Row::allocateColIDs($capacity));
     }
 
-    /**
-     * Get next column name, incrementing the internal state
-     *
-     * @return string
-     */
-    public function next()
+    protected function getKey($index)
     {
-        if ($this->next > $this->last)
-            throw new \OutOfBoundsException("Column numerator has run out of allocated GCID slots");
-
-        $key = Utils::alphabetEncode($this->next, self::ALPHABET);
-        $this->next++;
-        return $key;
+        return Utils::alphabetEncode($index, self::ALPHABET);
     }
 }

app/Tables/RowNumerator.php

@@ -5,28 +5,15 @@ namespace App\Tables;
 
 use App\Models\Row;
 
-class RowNumerator
+class RowNumerator extends BaseNumerator
 {
-    /** @var int */
-    private $next;
-    /** @var int */
-    private $last;
-
     /**
      * Create a numerator for the given number of rows.
      *
-     * @param int $capacity
+     * @param int $capacity - how many
      */
     public function __construct($capacity)
     {
-        list($this->next, $this->last) = Row::allocateRowIDs($capacity);
-    }
-
-    public function next()
-    {
-        if ($this->next > $this->last)
-            throw new \OutOfBoundsException("Row numerator has run out of allocated GRID slots");
-
-        return $this->next++;
+        parent::__construct(Row::allocateRowIDs($capacity));
     }
 }

app/helpers.php

@@ -88,10 +88,45 @@ function old_json($name, $default) {
 }
 
 // Safe JSON funcs
-function toJSON($object) {
+function toJSON($object, $emptyObj=false) {
+    if ($emptyObj) {
+        if ((empty($object) || ($object instanceof \Illuminate\Support\Collection && $object->count()==0))) {
+            return '{}';
+        }
+    }
+
+    if (!$object instanceof JsonSerializable && $object instanceof \Illuminate\Contracts\Support\Arrayable) {
+        $object = $object->toArray();
+    }
+
     return \GuzzleHttp\json_encode($object, JSON_UNESCAPED_SLASHES + JSON_UNESCAPED_UNICODE);
 }
 
 function fromJSON($object, $assoc=false) {
     return \GuzzleHttp\json_decode($object, $assoc);
 }
+
+/**
+ * @param \Illuminate\Support\Collection|array $items
+ * @param int $per_page
+ * @param null|\Closure $mapFn
+ * @return \Illuminate\Pagination\LengthAwarePaginator
+ */
+function collection_paginate($items, $per_page, $mapFn = null)
+{
+    if (!$items instanceof \Illuminate\Support\Collection) {
+        $items = collect($items);
+    }
+
+    $page = Request::get('page', 1);
+
+    $pageItems = $items->forPage($page, $per_page)->values();
+
+    return new Illuminate\Pagination\LengthAwarePaginator(
+        $mapFn ? $pageItems->map($mapFn) : $pageItems,
+        $items->count(),
+        $per_page,
+        Illuminate\Pagination\Paginator::resolveCurrentPage(),
+        ['path' => Illuminate\Pagination\Paginator::resolveCurrentPath()]
+    );
+}