datatable.directory/app/Tables/Changeset.php

<?php


namespace App\Tables;

use App\Models\Revision;
use App\Models\Row;
use App\Models\Table;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use MightyPork\Exceptions\NotApplicableException;
use MightyPork\Exceptions\NotExistException;
use ReflectionClass;

/**
 * Object representing a set of table modifications
 */
class Changeset
{
    use SerializesModels;

    const object_excluded_properties = [
        'revision',
        'table',
        'note',
    ];

    /**
     * @var Revision - base revision this changeset belongs to
     */
    public $revision;

    /**
     * @var Table - table this changeset belongs to
     */
    public $table;

    /**
     * @var string - user's note attached to this changeset (future proposal)
     */
    public $note = '';

    /**
     * Rows whose content changed, identified by _id.
     * Only changed values are to be filled. Columns are identified by GCIDs
     *
     * Key'd by _id
     *
     * @var array|null - [_id -> [_id:…, cid:…, cid:…], …]
     */
    public $rowUpdates = [];

    /**
     * New rows in the full Row::data format, including GRIDs.
     * Values are identified by GCIDs from previously defined, or new columns.
     *
     * @var array|null - [_id -> [_id:…, cid:…, cid:…], …]
     */
    public $newRows = [];

    /**
     * Rows to be removed
     *
     * @var int[]|null - GRIDs
     */
    public $removedRows = [];

    /**
     * Values changed in column specifications, such as name, title, etc.
     * This does not affect the table rows in any way.
     *
     * Key'd by id
     *
     * @var array[] - column specification objects, with GCIDs, key'd by CID
     */
    public $columnUpdates = [];

    /**
     * New columns in the full format, including GCIDs
     *
     * @var array|null - [id -> [id:…, …], …]
     */
    public $newColumns = [];

    /**
     * When reordering columns, here is the column IDs array
     * in the new order. Columns meanwhile removed from the table
     * or added to it are to be ignored or appended at the end.
     *
     * This shall not be filled if merely editing or adding columns,
     * unless the order is explicitly adjusted by the user.
     *
     * @var string[]|null - GCIDs
     */
    public $columnOrder = [];

    /**
     * Columns to be removed
     *
     * The data associated to those columns may or may not be removed from the Rows.
     * It does not matter, other than in regard to the table size.
     *
     * @var int[]|null - GCIDs
     */
    public $removedColumns = [];

    /**
     * Draft row numerator state holder; numbers grow to negative,
     * and are replaced with real unique row IDs when the proposal is submitted.
     *
     * @var int
     */
    public $nextRowID = -1;

    /**
     * Generator iterating all properties, used internally for serialization to array
     *
     * @return \Generator
     * @throws \ReflectionException
     */
    private function walkProps()
    {
        $properties = (new ReflectionClass($this))->getProperties();

        foreach ($properties as $property) {
            if (in_array($property->name, self::object_excluded_properties)) {
                continue;
            }

            yield $property->name;
        }
    }

    /**
     * Reconstruct from a object, such as one found in Proposal.
     * Note that the fromProposal() method should be used when the
     * proposal is available, as it populates additional fields.
     *
     * @param \stdClass $object
     * @return Changeset
     */
    public static function fromObject($object)
    {
        $object = (array)$object;
        $changeset = new Changeset();

        foreach ($changeset->walkProps() as $prop) {
            if (isset($object[$prop])) {
                $changeset->$prop = $object[$prop];
            }
        }

        return $changeset;
    }

    /**
     * Serialize to an object format that may be stored in a Proposal.
     *
     * @return \stdClass
     */
    public function toObject()
    {
        $object = new \stdClass();

        foreach ($this->walkProps() as $prop) {
            $object->$prop = $this->$prop;
        }

        return $object;
    }

    /**
     * Decorate / transform a single row for the editor view
     *
     * @param object|\DecoratedRow $row - row, must be key'd by column ids
     * @param bool $decorate - to add extra underscored info for the editor
     * @return \DecoratedRow|object|null - null if not decorating and the row was removed
     */
    public function transformRow($row, $decorate)
    {
        if ($row instanceof Row) $row = (object)$row->getAttributes(); // row must be in the rowData() format

        if ($decorate) {
            $row->_remove = false;
            $row->_changed = [];
            $row->_orig = [];
        }

        if ($decorate) {
            $row->_orig = array_diff((array)$row, []);
            // remove junk
            unset($row->_orig['_id']);
            unset($row->_orig['_new']);
            unset($row->_orig['_remove']);
            unset($row->_orig['_changed']);
            unset($row->_orig['_orig']);
        }

        if ($this->isNewRow($row->_id)) {
            if ($decorate) {
                $row->_new = true;
            }
            return $row;
        }

        // Removed rows - return as null
        if (in_array($row->_id, $this->removedRows)) {
            if ($decorate) {
                $row->_remove = true;
            } else {
                return null;
            }
        }

        // if marked for removal, hide changes
        if (!isset($row->_remove) || !$row->_remove) {
            // Changed values
            if (isset($this->rowUpdates[$row->_id])) {
                $newVals = $this->rowUpdates[$row->_id];

                if ($decorate) {
                    $row->_changed = array_keys($newVals);
                    $row->_orig = array_diff((array)$row, []);
                    unset($row->_orig['_id']);
                    unset($row->_orig['_new']);
                    unset($row->_orig['_remove']);
                    unset($row->_orig['_changed']);
                    unset($row->_orig['_orig']);
                }

                $row = (object)array_merge((array)$row, $newVals);
            }
        }

        // Drop deleted columns
        if (!$decorate) {
            foreach ($this->removedColumns as $colId) {
                unset($row->$colId);
                unset($row->_orig[$colId]);
            }
        }

        // move junk left over from the select
        unset($row->_row_pivot);

        if ($decorate) {
            $row->_loadvals = array_diff((array)$row, []);
            // remove junk
            unset($row->_loadvals['_id']);
            unset($row->_loadvals['_new']);
            unset($row->_loadvals['_remove']);
            unset($row->_loadvals['_changed']);
            unset($row->_loadvals['_orig']);
        }

        return $row;
    }

    /**
     * Decorate / transform columns (loaded from the source revision)
     *
     * @return Column[]
     */
    public function fetchAndTransformColumns()
    {
        /** @var Column[] - loaded and transformed columns, cached from previous call to transformColumns() */
        static $cachedColumns = [];

        if ($cachedColumns) return $cachedColumns;
        $columns = Column::columnsFromJson($this->revision->columns);

        // Modify columns
        foreach ($columns as $column) {
            if (isset($this->columnUpdates[$column->id])) {
                $column->modifyByChangeset($this->columnUpdates[$column->id]);
            }

            if (in_array($column->id, $this->removedColumns)) {
                $column->markForRemoval();
            }
        }

        // Append new columns
        foreach ($this->newColumns as $newColumn) {
            $columns[] = $c = new Column($newColumn);
            $c->markAsNew();
        }

        // Reorder
        $colsById = collect($columns)->keyBy('id')->all();
        $newOrder = [];
        foreach ($this->columnOrder as $id) {
            if (isset($colsById[$id])) {
                $newOrder[] = $colsById[$id];
            }
        }

        $leftover_keys = array_diff(array_keys($colsById), $this->columnOrder);
        foreach ($leftover_keys as $id) {
            $newOrder[] = $colsById[$id];
        }

        return $cachedColumns = $newOrder;
    }

    /**
     * Retrieve a column by ID (even new columns)
     *
     * @param string $id
     * @return Column
     */
    public function fetchColumn(string $id)
    {
        if ($this->isNewColumn($id)) {
            $c = new Column($this->newColumns[$id]);
            $c->markAsNew();
            return $c;
        } else {
            $columns = collect($this->revision->columns)->keyBy('id');
            return new Column($columns[$id]);
        }
    }

    /**
     * Retrieve a column and modify it by the Changeset.
     *
     * @param string $id - column ID
     * @return Column
     */
    public function fetchAndTransformColumn(string $id)
    {
        $column = $this->fetchColumn($id);

        if (isset($this->columnUpdates[$column->id])) {
            $column->modifyByChangeset($this->columnUpdates[$column->id]);
        }

        if (in_array($column->id, $this->removedColumns)) {
            $column->markForRemoval();
        }

        return $column;
    }

    /**
     * Mark a row for removal
     *
     * @param int $id - row ID (_id)
     */
    public function rowRemove(int $id)
    {
        if ($this->isNewRow($id)) {
            unset($this->newRows[$id]);
        }
        else {
            $this->removedRows[] = $id;
        }
    }

    /**
     * Undo row removal
     *
     * @param int $id - row ID (_id)
     */
    public function rowRestore(int $id)
    {
        $this->removedRows = array_diff($this->removedRows, [$id]);
    }

    /**
     * Test if a row is new
     *
     * @param int $id - row ID (_id)
     * @return bool - is new
     */
    public function isNewRow(int $id)
    {
        return isset($this->newRows[$id]);
    }

    /**
     * Test if a column is new
     *
     * @param string $id - column ID
     * @return bool - is new
     */
    public function isNewColumn(string $id)
    {
        return isset($this->newColumns[$id]);
    }

    /**
     * Retrieve a single row and transform it by the Changeset
     *
     * @param int $id - row ID (_id)
     * @return \DecoratedRow|object
     */
    public function fetchAndTransformRow(int $id)
    {
        $r = $this->fetchRow($id);
        $transformed = $this->transformRow($r, true);
        return $transformed;
    }

    /**
     * Fetch columns from DB (not including new columns)
     *
     * @return Column[]
     */
    public function fetchRevisionColumns()
    {
        return Column::columnsFromJson($this->revision->columns);
    }

    /**
     * Fetch an existing row from DB, or a new row.
     *
     * @param $id
     * @return object
     */
    public function fetchRow(int $id)
    {
        if ($this->isNewRow($id)) {
            $nr = (object)$this->newRows[$id];
            $nr->_new = true;
            return $nr;
        }

        $r = $this->revision->rowsData($this->fetchRevisionColumns(), true, false)
            ->whereRaw("data->>'_id' = ?", $id)->first();

        if (!$r) throw new NotExistException("No such row _id = $id in this revision.");

        // remove junk
        unset($r->pivot_revision_id);
        unset($r->pivot_row_id);

        return (object)$r->getAttributes();
    }

    /**
     * Apply a row update, adding the row to the list of changes, or removing it
     * if all differences were undone.
     *
     * @param array|object $newVals - values of the new row
     * @return object - updated column
     */
    public function rowUpdate($newVals)
    {
        $newVals = (object)$newVals;

        $_id = $newVals->_id;
        $origRow = $this->fetchRow($_id);

        /** @var Column[]|Collection $cols */
        $cols = collect($this->fetchAndTransformColumns())->keyBy('id');

        $updateObj = [];

        foreach ($newVals as $colId => $value) {
            if (starts_with($colId, '_')) continue; // internals

            $col = $cols[$colId];
            $value = $col->cast($value);
            $origValue = isset($origRow->$colId) ? $origRow->$colId : null;
            $origValueCast = $col->cast($origValue);

            if ($value !== $origValueCast && !($origValue===null&&$value==="")) { // fix for null in numeric cols being forcibly populated and sticking as change
                $updateObj[$colId] = $value;
            }
        }

        if ($this->isNewRow($_id)) {
            $this->newRows[$_id] = array_merge($this->newRows[$_id], $updateObj);
        }
        else {
            if (!empty($updateObj)) {
                $this->rowUpdates[$_id] = $updateObj;
            } else {
                // remove possible old update record for this row, if nothing changes now
                unset($this->rowUpdates[$_id]);
            }
        }

        return $this->fetchAndTransformRow($_id);
    }

    /**
     * Update a column specification
     *
     * @param object $newVals - new values for the column
     * @return Column - the column, modified and decorated with _orig etc
     */
    public function columnUpdate($newVals)
    {
        $id = $newVals->id;
        $col = $this->fetchColumn($id);

        $updateObj = [];
        foreach ($newVals as $field => $value) {
            if (starts_with($field, '_')) continue; // internals

            if ($value !== $col->$field) {
                $updateObj[$field] = $value;
            }
        }

        // try creating a column with the new data
        new Column(array_merge($col->toArray(), $updateObj));

        if ($this->isNewColumn($id)) {
            $this->newColumns[$id] = array_merge($this->newColumns[$id], $updateObj);
        }
        else {
            if (!empty($updateObj)) {
                $this->columnUpdates[$id] = $updateObj;
            } else {
                // remove possible old update record for this row, if nothing changes now
                unset($this->columnUpdates[$id]);
            }
        }

        return $this->fetchAndTransformColumn($id);
    }

    /**
     * Mark a column for removal, or forget it if it's a new column
     *
     * @param string $id - column ID
     */
    public function columnRemove(string $id)
    {
        if ($this->isNewColumn($id)) {
            unset($this->newColumns[$id]);
            // remove it from order
            $this->columnOrder = array_values(array_diff($this->columnOrder, [$id]));
        }
        else {
            $this->removedColumns[] = $id;
        }

        $this->clearColumnOrderIfUnchanged();
    }

    /**
     * Restore a column previously marked for removal
     *
     * @param string $id - column ID
     */
    public function columnRestore(string $id)
    {
        $this->removedColumns = array_diff($this->removedColumns, [$id]);
    }

    /**
     * Get a page of new rows, for display in the editor
     *
     * @param int $perPage
     * @return \Illuminate\Pagination\LengthAwarePaginator|Collection|array
     */
    public function getAddedRows($perPage = 25)
    {
        return collection_paginate($this->newRows, $perPage);
    }

    /**
     * Convert a raw CSV array to a rows array, validating and casting the data.
     * All created rows are assigned globally unique IDs
     *
     * @param Column[] $columns - ordered array of columns to collect from all CSV rows
     * @param array $csvArray - CSV parsed to 2D array
     * @param bool $forTableInsert - if true, row data will be wrapped in ['data'=>...]
     *                               so it can be inserted directly into DB
     * @return Collection
     */
    public function csvToRowsArray($columns, $csvArray, $forTableInsert, $useDraftIds)
    {
        /** @var Collection $rows */
        $rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) {
            if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
            if (count($row) != count($columns)) {
                throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
            }

            $data = [];

            foreach ($row as $i => $val) {
                $col = $columns[$i];

                if (strlen($val) > 1000) {
                    // try to stop people inserting unstructured crap / malformed CSV
                    throw new NotApplicableException("Value for column {$col->name} too long.");
                }
                $data[$col->id] = $col->cast(trim($val));
            }

            if ($forTableInsert) {
                return ['data' => $data];
            } else {
                return $data;
            }
        })->filter();

        if ($useDraftIds) {
            $rowNumerator = new DraftRowNumerator($this, count($csvArray));
        } else {
            $rowNumerator = new RowNumerator(count($csvArray));
        }

        if ($forTableInsert) {
            return $rows->map(function ($row) use (&$rowNumerator) {
                $row['data']['_id'] = $rowNumerator->next();
                return $row;
            });
        }
        else {
            return $rows->map(function ($row) use (&$rowNumerator) {
                $row['_id'] = $rowNumerator->next();
                return $row;
            });
        }
    }

    /**
     * Append N blank rows
     *
     * @param int $count
     */
    public function addBlankRows(int $count)
    {
        $columns = $this->fetchAndTransformColumns();
        $template = [];
        foreach ($columns as $column) {
            $template[$column->id] = $column->cast(null);
        }

        $numerator = new DraftRowNumerator($this, $count);

        foreach ($numerator->generate() as $_id) {
            $row = $template;
            $row['_id'] = $_id;
            $this->newRows[$_id] = $row;
        }
    }

    /**
     * Add rows from imported CSV
     *
     * @param array $csvArray - CSV parsed to 2D array
     */
    public function addFilledRows($csvArray)
    {
        $columns = collect($this->fetchAndTransformColumns())->where('toRemove', false);

        /** @var Column[] $columns */
        $columns = $columns->values()->all();

        $rows = self::csvToRowsArray($columns, $csvArray, false, true)
            ->keyBy('_id');

        // using '+' to avoid renumbering
        $this->newRows = $this->newRows + $rows->toArray();
    }

    /**
     * Add a blank column (pre-filled with dummies to satisfy validation)
     *
     * @return array - column array
     */
    public function addBlankCol()
    {
        $cid = (new ColumnNumerator(1))->next();

        $allCols = $this->fetchAndTransformColumns();
        $num = count($allCols) + 1;

        $col = [
            'name' => "col_{$num}",
            'type' => "string",
            'title' => "Column {$num}",
            'id' => $cid,
            '_new' => true,
        ];

        $this->newColumns[$cid] = $col;
        return $col;
    }

    /**
     * Set a new column order
     *
     * @param array $order - array of column IDs
     */
    public function setColOrder(array $order)
    {
        $allCols = $this->fetchAndTransformColumns();
        $ids = collect($allCols)->pluck('id')->all();
        $order = array_intersect($order, $ids);
        $missing = array_diff($ids, $order);

        $this->columnOrder = array_values(array_merge($order, $missing));
        $this->clearColumnOrderIfUnchanged();
    }

    /**
     * Remove added rows that are not filled (user added too many with the Add Rows form)
     */
    public function removeEmptyNewRows()
    {
        $cols = $this->fetchRevisionColumns();
        $emptyTpl = collect($cols)->keyBy('id')->map(function(Column $c) {
            return $c->cast(null);
        })->all();

        foreach ($this->newColumns as $k => $obj) {
            $cols[] = $c = new Column($obj);
            $c->markAsNew();

            $emptyTpl[$k] = $c->cast(null);
        }

        $this->newRows = array_filter($this->newRows, function ($r) use ($emptyTpl) {
            foreach ($r as $k => $val) {
                if ($k[0] == '_') continue;
                if ($val != $emptyTpl[$k]) return true;
            }
            return false;
        });
    }

    /**
     * Discard the column order array if it's identical to the natural / default
     * ordering. This simplifies the Changeset and reduces possible conflicts
     */
    public function clearColumnOrderIfUnchanged()
    {
        $expected = collect($this->revision->columns)
            ->pluck('id')
            ->diff($this->removedColumns)
            ->merge(collect($this->newColumns)->pluck('id'))
            ->values()->all();

        $this->columnOrder = array_values($this->columnOrder);

        if ($expected == $this->columnOrder) {
            $this->columnOrder = [];
        }
    }

    /**
     * Clear the custom column order
     */
    public function resetColumnOrder()
    {
        $this->columnOrder = [];
    }

    /**
     * Restore all columns marked for removal
     */
    public function resetRemovedColumns()
    {
        $this->removedColumns = [];
    }

    /**
     * Discard all added columns
     */
    public function resetAddedColumns()
    {
        $this->columnOrder = array_values(
            array_diff($this->columnOrder,
                collect($this->newColumns)->pluck('id')->all()
            )
        );

        $this->newColumns = [];
        $this->clearColumnOrderIfUnchanged();
    }

    /**
     * Discard all column changes
     */
    public function resetUpdatedColumns()
    {
        $this->columnUpdates = [];
    }

    /**
     * Restore all rows marked for removal
     */
    public function resetRemovedRows()
    {
        $this->removedRows = [];
    }

    /**
     * Discard all added rows
     */
    public function resetAddedRows()
    {
        $this->newRows = [];
    }

    /**
     * Discard all row changes
     */
    public function resetUpdatedRows()
    {
        $this->rowUpdates = [];
    }

    /**
     * Get row change counts (used for the view)
     *
     * @return object
     */
    public function getRowChangeCounts()
    {
        $numChangedRows = count($this->rowUpdates);
        $numNewRows = count($this->newRows);
        $numRemovedRows = count($this->removedRows);

        return (object) [
            'changed' => $numChangedRows,
            'new' => $numNewRows,
            'removed' => $numRemovedRows,
            'any' => $numChangedRows || $numNewRows || $numRemovedRows,
        ];
    }

    /**
     * Get column change counts (used for the view)
     *
     * @return object
     */
    public function getColumnChangeCounts()
    {
        $numChangedColumns = count($this->columnUpdates);
        $numNewColumns = count($this->newColumns);
        $numRemovedColumns = count($this->removedColumns);
        $reordered = count($this->columnOrder) != 0;

        return (object) [
            'changed' => $numChangedColumns,
            'new' => $numNewColumns,
            'removed' => $numRemovedColumns,
            'reordered' => $reordered,
            'any' => $reordered || $numChangedColumns || $numNewColumns || $numRemovedColumns,
        ];
    }

    /**
     * Check if there is any change in this changeset
     *
     * @return bool - any found
     */
    public function hasAnyChanges()
    {
        $colChanges = $this->getColumnChangeCounts();
        $rowChanges = $this->getRowChangeCounts();

        return $colChanges->any || $rowChanges->any;
    }

    /**
     * Replace temporary negative row IDs with real unique row IDs
     */
    public function renumberRows()
    {
        if (count($this->newRows) == 0) return;

        $rows = [];
        $numerator = new RowNumerator(count($this->newRows));
        foreach ($this->newRows as $row) {
            if ($row['_id'] < 0) {
                $id = $numerator->next();
                $row['_id'] = $id;
                $rows[$id] = $row;
            } else {
                // keep ID
                $rows[$row['_id']] = $row;
            }
        }
        $this->newRows = $rows;
    }
}