MightyPork/datatable.directory
1
0
Fork 1
Code Issues 25 Pull Requests Releases Wiki Activity

add column unique numbering scheme, more efficient selects

Browse Source
This commit is contained in:
MightyPork
2018-08-04 19:57:59 +02:00
parent 69bda25bbb
commit fabc3ad24e
23 changed files with 452 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/ConfirmUser.php
-2
View File
@@ -41,7 +41,5 @@ class ConfirmUser extends Command
$u = User::resolve($this->argument('user'));
$u->update(['confirmed' => true]);
$this->info("User #$u->id with e-mail $u->email and handle @$u->name was confirmed.");
dd($u);
}
}
app/Http/Controllers/TableController.php
+79 -43
View File
@@ -3,15 +3,16 @@
namespace App\Http\Controllers;
use App\Models\Revision;
use App\Models\Row;
use App\Models\Table;
use App\Models\User;
use App\Tables\Column;
use App\Tables\ColumnNumerator;
use App\Tables\CStructArrayExporter;
use App\Tables\CsvExporter;
use App\Tables\CXMacroExporter;
use App\Tables\JsonExporter;
use App\Tables\PhpExporter;
use App\Tables\RowNumerator;
use Illuminate\Http\Request;
use Illuminate\Validation\Rule;
use MightyPork\Exceptions\NotApplicableException;
@@ -41,11 +42,15 @@ class TableController extends Controller
$this->countTableVisit($request, $tableModel);
$columns = Column::columnsFromJson($revision->columns);
$rows = $revision->rowsData($columns)->paginate(25, []);
return view('table.view', [
'table' => $tableModel,
'revision' => $revision,
'columns' => Column::columnsFromJson($revision->columns),
'rows' => $revision->rows()->paginate(25),
'columns' => $columns,
'rows' => $rows,
]);
}
@@ -78,6 +83,7 @@ class TableController extends Controller
"Patella vulgata,common limpet,20";
$columns = Column::columnsFromJson([
// fake 'id' to satisfy the check in Column constructor
['name' => 'latin', 'type' => 'string', 'title' => 'Latin Name'],
['name' => 'common', 'type' => 'string', 'title' => 'Common Name'],
['name' => 'lifespan', 'type' => 'int', 'title' => 'Lifespan (years)']
@@ -90,6 +96,9 @@ class TableController extends Controller
]);
}
/**
* Show table settings edit form
*/
public function settings(Request $request, User $user, string $table)
{
/** @var Table $tableModel */
@@ -102,6 +111,9 @@ class TableController extends Controller
]);
}
/**
* Store modified table settings
*/
public function storeSettings(Request $request, User $user, string $table)
{
/** @var Table $tableModel */
@@ -149,85 +161,100 @@ class TableController extends Controller
]);
// Check if table name is unique for user
if ($u->tables()->where('name', $input->name)->exists()) {
if ($u->hasTable($input->name)) {
return $this->backWithErrors([
'name' => "A table called \"$input->name\" already exists in your account.",
]);
}
// --- COLUMNS ---
// Parse and validate the columns specification
/** @var Column[] $columns */
$columns = [];
$column_keys = []; // for checking duplicates
$col_names = []; // for checking duplicates
$colsArray = fromJSON($input->columns);
// prevent griefing via long list of columns
if (count($colsArray) > 100) return $this->backWithErrors(['columns' => "Too many columns"]);
foreach ($colsArray as $col) {
if (!isset($col->name) || !isset($col->type) || empty($col->name) || empty($col->type)) {
return $this->backWithErrors(['columns' => "All columns must have at least name and type."]);
foreach ($colsArray as $colObj) {
// ensure column has a title
if (!isset($colObj->title)) {
$colObj->title = $colObj->name;
}
try {
if (in_array($col->name, $column_keys)) {
return $this->backWithErrors(['columns' => "Duplicate column: $col->name"]);
if (in_array($colObj->name, $col_names)) {
return $this->backWithErrors(['columns' => "Duplicate column: $colObj->name"]);
}
$column_keys[] = $col->name;
if (!isset($col->title)) $col->title = $col->name;
$columns[] = new Column($col);
$col_names[] = $colObj->name;
$columns[] = new Column($colObj);
} catch (\Exception $e) {
// validation errors from the Column constructor
return $this->backWithErrors(['columns' => $e->getMessage()]);
}
}
if (count($columns) == 0) return $this->backWithErrors(['columns' => "Define at least one column"]);
if (count($columns) == 0) {
return $this->backWithErrors(['columns' => "Define at least one column"]);
}
$rowTable = array_map('str_getcsv', explode("\n", $input->data));
// Now assign column IDs
$columnNumerator = new ColumnNumerator(count($columns));
foreach ($columns as $column) {
$column->setID($columnNumerator->next());
}
// --- DATA ---
$dataCsvLines = array_map('str_getcsv', explode("\n", $input->data));
// Preparing data to insert into the Rows table
$rowsData = null;
$rowsToInsert = null;
$rowNumerator = null;
try {
$grid_range = Row::allocateGRIDs(count($rowTable));
$grid_cnt = $grid_range[0];
$rowsData = array_map(function ($row) use ($columns, &$grid_cnt) {
if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null;
$rowsToInsert = collect($dataCsvLines)->map(function ($row) use ($columns) {
if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
if (count($row) != count($columns)) {
throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
}
$parsed = [
'_grid' => $grid_cnt++
];
$data = [];
foreach ($row as $i => $val) {
$key = $columns[$i]->name;
if (strlen($val) > 255) {
// try to stop people inserting unstructured crap / malformed CSV
throw new NotApplicableException("Value for column $key too long.");
}
$parsed[$key] = $columns[$i]->cast($val);
}
return [
'data' => $parsed,
];
}, $rowTable);
$col = $columns[$i];
$rowsData = array_filter($rowsData);
if (strlen($val) > 1000) {
// try to stop people inserting unstructured crap / malformed CSV
throw new NotApplicableException("Value for column {$col->name} too long.");
}
$data[$col->id] = $col->cast($val);
}
// return in a format prepared for filling eloquent
return ['data' => $data];
})->filter()->all(); // remove nulls, to array
// attach _id labels to all rows
$rowNumerator = new RowNumerator(count($dataCsvLines));
foreach ($rowsToInsert as &$item) {
$item['data']['_id'] = $rowNumerator->next();
}
} catch (\Exception $e) {
return $this->backWithErrors(['data' => $e->getMessage()]);
}
// --- STORE TO DB ---
/* Fields for the new Revision instance */
$revisionFields = [
'note' => "Initial revision of table $u->name/$input->name",
'columns' => $columns,
'row_count' => count($rowsData),
'row_count' => count($rowsToInsert),
];
/* Fields for the new Table instance */
$tableFields = [
'owner_id' => $u->id,
'revision_id' => 0,
'name' => $input->name,
'title' => $input->title,
'description' => $input->description,
@@ -235,10 +262,12 @@ class TableController extends Controller
'origin' => $input->origin,
];
\DB::transaction(function () use ($revisionFields, $tableFields, $rowsData) {
/** @var Table $table */
$table = null;
\DB::transaction(function () use ($revisionFields, $tableFields, $rowsToInsert, &$table) {
$revision = Revision::create($revisionFields);
$tableFields['revision_id'] = $revision->id; // to meet the not-null constraint
$tableFields['revision_id'] = $revision->id; // current revision (not-null constraint on this FK)
$table = Table::create($tableFields);
// Attach the revision to the table, set as current
@@ -246,10 +275,10 @@ class TableController extends Controller
$table->revision()->associate($revision);
// Spawn rows, linked to the revision
$revision->rows()->createMany($rowsData);
$revision->rows()->createMany($rowsToInsert);
});
return redirect(route('table.view', ['user' => $u, 'table' => $input->name]));
return redirect($table->viewRoute);
}
/**
@@ -260,7 +289,7 @@ class TableController extends Controller
*/
private function countTableVisit(Request $request, Table $table)
{
$cookieName = "view_$table->id";
$cookieName = "view_{$table->owner->name}_{$table->name}";
if (!$request->cookie($cookieName, false)) {
$ua = $request->userAgent();
// Filter out suspicious user agents
@@ -271,6 +300,13 @@ class TableController extends Controller
}
}
/**
* Simple export via a preset
*
* @param Request $request
* @param User $user
* @param string $table
*/
public function export(Request $request, User $user, string $table)
{
/** @var Table $tableModel */
app/Models/BaseModel.php
+2 -2
View File
@@ -10,7 +10,7 @@ class BaseModel extends Model
{
public function getAttribute($key)
{
if (! $key) {
if ($this->exists && ! $key) {
throw new \LogicException("No attribute ".var_export($key, true));
}
@@ -25,7 +25,7 @@ class BaseModel extends Model
*/
public function getRelationValue($key)
{
if (!method_exists($this, $key)) {
if ($this->exists && !method_exists($this, $key)) {
throw new \LogicException("No attribute or relation ".var_export($key, true));
}
app/Models/Revision.php
+17 -1
View File
@@ -2,6 +2,7 @@
namespace App\Models;
use App\Tables\Column;
use Illuminate\Support\Collection;
use Riesjart\Relaquent\Model\Concerns\HasRelaquentRelationships;
@@ -32,7 +33,22 @@ class Revision extends BaseModel
/** Included rows */
public function rows()
{
return $this->belongsToMany(Row::class, 'revision_row_pivot');
return $this->belongsToMany(Row::class, 'revision_row_pivot')->as('_row_pivot');
}
/** Included rows
* @param Column[] $columns
* @return \Illuminate\Database\Query\Builder|static
*/
public function rowsData($columns, $withId=true)
{
$selects = $withId ? ["data->>'_id' as _id"] : [];
foreach ($columns as $col) {
$selects[] = "data->>'$col->id' as $col->name";
}
return $this->rows()->select([])->selectRaw(implode(', ', $selects));
}
/** Proposal that lead to this revision */
app/Models/Row.php
+37 -10
View File
@@ -6,7 +6,7 @@ namespace App\Models;
* Row in a data table
*
* @property int $id
* @property string $data - JSONB, always containing _grid
* @property string $data - JSONB, always containing _id
*/
class Row extends BaseModel
{
@@ -17,12 +17,12 @@ class Row extends BaseModel
protected $guarded = [];
public $timestamps = false;
public function getGridAttribute() {
return $this->data->_grid;
public function getRowIdAttribute() {
return $this->data->_id;
}
public function setGridAttribute($value) {
$this->data->_grid = $value;
public function setRowIdAttribute($value) {
$this->data->_id = $value;
}
/**
@@ -34,21 +34,48 @@ class Row extends BaseModel
*
* @return int
*/
public static function allocateGRID()
public static function allocateRowID()
{
return \DB::selectOne("SELECT nextval('global_row_id_seq') AS grid;")->grid;
return \DB::selectOne("SELECT nextval('global_row_id_seq') AS rid;")->rid;
}
/**
* Allocate a block of Global Row IDs, application-unique.
*
* @see Row::allocateGRID()
* @see Row::allocateRowID()
*
* @return int[] first and last
*/
public static function allocateGRIDs($count)
public static function allocateRowIDs($count)
{
$last = \DB::selectOne("SELECT multi_nextval('global_row_id_seq', ?) AS last_grid;", [$count])->last_grid;
$last = \DB::selectOne("SELECT multi_nextval('global_row_id_seq', ?) AS last_id;", [$count])->last_id;
return [$last - $count + 1, $last];
}
/**
* Allocate a single Global Column ID, application-unique.
*
* GCIDs are used to uniquely identify existing or proposed new columns,
* and are preserved after column modifications, to ensure change proposals have
* a clear target. This is the column equivalent of GRID.
*
* @return int
*/
public static function allocateColID()
{
return \DB::selectOne("SELECT nextval('global_column_id_seq') AS cid;")->cid;
}
/**
* Allocate a block of Global Column IDs, application-unique.
*
* @see Row::allocateColID()
*
* @return int[] first and last
*/
public static function allocateColIDs($count)
{
$last = \DB::selectOne("SELECT multi_nextval('global_column_id_seq', ?) AS last_id;", [$count])->last_id;
return [$last - $count + 1, $last];
}
}
app/Models/User.php
+10
View File
@@ -226,4 +226,14 @@ class User extends BaseModel implements
$this->notify(new ConfirmEmail($newEmail, $confirmation->token));
}
/**
* Check if this user has a table with the givenname
*
* @param string $name
*/
public function hasTable(string $name)
{
$this->tables()->where('name', $name)->exists();
}
}
app/Tables/BaseExporter.php
+5 -6
View File
@@ -128,14 +128,13 @@ abstract class BaseExporter
$start = 0;
while ($start < $count) {
$rows = $revision->rows()->offset($start)->limit($chunkSize)->get();
// TODO raw query to allow selecting aggregates, column subsets, etc
$rows = $revision->rowsData($this->columns, false)
->offset($start)->limit($chunkSize)->get()->toArray();
foreach ($rows as $row) {
$data = $row->data;
// column renaming, value formatting...
yield $data;
unset($row['_row_pivot']);
yield $row;
}
$start += $chunkSize;
app/Tables/CStructArrayExporter.php
+1 -14
View File
@@ -99,20 +99,7 @@ class CStructArrayExporter extends BaseExporter
echo ",";
}
$val = 0;
switch ($field->type) {
case 'string':
$val = "";
break;
case 'bool':
$val = false;
break;
}
if (isset($row->{$field->name})) {
$val = $row->{$field->name};
}
$val = $row[$field->name];
// export to C format
switch ($field->type) {
app/Tables/Column.php
+32 -15
View File
@@ -11,9 +11,10 @@ use MightyPork\Utils\Utils;
/**
* Helper class representing one column in a data table.
*
* @property-read string $id
* @property-read string $type
* @property-read string $name
* @property-read string $title
* @property-read string $type
*/
class Column implements JsonSerializable
{
@@ -21,9 +22,10 @@ class Column implements JsonSerializable
'int', 'bool', 'float', 'string'
];
private $id;
private $type;
private $name;
private $title;
private $type;
public static function columnsFromJson($columns)
{
@@ -36,13 +38,13 @@ class Column implements JsonSerializable
}, $columns);
}
public function __get($name)
/**
* Set column ID
* @param string $id - GCID
*/
public function setID($id)
{
if (property_exists($this, $name)) {
return $this->$name;
}
throw new NotApplicableException("No such column property");
$this->id = $id;
}
/**
@@ -52,18 +54,32 @@ class Column implements JsonSerializable
*/
public function __construct($obj)
{
$b = new \objBag($obj);
$b = objBag($obj);
if (!$b->has('name')) throw new NotApplicableException('Missing name in column');
if (!$b->has('type')) throw new NotApplicableException('Missing type in column');
if ($b->name[0] == '_') { // global row ID
throw new NotApplicableException("Column name can't start with underscore.");
}
if (!in_array($b->type, self::colTypes)) {
throw new NotApplicableException("\"$b->type\" is not a valid column type.");
}
$this->id = $b->get('id', null);
$this->name = $b->name;
$this->title = $b->title;
$this->type = $b->type;
$this->title = $b->title ?: $b->name;
}
if ($this->name == '_grid') { // global row ID
throw new NotApplicableException("_grid is a reserved column name.");
public function __get($name)
{
if (property_exists($this, $name)) {
return $this->$name;
}
if (!in_array($this->type, self::colTypes)) {
throw new NotApplicableException("\"$this->type\" is not a valid column type.");
}
throw new NotApplicableException("No such column property: $name");
}
/**
@@ -72,6 +88,7 @@ class Column implements JsonSerializable
public function toArray()
{
return [
'id' => $this->id,
'name' => $this->name,
'title' => $this->title,
'type' => $this->type,
app/Tables/ColumnNumerator.php
+52
View File
@@ -0,0 +1,52 @@
<?php
namespace App\Tables;
use App\Models\Row;
use MightyPork\Utils\Utils;
/**
* Generates a sequence of string codes for internal naming of columns
*
* Produces database-unique lowercase identifiers for created or proposed columns.
* Column identifiers are used in row objects to uniquely identify data even if the
* revision header (columns object) is modified, such as reordering, changing name,
* adding new column in concurrent proposals, etc.
*
* Thanks to this uniqueness, it could even be possible to compare or merge forks
* of the same table.
*/
class ColumnNumerator
{
const ALPHABET = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'];
/** @var int */
private $next;
/** @var int */
private $last;
/**
* Create numerator for the given number of columns
*/
public function __construct($capacity)
{
list($this->next, $this->last) = Row::allocateColIDs($capacity);
}
/**
* Get next column name, incrementing the internal state
*
* @return string
*/
public function next()
{
if ($this->next > $this->last)
throw new \OutOfBoundsException("Column numerator has run out of allocated GCID slots");
$key = Utils::alphabetEncode($this->next, self::ALPHABET);
$this->next++;
return $key;
}
}
app/Tables/CsvExporter.php
+1 -9
View File
@@ -44,15 +44,7 @@ class CsvExporter extends BaseExporter
fputcsv($handle, $columnNames, $this->delimiter);
foreach ($this->iterateRows() as $row) {
$items = [];
foreach ($this->columns as $column) {
if (isset($row->{$column->name})) {
$items[] = $row->{$column->name};
} else {
$items[] = null;
}
}
fputcsv($handle, $items, $this->delimiter);
fputcsv($handle, array_values($row), $this->delimiter);
}
fclose($handle);
app/Tables/PhpExporter.php
+1 -1
View File
@@ -53,7 +53,7 @@ class PhpExporter extends BaseExporter
var_export($column->name);
echo ' => ';
var_export($row->{$column->name});
var_export($row[$column->name]);
}
echo ']';
app/Tables/RowNumerator.php
+32
View File
@@ -0,0 +1,32 @@
<?php
namespace App\Tables;
use App\Models\Row;
class RowNumerator
{
/** @var int */
private $next;
/** @var int */
private $last;
/**
* Create a numerator for the given number of rows.
*
* @param int $capacity
*/
public function __construct($capacity)
{
list($this->next, $this->last) = Row::allocateRowIDs($capacity);
}
public function next()
{
if ($this->next > $this->last)
throw new \OutOfBoundsException("Row numerator has run out of allocated GRID slots");
return $this->next++;
}
}