From c76fc218208c315c8fd964b89cef78ace590e9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hru=C5=A1ka?= Date: Sun, 12 Aug 2018 23:49:52 +0200 Subject: [PATCH] Brutal insert speed optimization by bypassing the ORM completely --- app/Http/Controllers/TableController.php | 28 +++++++++++++++++++----- app/Models/Proposal.php | 17 ++++++++++---- app/Tables/Changeset.php | 28 +++++++----------------- app/Tables/Column.php | 9 ++++---- 4 files changed, 48 insertions(+), 34 deletions(-) diff --git a/app/Http/Controllers/TableController.php b/app/Http/Controllers/TableController.php index 3a747ab..cf2e163 100644 --- a/app/Http/Controllers/TableController.php +++ b/app/Http/Controllers/TableController.php @@ -14,6 +14,7 @@ use App\Tables\CXMacroExporter; use App\Tables\JsonExporter; use App\Tables\PhpExporter; use App\Tables\RowNumerator; +use DB; use Illuminate\Http\Request; use Illuminate\Validation\Rule; use MightyPork\Exceptions\NotApplicableException; @@ -243,7 +244,7 @@ class TableController extends Controller $rowsToInsert = null; $rowNumerator = null; try { - $rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, true, false)->all(); + $rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, false)->all(); } catch (\Exception $e) { return $this->backWithErrors(['data' => $e->getMessage()]); } @@ -270,17 +271,32 @@ class TableController extends Controller /** @var Table $table */ $table = null; \DB::transaction(function () use ($revisionFields, $tableFields, $rowsToInsert, &$table) { - $revision = Revision::create($revisionFields); + $newRevision = Revision::create($revisionFields); - $tableFields['revision_id'] = $revision->id; // current revision (not-null constraint on this FK) + $tableFields['revision_id'] = $newRevision->id; // current revision (not-null constraint on this FK) $table = Table::create($tableFields); // Attach the revision to the table, set as current - $table->revisions()->attach($revision); - $table->revision()->associate($revision); + $table->revisions()->attach($newRevision); + $table->revision()->associate($newRevision); // Spawn rows, linked to the revision - $revision->rows()->createMany($rowsToInsert); + if (!empty($rowsToInsert)) { + // this replicates the code in Proposal->toRevision + $conn = DB::connection(); + $prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)'); + foreach (array_chunk($rowsToInsert, 10000) as $i => $chunk) { + $ids = []; + foreach ($chunk as $newRow) { + $prepared->execute([json_encode($newRow)]); + $ids[] = $conn->getPdo()->lastInsertId(); + } + + $qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ','); + $conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';', + $ids); + } + } }); return redirect($table->viewRoute); diff --git a/app/Models/Proposal.php b/app/Models/Proposal.php index 71eb2d5..7b30951 100644 --- a/app/Models/Proposal.php +++ b/app/Models/Proposal.php @@ -193,11 +193,20 @@ class Proposal extends BaseModel // --- Insert new rows --- if ($changeset->newRows) { - $newRowData = []; - foreach ($changeset->newRows as $newRow) { - $newRowData[] = new Row(['data' => $newRow]); + $conn = DB::connection(); + $prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)'); + + foreach (array_chunk($changeset->newRows, 10000) as $i => $chunk) { + $ids = []; + foreach ($chunk as $newRow) { + $prepared->execute([json_encode($newRow)]); + $ids[] = $conn->getPdo()->lastInsertId(); + } + + $qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ','); + $conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';', + $ids); } - $newRevision->rows()->saveMany($newRowData); } $newRevision->update(['row_count' => $newRevision->rows()->count()]); diff --git a/app/Tables/Changeset.php b/app/Tables/Changeset.php index 7227eea..864d1ef 100644 --- a/app/Tables/Changeset.php +++ b/app/Tables/Changeset.php @@ -573,10 +573,10 @@ class Changeset * so it can be inserted directly into DB * @return Collection */ - public function csvToRowsArray($columns, $csvArray, $forTableInsert, $useDraftIds) + public function csvToRowsArray($columns, $csvArray, $useDraftIds) { /** @var Collection $rows */ - $rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) { + $rows = collect($csvArray)->map(function ($row) use ($columns) { if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines if (count($row) != count($columns)) { throw new NotApplicableException("All rows must have " . count($columns) . " fields."); @@ -594,11 +594,7 @@ class Changeset $data[$col->id] = $col->cast(trim($val)); } - if ($forTableInsert) { - return ['data' => $data]; - } else { - return $data; - } + return $data; })->filter(); if ($useDraftIds) { @@ -607,18 +603,10 @@ class Changeset $rowNumerator = new RowNumerator(count($csvArray)); } - if ($forTableInsert) { - return $rows->map(function ($row) use (&$rowNumerator) { - $row['data']['_id'] = $rowNumerator->next(); - return $row; - }); - } - else { - return $rows->map(function ($row) use (&$rowNumerator) { - $row['_id'] = $rowNumerator->next(); - return $row; - }); - } + return $rows->map(function ($row) use (&$rowNumerator) { + $row['_id'] = $rowNumerator->next(); + return $row; + }); } /** @@ -655,7 +643,7 @@ class Changeset /** @var Column[] $columns */ $columns = $columns->values()->all(); - $rows = self::csvToRowsArray($columns, $csvArray, false, true) + $rows = self::csvToRowsArray($columns, $csvArray, true) ->keyBy('_id'); // using '+' to avoid renumbering diff --git a/app/Tables/Column.php b/app/Tables/Column.php index d48a339..9f4522a 100644 --- a/app/Tables/Column.php +++ b/app/Tables/Column.php @@ -187,20 +187,21 @@ class Column implements JsonSerializable, Arrayable { switch ($this->type) { case 'int': - if (is_null($value)) return 0; + if (is_null($value) || $value === "") return 0; if (is_int($value)) return $value; + if (is_bool($value)) return (int)$value; if (is_float($value)) return round($value); if (is_numeric($value)) return intval($value); throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to int!"); case 'float': - if (is_null($value)) return 0; - if (is_int($value) || is_float($value)) return (float)$value; + if (is_null($value) || $value === "") return 0; + if (is_int($value) || is_float($value) || is_bool($value)) return (float)$value; if (is_numeric($value)) return floatval($value); throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to float!"); case 'bool': - if (is_null($value)) return false; + if (is_null($value) || $value === "") return false; return Utils::parseBool($value); case 'string':