Brutal insert speed optimization by bypassing the ORM completely

master
Ondřej Hruška 6 years ago
parent 13b22f7cb4
commit c76fc21820
Signed by: MightyPork
GPG Key ID: 2C5FD5035250423D
  1. 28
      app/Http/Controllers/TableController.php
  2. 17
      app/Models/Proposal.php
  3. 18
      app/Tables/Changeset.php
  4. 9
      app/Tables/Column.php

@ -14,6 +14,7 @@ use App\Tables\CXMacroExporter;
use App\Tables\JsonExporter; use App\Tables\JsonExporter;
use App\Tables\PhpExporter; use App\Tables\PhpExporter;
use App\Tables\RowNumerator; use App\Tables\RowNumerator;
use DB;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Validation\Rule; use Illuminate\Validation\Rule;
use MightyPork\Exceptions\NotApplicableException; use MightyPork\Exceptions\NotApplicableException;
@ -243,7 +244,7 @@ class TableController extends Controller
$rowsToInsert = null; $rowsToInsert = null;
$rowNumerator = null; $rowNumerator = null;
try { try {
$rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, true, false)->all(); $rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, false)->all();
} catch (\Exception $e) { } catch (\Exception $e) {
return $this->backWithErrors(['data' => $e->getMessage()]); return $this->backWithErrors(['data' => $e->getMessage()]);
} }
@ -270,17 +271,32 @@ class TableController extends Controller
/** @var Table $table */ /** @var Table $table */
$table = null; $table = null;
\DB::transaction(function () use ($revisionFields, $tableFields, $rowsToInsert, &$table) { \DB::transaction(function () use ($revisionFields, $tableFields, $rowsToInsert, &$table) {
$revision = Revision::create($revisionFields); $newRevision = Revision::create($revisionFields);
$tableFields['revision_id'] = $revision->id; // current revision (not-null constraint on this FK) $tableFields['revision_id'] = $newRevision->id; // current revision (not-null constraint on this FK)
$table = Table::create($tableFields); $table = Table::create($tableFields);
// Attach the revision to the table, set as current // Attach the revision to the table, set as current
$table->revisions()->attach($revision); $table->revisions()->attach($newRevision);
$table->revision()->associate($revision); $table->revision()->associate($newRevision);
// Spawn rows, linked to the revision // Spawn rows, linked to the revision
$revision->rows()->createMany($rowsToInsert); if (!empty($rowsToInsert)) {
// this replicates the code in Proposal->toRevision
$conn = DB::connection();
$prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)');
foreach (array_chunk($rowsToInsert, 10000) as $i => $chunk) {
$ids = [];
foreach ($chunk as $newRow) {
$prepared->execute([json_encode($newRow)]);
$ids[] = $conn->getPdo()->lastInsertId();
}
$qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ',');
$conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';',
$ids);
}
}
}); });
return redirect($table->viewRoute); return redirect($table->viewRoute);

@ -193,11 +193,20 @@ class Proposal extends BaseModel
// --- Insert new rows --- // --- Insert new rows ---
if ($changeset->newRows) { if ($changeset->newRows) {
$newRowData = []; $conn = DB::connection();
foreach ($changeset->newRows as $newRow) { $prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)');
$newRowData[] = new Row(['data' => $newRow]);
foreach (array_chunk($changeset->newRows, 10000) as $i => $chunk) {
$ids = [];
foreach ($chunk as $newRow) {
$prepared->execute([json_encode($newRow)]);
$ids[] = $conn->getPdo()->lastInsertId();
}
$qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ',');
$conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';',
$ids);
} }
$newRevision->rows()->saveMany($newRowData);
} }
$newRevision->update(['row_count' => $newRevision->rows()->count()]); $newRevision->update(['row_count' => $newRevision->rows()->count()]);

@ -573,10 +573,10 @@ class Changeset
* so it can be inserted directly into DB * so it can be inserted directly into DB
* @return Collection * @return Collection
*/ */
public function csvToRowsArray($columns, $csvArray, $forTableInsert, $useDraftIds) public function csvToRowsArray($columns, $csvArray, $useDraftIds)
{ {
/** @var Collection $rows */ /** @var Collection $rows */
$rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) { $rows = collect($csvArray)->map(function ($row) use ($columns) {
if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
if (count($row) != count($columns)) { if (count($row) != count($columns)) {
throw new NotApplicableException("All rows must have " . count($columns) . " fields."); throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
@ -594,11 +594,7 @@ class Changeset
$data[$col->id] = $col->cast(trim($val)); $data[$col->id] = $col->cast(trim($val));
} }
if ($forTableInsert) {
return ['data' => $data];
} else {
return $data; return $data;
}
})->filter(); })->filter();
if ($useDraftIds) { if ($useDraftIds) {
@ -607,19 +603,11 @@ class Changeset
$rowNumerator = new RowNumerator(count($csvArray)); $rowNumerator = new RowNumerator(count($csvArray));
} }
if ($forTableInsert) {
return $rows->map(function ($row) use (&$rowNumerator) {
$row['data']['_id'] = $rowNumerator->next();
return $row;
});
}
else {
return $rows->map(function ($row) use (&$rowNumerator) { return $rows->map(function ($row) use (&$rowNumerator) {
$row['_id'] = $rowNumerator->next(); $row['_id'] = $rowNumerator->next();
return $row; return $row;
}); });
} }
}
/** /**
* Append N blank rows * Append N blank rows
@ -655,7 +643,7 @@ class Changeset
/** @var Column[] $columns */ /** @var Column[] $columns */
$columns = $columns->values()->all(); $columns = $columns->values()->all();
$rows = self::csvToRowsArray($columns, $csvArray, false, true) $rows = self::csvToRowsArray($columns, $csvArray, true)
->keyBy('_id'); ->keyBy('_id');
// using '+' to avoid renumbering // using '+' to avoid renumbering

@ -187,20 +187,21 @@ class Column implements JsonSerializable, Arrayable
{ {
switch ($this->type) { switch ($this->type) {
case 'int': case 'int':
if (is_null($value)) return 0; if (is_null($value) || $value === "") return 0;
if (is_int($value)) return $value; if (is_int($value)) return $value;
if (is_bool($value)) return (int)$value;
if (is_float($value)) return round($value); if (is_float($value)) return round($value);
if (is_numeric($value)) return intval($value); if (is_numeric($value)) return intval($value);
throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to int!"); throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to int!");
case 'float': case 'float':
if (is_null($value)) return 0; if (is_null($value) || $value === "") return 0;
if (is_int($value) || is_float($value)) return (float)$value; if (is_int($value) || is_float($value) || is_bool($value)) return (float)$value;
if (is_numeric($value)) return floatval($value); if (is_numeric($value)) return floatval($value);
throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to float!"); throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to float!");
case 'bool': case 'bool':
if (is_null($value)) return false; if (is_null($value) || $value === "") return false;
return Utils::parseBool($value); return Utils::parseBool($value);
case 'string': case 'string':

Loading…
Cancel
Save