MightyPork/datatable.directory
1
0
Fork 1
Code Issues 25 Pull Requests Releases Wiki Activity

Brutal insert speed optimization by bypassing the ORM completely

Browse Source
This commit is contained in:
MightyPork
2018-08-12 23:49:52 +02:00
parent 13b22f7cb4
commit c76fc21820
4 changed files with 48 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Http/Controllers/TableController.php
+22 -6
View File
@@ -14,6 +14,7 @@ use App\Tables\CXMacroExporter;
use App\Tables\JsonExporter;
use App\Tables\PhpExporter;
use App\Tables\RowNumerator;
use DB;
use Illuminate\Http\Request;
use Illuminate\Validation\Rule;
use MightyPork\Exceptions\NotApplicableException;
@@ -243,7 +244,7 @@ class TableController extends Controller
$rowsToInsert = null;
$rowNumerator = null;
try {
$rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, true, false)->all();
$rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, false)->all();
} catch (\Exception $e) {
return $this->backWithErrors(['data' => $e->getMessage()]);
}
@@ -270,17 +271,32 @@ class TableController extends Controller
/** @var Table $table */
$table = null;
\DB::transaction(function () use ($revisionFields, $tableFields, $rowsToInsert, &$table) {
$revision = Revision::create($revisionFields);
$newRevision = Revision::create($revisionFields);
$tableFields['revision_id'] = $revision->id; // current revision (not-null constraint on this FK)
$tableFields['revision_id'] = $newRevision->id; // current revision (not-null constraint on this FK)
$table = Table::create($tableFields);
// Attach the revision to the table, set as current
$table->revisions()->attach($revision);
$table->revision()->associate($revision);
$table->revisions()->attach($newRevision);
$table->revision()->associate($newRevision);
// Spawn rows, linked to the revision
$revision->rows()->createMany($rowsToInsert);
if (!empty($rowsToInsert)) {
// this replicates the code in Proposal->toRevision
$conn = DB::connection();
$prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)');
foreach (array_chunk($rowsToInsert, 10000) as $i => $chunk) {
$ids = [];
foreach ($chunk as $newRow) {
$prepared->execute([json_encode($newRow)]);
$ids[] = $conn->getPdo()->lastInsertId();
}
$qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ',');
$conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';',
$ids);
}
}
});
return redirect($table->viewRoute);