Brutal insert speed optimization by bypassing the ORM completely

app/Http/Controllers/TableController.php

@@ -14,6 +14,7 @@ use App\Tables\CXMacroExporter;
 use App\Tables\JsonExporter;
 use App\Tables\PhpExporter;
 use App\Tables\RowNumerator;
+use DB;
 use Illuminate\Http\Request;
 use Illuminate\Validation\Rule;
 use MightyPork\Exceptions\NotApplicableException;
@@ -243,7 +244,7 @@ class TableController extends Controller
         $rowsToInsert = null;
         $rowNumerator = null;
         try {
-            $rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, true, false)->all();
+            $rowsToInsert = (new Changeset)->csvToRowsArray($columns, $dataCsvLines, false)->all();
         } catch (\Exception $e) {
             return $this->backWithErrors(['data' => $e->getMessage()]);
         }
@@ -270,17 +271,32 @@ class TableController extends Controller
         /** @var Table $table */
         $table = null;
         \DB::transaction(function () use ($revisionFields, $tableFields, $rowsToInsert, &$table) {
-            $revision = Revision::create($revisionFields);
+            $newRevision = Revision::create($revisionFields);
 
-            $tableFields['revision_id'] = $revision->id; // current revision (not-null constraint on this FK)
+            $tableFields['revision_id'] = $newRevision->id; // current revision (not-null constraint on this FK)
             $table = Table::create($tableFields);
 
             // Attach the revision to the table, set as current
-            $table->revisions()->attach($revision);
+            $table->revisions()->attach($newRevision);
-            $table->revision()->associate($revision);
+            $table->revision()->associate($newRevision);
 
             // Spawn rows, linked to the revision
-            $revision->rows()->createMany($rowsToInsert);
+            if (!empty($rowsToInsert)) {
+                // this replicates the code in Proposal->toRevision
+                $conn = DB::connection();
+                $prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)');
+                foreach (array_chunk($rowsToInsert, 10000) as $i => $chunk) {
+                    $ids = [];
+                    foreach ($chunk as $newRow) {
+                        $prepared->execute([json_encode($newRow)]);
+                        $ids[] = $conn->getPdo()->lastInsertId();
+                    }
+
+                    $qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ',');
+                    $conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';',
+                        $ids);
+                }
+            }
         });
 
         return redirect($table->viewRoute);

app/Models/Proposal.php

@@ -193,11 +193,20 @@ class Proposal extends BaseModel
 
             // --- Insert new rows ---
             if ($changeset->newRows) {
-                $newRowData = [];
+                $conn = DB::connection();
-                foreach ($changeset->newRows as $newRow) {
+                $prepared = $conn->getPdo()->prepare('INSERT INTO rows (data) VALUES (?)');
-                    $newRowData[] = new Row(['data' => $newRow]);
+
+                foreach (array_chunk($changeset->newRows, 10000) as $i => $chunk) {
+                    $ids = [];
+                    foreach ($chunk as $newRow) {
+                        $prepared->execute([json_encode($newRow)]);
+                        $ids[] = $conn->getPdo()->lastInsertId();
+                    }
+
+                    $qms = rtrim(str_repeat('('.$newRevision->id.', ?),', count($ids)), ',');
+                    $conn->statement('INSERT INTO revision_row_pivot (revision_id, row_id) VALUES '.$qms.';',
+                        $ids);
                 }
-                $newRevision->rows()->saveMany($newRowData);
             }
 
             $newRevision->update(['row_count' => $newRevision->rows()->count()]);

app/Tables/Changeset.php

@@ -573,10 +573,10 @@ class Changeset
      *                               so it can be inserted directly into DB
      * @return Collection
      */
-    public function csvToRowsArray($columns, $csvArray, $forTableInsert, $useDraftIds)
+    public function csvToRowsArray($columns, $csvArray, $useDraftIds)
     {
         /** @var Collection $rows */
-        $rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) {
+        $rows = collect($csvArray)->map(function ($row) use ($columns) {
             if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
             if (count($row) != count($columns)) {
                 throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
@@ -594,11 +594,7 @@ class Changeset
                 $data[$col->id] = $col->cast(trim($val));
             }
 
-            if ($forTableInsert) {
+            return $data;
-                return ['data' => $data];
-            } else {
-                return $data;
-            }
         })->filter();
 
         if ($useDraftIds) {
@@ -607,18 +603,10 @@ class Changeset
             $rowNumerator = new RowNumerator(count($csvArray));
         }
 
-        if ($forTableInsert) {
+        return $rows->map(function ($row) use (&$rowNumerator) {
-            return $rows->map(function ($row) use (&$rowNumerator) {
+            $row['_id'] = $rowNumerator->next();
-                $row['data']['_id'] = $rowNumerator->next();
+            return $row;
-                return $row;
+        });
-            });
-        }
-        else {
-            return $rows->map(function ($row) use (&$rowNumerator) {
-                $row['_id'] = $rowNumerator->next();
-                return $row;
-            });
-        }
     }
 
     /**
@@ -655,7 +643,7 @@ class Changeset
         /** @var Column[] $columns */
         $columns = $columns->values()->all();
 
-        $rows = self::csvToRowsArray($columns, $csvArray, false, true)
+        $rows = self::csvToRowsArray($columns, $csvArray, true)
             ->keyBy('_id');
 
         // using '+' to avoid renumbering

app/Tables/Column.php

@@ -187,20 +187,21 @@ class Column implements JsonSerializable, Arrayable
     {
         switch ($this->type) {
             case 'int':
-                if (is_null($value)) return 0;
+                if (is_null($value) || $value === "") return 0;
                 if (is_int($value)) return $value;
+                if (is_bool($value)) return (int)$value;
                 if (is_float($value)) return round($value);
                 if (is_numeric($value)) return intval($value);
                 throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to int!");
 
             case 'float':
-                if (is_null($value)) return 0;
+                if (is_null($value) || $value === "") return 0;
-                if (is_int($value) || is_float($value)) return (float)$value;
+                if (is_int($value) || is_float($value) || is_bool($value)) return (float)$value;
                 if (is_numeric($value)) return floatval($value);
                 throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to float!");
 
             case 'bool':
-                if (is_null($value)) return false;
+                if (is_null($value) || $value === "") return false;
                 return Utils::parseBool($value);
 
             case 'string':