MightyPork/datatable.directory
1
0
Fork 1
Code Issues 25 Pull Requests Releases Wiki Activity

Brutal insert speed optimization by bypassing the ORM completely

Browse Source
This commit is contained in:
MightyPork
2018-08-12 23:49:52 +02:00
parent 13b22f7cb4
commit c76fc21820
4 changed files with 48 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Tables/Changeset.php
+8 -20
View File
@@ -573,10 +573,10 @@ class Changeset
* so it can be inserted directly into DB
* @return Collection
*/
public function csvToRowsArray($columns, $csvArray, $forTableInsert, $useDraftIds)
public function csvToRowsArray($columns, $csvArray, $useDraftIds)
{
/** @var Collection $rows */
$rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) {
$rows = collect($csvArray)->map(function ($row) use ($columns) {
if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
if (count($row) != count($columns)) {
throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
@@ -594,11 +594,7 @@ class Changeset
$data[$col->id] = $col->cast(trim($val));
}
if ($forTableInsert) {
return ['data' => $data];
} else {
return $data;
}
return $data;
})->filter();
if ($useDraftIds) {
@@ -607,18 +603,10 @@ class Changeset
$rowNumerator = new RowNumerator(count($csvArray));
}
if ($forTableInsert) {
return $rows->map(function ($row) use (&$rowNumerator) {
$row['data']['_id'] = $rowNumerator->next();
return $row;
});
}
else {
return $rows->map(function ($row) use (&$rowNumerator) {
$row['_id'] = $rowNumerator->next();
return $row;
});
}
return $rows->map(function ($row) use (&$rowNumerator) {
$row['_id'] = $rowNumerator->next();
return $row;
});
}
/**
@@ -655,7 +643,7 @@ class Changeset
/** @var Column[] $columns */
$columns = $columns->values()->all();
$rows = self::csvToRowsArray($columns, $csvArray, false, true)
$rows = self::csvToRowsArray($columns, $csvArray, true)
->keyBy('_id');
// using '+' to avoid renumbering