MightyPork/datatable.directory
1
0
Fork 1
Code Issues 25 Pull Requests Releases Wiki Activity

Brutal insert speed optimization by bypassing the ORM completely

Browse Source
This commit is contained in:
MightyPork
2018-08-12 23:49:52 +02:00
parent 13b22f7cb4
commit c76fc21820
4 changed files with 48 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Tables/Changeset.php
+8 -20
View File
@@ -573,10 +573,10 @@ class Changeset
* so it can be inserted directly into DB
* @return Collection
*/
public function csvToRowsArray($columns, $csvArray, $forTableInsert, $useDraftIds)
public function csvToRowsArray($columns, $csvArray, $useDraftIds)
{
/** @var Collection $rows */
$rows = collect($csvArray)->map(function ($row) use ($columns, $forTableInsert) {
$rows = collect($csvArray)->map(function ($row) use ($columns) {
if (count($row) == 0 || count($row) == 1 && $row[0] == '') return null; // discard empty lines
if (count($row) != count($columns)) {
throw new NotApplicableException("All rows must have " . count($columns) . " fields.");
@@ -594,11 +594,7 @@ class Changeset
$data[$col->id] = $col->cast(trim($val));
}
if ($forTableInsert) {
return ['data' => $data];
} else {
return $data;
}
return $data;
})->filter();
if ($useDraftIds) {
@@ -607,18 +603,10 @@ class Changeset
$rowNumerator = new RowNumerator(count($csvArray));
}
if ($forTableInsert) {
return $rows->map(function ($row) use (&$rowNumerator) {
$row['data']['_id'] = $rowNumerator->next();
return $row;
});
}
else {
return $rows->map(function ($row) use (&$rowNumerator) {
$row['_id'] = $rowNumerator->next();
return $row;
});
}
return $rows->map(function ($row) use (&$rowNumerator) {
$row['_id'] = $rowNumerator->next();
return $row;
});
}
/**
@@ -655,7 +643,7 @@ class Changeset
/** @var Column[] $columns */
$columns = $columns->values()->all();
$rows = self::csvToRowsArray($columns, $csvArray, false, true)
$rows = self::csvToRowsArray($columns, $csvArray, true)
->keyBy('_id');
// using '+' to avoid renumbering
app/Tables/Column.php
+5 -4
View File
@@ -187,20 +187,21 @@ class Column implements JsonSerializable, Arrayable
{
switch ($this->type) {
case 'int':
if (is_null($value)) return 0;
if (is_null($value) || $value === "") return 0;
if (is_int($value)) return $value;
if (is_bool($value)) return (int)$value;
if (is_float($value)) return round($value);
if (is_numeric($value)) return intval($value);
throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to int!");
case 'float':
if (is_null($value)) return 0;
if (is_int($value) || is_float($value)) return (float)$value;
if (is_null($value) || $value === "") return 0;
if (is_int($value) || is_float($value) || is_bool($value)) return (float)$value;
if (is_numeric($value)) return floatval($value);
throw new SimpleValidationException($this->id, "Could not convert value \"$value\" to float!");
case 'bool':
if (is_null($value)) return false;
if (is_null($value) || $value === "") return false;
return Utils::parseBool($value);
case 'string':