CsvSeeder::seedFromCSV() - Code Metrics - Inspection of "Extra tests, seedFromCSV now returns success bool" - Flynsarmy/laravel-csv-seeder - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( f59d68...c4db8c )

by Flyn

created 2020-09-16 11:48 UTC

CsvSeeder::seedFromCSV() C

↳ Parent: CsvSeeder

Complexity

Conditions	12
Paths	43

Size

Total Lines	63
Code Lines	29

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	12
eloc	29
nc	43
nop	2
dl	0
loc	63
rs	6.9666
c	0
b	0
f	0

How to fix Long Method Complexity

<?php

namespace Flynsarmy\CsvSeeder;

use App;
use Log;
use DB;
use Carbon\Carbon;
use Illuminate\Database\Seeder;
use Illuminate\Database\Schema;
use Illuminate\Support\Facades\Hash;

/**
 * Taken from http://laravelsnippets.com/snippets/seeding-database-with-csv-files-cleanly
 * and modified to include insert chunking
 */
class CsvSeeder extends Seeder
{
    /**
     * DB table name
     *
     * @var string
     */
    public string $table = '';

    /**
     * CSV filename
     *
     * @var string
     */
    public string $filename = '';

    /**
     * DB connection to use. Leave empty for default connection
     */
    public string $connection = '';

    /**
     * DB fields to be hashed before import, For example a password field.
     */
    public array $hashable = ['password'];

    /**
     * An SQL INSERT query will execute every time this number of rows
     * are read from the CSV. Without this, large INSERTS will silently
     * fail.
     */
    public int $insert_chunk_size = 50;

    /**
     * CSV delimiter (defaults to ,)
     */
    public string $csv_delimiter = ',';

    /**
     * Number of rows to skip at the start of the CSV
     */
    public int $offset_rows = 0;

    /**
     * Can be used to tell the import to trim any leading or trailing white space from the column;
     */
    public bool $should_trim = false;

    /**
     * Add created_at and updated_at to rows
     */
    public bool $timestamps = false;
    /**
     * created_at and updated_at values to be added to each row. Only used if
     * $this->timestamps is true
     */
    public string $created_at = '';
    public string $updated_at = '';

    /**
     * The mapping of CSV to DB column. If not specified manually, the first
     * row (after offset_rows) of your CSV will be read as your DB columns.
     *
     * Mappings take the form of csvColNumber => dbColName.
     *
     * IE to read the first, third and fourth columns of your CSV only, use:
     * array(
     *   0 => id,
     *   2 => name,
     *   3 => description,
     * )
     */
    public array $mapping = [];


    /**
     * Run DB seed
     */
    public function run()
    {
        // Cache created_at and updated_at if we need to
        if ($this->timestamps) {
            if (!$this->created_at) {
                $this->created_at = Carbon::now()->toString();
            }
            if (!$this->updated_at) {
                $this->updated_at = Carbon::now()->toString();
            }
        }

        $this->seedFromCSV($this->filename, $this->csv_delimiter);
    }

    /**
     * Strip UTF-8 BOM characters from the start of a string
     *
     * @param  string $text
     * @return string       String with BOM stripped
     */
    public function stripUtf8Bom(string $text): string
    {
        $bom = pack('H*', 'EFBBBF');
        $text = preg_replace("/^$bom/", '', $text);

        return $text;
    }

    /**
     * Opens a CSV file and returns it as a resource
     *
     * @param string $filename
     * @return FALSE|resource
     */
    public function openCSV(string $filename)
    {
        if (!file_exists($filename) || !is_readable($filename)) {
            Log::error("CSV insert failed: CSV " . $filename . " does not exist or is not readable.");
            return false;
        }

        // check if file is gzipped
        $finfo = finfo_open(FILEINFO_MIME_TYPE);
        $file_mime_type = finfo_file($finfo, $filename);
        finfo_close($finfo);
        $gzipped = strcmp($file_mime_type, "application/x-gzip") == 0;

        $handle = $gzipped ? gzopen($filename, 'r') : fopen($filename, 'r');

        return $handle;
    }

    /**
     * Reads all rows of a given CSV and imports the data.
     *
     * @param string $filename
     * @param string $deliminator
     * @return bool  Whether or not the import completed successfully.
     */
    public function seedFromCSV(string $filename, string $deliminator = ","): bool
    {
        $handle = $this->openCSV($filename);

        // CSV doesn't exist or couldn't be read from.
        if ($handle === false) {
            return false;
        }

        $success = true;
        $row_count = 0;
        $data = [];
        $mapping = $this->mapping ?: [];
        $offset = $this->offset_rows;

        if ($mapping) {
            $this->hashable = $this->removeUnusedHashColumns($mapping);
        }

        while (($row = fgetcsv($handle, 0, $deliminator)) !== false) {
            // Offset the specified number of rows

            while ($offset-- > 0) {
                continue 2;
            }

            // No mapping specified - the first row will be used as the mapping
            // ie it's a CSV title row. This row won't be inserted into the DB.
            if (!$mapping) {
                $mapping = $this->createMappingFromRow($row);
                $this->hashable = $this->removeUnusedHashColumns($mapping);
                continue;
            }

            $row = $this->readRow($row, $mapping);

            // insert only non-empty rows from the csv file
            if (empty($row)) {
                continue;
            }

            $data[$row_count] = $row;

            // Chunk size reached, insert
            if (++$row_count == $this->insert_chunk_size) {
                $success = $success && $this->insert($data);
                $row_count = 0;
                // clear the data array explicitly when it was inserted so
                // that nothing is left, otherwise a leftover scenario can
                // cause duplicate inserts
                $data = [];
            }
        }

        // Insert any leftover rows
        //check if the data array explicitly if there are any values left to be inserted, if insert them
        if (count($data)) {
            $success = $success && $this->insert($data);
        }

        fclose($handle);

        return $success;
    }

    /**
     * Creates a CSV->DB column mapping from the given CSV row.
     *
     * @param array $row
     * @return array
     */
    public function createMappingFromRow(array $row): array
    {
        $mapping = $row;
        $mapping[0] = $this->stripUtf8Bom($mapping[0]);

        // skip csv columns that don't exist in the database
        foreach ($mapping as $index => $fieldname) {
            if (!DB::getSchemaBuilder()->hasColumn($this->table, $fieldname)) {
                if (isset($mapping[$index])) {
                    unset($mapping[$index]);
                }
            }
        }

        return $mapping;
    }

    /**
     * Removes fields from the hashable array that don't exist in our mapping.
     *
     * This function acts as a performance enhancement - we don't want
     * to search for hashable columns on every row imported when we already
     * know they don't exist.
     *
     * @param array $mapping
     * @return array
     */
    public function removeUnusedHashColumns(array $mapping)
    {
        $hashables = $this->hashable;

        foreach ($hashables as $key => $field) {
            if (!in_array($field, $mapping)) {
                unset($hashables[$key]);
            }
        }

        return $hashables;
    }

    /**
     * Read a CSV row into a DB insertable array
     *
     * @param array $row        A row of data to read
     * @param array $mapping    Array of csvCol => dbCol
     * @return array
     */
    public function readRow(array $row, array $mapping): array
    {
        $row_values = [];

        foreach ($mapping as $csvCol => $dbCol) {
            if (!isset($row[$csvCol]) || $row[$csvCol] === '') {
                $row_values[$dbCol] = null;
            } else {
                $row_values[$dbCol] = $this->should_trim ? trim($row[$csvCol]) : $row[$csvCol];
            }
        }

        if (!empty($this->hashable)) {
            foreach ($this->hashable as $columnToHash) {
                if (isset($row_values[$columnToHash])) {
                    $row_values[$columnToHash] = Hash::make($row_values[$columnToHash]);
                }
            }
        }

        if ($this->timestamps) {
            $row_values['created_at'] = $this->created_at;
            $row_values['updated_at'] = $this->updated_at;
        }

        return $row_values;
    }

    /**
     * Seed a given set of data to the DB
     *
     * @param array $seedData
     * @return bool   TRUE on success else FALSE
     */
    public function insert(array $seedData): bool
    {
        try {
            DB::connection($this->connection)->table($this->table)->insert($seedData);
        } catch (\Exception $e) {
            Log::error("CSV insert failed: " . $e->getMessage() . " - CSV " . $this->filename);
            return false;
        }

        return true;
    }
}


1			<?php
2
3			namespace Flynsarmy\CsvSeeder;
4
5			use App;
6			use Log;
7			use DB;
8			use Carbon\Carbon;
9			use Illuminate\Database\Seeder;
10			use Illuminate\Database\Schema;
11			use Illuminate\Support\Facades\Hash;
12
13			/**
14			* Taken from http://laravelsnippets.com/snippets/seeding-database-with-csv-files-cleanly
15			* and modified to include insert chunking
16			*/
17			class CsvSeeder extends Seeder
18			{
19			/**
20			* DB table name
21			*
22			* @var string
23			*/
24			public string $table = '';
25
26			/**
27			* CSV filename
28			*
29			* @var string
30			*/
31			public string $filename = '';
32
33			/**
34			* DB connection to use. Leave empty for default connection
35			*/
36			public string $connection = '';
37
38			/**
39			* DB fields to be hashed before import, For example a password field.
40			*/
41			public array $hashable = ['password'];
42
43			/**
44			* An SQL INSERT query will execute every time this number of rows
45			* are read from the CSV. Without this, large INSERTS will silently
46			* fail.
47			*/
48			public int $insert_chunk_size = 50;
49
50			/**
51			* CSV delimiter (defaults to ,)
52			*/
53			public string $csv_delimiter = ',';
54
55			/**
56			* Number of rows to skip at the start of the CSV
57			*/
58			public int $offset_rows = 0;
59
60			/**
61			* Can be used to tell the import to trim any leading or trailing white space from the column;
62			*/
63			public bool $should_trim = false;
64
65			/**
66			* Add created_at and updated_at to rows
67			*/
68			public bool $timestamps = false;
69			/**
70			* created_at and updated_at values to be added to each row. Only used if
71			* $this->timestamps is true
72			*/
73			public string $created_at = '';
74			public string $updated_at = '';
75
76			/**
77			* The mapping of CSV to DB column. If not specified manually, the first
78			* row (after offset_rows) of your CSV will be read as your DB columns.
79			*
80			* Mappings take the form of csvColNumber => dbColName.
81			*
82			* IE to read the first, third and fourth columns of your CSV only, use:
83			* array(
84			* 0 => id,
85			* 2 => name,
86			* 3 => description,
87			* )
88			*/
89			public array $mapping = [];
90
91
92			/**
93			* Run DB seed
94			*/
95			public function run()
96			{
97			// Cache created_at and updated_at if we need to
98			if ($this->timestamps) {
99			if (!$this->created_at) {
100			$this->created_at = Carbon::now()->toString();
101			}
102			if (!$this->updated_at) {
103			$this->updated_at = Carbon::now()->toString();
104			}
105			}
106
107			$this->seedFromCSV($this->filename, $this->csv_delimiter);
108			}
109
110			/**
111			* Strip UTF-8 BOM characters from the start of a string
112			*
113			* @param string $text
114			* @return string String with BOM stripped
115			*/
116			public function stripUtf8Bom(string $text): string
117			{
118			$bom = pack('H*', 'EFBBBF');
119			$text = preg_replace("/^$bom/", '', $text);
120
121			return $text;
122			}
123
124			/**
125			* Opens a CSV file and returns it as a resource
126			*
127			* @param string $filename
128			* @return FALSE\|resource
129			*/
130			public function openCSV(string $filename)
131			{
132			if (!file_exists($filename) \|\| !is_readable($filename)) {
133			Log::error("CSV insert failed: CSV " . $filename . " does not exist or is not readable.");
134			return false;
135			}
136
137			// check if file is gzipped
138			$finfo = finfo_open(FILEINFO_MIME_TYPE);
139			$file_mime_type = finfo_file($finfo, $filename);
140			finfo_close($finfo);
141			$gzipped = strcmp($file_mime_type, "application/x-gzip") == 0;
142
143			$handle = $gzipped ? gzopen($filename, 'r') : fopen($filename, 'r');
144
145			return $handle;
146			}
147
148			/**
149			* Reads all rows of a given CSV and imports the data.
150			*
151			* @param string $filename
152			* @param string $deliminator
153			* @return bool Whether or not the import completed successfully.
154			*/
155			public function seedFromCSV(string $filename, string $deliminator = ","): bool
156			{
157			$handle = $this->openCSV($filename);
158
159			// CSV doesn't exist or couldn't be read from.
160			if ($handle === false) {
161			return false;
162			}
163
164			$success = true;
165			$row_count = 0;
166			$data = [];
167			$mapping = $this->mapping ?: [];
168			$offset = $this->offset_rows;
169
170			if ($mapping) {
171			$this->hashable = $this->removeUnusedHashColumns($mapping);
172			}
173
174			while (($row = fgetcsv($handle, 0, $deliminator)) !== false) {
175			// Offset the specified number of rows
176
177			while ($offset-- > 0) {
178			continue 2;
179			}
180
181			// No mapping specified - the first row will be used as the mapping
182			// ie it's a CSV title row. This row won't be inserted into the DB.
183			if (!$mapping) {
184			$mapping = $this->createMappingFromRow($row);
185			$this->hashable = $this->removeUnusedHashColumns($mapping);
186			continue;
187			}
188
189			$row = $this->readRow($row, $mapping);
190
191			// insert only non-empty rows from the csv file
192			if (empty($row)) {
193			continue;
194			}
195
196			$data[$row_count] = $row;
197
198			// Chunk size reached, insert
199			if (++$row_count == $this->insert_chunk_size) {
200			$success = $success && $this->insert($data);
201			$row_count = 0;
202			// clear the data array explicitly when it was inserted so
203			// that nothing is left, otherwise a leftover scenario can
204			// cause duplicate inserts
205			$data = [];
206			}
207			}
208
209			// Insert any leftover rows
210			//check if the data array explicitly if there are any values left to be inserted, if insert them
211			if (count($data)) {
212			$success = $success && $this->insert($data);
213			}
214
215			fclose($handle);
216
217			return $success;
218			}
219
220			/**
221			* Creates a CSV->DB column mapping from the given CSV row.
222			*
223			* @param array $row
224			* @return array
225			*/
226			public function createMappingFromRow(array $row): array
227			{
228			$mapping = $row;
229			$mapping[0] = $this->stripUtf8Bom($mapping[0]);
230
231			// skip csv columns that don't exist in the database
232			foreach ($mapping as $index => $fieldname) {
233			if (!DB::getSchemaBuilder()->hasColumn($this->table, $fieldname)) {
234			if (isset($mapping[$index])) {
235			unset($mapping[$index]);
236			}
237			}
238			}
239
240			return $mapping;
241			}
242
243			/**
244			* Removes fields from the hashable array that don't exist in our mapping.
245			*
246			* This function acts as a performance enhancement - we don't want
247			* to search for hashable columns on every row imported when we already
248			* know they don't exist.
249			*
250			* @param array $mapping
251			* @return array
252			*/
253			public function removeUnusedHashColumns(array $mapping)
254			{
255			$hashables = $this->hashable;
256
257			foreach ($hashables as $key => $field) {
258			if (!in_array($field, $mapping)) {
259			unset($hashables[$key]);
260			}
261			}
262
263			return $hashables;
264			}
265
266			/**
267			* Read a CSV row into a DB insertable array
268			*
269			* @param array $row A row of data to read
270			* @param array $mapping Array of csvCol => dbCol
271			* @return array
272			*/
273			public function readRow(array $row, array $mapping): array
274			{
275			$row_values = [];
276
277			foreach ($mapping as $csvCol => $dbCol) {
278			if (!isset($row[$csvCol]) \|\| $row[$csvCol] === '') {
279			$row_values[$dbCol] = null;
280			} else {
281			$row_values[$dbCol] = $this->should_trim ? trim($row[$csvCol]) : $row[$csvCol];
282			}
283			}
284
285			if (!empty($this->hashable)) {
286			foreach ($this->hashable as $columnToHash) {
287			if (isset($row_values[$columnToHash])) {
288			$row_values[$columnToHash] = Hash::make($row_values[$columnToHash]);
289			}
290			}
291			}
292
293			if ($this->timestamps) {
294			$row_values['created_at'] = $this->created_at;
295			$row_values['updated_at'] = $this->updated_at;
296			}
297
298			return $row_values;
299			}
300
301			/**
302			* Seed a given set of data to the DB
303			*
304			* @param array $seedData
305			* @return bool TRUE on success else FALSE
306			*/
307			public function insert(array $seedData): bool
308			{
309			try {
310			DB::connection($this->connection)->table($this->table)->insert($seedData);
311			} catch (\Exception $e) {
312			Log::error("CSV insert failed: " . $e->getMessage() . " - CSV " . $this->filename);
313			return false;
314			}
315
316			return true;
317			}
318			}
319

Flynsarmy / laravel-csv-seeder

Push — master ( f59d68...c4db8c )

CsvSeeder::seedFromCSV() C

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Duplication Side-by-Side

Filter issues like