Updated CSV with improved whitespace/empty line handling and added re…

…gex white/blacklist to CSV implementation
hwperkins · Oct 22, 2015 · 16435d8 · 16435d8
1 parent d6d2b38
commit 16435d8
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 11 deletions.
diff --git a/src/IO/CSV.php b/src/IO/CSV.php
@@ -34,8 +34,9 @@ final class CSV
         'colmap' => null,
         'quote' => "\"",
         'escape' => "\\",
-
-        'overwrite' => false
+        'overwrite' => false,
+        'include' => null,
+        'exclude' => null
     ];
 
     public function __construct($fileName)
@@ -54,6 +55,8 @@ public function __construct($fileName)
      *               to be once loaded into memory (default: null)
      *      quote:   The character used to specify literal quoted segments (default: ")
      *      escape:  The character used to escape quotes or other special characters (default: \)
+     *      include: Whitelist Regular Expression
+     *      exclude: Blacklist Regular Expression
      * @param  array $options The option map.
      * @return array         Returns multi-dimensional array of row-column strings.
      * @throws \Archon\Exceptions\UnknownOptionException
@@ -71,10 +74,18 @@ public function loadFile(array $options = [])
         $colmapOpt = $options['colmap'];
         $quoteOpt = $options['quote'];
         $escapeOpt = $options['escape'];
+        $includeRegexOpt = $options['include'];
+        $excludeRegexOpt = $options['exclude'];
 
         $fileData = file_get_contents($fileName);
         $fileData = explode($nlsepOpt, $fileData);
 
+        // Remove whitespace/empty lines
+        $fileData = preg_grep('/^\s*$/', $fileData, PREG_GREP_INVERT);
+
+        $fileData = $includeRegexOpt ? preg_grep($includeRegexOpt, $fileData) : $fileData;
+        $fileData = $excludeRegexOpt ? preg_grep($excludeRegexOpt, $fileData, PREG_GREP_INVERT) : $fileData;
+
         /**
          * Determines how to assign columns of the CSV
          * First checks if options specify a line of the file to use
@@ -107,15 +118,8 @@ public function loadFile(array $options = [])
         foreach ($fileData as $i => $line) {
             $line = trim($line);
 
-            if ($line === '') {
-                unset($fileData[$i]);
-                continue;
-            }
-
-            if ($line !== '') {
-                $line = str_getcsv($line, $sepOpt, $quoteOpt, $escapeOpt);
-                $fileData[$i] = $this->applyColMapToRowKeys($line, $columns);
-            }
+            $line = str_getcsv($line, $sepOpt, $quoteOpt, $escapeOpt);
+            $fileData[$i] = $this->applyColMapToRowKeys($line, $columns);
         }
 
         $fileData = array_values($fileData);

diff --git a/tests/DataFrame/CSV/CSVDataFrameUnitTest.php b/tests/DataFrame/CSV/CSVDataFrameUnitTest.php
@@ -17,6 +17,21 @@ public function testFromCSV()
         ], $df->toArray());
     }
 
+    public function testFromCSVDirty()
+    {
+        $fileName = __DIR__.DIRECTORY_SEPARATOR.'TestFiles'.DIRECTORY_SEPARATOR.'testCSVdirty.csv';
+
+        $df = DataFrame::fromCSV($fileName, [
+            'include' => '/^([1-9]|a)/',
+            'exclude' => '/^([7]|junk)/'
+        ]);
+
+        $this->assertEquals([
+            ['a' => 1, 'b' => 2, 'c' => 3],
+            ['a' => 4, 'b' => 5, 'c' => 6],
+        ], $df->toArray());
+    }
+
     public function testFromCSVNoHeader()
     {
         $fileName = __DIR__.DIRECTORY_SEPARATOR.'TestFiles'.DIRECTORY_SEPARATOR.'testCSV.csv';

diff --git a/tests/DataFrame/CSV/TestFiles/testCSVdirty.csv b/tests/DataFrame/CSV/TestFiles/testCSVdirty.csv
@@ -0,0 +1,7 @@
+a,b,c
+1,2,3
+
+4,5,6
+7,8,9
+
+junk
-Original file line number
+Diff line change
@@ -0,0 +1,7 @@
+    a,b,c
+,2,3
+,5,6
+,8,9
+    junk