Merge pull request #43 from theodi/info-messages-for-line-breaks

Info messages for line breaks

features/fixtures/cr-line-endings.csv

@@ -0,0 +1 @@
+"Foo","Bar","Baz""Biff","Baff","Boff""Qux","Teaspoon","Doge"

features/fixtures/crlf-line-endings.csv

@@ -0,0 +1,3 @@
+"Foo","Bsr","Baz"
+"Biff","Baff","Boff"
+"Qux","Teaspoon","Doge"

features/fixtures/inconsistent-line-endings.csv

@@ -0,0 +1,2 @@
+"Foo","Bsr","Baz""Biff","Baff","Boff"
+"Qux","Teaspoon","Doge"

features/fixtures/incorrect-line-endings.csv → features/fixtures/lf-line-endings.csv

@@ -1,2 +1,3 @@
 "Foo","Bsr","Baz"
 "Biff","Baff","Boff"
+"Qux","Teaspoon","Doge"

features/step_definitions/parse_csv_steps.rb

@@ -1,5 +1,5 @@
 Given(/^I have a CSV with the following content:$/) do |string|
-  @csv = string.gsub("\n", "\r\n")
+  @csv = string
 end
 
 Given(/^it is stored at the url "(.*?)"$/) do |url|

features/step_definitions/validation_errors_steps.rb

@@ -40,8 +40,4 @@
 
 Then(/^there should be no "(.*?)" errors$/) do |type|
   @errors.each do |error| error.type.should_not == type.to_sym end
-end
-
-Given(/^I have a CSV with carriage returns in fields$/) do
-  @csv = "\"Foo\",\"Bar\",\"Baz\"\r\n\"Bing\",\"Bang\nBung\",\"Bong\""
 end

features/step_definitions/validation_info_steps.rb

@@ -0,0 +1,18 @@
+Given(/^I ask if there are info messages$/) do
+  @csv_options ||= default_csv_options
+
+  if @schema_json
+    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+  end
+
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+  @info_messages = @validator.info_messages
+end
+
+Then(/^there should be (\d+) info messages?$/) do |num|
+  @info_messages.count.should == num.to_i
+end
+
+Then(/^that message should have the type "(.*?)"$/) do |msg_type|
+  @info_messages.first.type.should == msg_type.to_sym
+end

features/support/env.rb

@@ -21,7 +21,6 @@
 class CustomWorld
   def default_csv_options
     return {
-      "lineTerminator" => "\r\n"
     }
   end
 end

features/validation_errors.feature

@@ -21,7 +21,7 @@ Feature: Get validation errors
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if there are errors
     Then there should be 1 error
-    And that error should have the type "quoting"
+    And that error should have the type "unclosed_quote"
     And that error should have the row "1"
     And that error should have the content ""Foo","Bar","Baz"
     
@@ -130,25 +130,16 @@ Feature: Get validation errors
     And that error should have the type "not_found"
     
   Scenario: Incorrect line endings specified in settings
-    Given I have a CSV with the following content:
-    """
-"abc","2","3"
-    """
+    Given I have a CSV file called "cr-line-endings.csv"
     And I set the line endings to linefeed
     And it is stored at the url "http://example.com/example1.csv"
     And I ask if there are errors
     Then there should be 1 error
     And that error should have the type "line_breaks"
   
-  Scenario: Incorrect line endings in file
-    Given I have a CSV file called "incorrect-line-endings.csv"
+  Scenario: inconsistent line endings in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings.csv"
     And it is stored at the url "http://example.com/example1.csv"
     And I ask if there are errors
     Then there should be 1 error
-    And that error should have the type "line_breaks"
-
-  Scenario: Incorrect line endings in file
-    Given I have a CSV with carriage returns in fields
-    And it is stored at the url "http://example.com/example1.csv"
-    And I ask if there are errors
-    Then there should be 0 error
+    And that error should have the type "line_breaks"

features/validation_info.feature

@@ -0,0 +1,21 @@
+Feature: Get validation information messages
+
+  Scenario: LF line endings in file give an info message
+    Given I have a CSV file called "lf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are info messages
+    Then there should be 1 info message
+    And that message should have the type "nonrfc_line_breaks"
+
+  Scenario: CR line endings in file give an info message
+    Given I have a CSV file called "cr-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are info messages
+    Then there should be 1 info message
+    And that message should have the type "nonrfc_line_breaks"
+
+  Scenario: CRLF line endings in file produces no info messages
+    Given I have a CSV file called "crlf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are info messages
+    Then there should be 0 info messages

lib/csvlint/error_collector.rb

@@ -1,9 +1,7 @@
 module Csvlint
 
   module ErrorCollector
-
-    attr_reader :errors, :warnings
-
+
     def build_message(type, category, row, column, content)
       Csvlint::ErrorMessage.new({
                                   :type => type,
@@ -14,21 +12,30 @@ def build_message(type, category, row, column, content)
                                 })
     end
 
-    def build_errors(type, category = nil, row = nil, column = nil, content = nil)
-      @errors << build_message(type, category, row, column, content)
-    end
+    MESSAGE_LEVELS = [
+      :errors,
+      :warnings,
+      :info_messages
+    ]
 
-    def build_warnings(type, category = nil, row = nil, column = nil, content = nil)
-      @warnings << build_message(type, category, row, column, content)
+    MESSAGE_LEVELS.each do |level|
+
+      attr_reader level
+
+      define_method "build_#{level}" do |type, category = nil, row = nil, column = nil, content = nil|
+        instance_variable_get("@#{level}") << build_message(type, category, row, column, content)
+      end
+
     end
 
     def valid?
       errors.empty?
     end
 
     def reset
-      @errors = []
-      @warnings = []
+      MESSAGE_LEVELS.each do |level|
+        instance_variable_set("@#{level}", [])
+      end
     end
 
   end

lib/csvlint/validate.rb

@@ -7,17 +7,16 @@ class Validator
     attr_reader :encoding, :content_type, :extension, :headers
 
     ERROR_MATCHERS = {
-      "Missing or stray quote" => :quoting,
+      "Missing or stray quote" => :stray_quote,
       "Illegal quoting" => :whitespace,
-      "Unclosed quoted field" => :quoting,
+      "Unclosed quoted field" => :unclosed_quote,
     }
 
     def initialize(source, dialect = nil, schema = nil)      
       @source = source
       @formats = []
       @schema = schema  
       @csv_options = dialect_to_csv_options(dialect)
-      @csv_options[:row_sep] == nil ? @line_terminator = $/ : @line_terminator = @csv_options[:row_sep]
 
       @extension = parse_extension(source)
       reset
@@ -54,7 +53,6 @@ def validate_metadata(io)
         build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
         build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)
       end
-      build_errors(:line_breaks, :structure) unless @line_terminator == "\r\n"
     end
 
     def parse_csv(io)
@@ -64,10 +62,14 @@ def parse_csv(io)
 
       @csv_options[:encoding] = @encoding  
 
-      wrapper = WrappedIO.new( io )        
-      csv = CSV.new( wrapper , @csv_options )
-      row = nil
-      loop do
+      begin
+        wrapper = WrappedIO.new( io )
+        csv = CSV.new( wrapper, @csv_options )
+        if csv.row_sep != "\r\n"
+          build_info_messages(:nonrfc_line_breaks, :structure)
+        end
+        row = nil
+        loop do
          current_line = current_line + 1
          begin
            row = csv.shift
@@ -90,23 +92,23 @@ def parse_csv(io)
          rescue CSV::MalformedCSVError => e
            wrapper.finished
            type = fetch_error(e)
-           if type == :quoting && wrapper.line.match(/[^\r]\n/)
+           if type == :stray_quote && !wrapper.line.match(csv.row_sep)
              build_errors(:line_breaks, :structure)
            else
              build_errors(type, :structure, current_line, nil, wrapper.line)
            end
-         rescue ArgumentError => ae
-           wrapper.finished           
-           build_errors(:invalid_encoding, :structure, current_line, wrapper.line) unless reported_invalid_encoding
-           reported_invalid_encoding = true
          end
       end
+      rescue ArgumentError => ae
+        wrapper.finished           
+        build_errors(:invalid_encoding, :structure, current_line, wrapper.line) unless reported_invalid_encoding
+        reported_invalid_encoding = true
+      end
       return expected_columns        
     end          
 
 
     def fetch_error(error)
-      return :quoting if error.message.start_with?("Unquoted fields do not allow")
       e = error.message.match(/^([a-z ]+) (i|o)n line ([0-9]+)\.?$/i)
       ERROR_MATCHERS.fetch(e[1], :unknown_error)
     end
@@ -119,7 +121,7 @@ def dialect_to_csv_options(dialect)
         delimiter = delimiter + " " if !skipinitialspace
         return {
             :col_sep => delimiter,
-            :row_sep => ( dialect["lineTerminator"] || "\r\n" ),
+            :row_sep => ( dialect["lineTerminator"] || :auto),
             :quote_char => ( dialect["quoteChar"] || '"'),
             :skip_blanks => false
         }

lib/csvlint/wrapped_io.rb

@@ -5,13 +5,18 @@ def initialize(io)
       @line = ""
     end
 
-    def gets(delim)
-      @line = "" if @new_line
-      s = @io.gets(delim)
-      if s != nil
-        @line << s 
+    def gets(*args)
+      if args.count == 1 && args[0].is_a?(String)
+        delim = args[0]
+        @line = "" if @new_line
+        s = @io.gets(delim)
+        if s != nil
+          @line << s 
+        end
+        return s
+      else
+        @io.gets(*args)
       end
-      return s
     end
 
     def eof?
@@ -25,5 +30,10 @@ def finished
     def line
       @line
     end
+
+    def method_missing(method, *args)
+      @io.send(method, *args)
+    end
+
   end
 end

spec/validator_spec.rb

@@ -12,7 +12,7 @@
       opts = validator.dialect_to_csv_options( nil )
       opts.should == {
         :col_sep => ",",
-        :row_sep => "\r\n",
+        :row_sep => :auto,
         :quote_char => '"',
         :skip_blanks => false  
       }