From 340bce25a575427ab6f3cd664d64f64aabd83d1b Mon Sep 17 00:00:00 2001 From: Leigh Dodds Date: Tue, 18 Feb 2014 11:47:44 +0000 Subject: [PATCH 1/3] Added date,dateTime,time types and support for pattern --- lib/csvlint/field.rb | 41 +++++++++++++++++++++++++++---------- spec/field_spec.rb | 48 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/lib/csvlint/field.rb b/lib/csvlint/field.rb index 8f109039..abf3a565 100644 --- a/lib/csvlint/field.rb +++ b/lib/csvlint/field.rb @@ -1,4 +1,5 @@ require 'set' +require 'date' module Csvlint @@ -8,39 +9,57 @@ class Field attr_reader :name, :constraints, :title, :description TYPE_VALIDATIONS = { - 'http://www.w3.org/2001/XMLSchema#int' => lambda { |value| Integer value }, - 'http://www.w3.org/2001/XMLSchema#float' => lambda { |value| Float value }, - 'http://www.w3.org/2001/XMLSchema#double' => lambda { |value| Float value }, - 'http://www.w3.org/2001/XMLSchema#anyURI' => lambda do |value| + 'http://www.w3.org/2001/XMLSchema#int' => lambda { |value, constraints| Integer value }, + 'http://www.w3.org/2001/XMLSchema#float' => lambda { |value, constraints| Float value }, + 'http://www.w3.org/2001/XMLSchema#double' => lambda { |value, constraints| Float value }, + 'http://www.w3.org/2001/XMLSchema#anyURI' => lambda do |value, constraints| u = URI.parse value raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS) u end, - 'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value| + 'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints| return true if ['true', '1'].include? value return false if ['false', '0'].include? value raise ArgumentError end, - 'http://www.w3.org/2001/XMLSchema#nonPositiveInteger' => lambda do |value| + 'http://www.w3.org/2001/XMLSchema#nonPositiveInteger' => lambda do |value, constraints| i = Integer value raise ArgumentError unless i <= 0 i end, - 'http://www.w3.org/2001/XMLSchema#negativeInteger' => lambda do |value| + 'http://www.w3.org/2001/XMLSchema#negativeInteger' => lambda do |value, constraints| i = Integer value raise ArgumentError unless i < 0 i end, - 'http://www.w3.org/2001/XMLSchema#nonNegativeInteger' => lambda do |value| + 'http://www.w3.org/2001/XMLSchema#nonNegativeInteger' => lambda do |value, constraints| i = Integer value raise ArgumentError unless i >= 0 i end, - 'http://www.w3.org/2001/XMLSchema#positiveInteger' => lambda do |value| + 'http://www.w3.org/2001/XMLSchema#positiveInteger' => lambda do |value, constraints| i = Integer value raise ArgumentError unless i > 0 i - end + end, + 'http://www.w3.org/2001/XMLSchema#dateTime' => lambda do |value, constraints| + date_pattern = constraints["datePattern"] || "%Y-%m-%dT%H:%M:%SZ" + d = DateTime.strptime(value, date_pattern) + raise ArgumentError unless d.strftime(date_pattern) == value + d + end, + 'http://www.w3.org/2001/XMLSchema#date' => lambda do |value, constraints| + date_pattern = constraints["datePattern"] || "%Y-%m-%d" + d = Date.strptime(value, date_pattern) + raise ArgumentError unless d.strftime(date_pattern) == value + d + end, + 'http://www.w3.org/2001/XMLSchema#time' => lambda do |value, constraints| + date_pattern = constraints["datePattern"] || "%H:%M:%S" + d = DateTime.strptime(value, date_pattern) + raise ArgumentError unless d.strftime(date_pattern) == value + d + end } def initialize(name, constraints={}, title=nil, description=nil) @@ -121,7 +140,7 @@ def convert_to_type(value) tv = TYPE_VALIDATIONS[constraints["type"]] if tv begin - parsed = tv.call value + parsed = tv.call value, constraints rescue ArgumentError end end diff --git a/spec/field_spec.rb b/spec/field_spec.rb index 5020caaf..c4ace278 100644 --- a/spec/field_spec.rb +++ b/spec/field_spec.rb @@ -141,9 +141,53 @@ expect( field.validate_column("41")).to be(false) expect( field.errors.first.type ).to eql(:out_of_range) + end + end + + context "when validating dates" do + it "should validate a date time" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#dateTime" + }) + expect( field.validate_column("2014-02-17T11:09:00Z")).to be(true) + expect( field.validate_column("invalid-date")).to be(false) + expect( field.validate_column("2014-02-17")).to be(false) + end + it "should validate a date" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#date" + }) + expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false) + expect( field.validate_column("invalid-date")).to be(false) + expect( field.validate_column("2014-02-17")).to be(true) + end + it "should validate a time" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#time" + }) + expect( field.validate_column("11:09:00")).to be(true) + expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false) + expect( field.validate_column("not-a-time")).to be(false) + expect( field.validate_column("27:97:00")).to be(false) + end + it "should allow user to specify custom date time pattern" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#dateTime", + "datePattern" => "%Y-%m-%d %H:%M:%S" + }) + expect( field.validate_column("1999-12-01 10:00:00")).to be(true) + expect( field.validate_column("invalid-date")).to be(false) + expect( field.validate_column("2014-02-17")).to be(false) + end + it "should allow user to compare dates" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#dateTime", + "datePattern" => "%Y-%m-%d %H:%M:%S", + "minimum" => "1990-01-01 10:00:00" + }) + expect( field.validate_column("1999-12-01 10:00:00")).to be(true) + expect( field.validate_column("1989-12-01 10:00:00")).to be(false) end - - end end end \ No newline at end of file From 5bbdcf72f258ab8ad85031e03ef437c1f7e5f6bf Mon Sep 17 00:00:00 2001 From: Leigh Dodds Date: Tue, 18 Feb 2014 11:54:49 +0000 Subject: [PATCH 2/3] Added year and year-month --- lib/csvlint/field.rb | 14 +++++++++++++- spec/field_spec.rb | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/csvlint/field.rb b/lib/csvlint/field.rb index abf3a565..7ac09759 100644 --- a/lib/csvlint/field.rb +++ b/lib/csvlint/field.rb @@ -59,7 +59,19 @@ class Field d = DateTime.strptime(value, date_pattern) raise ArgumentError unless d.strftime(date_pattern) == value d - end + end, + 'http://www.w3.org/2001/XMLSchema#gYear' => lambda do |value, constraints| + date_pattern = constraints["datePattern"] || "%Y" + d = Date.strptime(value, date_pattern) + raise ArgumentError unless d.strftime(date_pattern) == value + d + end, + 'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints| + date_pattern = constraints["datePattern"] || "%Y-%m" + d = Date.strptime(value, date_pattern) + raise ArgumentError unless d.strftime(date_pattern) == value + d + end } def initialize(name, constraints={}, title=nil, description=nil) diff --git a/spec/field_spec.rb b/spec/field_spec.rb index c4ace278..466bcd15 100644 --- a/spec/field_spec.rb +++ b/spec/field_spec.rb @@ -170,6 +170,27 @@ expect( field.validate_column("not-a-time")).to be(false) expect( field.validate_column("27:97:00")).to be(false) end + it "should validate a year" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#gYear" + }) + expect( field.validate_column("1999")).to be(true) + expect( field.validate_column("2525")).to be(true) + expect( field.validate_column("0001")).to be(true) + expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false) + expect( field.validate_column("not-a-time")).to be(false) + expect( field.validate_column("27:97:00")).to be(false) + end + it "should validate a year-month" do + field = Csvlint::Field.new("test", { + "type" => "http://www.w3.org/2001/XMLSchema#gYearMonth" + }) + expect( field.validate_column("1999-12")).to be(true) + expect( field.validate_column("2525-01")).to be(true) + expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false) + expect( field.validate_column("not-a-time")).to be(false) + expect( field.validate_column("27:97:00")).to be(false) + end it "should allow user to specify custom date time pattern" do field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#dateTime", From 3b7a0fe6de3377dc62c720fd4ac92242aaae9204 Mon Sep 17 00:00:00 2001 From: Leigh Dodds Date: Tue, 18 Feb 2014 11:58:31 +0000 Subject: [PATCH 3/3] Updated list of data types --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index db918876..38eb2be0 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ Supported constraints: * `type` -- specifies an XML Schema data type. Values of the column must be a valid value for that type * `minimum` -- specify a minimum range for values, the value will be parsed as specified by `type` * `maximum` -- specify a maximum range for values, the value will be parsed as specified by `type` +* `datePattern` -- specify a `strftime` compatible date pattern to be used when parsing date values and min/max constraints Supported data types (this is still a work in progress): @@ -180,6 +181,11 @@ Supported data types (this is still a work in progress): * Positive Integer -- `http://www.w3.org/2001/XMLSchema#positiveInteger` * Non Negative Integer -- `http://www.w3.org/2001/XMLSchema#nonNegativeInteger` * Negative Integer -- `http://www.w3.org/2001/XMLSchema#negativeInteger` +* Date -- `http://www.w3.org/2001/XMLSchema#date` +* Date Time -- `http://www.w3.org/2001/XMLSchema#dateTime` +* Year -- `http://www.w3.org/2001/XMLSchema#gYear` +* Year Month -- `http://www.w3.org/2001/XMLSchema#gYearMonth` +* Time -- `http://www.w3.org/2001/XMLSchema#time` Schema validation provides some additional types of error and warning messages: