Fix CsvSource quotes parser

teritorio · Dec 2, 2024 · 71e92b7 · 71e92b7
1 parent 99310c6
commit 71e92b7
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/datasources/sources/csv.rb b/datasources/sources/csv.rb
@@ -16,6 +16,7 @@ class Settings < Source::SourceSettings
     const :url, String
     const :uncompress, T.nilable(String)
     const :col_sep, String, default: ','
+    const :quote_char, String, default: '"'
     const :id, T::Array[String]
     const :lon, String
     const :lat, String
@@ -25,7 +26,7 @@ class Settings < Source::SourceSettings
   extend T::Generic
   SettingsType = type_member{ { upper: Settings } } # Generic param
 
-  def fetch(url, col_sep)
+  def fetch(url, col_sep, quote_char)
     resp = HTTP.follow.get(url)
     if !resp.status.success?
       raise [url, resp].inspect
@@ -36,11 +37,11 @@ def fetch(url, col_sep)
       reader = Bzip2::FFI::Reader.read(StringIO.new(reader))
     end
 
-    CSV.parse(reader, headers: true, col_sep: col_sep, quote_char: nil).each(&:to_h)
+    CSV.parse(reader, headers: true, col_sep: col_sep, quote_char: quote_char).each(&:to_h)
   end
 
   def each
-    super(ENV['NO_DATA'] ? [] : fetch(@settings.url, @settings.col_sep))
+    super(ENV['NO_DATA'] ? [] : fetch(@settings.url, @settings.col_sep, @settings.quote_char))
   end
 
   def map_id(feat)