File harvester and audio modification

removed some gems
added tests for wvunpack and other console apps
harvester is almost working for geting info from a file, sending it to a
REST api, then moving it to the correct folder

app/controllers/home_controller.rb

@@ -1,30 +1,4 @@
-require './lib/external/original_audio_harvester'
-
 class HomeController < ApplicationController
-  include OriginalAudioHarvester
-
   def index
-=begin
-    topdir = File.join(Rails.root,'media','harvestwaiting')
-
-    harvest_paths = OriginalAudioHarvester.directory_list topdir
-    config_files = harvest_paths.collect {|dir| OriginalAudioHarvester.config_file_path dir }
-    file_lists = harvest_paths.collect {|dir| OriginalAudioHarvester.file_list dir}
-
-    config_file_objects = config_files.collect { |file| OriginalAudioHarvester.read_config_file file }
-
-    result = {}
-
-    file_lists.each do |files|
-      files.each do |file|
-        result[file] = OriginalAudioHarvester.file_info file
-      end
-    end
-
-    params[:testing] = result
-
-
-    raise RuntimeError
-=end
   end
 end

app/controllers/media_controller.rb

@@ -1,5 +1,7 @@
+require './lib/modules/mime'
+
 class MediaController < ApplicationController
-  include FileCacher
+  include FileCacher, Mime
 
   #respond_to :xml, :json, :html, :png, :ogg, :oga, :webm, :webma, :mp3
 
@@ -113,22 +115,4 @@ def read_dir(dir)
       end
     end
   end
-end
-
-module Mime
-  class Type
-    class << self
-      # Lookup, guesstimate if fail, the file extension
-      # for a given mime string. For example:
-      #
-      # >> Mime::Type.file_extension_of 'text/rss+xml'
-      # => "xml"
-      def file_extension_of(mime_string)
-        set = Mime::LOOKUP[mime_string]
-        sym = set.instance_variable_get("@symbol") if set
-        return sym.to_s if sym
-        return $1 if mime_string =~ /(\w+)$/
-      end
-    end
-  end
 end

app/models/audio_recording.rb

@@ -12,7 +12,10 @@ class AudioRecording < ActiveRecord::Base
   # attr
   attr_accessible :bit_rate_bps, :channels, :data_length_bytes,
                   :duration_seconds, :file_hash, :media_type, :notes,
-                  :recorded_date, :sample_rate_hertz, :status, :uploader_id
+                  :recorded_date, :sample_rate_hertz, :status, :uploader_id,
+                  :site_id
+
+  accepts_nested_attributes_for :site
 
   # userstamp
   stampable

app/models/site.rb

@@ -13,6 +13,8 @@ class Site < ActiveRecord::Base
   # attr
   attr_accessible :name, :latitude, :longitude, :notes
 
+  accepts_nested_attributes_for :audio_recordings
+
   # userstamp
   stampable
   belongs_to :user

lib/external/original_audio_harvester.rb

@@ -1,26 +1,41 @@
 require 'yaml'
-require 'rest_client'
+require 'net/http'
+require 'json'
+require 'digest'
+
+=begin
+require './lib/modules/OS'
+require './lib/modules/audio_sox'
+require './lib/modules/audio_wavpack'
+require './lib/modules/audio_ffmpeg'
+require './lib/modules/audio_mp3splt'
+require './lib/modules/hash'
+
+require './lib/modules/audio'
+require './lib/modules/cache'
+require './lib/exceptions'
+=end
+
+require '../modules/OS'
+require '../modules/audio_sox'
+require '../modules/audio_wavpack'
+require '../modules/audio_ffmpeg'
+require '../modules/audio_mp3splt'
+require '../modules/hash'
+
+require '../modules/audio'
+require '../modules/cache'
+require '../exceptions'
+
 
 module OriginalAudioHarvester
-  include Audio, Cache
+  include Audio, Cache, Exceptions
 
   @folder_config = 'folder_config.yml'
 
 =begin
   Usage:
-  # get get all the sub dirs in the top level harvest directory (only one level deep, not recursive)
-  harvest_paths = OriginalAudioHarvester.directory_list topdir
-
-  # get the config files in each sub dir
-  config_files = harvest_paths.collect {|dir| OriginalAudioHarvester.config_file_path dir }
-  config_file_objects = config_files.collect { |file| OriginalAudioHarvester.read_config_file file }
-
-  # get the other files in the sub dirs (excluding config files)
-  file_lists = harvest_paths.collect {|dir| OriginalAudioHarvester.file_list dir}
 
-  # get the information about the files into a hash
-  # including calculating the recording start dates using either a recognised file name format,
-  # or file accessed/modified/created date
   # e.g.
 {
   file name => {
@@ -86,31 +101,133 @@ def self.file_info(full_file_path)
   end
 
   # calculate the audio recording start date and time
-  def self.recording_start_datetime(full_path)
+  def self.recording_start_datetime(full_path, utc_offset)
     if File.exists? full_path
       access_time = File.atime full_path
       change_time = File.ctime full_path
       modified_time = File.mtime full_path
 
       file_name = File.basename full_path
 
-      datetime_from_file = DateTime.new
+      datetime_from_file = modified_time
 
       # _yyyyMMdd_HHmmss.
-      file_name.scan(/.*_(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})\..+/) do |year,month, day, hour, min ,sec|
-        #datetime_from_file = DateTime.strptime
+      file_name.scan(/.*_(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})\..+/) do |year, month, day, hour, min ,sec|
+        datetime_from_file = DateTime.new(year.to_i, month.to_i, day.to_i, hour.to_i, min.to_i, sec.to_i, utc_offset)
       end
     end
+
+    datetime_from_file
   end
 
   # constructs the full path that a file will be moved to
   def self.create_target_path(original_base_path, uuid, audio_info)
 
   end
 
+  def self.generate_hash(file_path)
+    incr_hash = Digest::SHA256.new
+
+    File.open(file_path) do|file|
+      buffer = ''
+
+      # Read the file 512 bytes at a time
+      until file.eof
+        file.read(512, buffer)
+        incr_hash.update(buffer)
+      end
+    end
+
+    incr_hash
+  end
+
+  def self.create_params(file_path, file_info, config_file_object)
+    # info we need to send, construct based on mime type
+
+    puts file_info
+
+    media_type = file_info[:info][:ffmpeg]['STREAM codec_type']+'/'+file_info[:info][:ffmpeg]['STREAM codec_name']
+    recording_start = recording_start_datetime(file_path, config_file_object['utc_offset'])
+
+    to_send = {
+        :file_hash => 'SHA256::'+OriginalAudioHarvester.generate_hash(file_path).hexdigest,
+        :sample_rate_hertz => file_info[:info][:ffmpeg]['STREAM sample_rate'],
+        :media_type => media_type,
+        :uploader_id => config_file_object['uploader_id'],
+        :site_id => config_file_object['site_id'],
+        :recorded_date => recording_start
+    }
+
+    if media_type == 'audio/wavpack'
+      to_send[:bit_rate_bps] = file_info[:info][:wavpack]['ave bitrate']
+      to_send[:data_length_bytes] = file_info[:info][:wavpack]['file size']
+      to_send[:channels] =  file_info[:info][:wavpack]['channels']
+      to_send[:duration_seconds] =  file_info[:info][:wavpack]['duration']
+    else
+      to_send[:bit_rate_bps] = file_info[:info][:ffmpeg]['FORMAT bit_rate']
+      to_send[:data_length_bytes] =file_info[:info][:ffmpeg]['FORMAT size']
+      to_send[:channels] = file_info[:info][:ffmpeg]['STREAM channels']
+      to_send[:duration_seconds] = AudioFfmpeg::parse_duration(file_info[:info][:ffmpeg]['FORMAT duration'])
+    end
+
+    to_send
+  end
+
   # get uuid for audio recording from website via REST API
-  def self.new_uuid()
+  # If you post to a Ruby on Rails REST API endpoint, then you'll get an
+  # InvalidAuthenticityToken exception unless you set a different
+  # content type in the request headers, since any post from a form must
+  # contain an authenticity token.
+  def self.rails_post(endpoint, post_params)
+    uri = URI.parse(endpoint)
+
+    request = Net::HTTP::Post.new uri.path
+    request.body = post_params.to_json
+    request["Content-Type"] = "application/json"
+
+    http1 = Net::HTTP.new uri.hostname, uri.port
+    http1.set_debug_output $stderr
+
+   response = http1.start() do |http|
+    #  http
+    #  http.request request
+    http.request request
+    end
+
+    response
+  end
+
+  def self.run_once_dir(top_dir)
+    # get a sub dir in the top level harvest directory (only one level deep, not recursive)
+    available_dir = OriginalAudioHarvester.directory_list(top_dir).first
+
+    # get a non-config file from the sub dir
+    file_to_process = OriginalAudioHarvester.file_list(available_dir).first
 
+    OriginalAudioHarvester.run_once_file file_to_process
   end
 
-end
+  def self.run_once_file(file_path)
+    # get the config file in the same dir
+    config_file = OriginalAudioHarvester.config_file_path(File.dirname(file_path))
+
+    # load the config file
+    config_file_object = OriginalAudioHarvester.read_config_file config_file
+
+    # get info about the file to process
+    file_info = OriginalAudioHarvester.file_info(file_path)
+
+    # get the params to send
+    to_send = OriginalAudioHarvester.create_params(file_path, file_info, config_file_object)
+
+    post_result = rails_post('http://localhost:3000/audio_recordings.json',to_send)
+
+    post_result
+  end
+
+end
+
+
+# run the script
+#topdir = File.join(Rails.root,'media','harvestwaiting')
+#OriginalAudioHarvester.run_once_file '20081202-07-koala-calls.mp3'

lib/modules/OS.rb

@@ -1,18 +1,19 @@
 # From http://stackoverflow.com/questions/170956/how-can-i-find-which-operating-system-my-ruby-program-is-running-on
 module OS
-  def self.windows?
+
+  def OS.windows?
     (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil
   end
 
-  def self.mac?
+  def OS.mac?
    (/darwin/ =~ RUBY_PLATFORM) != nil
   end
 
-  def self.unix?
+  def OS.unix?
     !OS.windows?
   end
 
-  def self.linux?
+  def OS.linux?
     OS.unix? and not OS.mac?
   end
 end

lib/modules/audio.rb

@@ -1,3 +1,5 @@
+require 'open3'
+
 module Audio
   include AudioSox, AudioMp3splt, AudioWavpack, AudioFfmpeg
 
@@ -7,13 +9,13 @@ def self.info(source)
     result = {}
 
     sox = AudioSox::info_sox source
-    result.deep_merge! sox
+    result = result.deep_merge sox
 
     ffmpeg = AudioFfmpeg::info_ffmpeg source
-    result.deep_merge!  ffmpeg
+    result = result.deep_merge  ffmpeg
 
     wavpack = AudioWavpack::info_wavpack source
-    result.deep_merge! wavpack
+    result = result.deep_merge wavpack
 
     # return the packaged info array
     result
@@ -37,6 +39,7 @@ def self.modify(source, target, modify_parameters)
 
       AudioWavpack::modify_wavpack(source, target_possible_paths.first, modify_parameters)
     end
+
   end
 
 end