diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb index e1810d0c..95f29929 100644 --- a/app/controllers/home_controller.rb +++ b/app/controllers/home_controller.rb @@ -1,30 +1,4 @@ -require './lib/external/original_audio_harvester' - class HomeController < ApplicationController - include OriginalAudioHarvester - def index -=begin - topdir = File.join(Rails.root,'media','harvestwaiting') - - harvest_paths = OriginalAudioHarvester.directory_list topdir - config_files = harvest_paths.collect {|dir| OriginalAudioHarvester.config_file_path dir } - file_lists = harvest_paths.collect {|dir| OriginalAudioHarvester.file_list dir} - - config_file_objects = config_files.collect { |file| OriginalAudioHarvester.read_config_file file } - - result = {} - - file_lists.each do |files| - files.each do |file| - result[file] = OriginalAudioHarvester.file_info file - end - end - - params[:testing] = result - - - raise RuntimeError -=end end end diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb index 960eeb99..e192a938 100644 --- a/app/controllers/media_controller.rb +++ b/app/controllers/media_controller.rb @@ -1,5 +1,7 @@ +require './lib/modules/mime' + class MediaController < ApplicationController - include FileCacher + include FileCacher, Mime #respond_to :xml, :json, :html, :png, :ogg, :oga, :webm, :webma, :mp3 @@ -113,22 +115,4 @@ def read_dir(dir) end end end -end - -module Mime - class Type - class << self - # Lookup, guesstimate if fail, the file extension - # for a given mime string. For example: - # - # >> Mime::Type.file_extension_of 'text/rss+xml' - # => "xml" - def file_extension_of(mime_string) - set = Mime::LOOKUP[mime_string] - sym = set.instance_variable_get("@symbol") if set - return sym.to_s if sym - return $1 if mime_string =~ /(\w+)$/ - end - end - end end \ No newline at end of file diff --git a/app/models/audio_recording.rb b/app/models/audio_recording.rb index df5d7918..f4327c41 100644 --- a/app/models/audio_recording.rb +++ b/app/models/audio_recording.rb @@ -12,7 +12,10 @@ class AudioRecording < ActiveRecord::Base # attr attr_accessible :bit_rate_bps, :channels, :data_length_bytes, :duration_seconds, :file_hash, :media_type, :notes, - :recorded_date, :sample_rate_hertz, :status, :uploader_id + :recorded_date, :sample_rate_hertz, :status, :uploader_id, + :site_id + + accepts_nested_attributes_for :site # userstamp stampable diff --git a/app/models/site.rb b/app/models/site.rb index 13985418..72d5f357 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -13,6 +13,8 @@ class Site < ActiveRecord::Base # attr attr_accessible :name, :latitude, :longitude, :notes + accepts_nested_attributes_for :audio_recordings + # userstamp stampable belongs_to :user diff --git a/lib/external/original_audio_harvester.rb b/lib/external/original_audio_harvester.rb index b98618d6..7d24f4c7 100644 --- a/lib/external/original_audio_harvester.rb +++ b/lib/external/original_audio_harvester.rb @@ -1,26 +1,41 @@ require 'yaml' -require 'rest_client' +require 'net/http' +require 'json' +require 'digest' + +=begin +require './lib/modules/OS' +require './lib/modules/audio_sox' +require './lib/modules/audio_wavpack' +require './lib/modules/audio_ffmpeg' +require './lib/modules/audio_mp3splt' +require './lib/modules/hash' + +require './lib/modules/audio' +require './lib/modules/cache' +require './lib/exceptions' +=end + +require '../modules/OS' +require '../modules/audio_sox' +require '../modules/audio_wavpack' +require '../modules/audio_ffmpeg' +require '../modules/audio_mp3splt' +require '../modules/hash' + +require '../modules/audio' +require '../modules/cache' +require '../exceptions' + module OriginalAudioHarvester - include Audio, Cache + include Audio, Cache, Exceptions @folder_config = 'folder_config.yml' =begin Usage: - # get get all the sub dirs in the top level harvest directory (only one level deep, not recursive) - harvest_paths = OriginalAudioHarvester.directory_list topdir - - # get the config files in each sub dir - config_files = harvest_paths.collect {|dir| OriginalAudioHarvester.config_file_path dir } - config_file_objects = config_files.collect { |file| OriginalAudioHarvester.read_config_file file } - - # get the other files in the sub dirs (excluding config files) - file_lists = harvest_paths.collect {|dir| OriginalAudioHarvester.file_list dir} - # get the information about the files into a hash - # including calculating the recording start dates using either a recognised file name format, - # or file accessed/modified/created date # e.g. { file name => { @@ -86,7 +101,7 @@ def self.file_info(full_file_path) end # calculate the audio recording start date and time - def self.recording_start_datetime(full_path) + def self.recording_start_datetime(full_path, utc_offset) if File.exists? full_path access_time = File.atime full_path change_time = File.ctime full_path @@ -94,13 +109,15 @@ def self.recording_start_datetime(full_path) file_name = File.basename full_path - datetime_from_file = DateTime.new + datetime_from_file = modified_time # _yyyyMMdd_HHmmss. - file_name.scan(/.*_(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})\..+/) do |year,month, day, hour, min ,sec| - #datetime_from_file = DateTime.strptime + file_name.scan(/.*_(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})\..+/) do |year, month, day, hour, min ,sec| + datetime_from_file = DateTime.new(year.to_i, month.to_i, day.to_i, hour.to_i, min.to_i, sec.to_i, utc_offset) end end + + datetime_from_file end # constructs the full path that a file will be moved to @@ -108,9 +125,109 @@ def self.create_target_path(original_base_path, uuid, audio_info) end + def self.generate_hash(file_path) + incr_hash = Digest::SHA256.new + + File.open(file_path) do|file| + buffer = '' + + # Read the file 512 bytes at a time + until file.eof + file.read(512, buffer) + incr_hash.update(buffer) + end + end + + incr_hash + end + + def self.create_params(file_path, file_info, config_file_object) + # info we need to send, construct based on mime type + + puts file_info + + media_type = file_info[:info][:ffmpeg]['STREAM codec_type']+'/'+file_info[:info][:ffmpeg]['STREAM codec_name'] + recording_start = recording_start_datetime(file_path, config_file_object['utc_offset']) + + to_send = { + :file_hash => 'SHA256::'+OriginalAudioHarvester.generate_hash(file_path).hexdigest, + :sample_rate_hertz => file_info[:info][:ffmpeg]['STREAM sample_rate'], + :media_type => media_type, + :uploader_id => config_file_object['uploader_id'], + :site_id => config_file_object['site_id'], + :recorded_date => recording_start + } + + if media_type == 'audio/wavpack' + to_send[:bit_rate_bps] = file_info[:info][:wavpack]['ave bitrate'] + to_send[:data_length_bytes] = file_info[:info][:wavpack]['file size'] + to_send[:channels] = file_info[:info][:wavpack]['channels'] + to_send[:duration_seconds] = file_info[:info][:wavpack]['duration'] + else + to_send[:bit_rate_bps] = file_info[:info][:ffmpeg]['FORMAT bit_rate'] + to_send[:data_length_bytes] =file_info[:info][:ffmpeg]['FORMAT size'] + to_send[:channels] = file_info[:info][:ffmpeg]['STREAM channels'] + to_send[:duration_seconds] = AudioFfmpeg::parse_duration(file_info[:info][:ffmpeg]['FORMAT duration']) + end + + to_send + end + # get uuid for audio recording from website via REST API - def self.new_uuid() + # If you post to a Ruby on Rails REST API endpoint, then you'll get an + # InvalidAuthenticityToken exception unless you set a different + # content type in the request headers, since any post from a form must + # contain an authenticity token. + def self.rails_post(endpoint, post_params) + uri = URI.parse(endpoint) + + request = Net::HTTP::Post.new uri.path + request.body = post_params.to_json + request["Content-Type"] = "application/json" + + http1 = Net::HTTP.new uri.hostname, uri.port + http1.set_debug_output $stderr + + response = http1.start() do |http| + # http + # http.request request + http.request request + end + + response + end + + def self.run_once_dir(top_dir) + # get a sub dir in the top level harvest directory (only one level deep, not recursive) + available_dir = OriginalAudioHarvester.directory_list(top_dir).first + + # get a non-config file from the sub dir + file_to_process = OriginalAudioHarvester.file_list(available_dir).first + OriginalAudioHarvester.run_once_file file_to_process end -end \ No newline at end of file + def self.run_once_file(file_path) + # get the config file in the same dir + config_file = OriginalAudioHarvester.config_file_path(File.dirname(file_path)) + + # load the config file + config_file_object = OriginalAudioHarvester.read_config_file config_file + + # get info about the file to process + file_info = OriginalAudioHarvester.file_info(file_path) + + # get the params to send + to_send = OriginalAudioHarvester.create_params(file_path, file_info, config_file_object) + + post_result = rails_post('http://localhost:3000/audio_recordings.json',to_send) + + post_result + end + +end + + +# run the script +#topdir = File.join(Rails.root,'media','harvestwaiting') +#OriginalAudioHarvester.run_once_file '20081202-07-koala-calls.mp3' \ No newline at end of file diff --git a/lib/modules/OS.rb b/lib/modules/OS.rb index 6e5be786..95197cb2 100644 --- a/lib/modules/OS.rb +++ b/lib/modules/OS.rb @@ -1,18 +1,19 @@ # From http://stackoverflow.com/questions/170956/how-can-i-find-which-operating-system-my-ruby-program-is-running-on module OS - def self.windows? + + def OS.windows? (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil end - def self.mac? + def OS.mac? (/darwin/ =~ RUBY_PLATFORM) != nil end - def self.unix? + def OS.unix? !OS.windows? end - def self.linux? + def OS.linux? OS.unix? and not OS.mac? end end \ No newline at end of file diff --git a/lib/modules/audio.rb b/lib/modules/audio.rb index 2b8559e9..571fe4a6 100644 --- a/lib/modules/audio.rb +++ b/lib/modules/audio.rb @@ -1,3 +1,5 @@ +require 'open3' + module Audio include AudioSox, AudioMp3splt, AudioWavpack, AudioFfmpeg @@ -7,13 +9,13 @@ def self.info(source) result = {} sox = AudioSox::info_sox source - result.deep_merge! sox + result = result.deep_merge sox ffmpeg = AudioFfmpeg::info_ffmpeg source - result.deep_merge! ffmpeg + result = result.deep_merge ffmpeg wavpack = AudioWavpack::info_wavpack source - result.deep_merge! wavpack + result = result.deep_merge wavpack # return the packaged info array result @@ -37,6 +39,7 @@ def self.modify(source, target, modify_parameters) AudioWavpack::modify_wavpack(source, target_possible_paths.first, modify_parameters) end + end end \ No newline at end of file diff --git a/lib/modules/audio_ffmpeg.rb b/lib/modules/audio_ffmpeg.rb index c816ee28..ce0da293 100644 --- a/lib/modules/audio_ffmpeg.rb +++ b/lib/modules/audio_ffmpeg.rb @@ -3,6 +3,20 @@ module AudioFfmpeg @ffmpeg_path = if OS.windows? then "./vendor/bin/ffmpeg/windows/ffmpeg.exe" else "ffmpeg" end @ffprobe_path = if OS.windows? then "./vendor/bin/ffmpeg/windows/ffprobe.exe" else "ffprobe" end + # constants + CODECS = { + :wav => { + :codec_name => 'pcm_s16le', + :codec_long_name => 'PCM signed 16-bit little-endian', + :codec_type => 'audio', + :format_long_name => 'WAV / WAVE (Waveform Audio)' + }, + :mp3 => { + + } + } + + # public methods public @@ -15,18 +29,19 @@ def self.info_ffmpeg(source) ffprobe_command = "#@ffprobe_path #{ffprobe_arguments_info} \"#{source}\"" ffprobe_stdout_str, ffprobe_stderr_str, ffprobe_status = Open3.capture3(ffprobe_command) - Rails.logger.debug "Ffprobe info return status #{ffprobe_status.exitstatus}. Command: #{ffprobe_command}" + puts "Ffprobe info return status #{ffprobe_status.exitstatus}. Command: #{ffprobe_command}" + #Rails.logger.debug "Ffprobe info return status #{ffprobe_status.exitstatus}. Command: #{ffprobe_command}" if ffprobe_status.exitstatus == 0 result[:info][:ffmpeg] = parse_ffprobe_output(ffprobe_stdout_str) else - Rails.logger.debug "Ffprobe info error. Return status #{ffprobe_status.exitstatus}. Command: #{ffprobe_command}" + #Rails.logger.debug "Ffprobe info error. Return status #{ffprobe_status.exitstatus}. Command: #{ffprobe_command}" result[:error][:ffmpeg] = parse_ffprobe_output(ffprobe_stdout_str) end - if result[:info][:ffmpeg]['FFPROBE STREAM codec_type'] != 'audio' + if result[:info][:ffmpeg]['STREAM codec_type'] != 'audio' result[:error][:ffmpeg][:file_type] = 'Not an audio file.' - Rails.logger.debug "Ffprobe gave info about a non-audio file." + #Rails.logger.debug "Ffprobe gave info about a non-audio file." end @@ -40,6 +55,16 @@ def self.modify_ffmpeg(source, target, modify_parameters = {}) end + # returns the duration in seconds (and fractions if present) + def self.parse_duration(duration_string) + duration_match = /(?\d+):(?\d+):(?[\d+\.]+)/i.match(duration_string) + duration = 0 + if !duration_match.nil? && duration_match.size == 4 + duration = (duration_match[:hour].to_f * 60 * 60) + (duration_match[:minute].to_f * 60) + duration_match[:second].to_f + end + duration + end + private def self.parse_ffprobe_output(raw) diff --git a/lib/modules/audio_mp3splt.rb b/lib/modules/audio_mp3splt.rb index 49627478..1084a51a 100644 --- a/lib/modules/audio_mp3splt.rb +++ b/lib/modules/audio_mp3splt.rb @@ -5,10 +5,6 @@ module AudioMp3splt # public methods public - def self.info_mp3splt(source) - [ [], [] ] - end - # @param [file path] source # @param [file path] target # @param [hash] modify_parameters @@ -17,27 +13,31 @@ def self.modify_mp3splt(source, target, modify_parameters = {}) raise ArgumentError, "Source is not a mp3 file: #{File.basename(source)}" unless source.match(/\.mp3$/) raise ArgumentError, "Target is not a mp3 file: : #{File.basename(target)}" unless target.match(/\.mp3$/) raise ArgumentError, "Source does not exist: #{File.basename(source)}" unless File.exists? source - raise ArgumentError, "Target exists: #{File.basename(target)}" unless !File.exists? source + raise ArgumentError, "Target exists: #{File.basename(target)}" if File.exists? target - info = [] - error = [] + result = { + :info => { :wavpack => {} }, + :error => { :wavpack => {} } + } + # mp3splt needs the file extension removed target_dirname = File.dirname target - target_no_ext = File.basename target - arguments = " -d \"#{target_dirname}\" -o #{target_no_ext} #{source}" + target_no_ext = File.basename(target, File.extname(target)) + arguments = " -d \"#{target_dirname}\" -o \"#{target_no_ext}\" \"#{source}\"" # WARNING: can't get more than an hour, since minutes only goes to 59. # formatted time: mm.ss.ss - if modify_parameters.include? :start_offset && modify_parameters[:start_offset] > 0 - start_offset_formatted = Time.at(modify_parameters[:start_offset]).utc.strftime('%M.%S.%2N') - start_offset = start_offset_formatted + start_offset_num = 0.0 + if modify_parameters.include?(:start_offset) && modify_parameters[:start_offset] > 0 + start_offset = ' '+Time.at(modify_parameters[:start_offset]).utc.strftime('%M.%S.%2N')+' ' + start_offset_num = modify_parameters[:start_offset] else - start_offset = 0 + start_offset = ' 0.0 ' end arguments += " #{start_offset} " - if modify_parameters.include? :end_offset && modify_parameters[:end_offset] > 0 && modify_parameters[:end_offset] > start_offset + if modify_parameters.include?(:end_offset) && modify_parameters[:end_offset] > 0 && modify_parameters[:end_offset] > start_offset_num end_offset_formatted = Time.at(modify_parameters[:end_offset]).utc.strftime('%M.%S.%2N') arguments += " #{end_offset_formatted} " else @@ -47,10 +47,12 @@ def self.modify_mp3splt(source, target, modify_parameters = {}) mp3splt_command = "#@mp3splt_path #{arguments}" # commands to get info from audio file mp3splt_stdout_str, mp3splt_stderr_str, mp3splt_status = Open3.capture3(mp3splt_command) # run the commands and wait for the result + Rails.logger.debug "mp3splt info return status #{mp3splt_status.exitstatus}. Command: #{mp3splt_command}" + if mp3splt_status.exitstatus != 0 || !File.exists?(target) raise "Mp3splt exited with an error: #{mp3splt_stderr_str}" end - [info, error] + result end end \ No newline at end of file diff --git a/lib/modules/audio_shared.rb b/lib/modules/audio_shared.rb new file mode 100644 index 00000000..3aee5796 --- /dev/null +++ b/lib/modules/audio_shared.rb @@ -0,0 +1,18 @@ +module AudioShared + def AudioShared.run_info(executable_file, command) + stdout_str, stderr_str, status = Open3.capture3("#{executable_file} #{command}") + if status.exitstatus != 0 + raise + end + [stdout_str, stderr_str, status] + end + + def AudioShared.run_convert(executable_file, command, target_file) + stdout_str, stderr_str, status = Open3.capture3("#{executable_file} #{command}") + + if status.exitstatus != 0 || !File.exists?(target) + + end + [stdout_str, stderr_str, status] + end +end \ No newline at end of file diff --git a/lib/modules/audio_sox.rb b/lib/modules/audio_sox.rb index ae95acad..f826b912 100644 --- a/lib/modules/audio_sox.rb +++ b/lib/modules/audio_sox.rb @@ -1,5 +1,5 @@ module AudioSox - include OS +include OS @sox_path = if OS.windows? then "./vendor/bin/sox/windows/sox.exe" else "sox" end # public methods @@ -15,12 +15,17 @@ def self.info_sox(source) sox_command = "#@sox_path #{sox_arguments_info} \"#{source}\"" # commands to get info from audio file sox_stdout_str, sox_stderr_str, sox_status = Open3.capture3(sox_command) # run the commands and wait for the result + + puts "sox info return status #{sox_status.exitstatus}. Command: #{sox_command}" + + #Rails.logger.debug "mp3splt info return status #{sox_status.exitstatus}. Command: #{sox_command}" + if sox_status.exitstatus == 0 # sox std out contains info (separate on first colon(:)) sox_stdout_str.strip.split(/\r?\n|\r/).each { |line| result[:info][:sox][ line[0,line.index(':')].strip] = line[line.index(':')+1,line.length].strip } # sox_stderr_str is empty else - Rails.logger.debug "Sox info error. Return status #{sox_status.exitstatus}. Command: #{sox_command}" + #Rails.logger.debug "Sox info error. Return status #{sox_status.exitstatus}. Command: #{sox_command}" result[:error][:sox][:stderror] = sox_stderr_str end diff --git a/lib/modules/audio_wavpack.rb b/lib/modules/audio_wavpack.rb index 11f290d1..03d7cef1 100644 --- a/lib/modules/audio_wavpack.rb +++ b/lib/modules/audio_wavpack.rb @@ -1,22 +1,25 @@ module AudioWavpack include OS + + # path to the wvunpack executable for different platforms @wvunpack_path = if OS.windows? then "./vendor/bin/wavpack/windows/wvunpack.exe" else "wvunpack" end - # public methods public - # @return [array] contains info and error arrays + # get information about a wav file. def self.info_wavpack(source) result = { :info => { :wavpack => {} }, :error => { :wavpack => {} } } - wvunpack_arguments_info = "-s" - wvunpack_command = "#@wvunpack_path #{wvunpack_arguments_info} \"#{source}\"" # commands to get info from audio file - wvunpack_stdout_str, wvunpack_stderr_str, wvunpack_status = Open3.capture3(wvunpack_command) # run the commands and wait for the result + # commands to get info from audio file + wvunpack_command = "#@wvunpack_path -s \"#{source}\"" + + # run the commands and wait for the result + wvunpack_stdout_str, wvunpack_stderr_str, wvunpack_status = Open3.capture3(wvunpack_command) - Rails.logger.debug "Wavpack info return status #{wvunpack_status.exitstatus}. Command: #{wvunpack_command}" + #Rails.logger.debug "Wavpack info return status #{wvunpack_status.exitstatus}." if wvunpack_status.exitstatus == 0 # wvunpack std out contains info (separate on first colon(:)) @@ -42,15 +45,17 @@ def self.modify_wavpack(source, target, modify_parameters = {}) raise ArgumentError, "Source is not a wavpack file: #{File.basename(source)}" unless source.match(/\.wv$/) raise ArgumentError, "Target is not a wav file: : #{File.basename(target)}" unless target.match(/\.wav$/) + result = { + :info => { :wavpack => {} }, + :error => { :wavpack => {} } + } + if File.exists? target - return [[],[]] + return result end raise ArgumentError, "Source does not exist: #{File.basename(source)}" unless File.exists? source - info = [] - error = [] - # formatted time: hh:mm:ss.ss arguments = '-t -q' if modify_parameters.include? :start_offset @@ -71,10 +76,13 @@ def self.modify_wavpack(source, target, modify_parameters = {}) wvunpack_stdout_str, wvunpack_stderr_str, wvunpack_status = Open3.capture3(wvunpack_command) # run the commands and wait for the result + #Rails.logger.debug "mp3splt info return status #{wvunpack_status.exitstatus}. Command: #{wvunpack_command}" + if wvunpack_status.exitstatus != 0 || !File.exists?(target) raise "Wvunpack command #{wvunpack_command} exited with an error: #{wvunpack_stderr_str}" end - [info, error] + result end + end diff --git a/lib/modules/file_cacher.rb b/lib/modules/file_cacher.rb index fcd3741e..69887d4e 100644 --- a/lib/modules/file_cacher.rb +++ b/lib/modules/file_cacher.rb @@ -43,7 +43,7 @@ def self.generate_spectrogram(modify_parameters = {}) target_existing_paths.first end - def self. create_audio_segment(modify_parameters = {}) + def self.create_audio_segment(modify_parameters = {}) # first check if a cached audio file matches the request target_file = Cache::cached_audio_file modify_parameters target_existing_paths = Cache::existing_paths(Cache::cached_audio_storage_paths,target_file) diff --git a/lib/modules/hash.rb b/lib/modules/hash.rb new file mode 100644 index 00000000..b6b0305f --- /dev/null +++ b/lib/modules/hash.rb @@ -0,0 +1,10 @@ +# http://qugstart.com/blog/uncategorized/ruby-multi-level-nested-hash-value/ +# user_hash.hash_val(:extra, :birthday, :year) => 1951 +class ::Hash + + # http://stackoverflow.com/a/9381776 + def deep_merge(second) + merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 } + self.merge(second, &merger) + end +end \ No newline at end of file