qf

#!/usr/bin/env lua

-- above expects to find  ~/my/repos/k_edit/lua  which should be in PATH

-- 20120102 kgoodwin wrote this (finally!)

require "strict"  -- can be found in same dir as this file: %~dp0strict.lua

local q1,q2 = "'", '"'
local hardcodedArg = { 'm:/ebooks' }  -- undefine or empty to allow passing cmdline args (in arg variable)

local isCygWin = os.getenv('SHELL') and os.getenv('SHELL'):match( '%.exe$' ) or false
local fnmdisp
do
   if isCygWin then
      fnmdisp = function( fnm )
         fnm = fnm:gsub('\\','/')
         fnm = fnm:gsub('^(%a):/', '/%1/')
         if fnm:match'[$%s"&!|{}()]' and not fnm:match"'" then
            fnm = "'"..fnm.."'"
         else
            if fnm:match"'" then
               fnm = '"'..fnm..'"'
               end
            end
         return fnm
         end
   else
      fnmdisp = function( fnm )
         fnm = fnm:gsub('/','\\')
         if fnm:match"[%s'&!|()]" then
            fnm = '"'..fnm..'"'
            end
         return fnm
         end
      end
end

local function ms_esc(fnm)
   return fnmdisp(fnm)
   end

local function splitfpath(fullnm)  -- io.write("fnm=",fullnm,"\n")
   local dn,fn = fullnm:match"(.+[\\/])([^\\/]+)$"
   if not fn then return nil,fullnm end
   return dn,fn
   end

fmt = string.format
local mtfile = assert( getmetatable( io.stdout ) )
function mtfile:fmt(...)    return self:write( fmt(...)  ) end

function table.invert( tbl ) local rv = {} for ky,val in pairs(tbl) do rv[ val ] = ky  end return rv end

local function a_grep(  fx,ary )   local rv = {} for ky,val in ipairs(ary) do if fx( val ) then rv[ #rv + 1 ] = val end end return rv end
local function a_filenames( dirnm, recurse )  -- filter out directories and files in .xxx directories
   return a_grep( function(nm) return not (nm:match("[\\/]$") or nm:match("[\\/]%.%w+[\\/]")) end, _dir.read_names( dirnm and #dirnm > 0 and dirnm or "./", recurse and 1 or 0 ) )
   end

local function pairsByKeys( tbl1, sortfxn )  -- a pairsByKeys using pairsByKeyFxn would be overkill since there's no chance of duplicate keys
   local iv = {}
   for key1,val1 in pairs(tbl1) do iv[ #iv + 1 ] = key1 end
   table.sort( iv, sortfxn )
   local idx = 0
   return function()
      idx = idx + 1
      local key1 = iv[ idx ]
      -- if key1 then print( "+++  ", key1 ) end
      return key1, tbl1[ key1 ]
      end
   end

local function count(base, pattern)  -- https://stackoverflow.com/a/51256340
   return select(2, string.gsub(base, pattern, ""))
   end


local last_year = (os.date"*t").year+1  -- io.write( 'last_year', ' ', last_year, '\n' )
local function monok(mon) return mon >= 1 and mon <= 12 end
local function ynok(yn) yn = tonumber(yn) return yn > 1912 and yn <= last_year end
local t_monlongs = table.invert{"January","February","March","April","May","June","July","August","September","October","November","December"}
local t_mons = table.invert{"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"}

local fs_ext     local function sv_ext  (st) fs_ext   = st  return "" end
local fs_cropd   local function sv_cropd(st) fs_cropd = st  return "" end

local function monconv(pat,tbl)
   local pat1 = "[-.]"..pat.."%.".."(%d%d%d%d)$"
   local pat2 = "[-.]"..pat.."%."..pat.."%.".."(%d%d%d%d)$"
                                              -- io.write( 'mc+', ' ', pat1, ' ', pat2, '\n' )
   return function(fn)                        -- io.write( 'mc+', ' ', fn, ' ', pat, '\n' )
      local m1,m2,yr = fn:match(pat2)
      if yr and ynok( yr ) and tbl[m1] and tbl[m2] then
         return fn:gsub(pat2,"."..fmt("%s.%02d",yr,tbl[m1]))
         end
      m1,yr = fn:match(pat1)
      if yr and ynok( yr ) and tbl[m1] then  -- io.write( 'mc#', ' ', yr, ' ', m1, '\n' )
         return fn:gsub(pat1,"."..fmt("%s.%02d",yr,tbl[m1]))
         end
      return fn
      end
   end

local function show(fxn)  -- debug filters by wrapping with this
   return function(fn)
      io.write( '+', fn, '\n' ) fn = fxn(fn) io.write( '-', fn, '\n' ) return fn end
   end

local filters = {

   --------------- preprocessing
   function(fn)  fs_ext=""  fs_cropd=""  return fn         end,
   -- convert exo-ASCII chars to punct
   function(fn)  return fn:gsub( "–"         , "-" )       end,
   function(fn)  return fn:gsub( ' '         , '.' )       end,

   function(fn) return fn:gsub( "^Letmeread%.net_"       , "" ) end,
   function(fn) return fn:gsub( '[%s%.]%(z%-lib%.org%)'  , "" ) end,
   function(fn) return fn:gsub( '%.mobilism%.org'        , "" ) end,

   function(fn)  return fn:gsub( "%.[^.]+$"  , sv_ext   )  end,
   function(fn)  return fn:gsub( "_cropped$" , sv_cropd )  end,
   function(fn)  return fn:gsub( "^[-.]+" , '' )  end,

   function(fn)  return count(fn, "[%.%_%s]")==0 and count(fn, "%-")>1 and fn:gsub("%-",'.') or fn  end,  -- some (e.g. magazine) file names are 100% '-' separated
   function(fn)  return count(fn, "[%-%.%_%s]")==0 and count(fn, "%u%l")>1 and fn:gsub("(%u%l+)",'%1.'):gsub("(%d+)",'%1.'):gsub('%.$','') or fn  end,  -- some (e.g. magazine) file names are 100% CamelCase-only

   function(fn)  return fn:match( '[^_]_[^_]' ) and fn:gsub( "__+", "-" ) or fn end,
   function(fn)  return fn:gsub( "[%s.,_!]+", "." ) end,

   function(fn)  return fn:gsub( "%.?%-%.?", "-" ) end,

   --------------- actual filters
   monconv("(%a+)"   , t_monlongs ),  -- replace (publication) Monthname-Yyyy with Yyyy-mm
   monconv("(%a%a%a)", t_mons     ),  -- replace (publication) Mnm-Yyyy with Yyyy-mm

   function(fn) -- replace LEADING publication Yyyy-mm-dd (or Yyyy.mm.dd) with TRAILING Yyyy.mm[.dd]
      local yr,_,mon,day,ix = fn:match"^(%d%d%d%d)([%-%.])(%d%d)%2(%d%d)[%-%+%._]?()%u"
      if ix then
         return fn:sub( ix ) .. ('.'.. yr..'.'..mon) .. (day == '01' and '' or ('.'..day))
         end
      return fn
      end,

   --[[ function(fn) -- replace (publication) Yyyy-mm with Yyyy in older fnms
      local yr,mon = fn:match"%.(%d%d%d%d)%-(%d%d)$"
      if yr then
         local yn = tonumber(yr)
         mon   = tonumber(mon)
         -- io.write( yr, " ? ", last_year,"\n" )
         if monok( mon ) and ynok( yn ) then
            return fn:gsub("%.(%d%d%d%d)%-%d%d$",".%1")
            end
         end
      return fn
      end, ]]

   function(fn) return fn:gsub( "%((%d%d%d%d)%)$",".%1" ) end,  -- alt yr-of-publ fn format

   function(fn) return fn:gsub( "['‘’“”]"  , "" ) end,
   function(fn) return fn:gsub( "[%._-]+$" , "" ) end,

   function(fn) return fn:gsub( "([CcFf])#%.", "%1sharp." ) end,
   function(fn) return fn:gsub( "^Black%.Decker"  , "BD." ) end,
   function(fn) return fn:gsub( "%.039%.s%."  , "s." ) end,

   function(fn) return fn:gsub( "U%.S%.", "US." ) end,                                      -- go
   function(fn) return fn:gsub( "%.#(%d+)", ".v%1" ) end,
   function(fn) return fn:gsub( "%.#%.", ".@." ) end,
   function(fn) return fn:gsub( "[Ww]orld%.[Ww]ar%.[I1]%([-.])", "WW1%1" ) end,
   function(fn) return fn:gsub( "[Ww]orld%.[Ww]ar%.2%([-.])" , "WW2%1" ) end,
   function(fn) return fn:gsub( "[Ww]orld%.[Ww]ar%.II([-.])", "WW2%1" ) end,
   function(fn) return fn:gsub( "%.[Ww][Ww]II", ".WW2" ) end,

   function(fn) return fn:gsub( "[-.]1st%.[Ee]dition([-%.])"  , ".1e%1" ) end,
   function(fn) return fn:gsub( "[-.][Ff]irst%.[Ee]dition([-%.])"  , ".1e%1" ) end,
   function(fn) return fn:gsub( "[-.][Ss]econd%.[Ee]dition([-%.])"  , ".2e%1" ) end,
   function(fn) return fn:gsub( "[-.](%d+)[nr]d%.[Ee]dition([-%.])"  , ".%1e%2" ) end,
   function(fn) return fn:gsub( "[-.](%d+)th%.[Ee]dition([-%.])"  , ".%1e%2" ) end,
   function(fn) return fn:gsub( "[-.]&%.", ".n." ) end,                                       -- go
   function(fn) return fn:gsub( '%.downmagaz%.net'       , "" ) end,

   --------------- un-preprocessing
   function(fn) return fn..fs_cropd end,  -- #########  MUST BE LAST-1 !!!
   function(fn) return fn..fs_ext   end,  -- #########  MUST BE LAST   !!!
   }

local destroy = true  -- true for production, false for no-op verification
local rename_file,remove_file,mvUndoLog_fnm,mvUndoLog,mvCount
do
   local function sms_esc(fn)
      local mfn = ms_esc(fn)
      if not mfn:match( '^['..q1..q2..']' ) then mfn = " " .. mfn end
      return mfn
      end
   local function rm_msg(fn)     io.write( "removing ",sms_esc(fn), "\n" ) end
   local function mv_msg(fn,nfn) io.write( "move.now ",sms_esc(fn), "\nmove.new ",sms_esc(nfn),"\n" ) end
   if destroy then
      mvUndoLog_fnm = assert( os.getenv( "APPDATA" ), "$APPDATA not defined?" ) .. '/qf_mv.log'
      mvUndoLog = assert( io.open( mvUndoLog_fnm, 'a+' ) )
      mvUndoLog:write( '\n' )
      -- io.write( 'mvUndoLog_fnm ', mvUndoLog_fnm, '\n' )
      mvCount = 0
      rename_file = function( fn,nfn ) mv_msg(fn,nfn)
                       local r,e = os.rename( fn,nfn )
                       if r then
                          mvUndoLog:write( 'mv -n ', sms_esc(nfn), ' ', sms_esc(fn), '\n' )
                          mvUndoLog:flush()
                          mvCount = 1+mvCount
                       else
                          io.stderr:write( e, '\n' )
                          end
                       end
      remove_file = function( fn )     rm_msg(fn)     assert( os.remove(fn)       ) end
   else
      rename_file = mv_msg
      remove_file = rm_msg
      end
end

local mtpdf = 'medtype.pdf'
local function humanExtOf( fnm )
   if fnm:match('%.medtype.pdf$') then return mtpdf end
   return fnm:match('%.([^.]+)$')
   end

-- io.write( humanExtOf( 'm:\\ebooks\\travel\\non-NorthAmerica\\Lonely.Planet.Peru.8e.2013.medtype.pdf' ), '\n' )
-- io.write( humanExtOf( 'm:\\ebooks\\travel\\non-NorthAmerica\\Lonely.Planet.Peru.8e.2013.mextype.pdf' ), '\n' )
-- os.exit(1)

local function processFilenames( fnm )
   local dn,fn = splitfpath( fnm )     -- ; io.write( "d=",dn,",f=",fn,"\n" )
   local nfn = fn
   local mb,xt = 0,false               -- ; io.write( "\n","nfn=",nfn,"\n" )
   for ix,filter in ipairs(filters) do
      local _fn = filter( nfn )
      if false and _fn ~= nfn then
         io.write( "old[",ix,"]=",nfn,"\n" )
         io.write( "new[",ix,"]=",_fn,"\n" )
         end
      nfn = _fn
      end
   if nfn ~= fn then                   --   io.write( "now=",fn,"\nnew=",nfn,"\n" )
      fn  = dn .. fn
      nfn = dn .. nfn                  -- ; io.write( "pnow=",fn,"\npnew=",nfn,"\n" )
      local fs  = io.stat( fn )
      local nfs = io.stat( nfn )
      if nfs then  -- something in the way?
         if fn:lower()==nfn:lower() then
            io.write( "recase ",ms_esc(fn)," ",ms_esc(nfn),"\n" )
            rename_file( fn, nfn )
         else
            io.stdout:fmt( "%8d %s\n",nfs.size, ms_esc(nfn) )
            io.stdout:fmt( "%8d %s\n", fs.size, ms_esc( fn) )
            if fs.size == nfs.size then
               remove_file(fn)
            else
               xt = humanExtOf( nfn )
               if xt == mtpdf then
                  if fs.size > nfs.size then
                     remove_file(fn)
                  else
                     remove_file(nfn)
                     rename_file( fn, nfn )
                     end
               else
                  io.write( "move blocked" )
                  if xt == 'epub' then
                     io.write( '\n' , 'diff_epub ', ms_esc(nfn), ' ', ms_esc(fn) )
                     end
                  mb = 1
                  end
               end
            end
         io.stdout:fmt( "\n" )
      else
         rename_file( fn, nfn )
         fnm = nfn
         end
      end

   return mb, xt
   end

local recurse = not (#arg > 0 and arg)
local myArg = (#arg > 0 and arg) or (hardcodedArg and #hardcodedArg > 0 and hardcodedArg)
assert( #myArg > 0, "missing rootpaths arg(s)" )
local path = table.concat( myArg, ';' )
io.write( "rootpaths=", path, "\n")
local sl = path:match("[\\/]") or '/'
local dirs = {}
for dirnm in path:gmatch"[^;]+" do
   local dnm = dirnm:gsub("[\\/]$",'')..sl  -- force (non-dup'd) trailing dirsep presence
   io.write( 'dnm ', dnm, '\n' )
   dirs[1+#dirs] = dnm
   end

-- assert(false)

local numFiles, mvBlocked = 0, 0
local t_fnms = {}
local function skippedDnm( dnm )
   local qfnoNm = '.qfno'
   local skipped =    _dir.name_isfile( dnm..qfnoNm )
                   or _dir.name_isdir ( dnm..qfnoNm )
   if skipped then
      io.write( dnm, qfnoNm, ' found, skipping\n' )
      end
   return skipped
   end
for _,dnm in ipairs(dirs) do
   if not skippedDnm( dnm ) then
      io.write("scanning ", dnm, ' ')
      local fnm_ = a_filenames( dnm, recurse )  -- filters out directories and files in .xxx directories
      t_fnms[dnm] = fnm_
      numFiles = numFiles + #fnm_
      if 0==#fnm_ then io.write("*** ERROR ***") end
                       io.write("discovered ", #fnm_," filenames")
      io.write("\n")
      end
   end

local countByExt = {}
for dnm,fnms in pairsByKeys(t_fnms) do
   if not skippedDnm( dnm ) then
      for _,nm in ipairs(fnms) do
         local mb, xt = processFilenames( nm, dnm )
         mvBlocked = mvBlocked + mb
         if xt then
            countByExt[ xt ] = 1 + (countByExt[ xt ] or 0)
            end
         end
      end
   end

io.write("examined ", numFiles," filenames, ", mvBlocked, " moves blocked\n" )
if mvUndoLog_fnm and mvUndoLog then
   mvUndoLog:close()
   if mvCount and mvCount > 0 then
      io.write( mvCount, ' files moved, undo instructions in ', mvUndoLog_fnm, '\n' )
      end
   end

local xt, ct
for xt, ct in pairsByKeys(countByExt) do
   io.stdout:fmt( "%-12s %4d\n", xt, ct )
   end