newexp wrapper updates

bin/wrap-fastframe

@@ -28,7 +28,7 @@ else:
     rank = 0
 
 #- The proceed with other imports
-import sys, os, glob, time
+import sys, os, glob, time, subprocess
 tstart = time.time()
 import numpy as np
 
@@ -89,8 +89,24 @@ if rank < len(simspecfiles):
 
         try:
             t0 = time.time()
-            desisim.scripts.fastframe.main(cmd.split()[1:])
+            logfile = filename.replace('simspec', 'fastframe').replace('.fits', '.log')
+            assert logfile != filename
+
+            #- Use subprocess.call instead of fastframe.main() to avoid
+            #- potential memory leaks and separate logging; this does incur
+            #- extra python interpreter startup time.
+            ### desisim.scripts.fastframe.main(cmd.split()[1:])
+
+            print('logging to {}'.format(logfile))
+            with open(logfile, 'w') as logx:
+                err = subprocess.call(cmd.split(), stdout=logx, stderr=logx)
+
             runtime = time.time() - t0
+            if err != 0:
+                print("ERROR: rank {} simspec {} error code {} after {:.1f} sec".format(
+                    rank, os.path.basename(filename), err, runtime))
+                raise RuntimeError
+
             print("rank {} took {:.1f} seconds for {} frame".format(rank, runtime, flavor))
             sys.stdout.flush()
         except:

bin/wrap-newexp

@@ -8,6 +8,10 @@ MPI wrapper for newexp
 #- from login node where MPI doesn't work
 from __future__ import absolute_import, division, print_function
 import argparse
+import subprocess
+import sys
+
+import desisim.io
 
 parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
@@ -49,7 +53,7 @@ from astropy.table import Table
 
 from desisim.util import dateobs2night
 import desisim.io
-import desisim.scripts.newexp
+import desisim.scripts.newexp_mock
 import desisim.scripts.newflat
 import desisim.scripts.newarc
 
@@ -166,7 +170,7 @@ if rank < len(todo):
         (flavor, night, expid, obsnum, obscond) = thisobs
         assert flavor in ('science', 'arc', 'flat')
         if flavor == 'science':
-            cmd = "newexp --obslist {} --fiberassign {} --mockdir {}".format(
+            cmd = "newexp-mock --obslist {} --fiberassign {} --mockdir {}".format(
                 args.obslist, args.fiberassign, args.mockdir)
             if args.outdir is not None:
                 cmd = cmd + " --outdir {}".format(args.outdir)
@@ -183,19 +187,33 @@ if rank < len(todo):
         #- TODO: what about logging?  Use desisim.pipe log redirection logic?
 
         print('RUNNING: {}'.format(cmd))
+        sys.stdout.flush()
         if args.dryrun:
             continue
 
         try:
             t0 = time.time()
-            if cmd.startswith('newexp'):
-                desisim.scripts.newexp.main(cmd.split()[1:])
-            elif cmd.startswith('newflat'):
-                desisim.scripts.newflat.main(cmd.split()[1:])
-            elif cmd.startswith('newarc'):
-                desisim.scripts.newarc.main(cmd.split()[1:])
-            else:
-                print('ERROR: unknown command {}'.format(cmd))
+            logfile = desisim.io.findfile(
+                'simspec', night, expid).replace('.fits', '.log')
+            print('logging {}-{} {} to {}'.format(night, expid, flavor, logfile))
+
+            #- Spawn call to avoid memory leak problems from repeated
+            #- desisim.scripts.newexp_mock.main() calls within same interpreter
+            with open(logfile, 'w') as logx:
+                err = subprocess.call(cmd.split(), stdout=logx, stderr=logx)
+            # if cmd.startswith('newexp'):
+            #     desisim.scripts.newexp_mock.main(cmd.split()[1:])
+            # elif cmd.startswith('newflat'):
+            #     desisim.scripts.newflat.main(cmd.split()[1:])
+            # elif cmd.startswith('newarc'):
+            #     desisim.scripts.newarc.main(cmd.split()[1:])
+            # else:
+            #     print('ERROR: unknown command {}'.format(cmd))
+            if err != 0:
+                print('ERROR: night {} expid {} failed with error {}'.format(
+                    night, expid, err))
+                raise RuntimeError
+
             runtime = time.time() - t0
             print("{} took {:.1f} seconds".format(cmd.split()[0], runtime))
             sys.stdout.flush()

doc/changes.rst

@@ -18,11 +18,13 @@ desisim change log
 * Miscellaneous polishing in QA (velocity, clip before RMS, extend [OII] flux, S/N per Ang)
 * Bug fix: correctly select both "bright" and "faint" BGS templates by default
   (`PR #257`_).  
+* Updates for newexp/fastframe wrappers for end-to-end sims (`PR #258`_).
 
 .. _`PR #250`: https://github.com/desihub/desisim/pull/250
 .. _`PR #252`: https://github.com/desihub/desisim/pull/252
 .. _`PR #254`: https://github.com/desihub/desisim/pull/254
 .. _`PR #257`: https://github.com/desihub/desisim/pull/257
+.. _`PR #258`: https://github.com/desihub/desisim/pull/258
 
 0.20.0 (2017-07-12)
 -------------------

py/desisim/io.py

@@ -60,6 +60,8 @@ def findfile(filetype, night, expid, camera=None, outdir=None, mkdir=True):
         simpix = '{outdir:s}/simpix-{expid:08d}.fits',
         simfibermap = '{outdir:s}/fibermap-{expid:08d}.fits',
         pix = '{outdir:s}/pix-{camera:s}-{expid:08d}.fits',
+        fastframelog = '{outdir:s}/fastframe-{expid:08d}.log',
+        newexplog = '{outdir:s}/newexp-{expid:08d}.log',
     )
 
     #- Do we know about this kind of file?
@@ -129,9 +131,15 @@ def write_simspec(sim, truth, fibermap, obs, expid, night, outdir=None, filename
     header['NIGHT'] = night
     header['EXPTIME'] = sim.observation.exposure_time.to('s').value
     if obs is not None:
-        for key in obs.keys():
-            if key not in header:
-                header[key] = obs[key]
+        try:
+            keys = obs.keys()
+        except AttributeError:
+            keys = obs.dtype.names
+
+        for key in keys:
+            shortkey = key[0:8]  #- FITS keywords can only be 8 char
+            if shortkey not in header:
+                header[shortkey] = obs[key]
     if 'DOSVER' not in header:
         header['DOSVER'] = 'SIM'
     if 'FEEVER' not in header:
@@ -880,7 +888,7 @@ def simdir(night='', mkdir=False):
     Return $DESI_SPECTRO_SIM/$PIXPROD/{night}
     If mkdir is True, create directory if needed
     """
-    dirname = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'), night)
+    dirname = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'), str(night))
     if mkdir and not os.path.exists(dirname):
         os.makedirs(dirname)
 

py/desisim/obs.py

@@ -175,7 +175,7 @@ def new_exposure(program, nspec=5000, night=None, expid=None, tileid=None,
     if exptime is not None:
         obsconditions['EXPTIME'] = exptime
 
-    sim = simulate_spectra(wave, 1e-17*flux, fibermap=fibermap, obsconditions=obsconditions)
+    sim = simulate_spectra(wave, flux, fibermap=fibermap, obsconditions=obsconditions)
 
     #- Override $DESI_SPECTRO_DATA in order to write to simulation area
     datadir_orig = os.getenv('DESI_SPECTRO_DATA')
@@ -444,7 +444,9 @@ def update_obslog(obstype='science', program='DARK', expid=None, dateobs=None,
     INSERT OR REPLACE INTO obslog(expid,dateobs,night,obstype,program,tileid,ra,dec)
     VALUES (?,?,?,?,?,?,?,?)
     """
-    db.execute(insert, (expid, time.mktime(dateobs), night, obstype.upper(), program.upper(), tileid, ra, dec))
+    db.execute(insert, (int(expid), time.mktime(dateobs), str(night),
+        str(obstype.upper()), str(program.upper()), int(tileid),
+        float(ra), float(dec)))
     db.commit()
 
     return expid, dateobs

py/desisim/scripts/fastframe.py

@@ -22,8 +22,10 @@ def parse(options=None):
     parser = argparse.ArgumentParser(usage = "{prog} [options]")
     parser.add_argument("--simspec", type=str,  help="input simspec file")
     parser.add_argument("--outdir", type=str,  help="output directory")
-    parser.add_argument("--firstspec", type=int, default=0, help="first spectrum to simulate")
-    parser.add_argument("--nspec", type=int, default=5000, help="number of spectra to simulate")
+    parser.add_argument("--firstspec", type=int, default=0,
+                        help="first spectrum to simulate")
+    parser.add_argument("--nspec", type=int, default=5000,
+                        help="number of spectra to simulate")
 
     if options is None:
         args = parser.parse_args()
@@ -61,20 +63,22 @@ def main(args=None):
     flux = simspec.flux
     ii = slice(firstspec, firstspec+nspec)
     if simspec.flavor == 'science':
-        sim = desisim.simexp.simulate_spectra(wave, 1e-17*flux[ii], fibermap=fibermap[ii],
-            obsconditions=obs, dwave_out=1.0)
+        sim = desisim.simexp.simulate_spectra(wave, flux[ii],
+                fibermap=fibermap[ii], obsconditions=obs, dwave_out=1.0)
     elif simspec.flavor in ['arc', 'flat', 'calib']:
         x = fibermap['X_TARGET']
         y = fibermap['Y_TARGET']
-        fiber_area = desisim.simexp.fiber_area_arcsec2(fibermap['X_TARGET'], fibermap['Y_TARGET'])
+        fiber_area = desisim.simexp.fiber_area_arcsec2(
+                fibermap['X_TARGET'], fibermap['Y_TARGET'])
         surface_brightness = (flux.T / fiber_area).T
         config = desisim.simexp._specsim_config_for_wave(wave, dwave_out=1.0)
         # sim = specsim.simulator.Simulator(config, num_fibers=nspec)
         sim = desisim.specsim.get_simulator(config, num_fibers=nspec)
         sim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
         sbunit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm ** 2 * u.arcsec ** 2)
         xy = np.vstack([x, y]).T * u.mm
-        sim.simulate(calibration_surface_brightness=surface_brightness[ii]*sbunit, focal_positions=xy[ii])
+        sim.simulate(calibration_surface_brightness=surface_brightness[ii]*sbunit,
+                     focal_positions=xy[ii])
     else:
         raise ValueError('Unknown simspec flavor {}'.format(simspec.flavor))
 
@@ -89,7 +93,8 @@ def main(args=None):
                 results['random_noise_electrons']).T
         ivar = 1.0 / results['variance_electrons'].T
         R = Resolution(sim.instrument.cameras[i].get_output_resolution_matrix())
-        Rdata = np.tile(R.data.T, nspec).T.reshape(nspec, R.data.shape[0], R.data.shape[1])
+        Rdata = np.tile(R.data.T, nspec).T.reshape(
+                        nspec, R.data.shape[0], R.data.shape[1])
         assert np.all(Rdata[0] == R.data)
         assert phot.shape == (nspec, len(wave))
         for spectro in range(10):
@@ -105,6 +110,7 @@ def main(args=None):
             meta['CAMERA'] = camera
             frame = Frame(wave, xphot, xivar, resolution_data=Rdata[0:imax-imin],
                 spectrograph=spectro, fibermap=xfibermap, meta=meta)
-            outfile = desispec.io.findfile('frame', night, expid, camera, outdir=args.outdir)
+            outfile = desispec.io.findfile('frame', night, expid, camera,
+                                           outdir=args.outdir)
             print('writing {}'.format(outfile))
             desispec.io.write_frame(outfile, frame)

py/desisim/scripts/newexp_mock.py

@@ -101,7 +101,12 @@ def main(args=None):
         args.nspec = len(fiberassign)
 
     log.info('Simulating night {} expid {} tile {}'.format(night, args.expid, tileid))
-    flux, wave, meta = get_mock_spectra(fiberassign, mockdir=args.mockdir)
+    try:
+        flux, wave, meta = get_mock_spectra(fiberassign, mockdir=args.mockdir)
+    except Exception as err:
+        log.fatal('Failed obsnum {} fiberassign {} tile {}'.format(args.obsnum, args.fiberassign, tileid))
+        raise err
+
     sim, fibermap = simscience((flux, wave, meta), fiberassign, obsconditions=obs, nspec=args.nspec)
 
     #- TODO: header keyword code is replicated from obs.new_exposure()
@@ -114,17 +119,17 @@ def main(args=None):
         FLAVOR = ('science', 'Flavor [arc, flat, science, zero, ...]'),
         TELRA = (telera, 'Telescope pointing RA [degrees]'),
         TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
-        AIRMASS = (obsconditions['AIRMASS'], 'Airmass at middle of exposure'),
-        EXPTIME = (obsconditions['EXPTIME'], 'Exposure time [sec]'),
-        SEEING = (obsconditions['SEEING'], 'Seeing FWHM [arcsec]'),
-        MOONFRAC = (obsconditions['MOONFRAC'], 'Moon illumination fraction 0-1; 1=full'),
-        MOONALT  = (obsconditions['MOONALT'], 'Moon altitude [degrees]'),
-        MOONSEP  = (obsconditions['MOONSEP'], 'Moon:tile separation angle [degrees]'),
+        AIRMASS = (obs['AIRMASS'], 'Airmass at middle of exposure'),
+        EXPTIME = (obs['EXPTIME'], 'Exposure time [sec]'),
+        SEEING = (obs['SEEING'], 'Seeing FWHM [arcsec]'),
+        MOONFRAC = (obs['MOONFRAC'], 'Moon illumination fraction 0-1; 1=full'),
+        MOONALT  = (obs['MOONALT'], 'Moon altitude [degrees]'),
+        MOONSEP  = (obs['MOONSEP'], 'Moon:tile separation angle [degrees]'),
         )
     header['DATE-OBS'] = (sim.observation.exposure_start.isot, 'Start of exposure')
 
     #- Write fibermap to $DESI_SPECTRO_SIM/$PIXPROD not $DESI_SPECTRO_DATA
-    fibermap.meta.extend(header)
+    fibermap.meta.update(header)
     fibermap.write(desisim.io.findfile('simfibermap', night, args.expid,
         outdir=args.outdir), overwrite=args.clobber)