Merge pull request #222 from YaleDHLab/originals

store vecs in kwargs, shrink cells, and shrink originals

pixplot/pixplot.py

@@ -67,7 +67,6 @@ def timestamp():
 try:
   from MulticoreTSNE import MulticoreTSNE as TSNE
 except:
-  print(timestamp(), 'MulticoreTSNE not available; using sklearn TSNE')
   from sklearn.manifold import TSNE
 
 try:
@@ -145,6 +144,7 @@ def process_images(**kwargs):
   kwargs['out_dir'] = join(kwargs['out_dir'], 'data')
   kwargs['image_paths'], kwargs['metadata'] = filter_images(**kwargs)
   kwargs['atlas_dir'] = get_atlas_data(**kwargs)
+  kwargs['vecs'] = get_inception_vectors(**kwargs)
   get_manifest(**kwargs)
   write_images(**kwargs)
   print(timestamp(), 'Done!')
@@ -574,7 +574,7 @@ def get_inception_vectors(**kwargs):
 
 def get_umap_layout(**kwargs):
   '''Get the x,y positions of images passed through a umap projection'''
-  vecs = get_inception_vectors(**kwargs)
+  vecs = kwargs['vecs']
   w = PCA(n_components=min(100, len(vecs))).fit_transform(vecs)
   print(timestamp(), 'Creating umap layout')
   # single model umap
@@ -584,12 +584,12 @@ def get_umap_layout(**kwargs):
     return process_multi_layout_umap(w, **kwargs)
 
 
-def process_single_layout_umap(vecs, **kwargs):
+def process_single_layout_umap(v, **kwargs):
   '''Create a single layout UMAP projection'''
   model = get_umap_model(**kwargs)
   out_path = get_path('layouts', 'umap', **kwargs)
   if cuml_ready:
-    z = model.fit(vecs).embedding_
+    z = model.fit(v).embedding_
   else:
     if os.path.exists(out_path) and kwargs['use_cache']: return out_path
     y = []
@@ -603,7 +603,7 @@ def process_single_layout_umap(vecs, **kwargs):
           else: y.append(d[i])
         y = np.array(y)
     # project the PCA space down to 2d for visualization
-    z = model.fit(vecs, y=y if np.any(y) else None).embedding_
+    z = model.fit(v, y=y if np.any(y) else None).embedding_
   return {
     'variants': [
       {
@@ -615,7 +615,7 @@ def process_single_layout_umap(vecs, **kwargs):
     ]
   }
 
-def process_multi_layout_umap(vecs, **kwargs):
+def process_multi_layout_umap(v, **kwargs):
   '''Create a multi-layout UMAP projection'''
   params = []
   for n_neighbors, min_dist in itertools.product(kwargs['n_neighbors'], kwargs['min_dist']):
@@ -628,7 +628,7 @@ def process_multi_layout_umap(vecs, **kwargs):
       'out_path': out_path,
     })
   # map each image's index to itself and create one copy of that map for each layout
-  relations_dict = {idx: idx for idx, _ in enumerate(vecs)}
+  relations_dict = {idx: idx for idx, _ in enumerate(v)}
   # determine the subset of params that have already been computed
   uncomputed_params = [i for i in params if not os.path.exists(i['out_path'])]
   # determine the filepath where this model will be saved
@@ -641,7 +641,7 @@ def process_multi_layout_umap(vecs, **kwargs):
   if os.path.exists(model_path):
     model = load_model(model_path)
     for i in uncomputed_params:
-      model.update(vecs, relations_dict.copy())
+      model.update(v, relations_dict.copy())
     # after updating, we can read the results from the end of the updated model
     for idx, i in enumerate(uncomputed_params):
       embedding = z.embeddings_[len(uncomputed_params)-idx]
@@ -655,7 +655,7 @@ def process_multi_layout_umap(vecs, **kwargs):
     )
     # fit the model on the data
     z = model.fit(
-      [vecs for _ in params],
+      [v for _ in params],
       relations=[relations_dict.copy() for _ in params[:-1]]
     )
     for idx, i in enumerate(params):
@@ -1161,7 +1161,7 @@ def get_hotspots(layouts={}, use_high_dimensional_vectors=True, **kwargs):
   '''Return the stable clusters from the condensed tree of connected components from the density graph'''
   print(timestamp(), 'Clustering data with {}'.format(cluster_method))
   if use_high_dimensional_vectors:
-    vecs = get_inception_vectors(**kwargs)
+    vecs = kwargs['vecs']
   else:
     vecs = read_json(layouts['umap']['variants'][0]['layout'], **kwargs)
   model = get_cluster_model(**kwargs)
@@ -1241,7 +1241,9 @@ def write_images(**kwargs):
     if not exists(out_dir): os.makedirs(out_dir)
     out_path = join(out_dir, filename)
     if not os.path.exists(out_path):
-      shutil.copy(i.path, out_path)
+      resized = i.resize_to_height(600)
+      resized = array_to_img(resized)
+      save_img(out_path, resized)
     # copy thumb for lod texture
     out_dir = join(kwargs['out_dir'], 'thumbs')
     if not exists(out_dir): os.makedirs(out_dir)

pixplot/web/assets/js/tsne.js

@@ -152,7 +152,7 @@ Data.prototype.parseManifest = function(json) {
   // update the point size DOM element
   world.elems.pointSize.min = 0;
   world.elems.pointSize.max = config.size.points.max;
-  world.elems.pointSize.value = config.size.points.initial;
+  world.elems.pointSize.value = config.size.points.initial / window.devicePixelRatio;
   // set number of atlases and textures
   this.atlasCount = json.atlas.count;
   this.textureCount = Math.ceil(json.atlas.count / config.atlasesPerTex);
@@ -762,7 +762,7 @@ Layout.prototype.setPointScalar = function() {
   if (l == 'geographic') size = config.size.points.geographic;
   if (l == 'date') size = config.size.points.date;
   if (size) {
-    world.elems.pointSize.value = size;
+    world.elems.pointSize.value = size / window.devicePixelRatio;
     var scale = world.getPointScale();
     world.setUniform('targetScale', scale);
   }
@@ -4024,7 +4024,7 @@ function worldToScreenCoords(pos) {
 **/
 
 window.location.href = '#';
-window.devicePixelRatio = Math.min(window.devicePixelRatio, 2);
+window.devicePixelRatio = window.devicePixelRatio || 1;
 var welcome = new Welcome();
 var webgl = new Webgl();
 var config = new Config();

setup.py

@@ -17,7 +17,7 @@
 
 setup(
   name='pixplot',
-  version='0.0.110',
+  version='0.0.111',
   packages=['pixplot'],
   package_data={
     'pixplot': web,