Retina 3D UMAP in Blender: Data Import and Scene Setup

retina_3d_umap_blender.py

# Here is the workflow in a nutshell:
#
# 1) Annotate cells in your favorite single cell framework or system
# 2) Perform your favorite dimensionality reduction into 3D.
# 3) Export annotation and 3D coordinates to .csv.
# 4) I imported these data into Blender and created the objects using the below ‘retina_3d_umap_blender.py’ script.  Some of it is automated, some of it is hard-coded and project specific.
#   - For each age (since cells here are colored by developmental age) I create a new mesh and add each datapoint for a given age to the mesh as a vertex.
# 5) I then created a single ’sample object’ for each mesh consisting of a UV sphere primitive.  Added a material to that to match the color scheme I had selected.  These were the ’templates’ to be used and applied to each vertex in the mesh using a particle system
# 6) Next created a particle system for each age with the params indicated in the python script.
# 7)  Finally I created an empty parent object to group all of the particles so I could track with the camera system
# 8)  I then ran the code in the second script ’set_scene.py’ which sets up the other important elements in the Blender system including lighting, cameras, and camera tracks.  
# 9) The rest was basically by feel to create animations that I thought would be appealing (but as I said, I’m not really a Blender artist, so it was mostly ad hoc)

import bpy
import bmesh
import random
import csv

# Set render engine to cycles
bpy.context.scene.render.engine = 'CYCLES'

#Read in retina UMAP data
filename = "retina_pdata.csv"
fields = ["barcode","sample","age","Size_Factor","num_genes_expressed","Total_mRNAs","CellType","raw_cluster","new_CellType","largeVis1","largeVis2","largeVis3","newCellType","umap_cluster","umap_coord1","umap_coord2","umap_coord3","umap_CellType","used_for_pseudotime"]

reader = csv.DictReader(open(filename), fields, delimiter=',')

data = [row for row in reader]
data.pop(0)

# Get unique list of cell types
ages = set()
for datum in data:
    ages.add(datum['age'])

#Create meshes with single vertices for each celltype
scene = bpy.context.scene
objs = {}
meshes = {}
for age in ages:
    mesh = bpy.data.meshes.new("mesh")
    obj = bpy.data.objects.new(age, mesh)
    scene.objects.link(obj)  # put the object into the scene (link)
    scene.objects.active = obj  # set as the active object in the scene
    obj.select = False
    mesh = bpy.context.object.data
    bm = bmesh.new()
    bm.verts.new([0,0,0])
    bm.to_mesh(mesh)  
    bm.free()
    obj.select = False
    objs.setdefault(age,obj)
    meshes.setdefault(age,bm)
    

#obj = bpy.data.objects['Circle']

#if obj.mode == 'EDIT':
#    bm = bmesh.from_edit_mesh(obj.data)
#for v in bm.verts:
#    if v.select:
#        print(v.co)
#    else:
#        print("Object is not in edit mode")

#bm = bmesh.from_edit_mesh(obj.data)


# Colorize?
#rnd_rgba = lambda: [random.random() for i in range(4)]


#for val in range(10000):
for age in ages:
    targetObj = objs[age]
    targetObj.select = True
    bpy.context.scene.objects.active = targetObj
    bpy.ops.object.mode_set(mode='EDIT')
    curBm = bmesh.from_edit_mesh(targetObj.data)
    for i in data:
        if i['age'] == age:
            coords = [float(i['umap_coord1']),float(i['umap_coord2']),float(i['umap_coord3'])]
            curBm.verts.new(coords)
        else:
            pass
    bmesh.update_edit_mesh(targetObj.data)
    bpy.ops.object.mode_set(mode='OBJECT')
    targetObj.select = False
    print(age)


# Add Floor
bpy.ops.mesh.primitive_plane_add(radius=1, view_align=False, enter_editmode=False, location=(0,0, -5), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))
bpy.ops.transform.resize(value=(12,12,12), constraint_axis=(False, False, False), constraint_orientation='GLOBAL', mirror=False, proportional='DISABLED', proportional_edit_falloff='SMOOTH', proportional_size=1)

obj = bpy.context.active_object

# Get material
mat = bpy.data.materials.get("Flooring")
if mat is None:
    # create material
    mat = bpy.data.materials.new(name="Flooring")

# Assign it to object
if obj.data.materials:
    # assign to 1st material slot
    obj.data.materials[0] = mat
else:
    # no slots
    obj.data.materials.append(mat)

bpy.context.object.active_material.diffuse_color = (0.085, 0.085, 0.085)




# Add base objects for each age to make particles from each mesh (this is hard coded currently but could be automated)
age_colors = {'E11':(1.0, 0.0, 0.0),
'E12':(1.0, 0.6, 0.0),
'E14':(0.8, 1.0, 0.0),
'E16':(0.2, 1.0, 0.0),
'E18':(0.0, 1.0, 0.4),
'P0':(0.0, 1.0, 1.0),
'P2':(0.0, 0.4, 1.0),
'P5':(0.2, 0.0, 1.0),
'P8':(0.8, 0.0, 1.0),
'P14':(1.0,0.0,0.6)
}

age_counts = {}
for datum in data:
    age_counts[datum['age']] = age_counts.setdefault(datum['age'],0)+1

for age in ages:
    bpy.ops.mesh.primitive_uv_sphere_add(size=1, view_align=False, enter_editmode=False, location=(10, 10,10), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))
    for obj in bpy.context.selected_objects:
        obj.name = "keys_%s" % (age)
    mat = bpy.data.materials.get("material_%s" % age)
    if mat is None:
        # create material
        mat = bpy.data.materials.new(name="material_%s" % age)
    obj = bpy.data.objects["keys_%s"% (age)]
    scene.objects.active = obj
    # Assign material to object
    if obj.data.materials:
        # assign to 1st material slot
        obj.data.materials[0] = mat
    else:
        # no slots
        obj.data.materials.append(mat)
    bpy.context.object.active_material.diffuse_color = age_colors[age]
    bpy.ops.transform.resize(value=(-0.0740146, -0.0740146, -0.0740146), constraint_axis=(False, False, False), constraint_orientation='GLOBAL', mirror=False, proportional='DISABLED', proportional_edit_falloff='SMOOTH', proportional_size=1)

# Create Particle systems for each cell type mesh
for age in ages:
    obj = objs[age]
    scene.objects.active = obj
    obj.select = True
    bpy.ops.object.particle_system_add()
    obj.particle_systems[0].name = age
    part = obj.particle_systems[age]
    obj.particle_systems[age].settings.type = 'HAIR'
    obj.particle_systems[age].settings.use_advanced_hair = True
    obj.particle_systems[age].settings.hair_length = 10
    obj.particle_systems[age].settings.count = age_counts[age]
    obj.particle_systems[age].settings.emit_from = 'VERT'
    obj.particle_systems[age].settings.use_emit_random = False
    obj.particle_systems[age].settings.use_render_emitter = False
    obj.particle_systems[age].settings.render_type = 'OBJECT'
    obj.particle_systems[age].settings.dupli_object = bpy.data.objects["keys_%s"% (age)]
    
#Create Empty parent object
bpy.ops.object.empty_add(type='SPHERE', view_align=False, location=(0, 0, 0), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))

retina_pdata.csv

#truncated
"barcode","sample","age","Size_Factor","num_genes_expressed","Total_mRNAs","CellType","raw_cluster","new_CellType","largeVis1","largeVis2","largeVis3","newCellType","umap_cluster","umap_coord1","umap_coord2","umap_coord3","umap_CellType","used_for_pseudotime"
"E11.AAACCTGAGATGTAAC-1","E11","E11",1.73336947467531,1549,2794,"RPCs",32,"Early RPCs",2.23381750394553,-2.88810484339682,19.6098717791926,NA,130,3.58145476834942,-4.24576963917367,-3.75250223212542,"Early RPCs","FALSE"
"E11.AAACCTGAGGCAATTA-1","E11","E11",2.20114348466285,1848,3548,"RPCs",9,"Early RPCs",-0.680810089641868,-1.94696336706416,20.6122880357731,NA,103,2.63415065878584,-5.01684553372587,-3.92695603161619,"Early RPCs","FALSE"
"E11.AAACCTGAGTAGCGGT-1","E11","E11",2.38105656542728,1935,3838,"RPCs",24,"RPE/Margin/Periocular Mesenchyme/Lens Epithelial Cells",1.60317575213534,-3.16879624867717,21.1391586023674,NA,130,3.26884759392871,-3.96717744752578,-3.94266390362768,"Early RPCs","FALSE"
"E11.AAACCTGAGTGGAGTC-1","E11","E11",2.86123837408823,2139,4612,"RPCs",9,"Early RPCs",-0.735996548765238,-2.75750490279714,20.0452302967444,NA,115,3.73762459995756,-4.92651558603825,-4.01205750142569,"Early RPCs","TRUE"
"E11.AAACCTGCAAGTTGTC-1","E11","E11",2.04480522137789,1723,3296,"Lens Epithelia",74,"RPE/Margin/Periocular Mesenchyme/Lens Epithelial Cells",2.74582377470435,-2.7980057113443,24.4341818579708,NA,77,3.85785355982862,-4.02729221590716,-2.71743740510086,"Early RPCs","FALSE"
"E11.AAACCTGCACCCATTC-1","E11","E11",2.28055339617267,1806,3676,"RPCs",32,"Early RPCs",1.95791339408462,-3.12737647170953,19.3257848565467,NA,109,3.38698469893636,-3.67884392529438,-2.93926777528061,"Early RPCs","TRUE"
"E11.AAACCTGCACGGATAG-1","E11","E11",2.24270961021877,1765,3615,"RPCs",17,"Early RPCs",3.41394996466218,-2.59680875164801,18.3774328358115,NA,77,4.05577499432004,-4.18541642225432,-3.24282593875812,"Early RPCs","TRUE"

set_scene.py

import bpy
import bmesh
import random
import csv

#Read in retina UMAP data
filename = "retina_pdata.csv"
fields = ["barcode","sample","age","Size_Factor","num_genes_expressed","Total_mRNAs","CellType","raw_cluster","new_CellType","largeVis1","largeVis2","largeVis3","newCellType","umap_cluster","umap_coord1","umap_coord2","umap_coord3","umap_CellType","used_for_pseudotime"]

reader = csv.DictReader(open(filename), fields, delimiter=',')

data = [row for row in reader]
data.pop(0)

# Get unique list of cell types
ages = set()
for datum in data:
    ages.add(datum['age'])

#Add lamps
bpy.ops.object.lamp_add(type='SUN', radius=1, view_align=False, location=(5.4, 4, 2.3), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))
bpy.ops.object.lamp_add(type='SUN', radius=1, view_align=False, location=(5.4, 4, 2.3), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))

#bpy.context.object.data.energy = 1

# Configure camera
camera = bpy.data.objects['Camera']
camera.location.x=22.17867
camera.location.y=6.21281
camera.location.z=6.12145


# Group Cell objects on empty sphere
bpy.ops.object.empty_add(type='SPHERE', view_align=False, location=(0, 0, 0), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))
bpy.context.object.name = "Cells"
for age in ages:
    bpy.data.objects[age].parent = bpy.data.objects["Cells"]


#Add Camera Target
bpy.ops.object.empty_add(type='SPHERE', view_align=False, location=(0, 0, 0), layers=(True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False))
bpy.context.object.name = "Camera Target"
bpy.data.objects["Camera"].parent = bpy.data.objects["Camera Target"]
camera.select = True
bpy.data.objects["Camera Target"].constraint_add(type='TRACK_TO')
bpy.data.objects['Camera Target'].constraints["Track To"].target = bpy.data.objects["Cells"]
bpy.data.objects['Camera Target'].constraints["Track To"].track_axis = 'TRACK_NEGATIVE_Z'
bpy.data.objects['Camera Target'].constraints["Track To"].up_axis = 'UP_Y'