Merge branch 'master' of github.com:seung-lab/igneous

README.md

@@ -458,7 +458,7 @@ tasks = create_meshing_tasks(             # First Pass
 tasks = create_mesh_manifest_tasks(layer_path, magnitude=3) # Second Pass
 ```
 
-The parameters above are mostly self explainatory, but the magnitude parameter of `create_mesh_manifest_tasks` is a bit odd. What a MeshManifestTask does is iterate through a proportion of the files defined by a filename prefix. `magnitude` splits up the work by 
+The parameters above are mostly self explanatory, but the magnitude parameter of `create_mesh_manifest_tasks` is a bit odd. What a MeshManifestTask does is iterate through a proportion of the files defined by a filename prefix. `magnitude` splits up the work by
 an additional 10^magnitude. A high magnitude (3-5+) is appropriate for horizontal scaling workloads while small magnitudes 
 (1-2) are more suited for small volumes locally processed since there is overhead introduced by splitting up the work.  
 
@@ -515,8 +515,6 @@ tasks = tc.create_skeletonizing_tasks(
     dust_threshold=1000, # Don't skeletonize below this physical distance
     progress=False, # Show a progress bar
     parallel=1, # Number of parallel processes to use (more useful locally)
-    spatial_index=True, # generate a spatial index for querying skeletons by bounding box
-    sharded=False, # generate intermediate shard fragments for later processing into sharded format
     cross_sectional_area=False, # Compute the cross sectional area for each vertex.
     cross_sectional_area_smoothing_window=5, # Rolling average of vertices.
   )

igneous/task_creation/common.py

@@ -212,7 +212,7 @@ def compute_shard_params_for_hashed(
 
   return (int(shard_bits), int(minishard_bits), 0)
 
-def set_encoding(cv, mip, encoding, encoding_level):
+def set_encoding(cv, mip, encoding, encoding_level, encoding_effort):
   scale = cv.meta.scale(mip)
   if encoding is not None:
     scale['encoding'] = encoding
@@ -226,8 +226,10 @@ def set_encoding(cv, mip, encoding, encoding_level):
 
   if encoding == "jpeg":
     scale["jpeg_quality"] = encoding_level
-  elif encoding == "jpegxl":
-    scale["jpegxl_quality"] = encoding_level
+  elif encoding == "jxl":
+    scale["jxl_quality"] = encoding_level
+    if encoding_effort is not None:
+      scale["jxl_effort"] = int(encoding_effort)
   elif encoding == "png":
     scale["png_level"] = encoding_level
   elif encoding == "fpzip":

igneous/task_creation/image.py

@@ -210,6 +210,7 @@ def create_downsampling_tasks(
   bounds_mip:int = 0,
   memory_target:int = MEMORY_TARGET,
   encoding_level:Optional[int] = None,
+  encoding_effort:Optional[int] = None,
   method:int = DownsampleMethods.AUTO,
 ):
   """
@@ -280,7 +281,7 @@ def ds_shape(mip, chunk_size=None, factor=None):
   )
 
   for mip_i in range(mip+1, min(mip + num_mips, len(vol.available_mips))):
-    set_encoding(vol, mip_i, encoding, encoding_level)
+    set_encoding(vol, mip_i, encoding, encoding_level, encoding_effort)
   vol.commit_info()
 
   if not preserve_chunk_size or chunk_size:
@@ -325,6 +326,8 @@ def on_finish(self):
           'chunk_size': (list(chunk_size) if chunk_size else None),
           'preserve_chunk_size': preserve_chunk_size,
           'encoding': encoding,
+          'encoding_level': encoding_level,
+          'encoding_effort': encoding_effort,
           'fill_missing': bool(fill_missing),
           'delete_black_uploads': bool(delete_black_uploads),
           'background_color': background_color,
@@ -483,7 +486,7 @@ def update_bits():
   if preshift_bits + shard_bits + minishard_bits > max_bits:
     raise ValueError(f"{preshift_bits} preshift_bits {shard_bits} shard_bits + {minishard_bits} minishard_bits must be <= {max_bits}. Try reducing the number of minishards.")
 
-  if encoding in ("jpeg", "png", "kempressed", "fpzip", "zfpc"):
+  if encoding in ("jpeg", "jpegxl", "jxl", "png", "kempressed", "fpzip", "zfpc"):
     data_encoding = "raw"
 
   return {
@@ -517,6 +520,7 @@ def create_image_shard_transfer_tasks(
   cutout:bool = False,
   minishard_index_encoding:str = "gzip",
   stop_layer:Optional[int] = None,
+  encoding_effort:Optional[int] = None,
 ):
   src_vol = CloudVolume(src_layer_path, mip=mip)
 
@@ -531,7 +535,7 @@ def create_image_shard_transfer_tasks(
     src_vol, dst_layer_path, 
     dest_voxel_offset, 
     mip, bounds_mip,
-    encoding, encoding_level,
+    encoding, encoding_level, encoding_effort,
     chunk_size, truncate_scales,
     clean_info, cutout, bounds
   )
@@ -599,6 +603,7 @@ def on_finish(self):
           "mip": mip,
           "encoding_level": encoding_level,
           "stop_layer": stop_layer,
+          "encoding_effort": encoding_effort,
         },
         "by": operator_contact(),
         "date": strftime("%Y-%m-%d %H:%M %Z"),
@@ -618,6 +623,7 @@ def create_image_shard_downsample_tasks(
   agglomerate=False, timestamp=None,
   factor=(2,2,1), bounds=None, bounds_mip=0,
   encoding_level:Optional[int] = None,
+  encoding_effort:Optional[int] = None,
   method=DownsampleMethods.AUTO,
 ):
   """
@@ -639,7 +645,7 @@ def create_image_shard_downsample_tasks(
     dtype=cv.dtype,
     uncompressed_shard_bytesize=int(memory_target),
   )
-  set_encoding(cv, mip + 1, encoding, encoding_level)
+  set_encoding(cv, mip + 1, encoding, encoding_level, encoding_effort)
   cv.commit_info()
 
   shape = image_shard_shape_from_spec(
@@ -682,6 +688,8 @@ def on_finish(self):
           "agglomerate": agglomerate,
           "timestamp": timestamp,
           "method": method,
+          "encoding_level": encoding_level,
+          "encoding_effort": encoding_effort,
         },
         "by": operator_contact(),
         "date": strftime("%Y-%m-%d %H:%M %Z"),
@@ -747,7 +755,7 @@ def create_transfer_cloudvolume(
   src_vol:CloudVolume, dst_cloudpath:str, 
   dest_voxel_offset:ShapeType, 
   mip:int, bounds_mip:int,
-  encoding:str, encoding_level:int,
+  encoding:str, encoding_level:int, encoding_effort:Optional[int],
   chunk_size:ShapeType, 
   truncate_scales:bool, clean_info:bool,
   cutout:bool, bounds:Bbox
@@ -784,7 +792,7 @@ def create_transfer_cloudvolume(
         (dest_voxel_offset + bounds.minpt) * (bounds_resolution / dest_vol.meta.resolution(i))
       )
 
-  set_encoding(dest_vol, mip, encoding, encoding_level)
+  set_encoding(dest_vol, mip, encoding, encoding_level, encoding_effort)
   if truncate_scales:
     dest_vol.info['scales'] = dest_vol.info['scales'][:mip+1]
   dest_vol.info['scales'][mip]['chunk_sizes'] = [ chunk_size.tolist() ]
@@ -823,6 +831,7 @@ def create_transfer_tasks(
   cutout:bool = False,
   stop_layer:Optional[int] = None,
   downsample_method:int = DownsampleMethods.AUTO,
+  encoding_effort:Optional[int] = None,
 ) -> Iterator:
   """
   Transfer data to a new data layer. You can use this operation
@@ -860,10 +869,12 @@ def create_transfer_tasks(
     image type-specific first stage of compression and the "compress" flag as the data
     agnostic second stage compressor. For example, compressed_segmentation and gzip work
     well together, but not jpeg and gzip.
-  encoding_level: Some encoding schemes (png,jpeg,fpzip) offer a simple scalar knob
+  encoding_level: Some encoding schemes (png,jpeg,jpegxl,fpzip,zfpc) offer a simple scalar knob
     to control encoding quality. This number corresponds to png level, jpeg quality,
     and fpzip precision. Other schemes might require more complex inputs and may
     require info file modifications.
+  encoding_effort: (jpeg xl only) Sets JPEG XL effort to hit the specified quality target. 
+    Higher values are slower, but more reliable.
   factor: (overrides axis) can manually specify what each downsampling round is
     supposed to do: e.g. (2,2,1), (2,2,2), etc
   fill_missing: Treat missing image tiles as zeroed for both src and dest.
@@ -919,7 +930,7 @@ def create_transfer_tasks(
     src_vol, dest_layer_path, 
     dest_voxel_offset, 
     mip, bounds_mip,
-    encoding, encoding_level,
+    encoding, encoding_level, encoding_effort,
     chunk_size, truncate_scales,
     clean_info, cutout, bounds
   )
@@ -1015,6 +1026,7 @@ def on_finish(self):
           'factor': (tuple(factor) if factor else None),
           'sparse': bool(sparse),
           'encoding_level': encoding_level,
+          'encoding_effort': encoding_effort,
           'stop_layer': stop_layer,
           'downsample_method': int(downsample_method),
         },
@@ -1064,6 +1076,7 @@ def create_reordering_tasks(
   background_color:int = 0,
   encoding:Optional[str] = None, 
   encoding_level:Optional[int] = None,
+  encoding_effort:Optional[int] = None,
 ):
   src_cv = CloudVolume(src, mip=mip)
   zstart, zend = src_cv.bounds.min[2], src_cv.bounds.max[2]
@@ -1080,7 +1093,11 @@ def create_reordering_tasks(
   dest_cv.info['scales'][mip]['chunk_sizes'] = [ chunk_size ]
   dest_cv.info = clean_xfer_info(dest_cv.info)
   if encoding:
-    set_encoding(dest_cv, mip, encoding, encoding_level=encoding_level)
+    set_encoding(
+      dest_cv, mip, encoding,
+      encoding_level=encoding_level, 
+      encoding_effort=encoding_effort,
+    )
 
   dest_cv.commit_info()
 
@@ -1131,6 +1148,7 @@ def on_finish(self):
           # 'factor': (tuple(factor) if factor else None),
           # 'sparse': bool(sparse),
           'encoding_level': encoding_level,
+          'encoding_effort': encoding_effort,
         },
         'by': operator_contact(),
         'date': strftime('%Y-%m-%d %H:%M %Z'),

igneous/task_creation/mesh.py

@@ -168,7 +168,10 @@ def create_meshing_tasks(
   vol = CloudVolume(layer_path, mip)
 
   if mesh_dir is None:
-    mesh_dir = 'mesh_mip_{}_err_{}'.format(mip, max_simplification_error)
+    if 'mesh' in vol.info:
+      mesh_dir = vol.info['mesh']
+    else:
+      mesh_dir = 'mesh_mip_{}_err_{}'.format(mip, max_simplification_error)
 
   if not 'mesh' in vol.info:
     vol.info['mesh'] = mesh_dir

igneous/task_creation/skeleton.py

@@ -48,8 +48,9 @@ def bounds_from_mesh(
   """Estimate the bounding box of a label from its mesh if available."""
   bbxes = []
   for label in labels:
-    mesh = vol.mesh.get(labels)[label]
-    if mesh is None:
+    try:
+      mesh = vol.mesh.get(label)
+    except ValueError:
       raise ValueError(f"Mesh {label} is not available.")
 
     bounds = Bbox.from_points(mesh.vertices // vol.resolution)

igneous_cli/cli.py

@@ -44,12 +44,15 @@ def normalize_encoding(encoding):
     return "crackle"
   elif encoding == "cpso":
     return "compresso"
+  elif encoding == "jpegxl":
+    return "jxl"
   elif encoding == "auto":
     return None
 
   return encoding
 
-ENCODING_HELP = "Which image encoding to use. Options: [all] raw, png; [images] jpeg; [segmentations] compressed_segmentation (cseg), compresso (cpso), crackle (ckl); [floats] fpzip, kempressed, zfpc"
+ENCODING_HELP = "Which image encoding to use. Options: [all] raw, png; [images] jpeg, jpegxl (jxl); [segmentations] compressed_segmentation (cseg), compresso (cpso), crackle (ckl); [floats] fpzip, kempressed, zfpc"
+ENCODING_EFFORT = 5
 
 def enqueue_tasks(ctx, queue, tasks):
   parallel = int(ctx.obj.get("parallel", 1))
@@ -220,7 +223,8 @@ def imagegroup():
 @click.option('--fill-missing', is_flag=True, default=False, help="Interpret missing image files as background instead of failing.")
 @click.option('--num-mips', default=None, type=int, help="Build this many additional pyramid levels. Each increment increases memory requirements per task 4-8x.")
 @click.option('--encoding', type=EncodingType(), default="auto", help=ENCODING_HELP, show_default=True)
-@click.option('--encoding-level', default=None, help="For some encodings (png level,jpeg quality,fpzip precision) a simple scalar value can adjust the compression efficiency.", show_default=True)
+@click.option('--encoding-level', default=None, help="For some encodings (png level, jpeg & jpeg xl quality, fpzip precision) a simple scalar value can adjust the compression efficiency.", show_default=True)
+@click.option('--encoding-effort', default=ENCODING_EFFORT, help="(JPEG XL) Set effort (1-10) used by JPEG XL to hit the quality target.", show_default=True)
 @click.option('--sparse', is_flag=True, default=False, help="Don't count black pixels in mode or average calculations. For images, eliminates edge ghosting in 2x2x2 downsample. For segmentation, prevents small objects from disappearing at high mip levels.")
 @click.option('--chunk-size', type=Tuple3(), default=None, help="Chunk size of new layers. e.g. 128,128,64")
 @click.option('--compress', default=None, help="Set the image compression scheme. Options: 'none', 'gzip', 'br'")
@@ -236,8 +240,8 @@ def imagegroup():
 @click.pass_context
 def downsample(
   ctx, path, queue, mip, fill_missing, 
-  num_mips, encoding, encoding_level, sparse, 
-  chunk_size, compress, volumetric,
+  num_mips, encoding, encoding_level, encoding_effort, 
+  sparse, chunk_size, compress, volumetric,
   delete_bg, bg_color, sharded, memory,
   xrange, yrange, zrange, method,
 ):
@@ -279,6 +283,7 @@ def downsample(
       encoding=encoding, memory_target=memory,
       factor=factor, bounds=bounds, bounds_mip=mip,
       encoding_level=encoding_level, method=method,
+      encoding_effort=encoding_effort,
     )
   else:
     tasks = tc.create_downsampling_tasks(
@@ -293,6 +298,7 @@ def downsample(
       memory_target=memory,
       encoding_level=encoding_level,
       method=method,
+      encoding_effort=encoding_effort,
     )
 
   enqueue_tasks(ctx, queue, tasks)
@@ -308,7 +314,8 @@ def downsample(
 @click.option('--memory', default=3.5e9, type=int, help="Task memory limit in bytes. Task shape will be chosen to fit and maximize downsamples.", show_default=True)
 @click.option('--max-mips', default=5, help="Maximum number of additional pyramid levels.", show_default=True)
 @click.option('--encoding', type=EncodingType(), default="auto", help=ENCODING_HELP, show_default=True)
-@click.option('--encoding-level', default=None, help="For some encodings (png level,jpeg quality,fpzip precision) a simple scalar value can adjust the compression efficiency.", show_default=True)
+@click.option('--encoding-level', default=None, help="For some encodings (png level,jpeg & jpegxl quality,fpzip precision) a simple scalar value can adjust the compression efficiency.", show_default=True)
+@click.option('--encoding-effort', default=ENCODING_EFFORT, help="(JPEG XL) Set effort (1-10) used by JPEG XL to hit the quality target.", show_default=True)
 @click.option('--sparse', is_flag=True, default=False, help="Don't count black pixels in mode or average calculations. For images, eliminates edge ghosting in 2x2x2 downsample. For segmentation, prevents small objects from disappearing at high mip levels.", show_default=True)
 @click.option('--shape', type=Tuple3(), default=None, help="(overrides --memory) Set the task shape in voxels. This also determines how many downsamples you get. e.g. 2048,2048,64", show_default=True)
 @click.option('--chunk-size', type=Tuple3(), default=None, help="Chunk size of destination layer. e.g. 128,128,64", show_default=True)
@@ -332,7 +339,8 @@ def xfer(
 	ctx, src, dest, queue, translate, 
   downsample, mip, fill_missing, 
   memory, max_mips, shape, sparse, 
-  encoding, encoding_level, chunk_size, compress,
+  encoding, encoding_level, encoding_effort,
+  chunk_size, compress,
   volumetric, delete_bg, bg_color, sharded,
   dest_voxel_offset, clean_info, no_src_update,
   truncate_scales, 
@@ -379,7 +387,7 @@ def xfer(
       encoding=encoding, memory_target=memory, clean_info=clean_info,
       encoding_level=encoding_level, truncate_scales=truncate_scales,
       compress=compress, bounds=bounds, bounds_mip=bounds_mip,
-      cutout=cutout,
+      cutout=cutout, encoding_effort=encoding_effort,
     )
   else:
     tasks = tc.create_transfer_tasks(
@@ -393,7 +401,7 @@ def xfer(
       clean_info=clean_info, no_src_update=no_src_update,
       encoding_level=encoding_level, truncate_scales=truncate_scales,
       bounds=bounds, bounds_mip=bounds_mip, cutout=cutout,
-      downsample_method=downsample_method,
+      downsample_method=downsample_method, encoding_effort=encoding_effort,
     )
 
   enqueue_tasks(ctx, queue, tasks)
@@ -425,6 +433,7 @@ def image_roi(src, progress, suppress_faint, dust, z_step, max_axial_len):
 @click.option('--fill-missing', is_flag=True, default=False, help="Interpret missing image files as background instead of failing.")
 @click.option('--encoding', type=EncodingType(), default="auto", help=ENCODING_HELP, show_default=True)
 @click.option('--encoding-level', default=None, help="For some encodings (png level,jpeg quality,fpzip precision) a simple scalar value can adjust the compression efficiency.", show_default=True)
+@click.option('--encoding-effort', default=ENCODING_EFFORT, help="(JPEG XL) Set effort (1-10) used by JPEG XL to hit the quality target.", show_default=True)
 @click.option('--compress', default="br", help="Set the image compression scheme. Options: 'none', 'gzip', 'br'", show_default=True)
 @click.option('--delete-bg', is_flag=True, default=False, help="Issue a delete instead of uploading a background tile. This is helpful on systems that don't like tiny files.")
 @click.option('--bg-color', default=0, help="Determines which color is regarded as background.", show_default=True)
@@ -434,7 +443,7 @@ def image_reorder(
   ctx, src, dest, 
   queue, mip,
   fill_missing, 
-  encoding, encoding_level,
+  encoding, encoding_level, encoding_effort,
   compress, 
   delete_bg, bg_color,
   clean_info, 
@@ -471,6 +480,7 @@ def image_reorder(
     background_color=bg_color,
     encoding=encoding, 
     encoding_level=encoding_level,
+    encoding_effort=encoding_effort,
   )
 
   enqueue_tasks(ctx, queue, tasks)
@@ -1689,14 +1699,17 @@ def view(path, browser, port, ng):
 @click.option('--offset', type=Tuple3(), default=(0,0,0), help="Voxel offset in x,y, and z.", show_default=True)
 @click.option('--seg', is_flag=True, default=False, help="Sets layer type to segmentation (default image).", show_default=True)
 @click.option('--encoding', type=EncodingType(), default="raw", help=ENCODING_HELP, show_default=True)
+@click.option('--encoding-level', default=None, help="For some encodings (png level, jpeg & jpeg xl quality, fpzip precision) a simple scalar value can adjust the compression efficiency.", show_default=True)
+@click.option('--encoding-effort', default=ENCODING_EFFORT, help="(JPEG XL) Set effort (1-10) used by JPEG XL to hit the quality target.", show_default=True)
 @click.option('--compress', type=CompressType(), default="br", help="Set the image compression scheme. Options: 'none', 'gzip', 'br'", show_default=True)
 @click.option('--chunk-size', type=Tuple3(), default=(128,128,64), help="Chunk size of new layers. e.g. 128,128,64", show_default=True)
 @click.option('--h5-dataset', default="main", help="Which h5 dataset to acccess (hdf5 imports only).", show_default=True)
 @click.pass_context
 def create(
   ctx, src, dest, 
   resolution, offset, 
-  seg, encoding,
+  seg, 
+  encoding, encoding_level, encoding_effort,
   compress, chunk_size,
   h5_dataset
 ):
@@ -1735,6 +1748,8 @@ def create(
     encoding=encoding,
     compress=compress,
     progress=True,
+    encoding_level=encoding_level,
+    encoding_effort=encoding_effort,
   )
 
 def normalize_file_ext(filename):