Changed entire model (main model and pre-processing) to compress usin…

…g DEFLATE instead of LZW. DEFLATE-compressed files are substantially smaller than LZW-compressed files, and don't seem to take longer to create or need more memory. In fact, tests of full sets of tiles using DEFLATE instead of COMPRESS had the same peak memory for both and DEFLATE ran very slightly slower, but not so much so that it's worth using larger files (which obviously upload and download from s3 much faster). Testing is here: https://3.basecamp.com/3656819/buckets/7989024/todos/3884849933 (#20)

I ran the model locally in Docker on 00N_110E and everything ran fine. Not running on s3 instance because this shouldn't affect anything on s3. One thing to note, though, is that since the files are smaller now, I probably don't have to delete as many intermediate tiles during the model run to have enough space on the ec2 instance. I don't know exactly how many more intermediate tiles I can keep on the instance, but I hope I can at least keep the intermediate tiles that I'll need to use later in the model (like mangrove biomass).

Also, I haven't tested this for the pre-processeing steps, like mp_prep_other_inputs.py. I assume this new compression will work fine for those; the pre-processing steps are more likely not to work because of other changes I've made, actually.

analyses/aggregate_results_to_4_km.py

@@ -136,7 +136,7 @@ def aggregate(tile, thresh, sensit_type, no_upload):
     # from the 2D array created by rasterio above
     # https://gis.stackexchange.com/questions/279953/numpy-array-to-gtiff-using-rasterio-without-source-raster
     with rasterio.open(out_raster, 'w',
-                                driver='GTiff', compress='LZW', nodata='0', dtype='float32', count=1,
+                                driver='GTiff', compress='DEFLATE', nodata='0', dtype='float32', count=1,
                                 height=250, width=250,
                                 crs='EPSG:4326', transform=from_origin(xmin,ymax,0.04,0.04)) as aggregated:
         aggregated.write(sum_array, 1)
@@ -208,9 +208,9 @@ def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
     perc_diff_outfilename = '{0}_{1}_{2}.tif'.format(cn.pattern_aggreg_sensit_perc_diff, sensit_type, date_formatted)
     perc_diff_outfilearg = '--outfile={}'.format(perc_diff_outfilename)
     # cmd = ['gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux, perc_diff_calc, perc_diff_outfilearg,
-    #        '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--quiet']
+    #        '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=DEFLATE', '--quiet']
     cmd = ['gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux, perc_diff_calc, perc_diff_outfilearg,
-           '--overwrite', '--co', 'COMPRESS=LZW', '--quiet']
+           '--overwrite', '--co', 'COMPRESS=DEFLATE', '--quiet']
     uu.log_subprocess_output_full(cmd)
 
     # Prints information about the tile that was just processed

analyses/create_supplementary_outputs.py

@@ -68,7 +68,7 @@ def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit
     kwargs.update(
         driver='GTiff',
         count=1,
-        compress='lzw',
+        compress='DEFLATE',
         nodata=0,
         dtype='float32'
     )

analyses/mp_aggregate_results_to_4_km.py

@@ -176,8 +176,8 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux
         out_pattern = uu.name_aggregated_output(download_pattern_name, thresh, sensit_type)
         uu.print_log(out_pattern)
 
-        # Produces a single raster of all the 10x10 tiles (0.4 degree resolution)
-        cmd = ['gdalwarp', '-t_srs', "EPSG:4326", '-overwrite', '-dstnodata', '0', '-co', 'COMPRESS=LZW',
+        # Produces a single raster of all the 10x10 tiles (0.04 degree resolution)
+        cmd = ['gdalwarp', '-t_srs', "EPSG:4326", '-overwrite', '-dstnodata', '0', '-co', 'COMPRESS=DEFLATE',
                '-tr', '0.04', '0.04',
                out_vrt, '{}.tif'.format(out_pattern)]
         uu.log_subprocess_output_full(cmd)

analyses/net_flux.py

@@ -51,7 +51,7 @@ def net_calc(tile_id, pattern, sensit_type, no_upload):
         kwargs.update(
             driver='GTiff',
             count=1,
-            compress='lzw',
+            compress='DEFLATE',
             nodata=0,
             dtype='float32'
         )

analyses/tile_statistics.py

@@ -50,7 +50,7 @@ def create_tile_statistics(tile, sensit_type, tile_stats_txt):
     out = '--outfile={}'.format(outname)
 
     uu.print_log("Converting {} from /ha to /pixel...".format(tile))
-    cmd = ['gdal_calc.py', '-A', tile, '-B', area_tile, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=LZW',
+    cmd = ['gdal_calc.py', '-A', tile, '-B', area_tile, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE',
            '--overwrite', '--quiet']
     uu.log_subprocess_output_full(cmd)
 

burn_date/clip_year_tiles.py

@@ -29,7 +29,7 @@ def clip_year_tiles(tile_year_list, no_upload):
     uu.print_log("Clipping burn year vrt to {0} for {1}".format(tile_id, year))
 
     clipped_raster = "ba_clipped_{0}_{1}.tif".format(year, tile_id)
-    cmd = ['gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '-a_nodata', '0']
+    cmd = ['gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=DEFLATE', '-a_nodata', '0']
     cmd += [vrt_name, clipped_raster, '-tr', '.00025', '.00025']
     cmd += ['-projwin', str(xmin), str(ymax), str(xmax), str(ymin)]
     uu.log_subprocess_output_full(cmd)
@@ -39,7 +39,7 @@ def clip_year_tiles(tile_year_list, no_upload):
     recoded_output = "ba_{0}_{1}.tif".format(year, tile_id)
     outfile = '--outfile={}'.format(recoded_output)
 
-    cmd = ['gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0', '--co', 'COMPRESS=LZW', '--quiet']
+    cmd = ['gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--quiet']
     uu.log_subprocess_output_full(cmd)
 
     # Only copies to s3 if the tile has data.

burn_date/hansen_burnyear_final.py

@@ -134,7 +134,7 @@ def hansen_burnyear(tile_id, no_upload):
             kwargs.update(
                 driver='GTiff',
                 count=1,
-                compress='lzw',
+                compress='DEFLATE',
                 nodata=0
             )
 

carbon_pools/create_carbon_pools.py

@@ -124,7 +124,7 @@ def create_AGC(tile_id, sensit_type, carbon_pool_extent, no_upload):
     kwargs.update(
         driver='GTiff',
         count=1,
-        compress='lzw',
+        compress='DEFLATE',
         nodata=0,
         dtype='float32'
     )
@@ -271,7 +271,7 @@ def create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type, no_
         AGC_2000 = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_AGC_2000)
         AGC_2000_src = rasterio.open(AGC_2000)
         kwargs = AGC_2000_src.meta
-        kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
+        kwargs.update(driver='GTiff', count=1, compress='DEFLATE', nodata=0)
         windows = AGC_2000_src.block_windows(1)
         output_pattern_list = [cn.pattern_BGC_2000]
         if sensit_type != 'std':
@@ -292,7 +292,7 @@ def create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type, no_
         AGC_emis_year = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_AGC_emis_year)
         AGC_emis_year_src = rasterio.open(AGC_emis_year)
         kwargs = AGC_emis_year_src.meta
-        kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
+        kwargs.update(driver='GTiff', count=1, compress='DEFLATE', nodata=0)
         windows = AGC_emis_year_src.block_windows(1)
         output_pattern_list = [cn.pattern_BGC_emis_year]
         if sensit_type != 'std':
@@ -398,7 +398,7 @@ def create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_rat
         AGC_2000 = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_AGC_2000)
         AGC_2000_src = rasterio.open(AGC_2000)
         kwargs = AGC_2000_src.meta
-        kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
+        kwargs.update(driver='GTiff', count=1, compress='DEFLATE', nodata=0)
         windows = AGC_2000_src.block_windows(1)
         output_pattern_list = [cn.pattern_deadwood_2000, cn.pattern_litter_2000]
         if sensit_type != 'std':
@@ -429,7 +429,7 @@ def create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_rat
         AGC_emis_year = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_AGC_emis_year)
         AGC_emis_year_src = rasterio.open(AGC_emis_year)
         kwargs = AGC_emis_year_src.meta
-        kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
+        kwargs.update(driver='GTiff', count=1, compress='DEFLATE', nodata=0)
         windows = AGC_emis_year_src.block_windows(1)
 
         output_pattern_list = [cn.pattern_deadwood_emis_year_2000, cn.pattern_litter_emis_year_2000]
@@ -713,7 +713,7 @@ def create_soil_emis_extent(tile_id, pattern, sensit_type, no_upload):
     kwargs.update(
         driver='GTiff',
         count=1,
-        compress='lzw',
+        compress='DEFLATE',
         nodata=0,
         dtype='uint16'
     )
@@ -780,7 +780,7 @@ def create_total_C(tile_id, carbon_pool_extent, sensit_type, no_upload):
             uu.print_log("    No soil C 2000 tile found for", tile_id)
 
         kwargs = AGC_2000_src.meta
-        kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
+        kwargs.update(driver='GTiff', count=1, compress='DEFLATE', nodata=0)
         windows = AGC_2000_src.block_windows(1)
         output_pattern_list = [cn.pattern_total_C_2000]
         if sensit_type != 'std':
@@ -814,7 +814,7 @@ def create_total_C(tile_id, carbon_pool_extent, sensit_type, no_upload):
             uu.print_log("    No soil C emission year tile found for", tile_id)
 
         kwargs = AGC_emis_year_src.meta
-        kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
+        kwargs.update(driver='GTiff', count=1, compress='DEFLATE', nodata=0)
         windows = AGC_emis_year_src.block_windows(1)
         output_pattern_list = [cn.pattern_total_C_emis_year]
         if sensit_type != 'std':

carbon_pools/create_inputs_for_C_pools.py

@@ -52,7 +52,7 @@ def create_input_files(tile_id, no_upload):
     kwargs.update(
         driver='GTiff',
         count=1,
-        compress='lzw',
+        compress='DEFLATE',
         nodata=0
     )
 

carbon_pools/create_soil_C.py

@@ -91,7 +91,7 @@ def create_combined_soil_C(tile_id, no_upload):
         kwargs.update(
             driver='GTiff',
             count=1,
-            compress='lzw',
+            compress='DEFLATE',
             nodata=0
         )
 

carbon_pools/mp_create_soil_C.py

@@ -220,7 +220,7 @@ def mp_create_soil_C(tile_id_list, no_upload=None):
     calc = '--calc=(A-B)/3'
     out_filearg = '--outfile={}'.format(soil_C_stdev_global)
     cmd = ['gdal_calc.py', '-A', vrt_CI95, '-B', vrt_CI05, calc, out_filearg,
-           '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type=Float32']
+           '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=DEFLATE', '--type=Float32']
     uu.log_subprocess_output_full(cmd)
 
     uu.print_log("{} created.".format(soil_C_stdev_global))

data_prep/model_extent.py

@@ -55,7 +55,7 @@ def model_extent(tile_id, pattern, sensit_type, no_upload):
         kwargs.update(
             driver='GTiff',
             count=1,
-            compress='lzw',
+            compress='DEFLATE',
             nodata=0
         )
 

data_prep/plantation_preparation.py

@@ -35,7 +35,7 @@ def rasterize_gadm_1x1(tile_id):
             tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1)
             uu.print_log("Rasterizing", tile_1x1)
             cmd = ['gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)),
-                   '-co', 'COMPRESS=LZW', '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1),
+                   '-co', 'COMPRESS=DEFLATE', '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1),
                    '-burn', '1', '-a_nodata', '0', cn.gadm_iso, tile_1x1]
             # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
             process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
@@ -101,15 +101,15 @@ def create_1x1_plantation_from_1x1_gadm(tile_1x1):
 
         # https://gis.stackexchange.com/questions/187224/how-to-use-gdal-rasterize-with-postgis-vector
         # For plantation gain rate
-        cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=LZW', 'PG:dbname=ubuntu', '-l', 'all_plant', 'plant_gain_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1), '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-a', 'growth', '-a_nodata', '0']
+        cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=DEFLATE', 'PG:dbname=ubuntu', '-l', 'all_plant', 'plant_gain_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1), '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-a', 'growth', '-a_nodata', '0']
         # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
         process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
         with process.stdout:
             uu.log_subprocess_output(process.stdout)
 
         # https://gis.stackexchange.com/questions/187224/how-to-use-gdal-rasterize-with-postgis-vector
         # For plantation type
-        cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=LZW', 'PG:dbname=ubuntu', '-l', 'all_plant', 'plant_type_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1), '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-a', 'type_reclass', '-a_nodata', '0']
+        cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=DEFLATE', 'PG:dbname=ubuntu', '-l', 'all_plant', 'plant_type_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1), '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-a', 'type_reclass', '-a_nodata', '0']
         # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
         process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
         with process.stdout:
@@ -139,7 +139,7 @@ def create_1x1_plantation_growth_from_1x1_planted(tile_1x1):
     uu.print_log("There are plantations in {}. Converting to raster...".format(tile_1x1))
 
     # https://gis.stackexchange.com/questions/187224/how-to-use-gdal-rasterize-with-postgis-vector
-    cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=LZW',
+    cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=DEFLATE',
            'PG:dbname=ubuntu', '-l', 'all_plant', 'plant_gain_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1), '-te',
            str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-a', 'growth', '-a_nodata', '0']
     # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
@@ -167,7 +167,7 @@ def create_1x1_plantation_type_from_1x1_planted(tile_1x1):
     uu.print_log("There are plantations in {}. Converting to raster...".format(tile_1x1))
 
     # https://gis.stackexchange.com/questions/187224/how-to-use-gdal-rasterize-with-postgis-vector
-    cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=LZW', 'PG:dbname=ubuntu',
+    cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=DEFLATE', 'PG:dbname=ubuntu',
            '-l', 'all_plant', 'plant_type_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1),
            '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1),
            '-a', 'type_reclass', '-a_nodata', '0', '-ot', 'Byte']
@@ -193,7 +193,7 @@ def create_1x1_plantation_stdev_from_1x1_planted(tile_1x1):
     print("There are plantations in {}. Converting to stdev raster...".format(tile_1x1))
 
     # https://gis.stackexchange.com/questions/187224/how-to-use-gdal-rasterize-with-postgis-vector
-    cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=LZW', 'PG:dbname=ubuntu',
+    cmd = ['gdal_rasterize', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-co', 'COMPRESS=DEFLATE', 'PG:dbname=ubuntu',
            '-l', 'all_plant', 'plant_stdev_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1),
            '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1),
            '-a', 'SD_error', '-a_nodata', '0']
@@ -211,7 +211,7 @@ def create_10x10_plantation_gain(tile_id, plant_gain_1x1_vrt):
     tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)
     uu.print_log("Rasterizing", tile_10x10)
     cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)),
-           '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
+           '-co', 'COMPRESS=DEFLATE', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
            '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Float32', plant_gain_1x1_vrt, tile_10x10]
     # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
     process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
@@ -242,7 +242,7 @@ def create_10x10_plantation_type(tile_id, plant_type_1x1_vrt):
     tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_planted_forest_type_unmasked)
     uu.print_log("Rasterizing", tile_10x10)
     cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)),
-           '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
+           '-co', 'COMPRESS=DEFLATE', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
            '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Byte', plant_type_1x1_vrt, tile_10x10]
     # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
     process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
@@ -273,7 +273,7 @@ def create_10x10_plantation_gain_stdev(tile_id, plant_stdev_1x1_vrt):
     tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_planted_forest_stdev_unmasked)
     print("Rasterizing", tile_10x10)
     cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)),
-           '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
+           '-co', 'COMPRESS=DEFLATE', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
            '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Float32', plant_stdev_1x1_vrt, tile_10x10]
     subprocess.check_call(cmd)
 

data_prep/prep_other_inputs.py

@@ -26,7 +26,7 @@ def rasterize_pre_2000_plantations(tile_id):
 
     out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_plant_pre_2000)
 
-    cmd= ['gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=LZW', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res),
+    cmd= ['gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=DEFLATE', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res),
           '-tap', '-ot', 'Byte', '-a_nodata', '0', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
           '{}.shp'.format(cn.pattern_plant_pre_2000_raw), out_tile]
     # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
@@ -51,7 +51,7 @@ def create_climate_zone_tiles(tile_id):
     # Rather than the usual 40000x1 windows, this creates 1024x1024 windows for filling in missing values (see below).
     # The output of gdalwarp ("climate_zone_intermediate") is not used anywhere else.
     uu.print_log("Warping climate zone tile", tile_id)
-    cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', str(cn.Hansen_res), str(cn.Hansen_res),
+    cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=DEFLATE', '-tr', str(cn.Hansen_res), str(cn.Hansen_res),
            '-tap', '-te',
            str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-ot', 'Byte', '-overwrite',
            '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=1024', '-co', 'BLOCKYSIZE=1024',
@@ -82,7 +82,7 @@ def create_climate_zone_tiles(tile_id):
     kwargs.update(
         driver='GTiff',
         count=1,
-        compress='lzw',
+        compress='DEFLATE',
         nodata=0
     )
 

emissions/cpp_util/calc_gross_emissions_convert_to_grassland.cpp

@@ -223,7 +223,7 @@ GDALRasterBand *OUTBAND20;
 OGRSpatialReference oSRS;
 char *OUTPRJ = NULL;
 char **papszOptions = NULL;
-papszOptions = CSLSetNameValue( papszOptions, "COMPRESS", "LZW" );
+papszOptions = CSLSetNameValue( papszOptions, "COMPRESS", "DEFLATE" );
 OUTDRIVER = GetGDALDriverManager()->GetDriverByName("GTIFF");
 if( OUTDRIVER == NULL ) {cout << "no driver" << endl; exit( 1 );};
 oSRS.SetWellKnownGeogCS( "WGS84" );

emissions/cpp_util/calc_gross_emissions_generic.cpp

@@ -277,7 +277,7 @@ GDALRasterBand *OUTBAND20;
 OGRSpatialReference oSRS;
 char *OUTPRJ = NULL;
 char **papszOptions = NULL;
-papszOptions = CSLSetNameValue( papszOptions, "COMPRESS", "LZW" );
+papszOptions = CSLSetNameValue( papszOptions, "COMPRESS", "DEFLATE" );
 OUTDRIVER = GetGDALDriverManager()->GetDriverByName("GTIFF");
 if( OUTDRIVER == NULL ) {cout << "no driver" << endl; exit( 1 );};
 oSRS.SetWellKnownGeogCS( "WGS84" );