Merge branch 'devel'

VERSION

@@ -1 +1 @@
-master-v0.9.1
+master-v0.9.2

flows/load_image.py

@@ -183,6 +183,7 @@ def load_image(FILENAME, target_coord=None):
 						filters_used.append(hdr.get(check_headers).strip())
 				if len(filters_used) == 1:
 					image.photfilter = {
+						'B_Bes 440_100': 'B',
 						'V_Bes 530_80': 'V',
 						'R_Bes 650_130': 'R',
 						"g'_SDSS 480_145": 'gp',

notes/update_all_catalogs.py

@@ -4,7 +4,7 @@
 import logging
 import sys
 import os.path
-from tqdm import tqdm
+import tqdm
 from astropy.coordinates import SkyCoord
 if os.path.abspath('..') not in sys.path:
 	sys.path.insert(0, os.path.abspath('..'))
@@ -37,12 +37,16 @@ def emit(self, record):
 		logger.addHandler(console)
 	logger.setLevel(logging.INFO)
 
-	for target in tqdm(flows.api.get_targets()):
-		#if target['targetid'] != 1942:
-		#	continue
-
-		donefile = f"{target['targetid']:05d}.done"
-		if not os.path.exists(donefile):
-			flows.catalogs.download_catalog(target['targetid'], update_existing=True)
-
-			open(donefile, 'w').close()
+	# Do it by status, just to prioritize things a bit:
+	for tgtstatus in ('target', 'candidate', 'rejected'):
+		targetids = sorted([tgt['targetid'] for tgt in flows.api.get_targets() if tgt['target_status'] == tgtstatus])[::-1]
+
+		for targetid in tqdm.tqdm(targetids, desc=tgtstatus):
+			donefile = f"catalog_updates/{targetid:05d}.done"
+			if not os.path.exists(donefile):
+				try:
+					flows.catalogs.download_catalog(targetid, update_existing=True)
+				except:
+					logger.exception("targetid=%d", targetid)
+				else:
+					open(donefile, 'w').close()

run_ingest.py

@@ -119,6 +119,10 @@ def ingest_from_inbox():
 		db.cursor.execute("SELECT archive,path FROM aadc.files_archives;")
 		archives_list = db.cursor.fetchall()
 
+		# Get list of all available filters:
+		db.cursor.execute("SELECT photfilter FROM flows.photfilters;")
+		all_filters = set([row['photfilter'] for row in db.cursor.fetchall()])
+
 		for inputtype in ('science', 'templates', 'subtracted', 'replace'): #
 			for fpath in glob.iglob(os.path.join(rootdir_inbox, '*', inputtype, '*')):
 				logger.info("="*72)
@@ -131,6 +135,7 @@ def ingest_from_inbox():
 					uploadlogid = row['logid']
 				else:
 					uploadlogid = None
+				logger.info("Uploadlog ID: %s", uploadlogid)
 
 				# Only accept FITS file, or already compressed FITS files:
 				if not fpath.endswith('.fits') and not fpath.endswith('.fits.gz'):
@@ -272,27 +277,52 @@ def ingest_from_inbox():
 				# Try to load the image using the same function as the pipeline would:
 				try:
 					img = load_image(fpath, target_coord=target_coord)
-				except: # noqa: E722, pragma: no cover
+				except Exception as e: # pragma: no cover
 					logger.exception("Could not load FITS image")
+					if uploadlogid:
+						errmsg = str(e) if hasattr(e, 'message') else str(e.message)
+						db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", ['Load Image Error: ' + errmsg, uploadlogid])
+						db.conn.commit()
 					continue
 
 				# Use the WCS in the file to calculate the pixel-positon of the target:
 				try:
 					target_pixels = img.wcs.all_world2pix(target_radec, 0).flatten()
 				except: # noqa: E722, pragma: no cover
 					logger.exception("Could not find target position using the WCS.")
+					if uploadlogid:
+						errmsg = "Could not find target position using the WCS."
+						db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+						db.conn.commit()
 					continue
 
 				# Check that the position of the target actually falls within
 				# the pixels of the image:
 				if target_pixels[0] < -0.5 or target_pixels[1] < -0.5 \
 					or target_pixels[0] > img.shape[1]-0.5 or target_pixels[1] > img.shape[0]-0.5:
 					logger.error("Target position does not fall within image. Check the WCS.")
+					if uploadlogid:
+						errmsg = "Target position does not fall within image. Check the WCS."
+						db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+						db.conn.commit()
 					continue
 
 				# Check that the site was found:
 				if img.site is None or img.site['siteid'] is None:
 					logger.error("Unknown SITE")
+					if uploadlogid:
+						errmsg = "Unknown site"
+						db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+						db.conn.commit()
+					continue
+
+				# Check that the extracted photometric filter is valid:
+				if img.photfilter not in all_filters:
+					logger.error("Unknown PHOTFILTER: %s", img.photfilter)
+					if uploadlogid:
+						errmsg = "Unknown PHOTFILTER: '" + str(img.photfilter) + "'"
+						db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid])
+						db.conn.commit()
 					continue
 
 				# Do a deep check to ensure that there is not already another file with the same

tests/test_load_image.py

@@ -26,6 +26,7 @@
 	#['SN2021rhu_59465.86130221_B.fits.gz', 22],
 	#['20200613_SN2020lao_u_stacked_meandiff.fits.gz', 1],
 	#['2021aess_20220104_K.fits.gz', 5],
+	#['2021aess_B01_20220207v1.fits.gz', 5],
 ])
 def test_load_image(fpath, siteid):
 	# Get list of all available filters:
@@ -54,11 +55,11 @@ def test_load_image(fpath, siteid):
 	assert isinstance(img.obstime, Time)
 	assert isinstance(img.exptime, float)
 	assert img.exptime > 0
-	assert isinstance(img.photfilter, str)
-	assert img.photfilter in all_filters
 	assert isinstance(img.wcs, WCS)
 	assert isinstance(img.site, dict)
 	assert img.site['siteid'] == siteid
+	assert isinstance(img.photfilter, str)
+	assert img.photfilter in all_filters
 
 #--------------------------------------------------------------------------------------------------
 if __name__ == '__main__':

tests/test_tns.py

@@ -7,12 +7,14 @@
 """
 
 import pytest
+import os
 import datetime
 from astropy.coordinates import SkyCoord
 from conftest import capture_cli
 from flows import tns
 
 #--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
 def test_tns_search(SETUP_CONFIG):
 
 	coo_centre = SkyCoord(
@@ -28,6 +30,7 @@ def test_tns_search(SETUP_CONFIG):
 	assert res[0]['prefix'] == 'SN'
 
 #--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
 def test_tns_get_obj(SETUP_CONFIG):
 
 	res = tns.tns_get_obj('2019yvr')
@@ -37,6 +40,7 @@ def test_tns_get_obj(SETUP_CONFIG):
 	assert res['name_prefix'] == 'SN'
 
 #--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
 @pytest.mark.parametrize('date_begin,date_end', [
 	('2019-01-01', '2019-02-01'),
 	(datetime.date(2019, 1, 1), datetime.date(2019, 2, 1)),
@@ -69,6 +73,7 @@ def test_tns_getnames_wronginput(SETUP_CONFIG):
 		)
 
 #--------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error")
 def test_run_querytns(SETUP_CONFIG):
 
 	# Run the command line interface: