From 09c34b02019decf7464af583a42179f9f5369a0f Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Tue, 8 Feb 2022 11:37:30 +0100 Subject: [PATCH 1/5] Disable tests of TNS query on GitHub Actions Until we set up a way to avoid "too many requests" errors during testing --- tests/test_tns.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_tns.py b/tests/test_tns.py index e90ab33..c16d2fd 100644 --- a/tests/test_tns.py +++ b/tests/test_tns.py @@ -7,12 +7,14 @@ """ import pytest +import os import datetime from astropy.coordinates import SkyCoord from conftest import capture_cli from flows import tns #-------------------------------------------------------------------------------------------------- +@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error") def test_tns_search(SETUP_CONFIG): coo_centre = SkyCoord( @@ -28,6 +30,7 @@ def test_tns_search(SETUP_CONFIG): assert res[0]['prefix'] == 'SN' #-------------------------------------------------------------------------------------------------- +@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error") def test_tns_get_obj(SETUP_CONFIG): res = tns.tns_get_obj('2019yvr') @@ -37,6 +40,7 @@ def test_tns_get_obj(SETUP_CONFIG): assert res['name_prefix'] == 'SN' #-------------------------------------------------------------------------------------------------- +@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error") @pytest.mark.parametrize('date_begin,date_end', [ ('2019-01-01', '2019-02-01'), (datetime.date(2019, 1, 1), datetime.date(2019, 2, 1)), @@ -69,6 +73,7 @@ def test_tns_getnames_wronginput(SETUP_CONFIG): ) #-------------------------------------------------------------------------------------------------- +@pytest.mark.skipif(os.environ.get('CI') == 'true', reason="Disabled on GitHub Actions to avoid too many requests HTTP error") def test_run_querytns(SETUP_CONFIG): # Run the command line interface: From 3d6e499fb9a675b3fab17795e734a4ce8038afa9 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Feb 2022 10:50:27 +0100 Subject: [PATCH 2/5] Bugfix ALFOSC B-filter --- flows/load_image.py | 1 + tests/test_load_image.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flows/load_image.py b/flows/load_image.py index 35073eb..3693582 100644 --- a/flows/load_image.py +++ b/flows/load_image.py @@ -183,6 +183,7 @@ def load_image(FILENAME, target_coord=None): filters_used.append(hdr.get(check_headers).strip()) if len(filters_used) == 1: image.photfilter = { + 'B_Bes 440_100': 'B', 'V_Bes 530_80': 'V', 'R_Bes 650_130': 'R', "g'_SDSS 480_145": 'gp', diff --git a/tests/test_load_image.py b/tests/test_load_image.py index bd64679..3dd3927 100644 --- a/tests/test_load_image.py +++ b/tests/test_load_image.py @@ -26,6 +26,7 @@ #['SN2021rhu_59465.86130221_B.fits.gz', 22], #['20200613_SN2020lao_u_stacked_meandiff.fits.gz', 1], #['2021aess_20220104_K.fits.gz', 5], + #['2021aess_B01_20220207v1.fits.gz', 5], ]) def test_load_image(fpath, siteid): # Get list of all available filters: @@ -54,11 +55,11 @@ def test_load_image(fpath, siteid): assert isinstance(img.obstime, Time) assert isinstance(img.exptime, float) assert img.exptime > 0 - assert isinstance(img.photfilter, str) - assert img.photfilter in all_filters assert isinstance(img.wcs, WCS) assert isinstance(img.site, dict) assert img.site['siteid'] == siteid + assert isinstance(img.photfilter, str) + assert img.photfilter in all_filters #-------------------------------------------------------------------------------------------------- if __name__ == '__main__': From c3f14eaa18a1b1ba66a61a827930c48210831799 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Feb 2022 11:19:18 +0100 Subject: [PATCH 3/5] Better logging of ingest errors --- run_ingest.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/run_ingest.py b/run_ingest.py index f510b0d..406f9ad 100644 --- a/run_ingest.py +++ b/run_ingest.py @@ -119,6 +119,10 @@ def ingest_from_inbox(): db.cursor.execute("SELECT archive,path FROM aadc.files_archives;") archives_list = db.cursor.fetchall() + # Get list of all available filters: + db.cursor.execute("SELECT photfilter FROM flows.photfilters;") + all_filters = set([row['photfilter'] for row in db.cursor.fetchall()]) + for inputtype in ('science', 'templates', 'subtracted', 'replace'): # for fpath in glob.iglob(os.path.join(rootdir_inbox, '*', inputtype, '*')): logger.info("="*72) @@ -131,6 +135,7 @@ def ingest_from_inbox(): uploadlogid = row['logid'] else: uploadlogid = None + logger.info("Uploadlog ID: %s", uploadlogid) # Only accept FITS file, or already compressed FITS files: if not fpath.endswith('.fits') and not fpath.endswith('.fits.gz'): @@ -272,8 +277,12 @@ def ingest_from_inbox(): # Try to load the image using the same function as the pipeline would: try: img = load_image(fpath, target_coord=target_coord) - except: # noqa: E722, pragma: no cover + except Exception as e: # pragma: no cover logger.exception("Could not load FITS image") + if uploadlogid: + errmsg = str(e) if hasattr(e, 'message') else str(e.message) + db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", ['Load Image Error: ' + errmsg, uploadlogid]) + db.conn.commit() continue # Use the WCS in the file to calculate the pixel-positon of the target: @@ -281,6 +290,10 @@ def ingest_from_inbox(): target_pixels = img.wcs.all_world2pix(target_radec, 0).flatten() except: # noqa: E722, pragma: no cover logger.exception("Could not find target position using the WCS.") + if uploadlogid: + errmsg = "Could not find target position using the WCS." + db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid]) + db.conn.commit() continue # Check that the position of the target actually falls within @@ -288,11 +301,28 @@ def ingest_from_inbox(): if target_pixels[0] < -0.5 or target_pixels[1] < -0.5 \ or target_pixels[0] > img.shape[1]-0.5 or target_pixels[1] > img.shape[0]-0.5: logger.error("Target position does not fall within image. Check the WCS.") + if uploadlogid: + errmsg = "Target position does not fall within image. Check the WCS." + db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid]) + db.conn.commit() continue # Check that the site was found: if img.site is None or img.site['siteid'] is None: logger.error("Unknown SITE") + if uploadlogid: + errmsg = "Unknown site" + db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid]) + db.conn.commit() + continue + + # Check that the extracted photometric filter is valid: + if img.photfilter not in all_filters: + logger.error("Unknown PHOTFILTER: %s", img.photfilter) + if uploadlogid: + errmsg = "Unknown PHOTFILTER: '" + str(img.photfilter) + "'" + db.cursor.execute("UPDATE flows.uploadlog SET status=%s WHERE logid=%s;", [errmsg, uploadlogid]) + db.conn.commit() continue # Do a deep check to ensure that there is not already another file with the same From 3757322b9a40d597fc603d3f1c3ddfd817692029 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Feb 2022 11:19:57 +0100 Subject: [PATCH 4/5] Bumped version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 870328a..76279d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -master-v0.9.1 \ No newline at end of file +master-v0.9.2 \ No newline at end of file From e3e464fd8a16d652444abbfe865a405336b291a8 Mon Sep 17 00:00:00 2001 From: Rasmus Handberg Date: Thu, 10 Feb 2022 11:21:45 +0100 Subject: [PATCH 5/5] Update notes --- notes/update_all_catalogs.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/notes/update_all_catalogs.py b/notes/update_all_catalogs.py index e5b1618..7356441 100644 --- a/notes/update_all_catalogs.py +++ b/notes/update_all_catalogs.py @@ -4,7 +4,7 @@ import logging import sys import os.path -from tqdm import tqdm +import tqdm from astropy.coordinates import SkyCoord if os.path.abspath('..') not in sys.path: sys.path.insert(0, os.path.abspath('..')) @@ -37,12 +37,16 @@ def emit(self, record): logger.addHandler(console) logger.setLevel(logging.INFO) - for target in tqdm(flows.api.get_targets()): - #if target['targetid'] != 1942: - # continue - - donefile = f"{target['targetid']:05d}.done" - if not os.path.exists(donefile): - flows.catalogs.download_catalog(target['targetid'], update_existing=True) - - open(donefile, 'w').close() + # Do it by status, just to prioritize things a bit: + for tgtstatus in ('target', 'candidate', 'rejected'): + targetids = sorted([tgt['targetid'] for tgt in flows.api.get_targets() if tgt['target_status'] == tgtstatus])[::-1] + + for targetid in tqdm.tqdm(targetids, desc=tgtstatus): + donefile = f"catalog_updates/{targetid:05d}.done" + if not os.path.exists(donefile): + try: + flows.catalogs.download_catalog(targetid, update_existing=True) + except: + logger.exception("targetid=%d", targetid) + else: + open(donefile, 'w').close()