tasks.py

import glob
import logging

import sqlalchemy as sa
from invoke import run, task

import app
import aws
import refresh_log
import utils
from refresh_log import AutoapiTableRefreshLog as RefreshLog

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


@task
def requirements(upgrade=True):
    cmd = 'pip install -r requirements.txt'
    if upgrade:
        cmd += ' --upgrade'
    run(cmd)


@task
def apify(filename, tablename=None):
    try:
        filenames = glob.glob(filename, recursive=True)
    except TypeError:  # recursive glob in Python 3.5+ only
        filenames = glob.glob(filename)
    if len(filenames) > 1 and tablename:
        raise Exception("Can't specify a `tablename` for >1 file")
    for filename in filenames:
        _tablename = tablename or utils.get_name(filename)
        logger.info('Importing {0} to table {1}'.format(filename, _tablename))
        try:
            utils.drop_table(_tablename)
        except sa.exc.OperationalError as e:
            logger.debug('DROP TABLE {} failed, may not exist?'.format(
                _tablename))
            logger.debug(str(e))
        try:
            utils.load_table(filename, _tablename)
        except Exception as e:
            logger.error('Failed to load table from file {}'.format(filename))
            logger.error(str(e))
        logger.info('Finished importing {0}'.format(filename))


@task
def delete_from_bucket(filename, bucket_name=None):
    aws.delete_from_bucket(filename, bucket_name)


@task
def copy_to_bucket(filename, bucket_name=None):
    aws.copy_to_bucket(filename, bucket_name)


@task
def list_bucket(bucket_name=None):
    aws.list_bucket(bucket_name)


@task
def fetch_bucket(bucket_name=None):
    aws.fetch_bucket(bucket_name)


@task
def serve():
    app.make_app().run(host='0.0.0.0')


@task
def refresh(clear_tables=True):
    logger.info('Refresh invoked')
    with app.make_app().app_context():
        rlog_id = RefreshLog.start()
        logger.info('refresh log id {} started'.format(rlog_id))
        try:
            aws.fetch_bucket(clear_tables=clear_tables)
            logger.debug('bucket fetched')
            utils.refresh_tables()
            logger.info('refresh complete')
            refresh_log.stop(rlog_id)
        except Exception as e:
            refresh_log.stop(rlog_id, err_msg=str(e))


@task
def quick_refresh():
    refresh(clear_tables=False)


@task
def clear():
    utils.clear_tables()