diff --git a/dags/app/finance_bot/__init__.py b/dags/app/finance_bot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/app/finance_bot/dag.py b/dags/app/finance_bot/dag.py new file mode 100644 index 0000000..d7ea928 --- /dev/null +++ b/dags/app/finance_bot/dag.py @@ -0,0 +1,31 @@ +""" +Send Google Search Report to Discord +""" +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.python_operator import PythonOperator +from app.finance import udf + +DEFAULT_ARGS = { + "owner": "qchwan", + "depends_on_past": False, + "start_date": datetime(2023, 8, 27), + "retries": 2, + "retry_delay": timedelta(minutes=5), + "on_failure_callback": lambda x: "Need to send notification to Discord", +} +dag = DAG( + "DISCORD_CHORES_REMINDER", + default_args=DEFAULT_ARGS, + schedule_interval="@daily", + max_active_runs=1, + catchup=False, +) +with dag: + REMINDER_OF_THIS_TEAM = PythonOperator( + task_id="FINANCE_REMINDER", python_callable=udf.main + ) + +if __name__ == "__main__": + dag.cli() diff --git a/dags/app/finance_bot/udf.py b/dags/app/finance_bot/udf.py new file mode 100644 index 0000000..e14c65a --- /dev/null +++ b/dags/app/finance_bot/udf.py @@ -0,0 +1,98 @@ +import os + +import numpy as np +import pandas as pd +import pygsheets +import requests +from app import discord +from google.cloud import bigquery + +session = requests.session() + + +def main() -> None: + # read xls from google doc to df. + df_xls = read_google_xls_to_df() + # read bigquery to df. + df_bigquery = read_bigquery_to_df() + # check difference between 2 df + df_diff = df_difference(df_xls, df_bigquery) + # link to bigquery and write xls file + write_to_bigquery(df_diff) + # push to discord + webhook_url = os.getenv("discord_data_stratagy_webhook") + username = "財務機器人" + msg = refine_diff_df_to_string(df_diff) + if msg != "no data": + discord.send_webhook_message(webhook_url, username, msg) + + +def df_difference(df_xls, df_bigquery) -> pd.DataFrame: + merged = pd.merge(df_xls, df_bigquery, how="outer", indicator=True) + return merged[merged["_merge"] == "left_only"].drop("_merge", axis=1) + + +def read_bigquery_to_df() -> pd.DataFrame: + client = bigquery.Client() + query = """ + SELECT * + FROM `pycontw-225217.test.pycontw_finance` + """ + query_job = client.query(query) + results = query_job.result() + schema = results.schema + column_names = [field.name for field in schema] + data = [list(row.values()) for row in results] + df = pd.DataFrame(data=data, columns=column_names) + + return df + + +def read_google_xls_to_df() -> pd.DataFrame: + gc = pygsheets.authorize(service_file=os.getenv("GOOGLE_APPLICATION_CREDENTIALS")) + sheet = gc.open_by_url(os.getenv("finance_xls_path")) + wks = sheet.sheet1 + df = wks.get_as_df(include_tailing_empty=False) + df.replace("", np.nan, inplace=True) + df.dropna(inplace=True) + df = df.astype(str) + df.columns = [ + "Reason", + "Price", + "Remarks", + "Team_name", + "Details", + "To_who", + "Yes_or_No", + ] + return df + + +def write_to_bigquery(df) -> None: + project_id = "pycontw-225217" + dataset_id = "test" + table_id = "pycontw_finance" + client = bigquery.Client(project=project_id) + table = client.dataset(dataset_id).table(table_id) + schema = [ + bigquery.SchemaField("Reason", "STRING", mode="REQUIRED"), + bigquery.SchemaField("Price", "STRING", mode="REQUIRED"), + bigquery.SchemaField("Remarks", "STRING", mode="REQUIRED"), + bigquery.SchemaField("Team_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("Details", "STRING", mode="REQUIRED"), + bigquery.SchemaField("To_who", "STRING", mode="REQUIRED"), + bigquery.SchemaField("Yes_or_No", "STRING", mode="REQUIRED"), + ] + job_config = bigquery.LoadJobConfig(schema=schema) + job = client.load_table_from_dataframe(df, table, job_config=job_config) + job.result() + + +def refine_diff_df_to_string(df) -> str: + msg = "" + if df.empty: + return "no data" + else: + for row in df.itertuples(index=False): + msg += f"{row[0]}, 花費: {row[1]}, {row[3]}, {row[4]}\n" + return msg diff --git a/poetry.lock b/poetry.lock index 0bb4626..0533e4c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -416,11 +416,11 @@ optional = false python-versions = ">=3.6" [package.extras] -trio = ["sniffio (>=1.1)", "trio (>=0.14.0)"] -curio = ["sniffio (>=1.1)", "curio (>=1.2)"] -idna = ["idna (>=2.1)"] -doh = ["requests-toolbelt", "requests"] dnssec = ["cryptography (>=2.6)"] +doh = ["requests", "requests-toolbelt"] +idna = ["idna (>=2.1)"] +curio = ["curio (>=1.2)", "sniffio (>=1.1)"] +trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] [[package]] name = "docutils" @@ -487,9 +487,9 @@ Jinja2 = ">=2.10.1,<3.0" Werkzeug = ">=0.15,<2.0" [package.extras] +dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] dotenv = ["python-dotenv"] -docs = ["sphinx-issues", "sphinxcontrib-log-cabinet", "pallets-sphinx-themes", "sphinx"] -dev = ["sphinx-issues", "sphinxcontrib-log-cabinet", "pallets-sphinx-themes", "sphinx", "tox", "coverage", "pytest"] [[package]] name = "flask-admin" @@ -693,56 +693,61 @@ typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\" [[package]] name = "google-api-core" -version = "2.0.0" +version = "2.11.1" description = "Google API client core library" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -google-auth = ">=1.25.0,<3.0dev" -googleapis-common-protos = ">=1.6.0,<2.0dev" +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""} -protobuf = ">=3.12.0" -requests = ">=2.18.0,<3.0.0dev" +grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""} +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -grpcio-gcp = ["grpcio-gcp (>=0.2.2)"] -grpcgcp = ["grpcio-gcp (>=0.2.2)"] -grpc = ["grpcio (>=1.33.2,<2.0dev)"] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.17.0" +version = "2.96.0" description = "Google API Client Library for Python" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -google-api-core = ">=1.21.0,<3.0.0dev" -google-auth = ">=1.16.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0.dev0" +google-auth = ">=1.19.0,<3.0.0.dev0" google-auth-httplib2 = ">=0.1.0" -httplib2 = ">=0.15.0,<1dev" -uritemplate = ">=3.0.0,<4dev" +httplib2 = ">=0.15.0,<1.dev0" +uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.0.1" +version = "2.22.0" description = "Google Authentication Library" category = "main" optional = false -python-versions = ">= 3.6" +python-versions = ">=3.6" [package.dependencies] -cachetools = ">=2.0.0,<5.0" +cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" rsa = ">=3.1.4,<5" +six = ">=1.9.0" +urllib3 = "<2.0" [package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise_cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["pyopenssl (>=20.0.0)", "cryptography (>=38.0.3)"] reauth = ["pyu2f (>=0.1.5)"] -pyopenssl = ["pyopenssl (>=20.0.0)"] -aiohttp = ["requests (>=2.20.0,<3.0.0dev)", "aiohttp (>=3.6.2,<4.0.0dev)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-auth-httplib2" @@ -759,14 +764,14 @@ six = "*" [[package]] name = "google-auth-oauthlib" -version = "0.4.5" +version = "1.0.0" description = "Google Authentication Library" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -google-auth = ">=1.0.0" +google-auth = ">=2.15.0" requests-oauthlib = ">=0.7.0" [package.extras] @@ -839,22 +844,22 @@ python-versions = ">= 3.6" google-crc32c = ">=1.0,<2.0dev" [package.extras] -requests = ["requests (>=2.18.0,<3.0.0dev)"] aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.53.0" +version = "1.60.0" description = "Common protobufs used in Google APIs" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -protobuf = ">=3.12.0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.0.0)"] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "graphviz" @@ -883,6 +888,19 @@ six = ">=1.5.2" [package.extras] protobuf = ["grpcio-tools (>=1.39.0)"] +[[package]] +name = "grpcio-status" +version = "1.39.0" +description = "Status proto mapping for gRPC" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.39.0" +protobuf = ">=3.6.0" + [[package]] name = "gunicorn" version = "20.1.0" @@ -928,8 +946,8 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" zipp = ">=0.5" [package.extras] -testing = ["importlib-resources (>=1.3)", "unittest2", "pep517", "packaging"] -docs = ["rst.linker", "sphinx"] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "pep517", "unittest2", "importlib-resources (>=1.3)"] [[package]] name = "iso8601" @@ -948,10 +966,10 @@ optional = false python-versions = ">=3.6.1,<4.0" [package.extras] -plugins = ["setuptools"] -colors = ["colorama (>=0.4.3,<0.5.0)"] pipfile_deprecated_finder = ["pipreqs", "requirementslib"] requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] [[package]] name = "itsdangerous" @@ -1247,7 +1265,7 @@ python-versions = "*" six = "*" [package.extras] -dev = ["twine", "pipreqs", "nose"] +dev = ["nose", "pipreqs", "twine"] [[package]] name = "proto-plus" @@ -1265,14 +1283,11 @@ testing = ["google-api-core[grpc] (>=1.22.2)"] [[package]] name = "protobuf" -version = "3.17.3" -description = "Protocol Buffers" +version = "4.24.0" +description = "" category = "main" optional = false -python-versions = "*" - -[package.dependencies] -six = ">=1.9" +python-versions = ">=3.7" [[package]] name = "psutil" @@ -1370,6 +1385,21 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "pygsheets" +version = "2.0.6" +description = "Google Spreadsheets Python API v4" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +google-api-python-client = ">=2.50.0" +google-auth-oauthlib = ">=0.7.1" + +[package.extras] +pandas = ["pandas (>=0.14.0)"] + [[package]] name = "pyjwt" version = "1.7.1" @@ -1451,7 +1481,7 @@ docutils = "*" lockfile = ">=0.10" [package.extras] -test = ["testtools", "testscenarios (>=0.4)", "docutils", "coverage"] +test = ["coverage", "docutils", "testscenarios (>=0.4)", "testtools"] [[package]] name = "python-dateutil" @@ -1941,7 +1971,7 @@ test = ["zope.testrunner"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "38ed366f6d7284daf57ee9fb3ca13ee3ef37456d780d86534980f9419e647d35" +content-hash = "c16f459e10c57d3db505e84eb2e1c8b1a6cc273f436f2916255ac1d2d1fca33d" [metadata.files] alembic = [] @@ -1958,10 +1988,7 @@ cached-property = [] cachelib = [] cachetools = [] cattrs = [] -certifi = [ - {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, - {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, -] +certifi = [] cffi = [] charset-normalizer = [] click = [] @@ -2005,12 +2032,10 @@ google-resumable-media = [] googleapis-common-protos = [] graphviz = [] grpcio = [] +grpcio-status = [] gunicorn = [] httplib2 = [] -idna = [ - {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, - {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, -] +idna = [] importlib-metadata = [] iso8601 = [] isort = [] @@ -2047,37 +2072,12 @@ py = [] pyarrow = [] pyasn1 = [] pyasn1-modules = [] -pycodestyle = [ - {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, - {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, -] +pycodestyle = [] pycparser = [] -pydantic = [ - {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, - {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, - {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, - {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, - {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, - {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, - {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, - {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, - {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, - {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, -] +pydantic = [] pyflakes = [] pygments = [] +pygsheets = [] pyjwt = [] pyparsing = [] pyrsistent = [] @@ -2085,10 +2085,7 @@ pytest = [] pytest-cov = [] python-daemon = [] python-dateutil = [] -python-dotenv = [ - {file = "python-dotenv-0.18.0.tar.gz", hash = "sha256:effaac3c1e58d89b3ccb4d04a40dc7ad6e0275fda25fd75ae9d323e2465e202d"}, - {file = "python_dotenv-0.18.0-py2.py3-none-any.whl", hash = "sha256:dd8fe852847f4fbfadabf6183ddd4c824a9651f02d51714fa075c95561959c7d"}, -] +python-dotenv = [] python-editor = [] python-fb-page-insights-client = [] python-nvd3 = [] @@ -2098,10 +2095,7 @@ pytz = [] pytzdata = [] pyyaml = [] regex = [] -requests = [ - {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, - {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, -] +requests = [] requests-oauthlib = [] rsa = [] safety = [] @@ -2119,19 +2113,13 @@ tenacity = [] text-unidecode = [] thrift = [] tinydb = [] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] +toml = [] typed-ast = [] typing-extensions = [] tzlocal = [] unicodecsv = [] uritemplate = [] -urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, -] +urllib3 = [] wcwidth = [] werkzeug = [] wtforms = [] diff --git a/pyproject.toml b/pyproject.toml index 50debdd..c2a0c62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ StrEnum = "^0.4.6" python-fb-page-insights-client = {git = "https://github.com/pycontw/python-fb-page-insights-client.git", rev = "bd221ff"} Flask-OAuthlib = "^0.9.6" Flask-OpenID = "1.3.0" +pygsheets = "^2.0.6" [tool.poetry.dev-dependencies] safety = "^1.9.0"