diff --git a/Back-end/Dockerfile b/Back-end/Dockerfile deleted file mode 100644 index 15d1392..0000000 --- a/Back-end/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python -RUN pip3 install pandas -RUN pip3 install openpyxl -RUN pip3 install flask pymongo -RUN mkdir /home/cop \ No newline at end of file diff --git a/Back-end/Readme.md b/Back-end/Readme.md index 5158f32..89fb332 100644 --- a/Back-end/Readme.md +++ b/Back-end/Readme.md @@ -1,16 +1,7 @@ -### **City of Portland** +### **City of Portland: Designing an Open Contracting Program Back-end** -#### **Docker Usage for backend**: -1. Execute the Dockerfile to build the image: -``` -docker build -t cop_backend . -``` -2. Execute the script to run the image: -``` -sh docker_run.sh -# Note: need to change the absolute path of your Cop directory -``` -3. Once inside the container, initiate the backend service. + +Once inside the container, initiate the backend service. ``` python3 app.py ``` diff --git a/Back-end/app.py b/Back-end/app.py index d81596f..218d21f 100644 --- a/Back-end/app.py +++ b/Back-end/app.py @@ -9,8 +9,9 @@ app = Flask(__name__) CORS(app) -uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" -cluster = MongoClient(uri) +# uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" +# cluster = MongoClient(uri) +cluster = MongoClient('localhost', 27017) db = cluster["COP"] mycol_general = db["General"] mycol_award = db["Award"] diff --git a/Back-end/docker_run.sh b/Back-end/docker_run.sh deleted file mode 100644 index 0b7118f..0000000 --- a/Back-end/docker_run.sh +++ /dev/null @@ -1,7 +0,0 @@ -docker run -it \ - -p 8080:8080 \ - -w /home/cop\ - -v /Users/maoyiru/CMU_Y1/practicum/cop:/home/cop\ - --name cop_backend cop_backend \ - /bin/bash - diff --git a/Back-end/mapping.py b/Back-end/mapping.py deleted file mode 100644 index 6a0542a..0000000 --- a/Back-end/mapping.py +++ /dev/null @@ -1,773 +0,0 @@ -import pandas as pd -import json -from datetime import datetime -import pytz -import pymongo -from pymongo import MongoClient -from pymongo.mongo_client import MongoClient -import urllib -import math -uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" - -# Create a new client and connect to the server -cluster = MongoClient(uri) -db = cluster["COP"] -mycol = db["Award"] -mycol_general = db["General"] -mycol_tender = db["Tender"] -mycol_party = db["Party"] -mycol_contract = db["Contract"] -mycol_vendor = db["Vendor"] -# mydb = myclient["admin"] -# mycol = mydb["Awards"] -# mycol_general = mydb["General"] -# mycol_tender = mydb["Tender"] -# mycol_party = mydb["Party"] -# mycol_contract = mydb["Contract"] - -def po_progress_data_transfer(date_string): - # datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S.%f") - date_object = datetime.strptime(date_string,"%Y-%m-%d %H:%M:%S.%f") - desired_timezone = pytz.timezone("US/Pacific") - localized_datetime = desired_timezone.localize(date_object.replace(hour=0, minute=0, second=0, microsecond=0)) - formatted_date = localized_datetime.isoformat() - return formatted_date - -def contract_date_transfer(date_string): - date_object = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") - desired_timezone = pytz.timezone("US/Pacific") - localized_datetime = desired_timezone.localize(date_object.replace(hour=0, minute=0, second=0)) - formatted_date = localized_datetime.isoformat() - return formatted_date - -def bid_date_transfer(date_string): - date_object = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") - desired_timezone = pytz.timezone("US/Pacific") - localized_datetime = desired_timezone.localize(date_object.replace(hour=10, minute=24, second=0, microsecond=0)) - formatted_date = localized_datetime.isoformat() - return formatted_date - -def bid_open_date_transfer(date_string): - date_object = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") - desired_timezone = pytz.timezone("US/Pacific") - localized_datetime = desired_timezone.localize(date_object.replace(hour=14, minute=0, second=0, microsecond=0)) - formatted_date = localized_datetime.isoformat() - return formatted_date - -def cal_duration_date(start_date_str, end_date_str): - - start_date = datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S") - end_date = datetime.strptime(end_date_str, "%Y-%m-%d %H:%M:%S") - delta = end_date - start_date - days_difference = delta.days - return days_difference - -def get_award_json(df_byspeed, df_sap_contract, df_sap_po): - idx = 0 - # awards/date - awards_date = po_progress_data_transfer(str(df_byspeed.iloc[0]["PO - In Progress Date"])) - # print(awards_date) - - # awards/id - awards_id = df_sap_contract.iloc[0]["Purchasing Document"] - # print(awards_id) - - # awards/status - if df_sap_po.iloc[0]["Delivery date"]: - awards_status = "active" - # print(awards_status) - - # awards/value/amount - awards_value_amount = df_sap_po.iloc[idx]["Order Quantity"] - # print(awards_value_amount) - - # awards/value/currency - awards_value_currency = "USD" - # print(awards_value_currency) - - # awards/suppliers/name - awards_suppliers_name = df_sap_po.iloc[idx]["Name 1"].title() - # print(awards_suppliers_name) - - # awards/suppliers/id - vendor_id = df_sap_po.iloc[idx]["Vendor"] - awards_suppliers_id = f"pdx-vendor-{vendor_id}" - # print(awards_suppliers_id) - - # awards/contractPeriod/startDate - awards_contractPeriod_startDate = contract_date_transfer(str(df_sap_contract.iloc[0]["Validity Per. Start"])) - # print(awards_contractPeriod_startDate) - - # awards/contractPeriod/endDate - awards_contractPeriod_endDate = contract_date_transfer(str(df_sap_contract.iloc[0]["Validity Period End"])) - # print(awards_contractPeriod_endDate) - - # awards/title - awards_title = df_sap_contract.iloc[0]["Short Text"].title() - # print(awards_title) - - # awards/description - awards_description = "Agreement for construction services." - # print(awards_description) - # Create awards library - - awards = { - "date": awards_date, - "id": str(awards_id), - "status": awards_status, - "value":{ - "amount": awards_value_amount, - "currency": awards_value_currency - }, - "suppliers":{ - "name": awards_suppliers_name, - "id": awards_suppliers_id - }, - "contractPeriod":{ - "startDate": awards_contractPeriod_startDate, - "endDate": awards_contractPeriod_endDate - }, - "title": awards_title, - "description": awards_description - } - awards_json = { - "contract_id": int(awards_id), - "awards": awards - } - # print(awards) - # file_name = "awards.json" - # insert it into the database - mycol.insert_one(awards_json) - # print(mycol.insert_one(awards)) - # Use a context manager to open the file and write the JSON data - # with open(file_name, 'w') as json_file: - # json.dump(awards, json_file, indent=4) # Serialize and write the data to the file with indentation - -def get_tender_json(contract_id, df_byspeed,df_bureau_ref, df_procurement_ref): - # tender/id - # print(df_byspeed.loc[idx,"Bid Number"]) - idx = 0 - tender_id = int(df_byspeed.iloc[idx]["Bid Number"]) - - # tender/tenderPeriod/startDate - # print(bid_date_transfer(df_byspeed.loc[0,"Bid - In Progress Date"])) - tender_startDate = bid_date_transfer(str(df_byspeed.iloc[0]["Bid - In Progress Date"])) - - # tender/tenderPeriod/endDate - tender_endDate = bid_open_date_transfer(str(df_byspeed.iloc[0]["Bid - Opened Date"])) - - # tender/tenderPeriod/durationInDays - # print(cal_duration_date(df_byspeed.loc[0,"Bid - In Progress Date"], df_byspeed.loc[0,"Bid - Opened Date"])) - tender_durationDay = cal_duration_date(str(df_byspeed.iloc[0]["Bid - In Progress Date"]), str(df_byspeed.iloc[0]["Bid - Opened Date"])) - - # tender/procuringEntity/name - # tender/procuringEntity/id - - search_value=df_byspeed.iloc[0][" Req Header Column 1 Value"] - matching_row = df_bureau_ref.loc[df_bureau_ref['BuySpeed Abbreviation '] == search_value] - if not matching_row.empty: - # print(matching_row.iloc[0]['Buyer/Name']) - # print(matching_row.iloc[0]['Buyer/id']) - tender_entity_name = matching_row.iloc[0]['Buyer/Name'] - tender_entity_id = matching_row.iloc[0]['Buyer/id'] - else: - print("NOTV FOUND") - - # tender/procurementMethod - # tender/procurementMethodDetails - # tender/mainProcurementCategory - tender_method = "Selective" - tender_methodDetail = "" - tender_category = "" - po_type_code = df_byspeed.iloc[0]['PO Type Code'].rstrip() - # print(po_type_code) - matching_row = df_procurement_ref.loc[df_procurement_ref["Description"] == po_type_code] - if not matching_row.empty: - tender_methodDetail = matching_row.iloc[0][2] - else: - print("NOTV FOUND") - - if "Construction" in po_type_code: - tender_category = "works" - elif "G&S" in po_type_code: - tender_category = "goods" - elif "PTE" in po_type_code: - tender_category = "services" - - # tender/title - tender_title = df_byspeed.iloc[0][4] - tender = { - "id": str(tender_id), - "tenderPeriod":{ - "startDate": tender_startDate, - "endDate": tender_endDate, - "durationInDays": tender_durationDay - }, - "procuringEntity":{ - "name": tender_entity_name, - "id": tender_entity_id - }, - "procurementMethod": tender_method, - "procurementMethodDetails": tender_methodDetail, - "mainProcurementCategory": tender_category, - "title": tender_title - } - - tender_json = { - "contract_id": contract_id, - "tender": tender - } - print(tender_json) - mycol_tender.insert_one(tender_json) - # file_name = "tender.json" - # # Use a context manager to open the file and write the JSON data - # with open(file_name, 'w') as json_file: - # json.dump(tender, json_file, indent=4) # Serialize and write the data to the file with indentation - -def get_general_json(contract_id, df_byspeed, df_bureau_ref): - # Open Contracting ID - req_num = int(df_byspeed.iloc[0]["REQ_NBR"]) - ocid_prefix = "{ocid-prefix}" # 将这里替换成你的实际前缀 - general_ocid = f"{ocid_prefix}-{req_num}-master" - # print(general_ocid) - # Release ID - general_id = "1" - # Release Date - general_date = "" - # tag - tag = [] - if(df_byspeed.iloc[0]["Bid Number"]): - tag.append("tender") - if(df_byspeed.iloc[0]["PO - In Progress Date"]): - tag.append("award") - if(df_byspeed.iloc[0]["Alternate Id"]): - tag.append("contract") - general_tag = ";".join(tag) - # print(general_tag) - # initiationType - general_initiationType = "" - if(df_byspeed.iloc[0]["Bid Number"]): - general_initiationType = "tender" - # print(general_initiationType) - # buyer/name & buyer/id - search_value=df_byspeed.iloc[0][" Req Header Column 1 Value"] - matching_row = df_bureau_ref.loc[df_bureau_ref['BuySpeed Abbreviation '] == search_value] - if not matching_row.empty: - # print(matching_row.iloc[0]['Buyer/Name']) - # print(matching_row.iloc[0]['Buyer/id']) - general_buyer_name = matching_row.iloc[0]['Buyer/Name'] - general_buyer_id = matching_row.iloc[0]['Buyer/id'] - # print(general_buyer_name) - # print(general_buyer_id) - else: - general_buyer_name = "" - general_buyer_id = "" - # print("NOTV FOUND") - # language - general_language = "en" - # print(general_language) - general = { - "ocid": general_ocid, - "id": general_id, - "date": general_date, - "tag": general_tag, - "initiationType": general_initiationType, - "buyer": { - "name": general_buyer_name, - "id": general_buyer_id - }, - "language": general_language - } - - general_json = { - "contract_id": contract_id, - "general": general - } - print(general_json) - # insert it into the database - mycol_general.insert_one(general_json) - # print(general) - # file_name = "general.json" - # # Use a context manager to open the file and write the JSON data - # with open(file_name, 'w') as json_file: - # json.dump(general, json_file, indent=4) # Serialize and write the data to the file with indentation - -def get_contract_json(contract_id_input, df_sap_contract, df_sap_po): - # contracts/id - contracts_id = df_sap_contract.iloc[0]["Purchasing Document"] - # print(contracts_id) - - # contracts/awardID - award_id = contracts_id - # print(award_id) - - # contracts/titlex - contracts_titlex = df_sap_contract.iloc[0]["Short Text"].title() - # print(contracts_titlex) - - # contracts/description - contract_description = "" - - # contract/status - if df_sap_po.iloc[0]["Delivery date"]: - contracts_status = "active" - # print(contracts_status) - - # contracts/period/startDate - contract_contractPeriod_startDate = contract_date_transfer(str(df_sap_contract.iloc[0]["Validity Per. Start"])) - # print(contract_contractPeriod_startDate) - - # contracts/period/endDate - contracts_contractPeriod_endDate = contract_date_transfer(str(df_sap_contract.iloc[0]["Validity Period End"])) - # print(contracts_contractPeriod_endDate) - - # contracts/period/durationInDays - contracts_duration = cal_duration_date(str(df_sap_contract.iloc[0]["Validity Per. Start"]), str(df_sap_contract.iloc[0]["Validity Period End"]) ) - # print(contracts_duration) - - # contracts/value/amount - contract_value_amount = df_sap_po.iloc[0]["Order Quantity"] - # print(contract_value_amount) - - # contracts/value/currency - contracts_value_currency = "USD" - # print(contracts_value_currency) - contracts = { - "id": str(contracts_id), - "awardID": str(award_id), - "title": contracts_titlex, - "description": contract_description, - "status": contracts_status, - "period":{ - "startDate": contract_contractPeriod_startDate, - "endDate": contracts_contractPeriod_endDate, - "durationInDays": contracts_duration - }, - - "value":{ - "amount": contract_value_amount, - "currency": contracts_value_currency - } - } - # print(contracts) - contract_json = { - "contract_id": contract_id_input, - "contracts": contracts - } - # print(contract_json) - mycol_contract.insert_one(contract_json) - -def get_party_json(contract_id, df_byspeed, df_sap_contract, df_bureau_ref, df_sap_po, df_B2G): - # parties 1 - # parties/name - # parties/roles - # parties/id - search_value=df_byspeed.iloc[0][" Req Header Column 1 Value"] - matching_row = df_bureau_ref.loc[df_bureau_ref['BuySpeed Abbreviation '] == search_value] - if not matching_row.empty: - party1__name = matching_row.iloc[0]['Buyer/Name'] - party1_id = matching_row.iloc[0]['Buyer/id'] - party1_role = "buyer; procuringEntity" - else: - party1__name = "" - party1_id = "" - party1_role = "" - - # print(party1__name) - # print(party1_id) - # print(party1_role) - # parties/identifier/scheme - party1_identifier_scheme = '-'.join(party1_id.rsplit('-', 1)[:-1]) - # print(party1_identifier_scheme) - # parties/identifier/id - party1_identifier_id = party1_id.split('-')[-1] - # print(party1_identifier_id) - # parties/contactPoint/name - party1_contract_name = df_sap_contract.iloc[0]["Project Manager"] - # print(party1_contract_name) - - # parties/name - party2_name = df_sap_po.iloc[0]["Name 1"].title() - # print(party2_name) - # parties/roles - party2_role="supplier; tenderer; payer; payee" - # print(party2_role) - # parties/id - party2_identifier_id = df_sap_po.iloc[0]["Vendor"] - party2_id = f"pdx-vendor-{party2_identifier_id}" - # print(party2_id) - # parties/identifier/scheme - party2_identifier_scheme = '-'.join(party2_id.rsplit('-', 1)[:-1]) - # print(party2_identifier_scheme) - # parties/identifier/id - party2_identifier_id = party2_id.split('-')[-1] - # print(party2_identifier_id) - # parties/details/classification/description - party2_class1_desc = df_B2G.iloc[0]["Ethnicity"] - # print(party2_class1_desc) - # parties/details/classification/scheme - party2_class1_scheme = "Race/Ethnicity" - # print(party2_class1_scheme) - #parties/details/classification/description - party2_class2_desc = df_B2G.iloc[0]["Gender"] - # print(party2_class2_desc) - # parties/details/classification/scheme - party2_class2_scheme = "Gender" - # print(party2_class2_scheme) - #parties/details/classification/description - if df_B2G.iloc[0]["Additional Certifications"]: - try: - goal_type_set = set(df_B2G.iloc[0]["Goal Type"].split(", ")) - additional_certifications_set = set(df_B2G.iloc[0]["Additional Certifications"].split(", ")) - merged_set = goal_type_set.union(additional_certifications_set) - party2_class3_desc = ", ".join(merged_set) - - except: - party2_class3_desc = df_B2G.iloc[0]["Goal Type"] - else: - party2_class3_desc = df_B2G.iloc[0]["Goal Type"] - print(party2_class3_desc) - # parties/details/classification/scheme - party2_class3_scheme = "Certification" - # print(party2_class3_scheme) - parties = { - "party1": { - "name": party1__name, - "role": party1_role, - "id": party1_id, - "identifier": { - "scheme": party1_identifier_scheme, - "id": party1_identifier_id - }, - "contactPoint": { - "name": party1_contract_name - } - }, - "party2": { - "name": party2_name, - "role": party2_role, - "id": party2_id, - "identifier": { - "scheme": party2_identifier_scheme, - "id": party2_identifier_id - }, - "details": { - "classfication1": { - "description": party2_class1_desc, - "scheme": party2_class1_scheme - }, - "classfication2": { - "description": party2_class2_desc, - "scheme": party2_class2_scheme - }, - "classfication3": { - "description": party2_class3_desc, - "scheme": party2_class3_scheme - } - } - } - } - parties_json = { - "contract_id": contract_id, - "parties": parties - } - print(parties_json) - # mycol_party.insert_one(parties_json) - -def get_json(contract_id): - df_bureau_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Bureau Reference Sheet') - df_procurement_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Procurement Method Details') - df_sap_po = pd.read_excel('SAP PO Listing Report FY2019-23.xlsx') - matching_row = df_sap_po.loc[(df_sap_po['Outline agreement'] == str(contract_id)) & (df_sap_po["Item"] == 10)] - - if not matching_row.empty: - df_sap_po = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 1") - return - - df_sap_contract = pd.read_excel('SAP Contract Listing Report FY2019-23 from PO Listing Report.xlsx') - matching_row = df_sap_contract.loc[(df_sap_contract['Purchasing Document'] == contract_id) & (df_sap_contract["Item"] == 10)] - if not matching_row.empty: - df_sap_contract = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 2") - - - df_byspeed = pd.read_excel('BuySpeed Report.xlsx') - matching_row = df_byspeed.loc[df_byspeed['Alternate Id'] == str(contract_id)] - if not matching_row.empty: - df_byspeed = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND3") - - df_B2G = pd.read_excel("B2G Contract Status Report.xlsx") - # print(df_B2G) - matching_row = df_B2G.loc[df_B2G['Contract Number'] == str(contract_id)] - if not matching_row.empty: - df_B2G = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND3") - # print(df_B2G) - # get_award_json(df_byspeed, df_sap_contract, df_sap_po) - - - # get_tender_json(df_byspeed,df_bureau_ref, df_procurement_ref) - # get_general_json(df_byspeed, df_bureau_ref) - # get_contract_json(df_byspeed, df_sap_contract, df_sap_po) - get_party_json(contract_id, df_byspeed, df_sap_contract, df_bureau_ref, df_sap_po,df_B2G) - -# contract_id = 30007897 -# get_json(30007897) -# get_award_json(30007897) - -def get_award_json_and_insert_db(): - df_sap_po = pd.read_excel('SAP PO Listing Report FY2019-23.xlsx') - df_sap_contract = pd.read_excel('SAP Contract Listing Report FY2019-23 from PO Listing Report.xlsx') - df_byspeed = pd.read_excel('BuySpeed Report.xlsx') - - contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) - - total_num = len(contract_ids) - print("total contract_id num:", total_num) - num = 0 - for contract_id in contract_ids: - num += 1 - try: - contract_id = int(contract_id) - except: - continue - matching_row = df_sap_po.loc[(df_sap_po['Outline agreement'] == str(contract_id)) & (df_sap_po["Item"] == 10)] - if not matching_row.empty: - df_sap_po_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 1 for contract_id:", contract_id) - continue - - - matching_row = df_sap_contract.loc[(df_sap_contract['Purchasing Document'] == contract_id) & (df_sap_contract["Item"] == 10)] - if not matching_row.empty: - df_sap_contract_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 2 for contract_id:", contract_id) - continue - - - matching_row = df_byspeed.loc[df_byspeed['Alternate Id'] == str(contract_id)] - if not matching_row.empty: - df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND3 for contract_id:", contract_id) - continue - - get_award_json(df_byspeed_matched, df_sap_contract_matched, df_sap_po_matched) - if num%100 == 0: - print(f"Finished: {num} / {total_num}!") - -def get_general_json_and_insert_db(): - df_bureau_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Bureau Reference Sheet') - df_sap_po = pd.read_excel('SAP PO Listing Report FY2019-23.xlsx') - df_byspeed = pd.read_excel('BuySpeed Report.xlsx') - - contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) - - total_num = len(contract_ids) - print("total contract_id num:", total_num) - num = 0 - for contract_id in contract_ids: - num += 1 - try: - contract_id = int(contract_id) - except: - continue - matching_row = df_byspeed.loc[df_byspeed['Alternate Id'] == str(contract_id)] - if not matching_row.empty: - df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND3 for contract_id:", contract_id) - continue - try: - get_general_json(contract_id, df_byspeed_matched, df_bureau_ref) - except Exception as e: - print(f"An error occurred: {e}") - - if num%100 == 0: - print(f"Finished: {num} / {total_num}!") - -def get_tender_json_and_insert_db(): - df_bureau_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Bureau Reference Sheet') - df_procurement_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Procurement Method Details') - df_sap_po = pd.read_excel('SAP PO Listing Report FY2019-23.xlsx') - df_byspeed = pd.read_excel('BuySpeed Report.xlsx') - - contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) - - total_num = len(contract_ids) - print("total contract_id num:", total_num) - num = 0 - for contract_id in contract_ids: - num += 1 - try: - contract_id = int(contract_id) - except: - continue - - matching_row = df_byspeed.loc[df_byspeed['Alternate Id'] == str(contract_id)] - if not matching_row.empty: - df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND3 for contract_id:", contract_id) - continue - try: - get_tender_json(contract_id, df_byspeed_matched,df_bureau_ref, df_procurement_ref) - except Exception as e: - print(f"An error occurred: {e}") - - if num%100 == 0: - print(f"Finished: {num} / {total_num}!") - -def get_party_json_and_insert_db(): - df_sap_contract = pd.read_excel('SAP Contract Listing Report FY2019-23 from PO Listing Report.xlsx') - df_bureau_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Bureau Reference Sheet') - df_sap_po = pd.read_excel('SAP PO Listing Report FY2019-23.xlsx') - df_byspeed = pd.read_excel('BuySpeed Report.xlsx') - df_B2G = pd.read_excel("B2G Contract Status Report.xlsx") - - contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) - - total_num = len(contract_ids) - print("total contract_id num:", total_num) - num = 0 - for contract_id in contract_ids: - num += 1 - try: - contract_id = int(contract_id) - except: - continue - - matching_row = df_sap_po.loc[(df_sap_po['Outline agreement'] == str(contract_id)) & (df_sap_po["Item"] == 10)] - if not matching_row.empty: - df_sap_po_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 1 for contract_id:", contract_id) - continue - - matching_row = df_sap_contract.loc[(df_sap_contract['Purchasing Document'] == contract_id) & (df_sap_contract["Item"] == 10)] - if not matching_row.empty: - df_sap_contract_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 2 for contract_id:", contract_id) - continue - - matching_row = df_byspeed.loc[df_byspeed['Alternate Id'] == str(contract_id)] - if not matching_row.empty: - df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND3 for contract_id:", contract_id) - continue - - matching_row = df_B2G.loc[df_B2G['Contract Number'] == str(contract_id)] - if not matching_row.empty: - df_B2G_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND4 for contract_id:", contract_id) - continue - - try: - get_party_json(contract_id, df_byspeed_matched, df_sap_contract_matched, df_bureau_ref, df_sap_po_matched, df_B2G_matched) - except Exception as e: - print(f"An error occurred: {e}") - - if num%100 == 0: - print(f"Finished: {num} / {total_num}!") - -def get_contract_json_and_insert_db(): - df_sap_po = pd.read_excel('SAP PO Listing Report FY2019-23.xlsx') - df_sap_contract = pd.read_excel('SAP Contract Listing Report FY2019-23 from PO Listing Report.xlsx') - df_byspeed = pd.read_excel('BuySpeed Report.xlsx') - - contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) - - total_num = len(contract_ids) - print("total contract_id num:", total_num) - num = 0 - for contract_id in contract_ids: - num += 1 - try: - contract_id = int(contract_id) - except: - continue - matching_row = df_sap_po.loc[(df_sap_po['Outline agreement'] == str(contract_id)) & (df_sap_po["Item"] == 10)] - if not matching_row.empty: - df_sap_po_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 1 for contract_id:", contract_id) - continue - - - matching_row = df_sap_contract.loc[(df_sap_contract['Purchasing Document'] == contract_id) & (df_sap_contract["Item"] == 10)] - if not matching_row.empty: - df_sap_contract_matched = pd.DataFrame(matching_row.iloc[0]).T - else: - print("NOT FOUND 2 for contract_id:", contract_id) - continue - - get_contract_json(contract_id, df_sap_contract_matched, df_sap_po_matched) - if num%100 == 0: - print(f"Finished: {num} / {total_num}!") - -def create_vendor_directory_in_db(): - cursor = mycol_party.find() - num = 0 - for data_party in cursor: - desired_contract_id = data_party["contract_id"] - data_general = mycol_general.find_one({'contract_id': int(desired_contract_id)}) - data_award = mycol.find_one({'contract_id': int(desired_contract_id)}) - - if data_general: - org_name = data_general["general"]["buyer"]["name"] - # print(org_name) - else: - continue - - if data_award: - vendor_name = data_award["awards"]["suppliers"]["name"] - contract_status = data_award["awards"]["status"] - else: - continue - - if data_party: - # certification - certification = data_party["parties"]["party2"]["details"]["classfication3"]["description"] - try: - if math.isnan(certification): - certification = "NaN" - except: - pass - else: - continue - num += 1 - contract_info = { - "key": str(num), - "name": vendor_name, - "org": org_name, - "contract": desired_contract_id, - "cert": certification, - "aval": "80%", - "tags": [contract_status] - } - print(contract_info) - mycol_vendor.insert_one(contract_info) - - -# get_award_json_and_insert_db() -# get_general_json_and_insert_db() -# get_tender_json_and_insert_db() -# get_party_json_and_insert_db() -# get_contract_json_and_insert_db() -create_vendor_directory_in_db() \ No newline at end of file diff --git a/Back-end/mongodb_connect.py b/Back-end/mongodb_connect.py deleted file mode 100644 index 73e167f..0000000 --- a/Back-end/mongodb_connect.py +++ /dev/null @@ -1,109 +0,0 @@ - -from pymongo.mongo_client import MongoClient -import urllib -import json -import sys - -uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" -cluster = MongoClient(uri) -db = cluster["COP"] -mycol_general = db["General"] -mycol_award = db["Award"] -mycol_tender = db["Tender"] -mycol_contract = db["Contract"] -mycol_party = db["Party"] -# Create a new client and connect to the server -client = MongoClient(uri) - -# Send a ping to confirm a successful connection -try: - client.admin.command('ping') - print("Pinged your deployment. You successfully connected to MongoDB!") -except Exception as e: - print(e) - -def get_general_json_by_id(contract_id): - desired_contract_id = contract_id - - data = mycol_general.find_one({'contract_id': int(desired_contract_id)}) - if data: - return data["general"] - else: - print('Data not found') - -def get_award_json_by_id(contract_id): - desired_contract_id = contract_id - - data = mycol_award.find_one({'contract_id': int(desired_contract_id)}) - if data: - return data["awards"] - else: - print('Data not found') - -def get_tender_json_by_id(contract_id): - desired_contract_id = contract_id - - data = mycol_tender.find_one({'contract_id': int(desired_contract_id)}) - if data: - return data["tender"] - else: - print('Data not found') - -def get_party_json_by_id(contract_id): - desired_contract_id = contract_id - - data = mycol_party.find_one({'contract_id': int(desired_contract_id)}) - if data: - return data["parties"] - else: - print('Data not found') - -def get_contract_json_by_id(contract_id): - desired_contract_id = contract_id - - data = mycol_contract.find_one({'contract_id': int(desired_contract_id)}) - if data: - return data["contracts"] - else: - print('Data not found') - -def get_contract_id_by_vendor_id(vender_id): - data = mycol_party.find({'parties.party2.identifier.id': vender_id}) - for doc in data: - print(doc["contract_id"]) - -def main(contract_id): - print("general") - general_json = get_general_json_by_id(contract_id) - with open(f"{contract_id}_general.json", 'w', encoding='utf-8') as file: - json.dump(general_json, file, ensure_ascii=False, indent=4) - - print("award") - award_json = get_award_json_by_id(contract_id) - with open(f"{contract_id}_award.json", 'w', encoding='utf-8') as file: - json.dump(award_json, file, ensure_ascii=False, indent=4) - - print("tender") - tender_json = get_tender_json_by_id(contract_id) - with open(f"{contract_id}_tender.json", 'w', encoding='utf-8') as file: - json.dump(tender_json, file, ensure_ascii=False, indent=4) - - print("party") - party_json = get_party_json_by_id(contract_id) - with open(f"{contract_id}_party.json", 'w', encoding='utf-8') as file: - json.dump(party_json, file, ensure_ascii=False, indent=4) - - print("contract") - contract_json = get_contract_json_by_id(contract_id) - with open(f"{contract_id}_contract.json", 'w', encoding='utf-8') as file: - json.dump(contract_json, file, ensure_ascii=False, indent=4) - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: python mongodb_connect.py ") - sys.exit(1) - contract_id = sys.argv[1] - try: - main(int(contract_id)) - except ValueError: - print("Please provide a valid contract_id as an integer.") \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1d4bdf3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM mongo:latest +RUN apt-get update && apt-get install -y python3 python3-pip +RUN apt-get update && apt-get install -y nodejs npm +RUN pip3 install pandas +RUN pip3 install openpyxl +RUN pip3 install flask pymongo +RUN pip3 install flask-cors +RUN mkdir /home/cop \ No newline at end of file diff --git a/Front-end/README.md b/Front-end/README.md index 58beeac..f45021b 100644 --- a/Front-end/README.md +++ b/Front-end/README.md @@ -1,70 +1,18 @@ -# Getting Started with Create React App +## **City of Portland: Designing an Open Contracting Program Front-end** -This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). -## Available Scripts +Once you have accessed the container, proceed to start the front-end service. Follow the steps below depending on whether it's your first time running the front-end code or if there have been updates to the packages. -In the project directory, you can run: +### Initial Run or After Package Updates +If this is your first time running the front-end code, or if there have been any updates to the packages, you need to install the necessary dependencies. Run the following command: +``` +npm install +``` +This command installs all the dependencies defined in the package.json file. It is a crucial step to ensure that your front-end service runs smoothly with all the required packages. -### `npm start` - -Runs the app in the development mode.\ -Open [http://localhost:3000](http://localhost:3000) to view it in your browser. - -The page will reload when you make changes.\ -You may also see any lint errors in the console. - -### `npm test` - -Launches the test runner in the interactive watch mode.\ -See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. - -### `npm run build` - -Builds the app for production to the `build` folder.\ -It correctly bundles React in production mode and optimizes the build for the best performance. - -The build is minified and the filenames include the hashes.\ -Your app is ready to be deployed! - -See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. - -### `npm run eject` - -**Note: this is a one-way operation. Once you `eject`, you can't go back!** - -If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. - -Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own. - -You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it. - -## Learn More - -You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). - -To learn React, check out the [React documentation](https://reactjs.org/). - -### Code Splitting - -This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting) - -### Analyzing the Bundle Size - -This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size) - -### Making a Progressive Web App - -This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app) - -### Advanced Configuration - -This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration) - -### Deployment - -This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment) - -### `npm run build` fails to minify - -This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify) +### Starting the Front-end Service +After successfully installing the dependencies, or if you have already installed them in the past, start the front-end service by executing: +``` +npm start +``` +This command will compile the front-end code and typically starts a development server. Once the server is running, you should be able to access the front-end application. \ No newline at end of file diff --git a/Mapping/Dockerfile b/Mapping/Dockerfile deleted file mode 100644 index 15d1392..0000000 --- a/Mapping/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python -RUN pip3 install pandas -RUN pip3 install openpyxl -RUN pip3 install flask pymongo -RUN mkdir /home/cop \ No newline at end of file diff --git a/Mapping/Readme.md b/Mapping/Readme.md index b6d23cb..a83402c 100644 --- a/Mapping/Readme.md +++ b/Mapping/Readme.md @@ -1,41 +1,10 @@ -### **City of Portland** -#### **Docker Usage for mapping**: -1. Pull the docker iamge from docker hub: -- For arm: -``` -docker pull yirum/data_mapping:v0 -``` -- For amd/x86: -``` -docker pull yirum/data_mapping:v1 -``` -2. Run docker by running the script: -``` -sh docker_run.sh -# Note: need to change the absolute path of your Cop directory and the image name(v1 or v0) -``` +### **City of Portland: Designing an Open Contracting Program Mapping** +The mapping.py script serves as a mapper to transform contract data into the Open Contracting Data Standard (OCDS) format and subsequently stores this converted data in a MongoDB database. -Run the mapping code: +The mongodb_connect.py script is a Python utility designed to facilitate the retrieval of JSON files based on the contract ID. For example, to fetch a specific contract's data, you can use a command like this in your terminal: ``` -python3 mapping.py +python3 mongodb_connect.py 30006409 ``` +This command will access the MongoDB database and return the JSON data associated with the contract ID 30006409. -#### **Docker Usage for backend**: -1. Execute the Dockerfile to build the image: -``` -docker build -t cop_backend . -``` -2. Execute the script to run the image: -``` -sh docker_run.sh -# Note: need to change the absolute path of your Cop directory -``` -3. Once inside the container, initiate the backend service. -``` -python3 app.py -``` -For those wishing to access the backend and utilize the API for sending requests, we've provided a front-end script for your convenience. Please ensure you use the appropriate port number. In this example, we've designated port 8080 for our backend. -``` -python test_api.py -``` \ No newline at end of file diff --git a/Mapping/date_transfer.py b/Mapping/date_transfer.py deleted file mode 100644 index c3a7e39..0000000 --- a/Mapping/date_transfer.py +++ /dev/null @@ -1,12 +0,0 @@ -from datetime import datetime -import pytz - -# Define the input date string -date_string = "June 14, 2022" - -date_object = datetime.strptime(date_string, "%B %d, %Y") -desired_timezone = pytz.timezone("US/Pacific") -localized_datetime = desired_timezone.localize(date_object.replace(hour=0, minute=0, second=0)) -formatted_date = localized_datetime.isoformat() - -print(formatted_date) \ No newline at end of file diff --git a/Mapping/db.py b/Mapping/db.py deleted file mode 100644 index 21b1cc0..0000000 --- a/Mapping/db.py +++ /dev/null @@ -1,8 +0,0 @@ -import pymongo -myclient = pymongo.MongoClient("mongodb://localhost:27037/") -mydb = myclient["admin"] -mycol = mydb["Awards"] - -x = mycol.find_one() - -print(x) \ No newline at end of file diff --git a/Mapping/docker_run.sh b/Mapping/docker_run.sh deleted file mode 100644 index 0b7118f..0000000 --- a/Mapping/docker_run.sh +++ /dev/null @@ -1,7 +0,0 @@ -docker run -it \ - -p 8080:8080 \ - -w /home/cop\ - -v /Users/maoyiru/CMU_Y1/practicum/cop:/home/cop\ - --name cop_backend cop_backend \ - /bin/bash - diff --git a/Mapping/mapping.py b/Mapping/mapping.py index 6a0542a..d3be03e 100644 --- a/Mapping/mapping.py +++ b/Mapping/mapping.py @@ -7,10 +7,10 @@ from pymongo.mongo_client import MongoClient import urllib import math -uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" +import warnings -# Create a new client and connect to the server -cluster = MongoClient(uri) +warnings.filterwarnings('ignore') +cluster = MongoClient('localhost', 27017) db = cluster["COP"] mycol = db["Award"] mycol_general = db["General"] @@ -18,15 +18,9 @@ mycol_party = db["Party"] mycol_contract = db["Contract"] mycol_vendor = db["Vendor"] -# mydb = myclient["admin"] -# mycol = mydb["Awards"] -# mycol_general = mydb["General"] -# mycol_tender = mydb["Tender"] -# mycol_party = mydb["Party"] -# mycol_contract = mydb["Contract"] + def po_progress_data_transfer(date_string): - # datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S.%f") date_object = datetime.strptime(date_string,"%Y-%m-%d %H:%M:%S.%f") desired_timezone = pytz.timezone("US/Pacific") localized_datetime = desired_timezone.localize(date_object.replace(hour=0, minute=0, second=0, microsecond=0)) @@ -171,7 +165,8 @@ def get_tender_json(contract_id, df_byspeed,df_bureau_ref, df_procurement_ref): tender_entity_name = matching_row.iloc[0]['Buyer/Name'] tender_entity_id = matching_row.iloc[0]['Buyer/id'] else: - print("NOTV FOUND") + # print("NOTV FOUND") + pass # tender/procurementMethod # tender/procurementMethodDetails @@ -185,7 +180,8 @@ def get_tender_json(contract_id, df_byspeed,df_bureau_ref, df_procurement_ref): if not matching_row.empty: tender_methodDetail = matching_row.iloc[0][2] else: - print("NOTV FOUND") + # print("NOTV FOUND") + pass if "Construction" in po_type_code: tender_category = "works" @@ -217,7 +213,6 @@ def get_tender_json(contract_id, df_byspeed,df_bureau_ref, df_procurement_ref): "contract_id": contract_id, "tender": tender } - print(tender_json) mycol_tender.insert_one(tender_json) # file_name = "tender.json" # # Use a context manager to open the file and write the JSON data @@ -283,7 +278,6 @@ def get_general_json(contract_id, df_byspeed, df_bureau_ref): "contract_id": contract_id, "general": general } - print(general_json) # insert it into the database mycol_general.insert_one(general_json) # print(general) @@ -426,7 +420,7 @@ def get_party_json(contract_id, df_byspeed, df_sap_contract, df_bureau_ref, df_s party2_class3_desc = df_B2G.iloc[0]["Goal Type"] else: party2_class3_desc = df_B2G.iloc[0]["Goal Type"] - print(party2_class3_desc) + # print(party2_class3_desc) # parties/details/classification/scheme party2_class3_scheme = "Certification" # print(party2_class3_scheme) @@ -471,8 +465,8 @@ def get_party_json(contract_id, df_byspeed, df_sap_contract, df_bureau_ref, df_s "contract_id": contract_id, "parties": parties } - print(parties_json) - # mycol_party.insert_one(parties_json) + # print(parties_json) + mycol_party.insert_one(parties_json) def get_json(contract_id): df_bureau_ref = pd.read_excel('master contract 30007897.xlsx', sheet_name='Bureau Reference Sheet') @@ -483,7 +477,7 @@ def get_json(contract_id): if not matching_row.empty: df_sap_po = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 1") + # print("NOT FOUND 1") return df_sap_contract = pd.read_excel('SAP Contract Listing Report FY2019-23 from PO Listing Report.xlsx') @@ -491,7 +485,8 @@ def get_json(contract_id): if not matching_row.empty: df_sap_contract = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 2") + # print("NOT FOUND 2") + pass df_byspeed = pd.read_excel('BuySpeed Report.xlsx') @@ -499,7 +494,8 @@ def get_json(contract_id): if not matching_row.empty: df_byspeed = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND3") + # print("NOT FOUND3") + pass df_B2G = pd.read_excel("B2G Contract Status Report.xlsx") # print(df_B2G) @@ -507,7 +503,8 @@ def get_json(contract_id): if not matching_row.empty: df_B2G = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND3") + # print("NOT FOUND3") + pass # print(df_B2G) # get_award_json(df_byspeed, df_sap_contract, df_sap_po) @@ -527,7 +524,7 @@ def get_award_json_and_insert_db(): df_byspeed = pd.read_excel('BuySpeed Report.xlsx') contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) + # print(contract_ids) total_num = len(contract_ids) print("total contract_id num:", total_num) @@ -542,7 +539,7 @@ def get_award_json_and_insert_db(): if not matching_row.empty: df_sap_po_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 1 for contract_id:", contract_id) + # print("NOT FOUND 1 for contract_id:", contract_id) continue @@ -550,7 +547,7 @@ def get_award_json_and_insert_db(): if not matching_row.empty: df_sap_contract_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 2 for contract_id:", contract_id) + # print("NOT FOUND 2 for contract_id:", contract_id) continue @@ -558,7 +555,7 @@ def get_award_json_and_insert_db(): if not matching_row.empty: df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND3 for contract_id:", contract_id) + # print("NOT FOUND3 for contract_id:", contract_id) continue get_award_json(df_byspeed_matched, df_sap_contract_matched, df_sap_po_matched) @@ -571,7 +568,7 @@ def get_general_json_and_insert_db(): df_byspeed = pd.read_excel('BuySpeed Report.xlsx') contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) + # print(contract_ids) total_num = len(contract_ids) print("total contract_id num:", total_num) @@ -586,12 +583,13 @@ def get_general_json_and_insert_db(): if not matching_row.empty: df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND3 for contract_id:", contract_id) + # print("NOT FOUND3 for contract_id:", contract_id) continue try: get_general_json(contract_id, df_byspeed_matched, df_bureau_ref) except Exception as e: - print(f"An error occurred: {e}") + pass + # print(f"An error occurred: {e}") if num%100 == 0: print(f"Finished: {num} / {total_num}!") @@ -603,7 +601,7 @@ def get_tender_json_and_insert_db(): df_byspeed = pd.read_excel('BuySpeed Report.xlsx') contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) + # print(contract_ids) total_num = len(contract_ids) print("total contract_id num:", total_num) @@ -619,12 +617,13 @@ def get_tender_json_and_insert_db(): if not matching_row.empty: df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND3 for contract_id:", contract_id) + # print("NOT FOUND3 for contract_id:", contract_id) continue try: get_tender_json(contract_id, df_byspeed_matched,df_bureau_ref, df_procurement_ref) except Exception as e: - print(f"An error occurred: {e}") + # print(f"An error occurred: {e}") + pass if num%100 == 0: print(f"Finished: {num} / {total_num}!") @@ -637,7 +636,7 @@ def get_party_json_and_insert_db(): df_B2G = pd.read_excel("B2G Contract Status Report.xlsx") contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) + # print(contract_ids) total_num = len(contract_ids) print("total contract_id num:", total_num) @@ -653,34 +652,35 @@ def get_party_json_and_insert_db(): if not matching_row.empty: df_sap_po_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 1 for contract_id:", contract_id) + # print("NOT FOUND 1 for contract_id:", contract_id) continue matching_row = df_sap_contract.loc[(df_sap_contract['Purchasing Document'] == contract_id) & (df_sap_contract["Item"] == 10)] if not matching_row.empty: df_sap_contract_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 2 for contract_id:", contract_id) + # print("NOT FOUND 2 for contract_id:", contract_id) continue matching_row = df_byspeed.loc[df_byspeed['Alternate Id'] == str(contract_id)] if not matching_row.empty: df_byspeed_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND3 for contract_id:", contract_id) + # print("NOT FOUND3 for contract_id:", contract_id) continue matching_row = df_B2G.loc[df_B2G['Contract Number'] == str(contract_id)] if not matching_row.empty: df_B2G_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND4 for contract_id:", contract_id) + # print("NOT FOUND4 for contract_id:", contract_id) continue try: get_party_json(contract_id, df_byspeed_matched, df_sap_contract_matched, df_bureau_ref, df_sap_po_matched, df_B2G_matched) except Exception as e: - print(f"An error occurred: {e}") + # print(f"An error occurred: {e}") + pass if num%100 == 0: print(f"Finished: {num} / {total_num}!") @@ -691,7 +691,7 @@ def get_contract_json_and_insert_db(): df_byspeed = pd.read_excel('BuySpeed Report.xlsx') contract_ids = df_sap_po['Outline agreement'].unique() - print(contract_ids) + # print(contract_ids) total_num = len(contract_ids) print("total contract_id num:", total_num) @@ -706,7 +706,7 @@ def get_contract_json_and_insert_db(): if not matching_row.empty: df_sap_po_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 1 for contract_id:", contract_id) + # print("NOT FOUND 1 for contract_id:", contract_id) continue @@ -714,7 +714,7 @@ def get_contract_json_and_insert_db(): if not matching_row.empty: df_sap_contract_matched = pd.DataFrame(matching_row.iloc[0]).T else: - print("NOT FOUND 2 for contract_id:", contract_id) + # print("NOT FOUND 2 for contract_id:", contract_id) continue get_contract_json(contract_id, df_sap_contract_matched, df_sap_po_matched) @@ -761,13 +761,25 @@ def create_vendor_directory_in_db(): "aval": "80%", "tags": [contract_status] } - print(contract_info) + # print(contract_info) mycol_vendor.insert_one(contract_info) - -# get_award_json_and_insert_db() -# get_general_json_and_insert_db() -# get_tender_json_and_insert_db() -# get_party_json_and_insert_db() -# get_contract_json_and_insert_db() -create_vendor_directory_in_db() \ No newline at end of file + +print("Award ...") +get_award_json_and_insert_db() +print("Award Finished") +print("General ...") +get_general_json_and_insert_db() +print("General Finished") +print("Tender ...") +get_tender_json_and_insert_db() +print("Tender Finished") +print("Party ...") +get_party_json_and_insert_db() +print("Party Finished") +print("Contract ...") +get_contract_json_and_insert_db() +print("Contract Finished") +print("Vendor ...") +create_vendor_directory_in_db() +print("Vendor Finished") \ No newline at end of file diff --git a/Mapping/mongodb_connect.py b/Mapping/mongodb_connect.py index 73e167f..f93ca34 100644 --- a/Mapping/mongodb_connect.py +++ b/Mapping/mongodb_connect.py @@ -4,16 +4,17 @@ import json import sys -uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" -cluster = MongoClient(uri) -db = cluster["COP"] +# uri = "mongodb+srv://cop:" + urllib.parse.quote("Cityofportland@123") + "@cop.9izbbfh.mongodb.net/?retryWrites=true&w=majority" +# cluster = MongoClient(uri) +client = MongoClient('localhost', 27017) +db = client["COP"] mycol_general = db["General"] mycol_award = db["Award"] mycol_tender = db["Tender"] mycol_contract = db["Contract"] mycol_party = db["Party"] # Create a new client and connect to the server -client = MongoClient(uri) + # Send a ping to confirm a successful connection try: diff --git a/Readme.md b/Readme.md index e4c2e38..607041f 100644 --- a/Readme.md +++ b/Readme.md @@ -1,2 +1,50 @@ -## City of Portland OCDS program +## City of Portland: Designing an Open Contracting Program +This document provides the steps for deploying and running the City of Portland: Designing an Open Contracting Program using Docker. This program involves converting contract data into the Open Contracting Data Standard (OCDS) and presenting it through a front-end and back-end application. +### **1. Build Docker Image** +First, build the Docker image containing the necessary dependencies: +``` +docker build -t cop . +``` +![Alt text](./src/docker_build.png) +### **2. Start Docker Container** +Start the Docker container using the following command: +``` +sh docker_run.sh +``` +To verify if the container has started successfully: +``` +docker ps +``` +![Alt text](./src/docker_run.png) + +### **3. Set up Mongodb Database** +Execute the following steps to initialize the database and run the mapper, which converts contract data into the OCDS standard and stores it in the MongoDB database inside the Docker container: +``` +docker exec -it cop /bin/bash +cd Mapping/ +python3 mapping.py +``` +![Alt text](./src/initiate_db.png) +### **4. Start back-end server** +Start the back-end server inside the container: +``` +docker exec -it cop /bin/bash +cd Back-end/ +python3 app.py +``` +![Alt text](./src/back_end.png) +### **4. Start Front-end** +Finally, set up and start the front-end application: +``` +docker exec -it cop /bin/bash +cd Front-end/ +npm install +npm start +``` +![Alt text](./src/front_end.png) +Access the dashboard at the following URL:http://localhost:3000 +![Alt text](./src/dash_board.png) +### **Notes** +- Ensure that Docker is installed and configured on your machine before running any Docker commands. +- Follow the steps in order to ensure all services are correctly configured and started. diff --git a/docker_run.sh b/docker_run.sh new file mode 100644 index 0000000..4eb50c3 --- /dev/null +++ b/docker_run.sh @@ -0,0 +1,8 @@ +docker run -itd \ + -p 8080:8080 \ + -p 27017:27017 \ + -p 3000:3000 \ + -w /home/cop\ + -v /Users/maoyiru/CMU_Y1/practicum/COP_main/cop:/home/cop\ + --name cop cop + diff --git a/read.md b/read.md deleted file mode 100644 index e4c2e38..0000000 --- a/read.md +++ /dev/null @@ -1,2 +0,0 @@ -## City of Portland OCDS program - diff --git a/src/back_end.png b/src/back_end.png new file mode 100644 index 0000000..74b431b Binary files /dev/null and b/src/back_end.png differ diff --git a/src/dash_board.png b/src/dash_board.png new file mode 100644 index 0000000..1118e5e Binary files /dev/null and b/src/dash_board.png differ diff --git a/src/docker_build.png b/src/docker_build.png new file mode 100644 index 0000000..0840f64 Binary files /dev/null and b/src/docker_build.png differ diff --git a/src/docker_run.png b/src/docker_run.png new file mode 100644 index 0000000..ec35ca3 Binary files /dev/null and b/src/docker_run.png differ diff --git a/src/front_end.png b/src/front_end.png new file mode 100644 index 0000000..10817ac Binary files /dev/null and b/src/front_end.png differ diff --git a/src/initiate_db.png b/src/initiate_db.png new file mode 100644 index 0000000..68b9ecd Binary files /dev/null and b/src/initiate_db.png differ diff --git a/src/into_docker.png b/src/into_docker.png new file mode 100644 index 0000000..e8586a4 Binary files /dev/null and b/src/into_docker.png differ