GenAI-Workshop-Demo1a.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1G80OeNRFnqtEmVAV_CgkfCuACPe8J0_n","timestamp":1708270469063}],"authorship_tag":"ABX9TyMR8LQ+ZV+PuzNgptkGcsny"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Load the dataset from Hugging Face (AIatMongoDB/)\n","Link: https://huggingface.co/datasets/AIatMongoDB/..."],"metadata":{"id":"TIsBaLX0TN3_"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"yG2dbTbwq0-d"},"outputs":[],"source":["!pip install pymongo\n","!pip install datasets"]},{"cell_type":"markdown","source":["# Update user and password in the URI\n","\n","It should look like this:\n","\n","~~~\n","os.environ['MONGODB_ATLAS_URI'] = 'mongodb+srv://AIOPS:password@cluster0.ozciyn7.mongodb.net/?retryWrites=true&w=majority'\n","~~~\n"],"metadata":{"id":"eOC5HDjoUX4B"}},{"cell_type":"code","source":["import os\n","os.environ['MONGODB_ATLAS_URI'] = \"\""],"metadata":{"id":"Jvq1brSA0p6_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from pymongo import MongoClient\n","import datasets\n","from datasets import load_dataset\n","from bson import json_util"],"metadata":{"id":"bBMZJlWOrR-b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["uri = os.environ.get('MONGODB_ATLAS_URI')\n","client = MongoClient(uri)\n","\n","#db_name = 'whatscooking'\n","#collection_name = 'restaurants'\n","#hf_dataset = 'AIatMongoDB/whatscooking.restaurants'\n","\n","db_name = 'sample_mflix'\n","collection_name = 'embedded_items'\n","hf_dataset = 'AIatMongoDB/embedded_movies'"],"metadata":{"id":"iEXD37PDr29k"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["embedded_items_collection = client[db_name][collection_name]"],"metadata":{"id":"T9nGB8uc7VNF"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Clear out the collection if you want to start from scratch"],"metadata":{"id":"uYPz4W2DGaZy"}},{"cell_type":"code","source":["doc_count = embedded_items_collection.count_documents (filter = {})\n","print (f\"Document count before delete : {doc_count:,}\")\n","\n","result = embedded_items_collection.delete_many(filter= {})\n","print (f\"Deleted docs : {result.deleted_count}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rGOvb-QDCi2K","executionInfo":{"status":"ok","timestamp":1708984809551,"user_tz":300,"elapsed":950,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"c75043fa-b926-4b90-e7c5-7fef341a2360"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Document count before delete : 0\n","Deleted docs : 0\n"]}]},{"cell_type":"code","source":["dataset = load_dataset(hf_dataset)"],"metadata":{"id":"WLo8HbFU7awh"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Let's tske a look at the huggingface dataset"],"metadata":{"id":"abfK19HHFqBq"}},{"cell_type":"code","source":["insert_data = []"],"metadata":{"id":"VEqpUoKs7k3J"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Process 1k at a time"],"metadata":{"id":"ckkJdvhxWfhq"}},{"cell_type":"code","source":["for item in dataset['train']:\n","    doc_items = json_util.loads(json_util.dumps(item))\n","    insert_data.append(doc_items)\n","    if len(insert_data) == 1000:\n","        embedded_items_collection.insert_many(insert_data)\n","        print(\"1000 records ingested\")\n","        insert_data = []\n","\n","if len(insert_data) > 0:\n","    embedded_items_collection.insert_many(insert_data)\n","    insert_data = []\n","\n","print(\"Data Ingested\")"],"metadata":{"id":"oEv-VViv7pQj","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1708984936818,"user_tz":300,"elapsed":20443,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"ffd08d97-ad63-4b6b-aeb9-d6f5237fd1f2"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["1000 records ingested\n","Data Ingested\n"]}]}]}