jobboard_crawler.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN3TNUprHpEPr3wH0RzU1JN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/baranouskiart/jobboard-crawler/blob/main/jobboard_crawler.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "ktzgPBw-e3Wx"
      },
      "outputs": [],
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import requests\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#url = \"https://rocketjobs.pl/wszystkie-lokalizacje/marketing/ecommerce\""
      ],
      "metadata": {
        "id": "fb7VBnuQpgI7"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_pagination_details(soup):\n",
        "  \"\"\"Extracts pagination details from the soup object\"\"\"\n",
        "  # Identify pagination element based on element classes and structure from category webpage\n",
        "  pagination = soup.find(\"ul\", class_=\"css-musp26\")\n",
        "\n",
        "  # Check if pagination exists\n",
        "  if not pagination:\n",
        "    return None\n",
        "\n",
        "  # Extract page links from pagination element\n",
        "  page_links = []\n",
        "  for link in pagination.find_all(\"a\"):\n",
        "    page_links.append(link[\"href\"])\n",
        "\n",
        "  return page_links"
      ],
      "metadata": {
        "id": "YZsQaWr1pqck"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_job_listings(soup):\n",
        "  \"\"\"Extracts job listing details from the soup object\"\"\"\n",
        "  # Identify job listing elements based on element classes and structure from category webpage\n",
        "  job_listings = soup.find_all(\"div\", class_=\"css-168kwli\")\n",
        "\n",
        "  job_data = []\n",
        "  for listing in job_listings:\n",
        "    # Extract details based on element classes and structure from category webpage\n",
        "    source = \"...\" # Implement logic to extract source information based on category webpage structure\n",
        "    title = listing.find(\"h2\", class_=\"css-xjbvlp\").text.strip()\n",
        "    # Extract other relevant details like company, location, etc. using similar logic\n",
        "\n",
        "    job_data.append({\n",
        "      \"source\": source,\n",
        "      \"title\": title,\n",
        "      # Add other extracted details here\n",
        "    })\n",
        "\n",
        "  return job_data"
      ],
      "metadata": {
        "id": "RDL0jZy1qMrv"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def crawl_rocketjobs(url):\n",
        "  \"\"\"Crawls the provided RocketJobs URL and extracts job listings\"\"\"\n",
        "  try:\n",
        "    response = requests.get(url)\n",
        "    response.raise_for_status()  # Raise exception for unsuccessful requests\n",
        "\n",
        "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
        "\n",
        "    pagination_links = get_pagination_details(soup)\n",
        "    job_listings = get_job_listings(soup)\n",
        "\n",
        "    # If pagination exists, recursively crawl other pages\n",
        "    if pagination_links:\n",
        "      for page_link in pagination_links[1:]:  # Skip the current page\n",
        "        job_listings.extend(crawl_rocketjobs(f\"{url}{page_link}\"))\n",
        "\n",
        "    return job_listings\n",
        "\n",
        "  except requests.exceptions.RequestException as e:\n",
        "    print(f\"An error occurred while crawling the URL: {e}\")\n",
        "    return []"
      ],
      "metadata": {
        "id": "A_WRPwzYqPE4"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Replace with the desired RocketJobs URL\n",
        "starting_url = \"https://rocketjobs.pl/wszystkie-lokalizacje/marketing/ecommerce\"\n",
        "def get_pagination_details(soup):\n",
        "  \"\"\"Extracts pagination details from the soup object\"\"\"\n",
        "  # Identify pagination element based on element classes and structure from category webpage\n",
        "  pagination = soup.find(\"ul\", class_=\"css-musp26\")\n",
        "\n",
        "  # Check if pagination exists\n",
        "  if not pagination:\n",
        "    return None\n",
        "\n",
        "  # Extract page links from pagination element\n",
        "  page_links = []\n",
        "  for link in pagination.find_all(\"a\"):\n",
        "    page_links.append(link[\"href\"])\n",
        "\n",
        "  return page_links"
      ],
      "metadata": {
        "id": "-HuWapK8qYZc"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_job_listings(soup):\n",
        "  \"\"\"Extracts job listing details from the soup object\"\"\"\n",
        "  # Identify job listing elements based on element classes and structure from category webpage\n",
        "  job_listings = soup.find_all(\"div\", class_=\"css-168kwli\")\n",
        "\n",
        "  job_data = []\n",
        "  for listing in job_listings:\n",
        "    # Extract details based on element classes and structure from category webpage\n",
        "    source = \"...\" # Implement logic to extract source information based on category webpage structure\n",
        "    title = listing.find(\"h2\", class_=\"css-xjbvlp\").text.strip()\n",
        "    # Extract other relevant details like company, location, etc. using similar logic\n",
        "\n",
        "    job_data.append({\n",
        "      \"source\": source,\n",
        "      \"title\": title,\n",
        "      # Add other extracted details here\n",
        "    })\n",
        "\n",
        "  return job_data"
      ],
      "metadata": {
        "id": "UzHWxaVzqxFk"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def crawl_rocketjobs(url):\n",
        "  \"\"\"Crawls the provided RocketJobs URL and extracts job listings\"\"\"\n",
        "  try:\n",
        "    response = requests.get(url)\n",
        "    response.raise_for_status()  # Raise exception for unsuccessful requests\n",
        "\n",
        "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
        "\n",
        "    pagination_links = get_pagination_details(soup)\n",
        "    job_listings = get_job_listings(soup)\n",
        "\n",
        "    # If pagination exists, recursively crawl other pages\n",
        "    if pagination_links:\n",
        "      for page_link in pagination_links[1:]:  # Skip the current page\n",
        "        job_listings.extend(crawl_rocketjobs(f\"{url}{page_link}\"))\n",
        "\n",
        "    return job_listings\n",
        "\n",
        "  except requests.exceptions.RequestException as e:\n",
        "    print(f\"An error occurred while crawling the URL: {e}\")\n",
        "    return []"
      ],
      "metadata": {
        "id": "9rCG5Uppqyzc"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Replace with the desired RocketJobs URL\n",
        "#starting_url = \"https://rocketjobs.pl/wszystkie-lokalizacje/marketing/ecommerce\"\n",
        "\n",
        "job_data = crawl_rocketjobs(starting_url)# Process and store the extracted job data (job_data) as desired"
      ],
      "metadata": {
        "id": "G0-WGCqKpw6w"
      },
      "execution_count": 26,
      "outputs": []
    }
  ]
}