fix: webscraper not getting links

l3vels · Nov 29, 2023 · 8e2ba04 · 8e2ba04
1 parent 5c15858
commit 8e2ba04
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/apps/server/tools/webscraper/webscraper.py b/apps/server/tools/webscraper/webscraper.py
@@ -92,14 +92,18 @@ def extract_with_bs4(self, url):
                     content_tags = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "a"]
                     content = " ".join(
                         [
-                            tag.text.strip()
+                            (tag.text.strip() + " " + tag.get("href", ""))
+                            if tag.name == "a"
+                            else tag.text.strip()
                             for tag in main_content.find_all(content_tags)
                         ]
                     )
                 else:
                     content = " ".join(
                         [
-                            tag.text.strip()
+                            (tag.text.strip() + " " + tag.get("href", ""))
+                            if tag.name == "a"
+                            else tag.text.strip()
                             for tag in soup.find_all(
                                 ["p", "h1", "h2", "h3", "h4", "h5", "h6"]
                             )