From 4445c4d168df1ab71431da9db1a053629ed4d0d9 Mon Sep 17 00:00:00 2001
From: Baitinq <manuelpalenzuelamerino@gmail.com>
Date: Sun, 23 Oct 2022 12:06:33 +0200
Subject: Crawler: Only crawl 2 urls per url

This makes it so that we dont get rate limited from websites.
---
 crawler/src/main.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/crawler/src/main.rs b/crawler/src/main.rs
index f8dc226..d1333fe 100644
--- a/crawler/src/main.rs
+++ b/crawler/src/main.rs
@@ -86,9 +86,11 @@ async fn crawl_url(http_client: &Client, url: &str) -> Result<(String, Vec<Strin
         .select(&link_selector)
         .filter_map(|link| link.value().attr("href"))
         .unique()
+        .take(2)
         .map(String::from)
         .collect();
 
+    //we need to not append http if already has it
     let fixup_urls = |us: Vec<String>| {
         us.into_iter()
             .map(|u| {
@@ -105,6 +107,10 @@ async fn crawl_url(http_client: &Client, url: &str) -> Result<(String, Vec<Strin
     };
 
     let next_urls = fixup_urls(next_urls);
+    //limit to 2 or smth for ram? or depth
+    //normalise words somewhere
+    //fuzzy?
+    //probs lots of places where we can borrow or not do stupid stuff
 
     Ok((response_text, next_urls))
 }
-- 
cgit 1.4.1