diff options
Diffstat (limited to 'crawler/src/main.rs')
| -rw-r--r-- | crawler/src/main.rs | 5 |
1 files changed, 2 insertions, 3 deletions
diff --git a/crawler/src/main.rs b/crawler/src/main.rs index 6161578..e8efe77 100644 --- a/crawler/src/main.rs +++ b/crawler/src/main.rs @@ -90,7 +90,7 @@ async fn crawl_url(http_client: &Client, url: &str) -> Result<(String, Vec<Strin //we need to not append http if already has it let fixup_urls = |us: Vec<String>| { - us.into_iter() + us.iter() .map(|u| { //https://stackoverflow.com/questions/9646407/two-forward-slashes-in-a-url-src-href-attribute if u.starts_with("//") { @@ -98,14 +98,13 @@ async fn crawl_url(http_client: &Client, url: &str) -> Result<(String, Vec<Strin } else if u.starts_with('/') { format!("{}{}", &url, &u) } else { - u + u.to_string() } }) .collect() }; let next_urls = fixup_urls(next_urls); - //limit to 2 or smth for ram? or depth //normalise words somewhere //fuzzy? //probs lots of places where we can borrow or not do stupid stuff |