Indexer: Add website title and description to the CrawledResource

We now parse the HTML and extract the title and description of the site.

3 files changed, 26 insertions, 1 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 0bd1e55..fd3a99f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -619,6 +619,7 @@ dependencies = [
  "gloo 0.8.0",
  "gloo-net",
  "itertools",
+ "scraper",
  "serde",
  "wasm-bindgen",
  "wasm-bindgen-futures",
diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml
index 106b818..23acba2 100644
--- a/frontend/Cargo.toml
+++ b/frontend/Cargo.toml
@@ -14,3 +14,4 @@ gloo-net = "0.2"
 serde = { version = "1.0", features = ["derive", "rc"] }
 wasm-bindgen-futures = "0.4"
 itertools = "0.10.5"
+scraper = "0.12.0"
diff --git a/indexer/src/main.rs b/indexer/src/main.rs
index e63533b..36eabff 100644
--- a/indexer/src/main.rs
+++ b/indexer/src/main.rs
@@ -9,7 +9,9 @@ use std::sync::{Arc, Mutex};
 #[derive(Debug, Clone, Serialize)]
 struct CrawledResource {
     url: String,
-    priority: u32, //how do we even calculate this
+    title: String,
+    description: String,
+    priority: u32,
     word: Arc<String>,
 }
 
@@ -56,6 +58,7 @@ async fn serve_http_endpoint(address: &str, port: u16) -> std::io::Result<()> {
     .await
 }
 
+//we need to rename stuff
 #[derive(Deserialize, Debug)]
 struct Resource {
     url: String,
@@ -65,6 +68,8 @@ struct Resource {
 #[post("/resource")]
 async fn add_resource(data: web::Data<AppState>, resource: web::Json<Resource>) -> impl Responder {
     //parse content
+    let document = scraper::Html::parse_document(resource.content.as_str());
+
     let text = html2text::from_read(resource.content.as_str().as_bytes(), resource.content.len());
 
     let split_words = text.split(' ');
@@ -78,6 +83,22 @@ async fn add_resource(data: web::Data<AppState>, resource: web::Json<Resource>)
 
     println!("xd: {:?}", fixed_words);
 
+    let title_selector = scraper::Selector::parse("title").unwrap();
+    let description_selector = scraper::Selector::parse("meta").unwrap();
+
+    let page_title: String = document
+        .select(&title_selector)
+        .map(|e| e.inner_html())
+        .take(1)
+        .collect();
+
+    let page_description: String = document
+        .select(&description_selector)
+        .filter(|e| e.value().attr("name") == Some("description"))
+        .filter_map(|e| e.value().attr("content"))
+        .take(1)
+        .collect();
+
     //and for each changed content word we add it to the db (word -> list.append(url))
     let mut database = data.database.lock().unwrap();
     for word in fixed_words {
@@ -85,6 +106,8 @@ async fn add_resource(data: web::Data<AppState>, resource: web::Json<Resource>)
             url: resource.url.clone(),
             priority: calculate_word_priority(&word, resource.content.as_str()),
             word: Arc::new(word.clone()),
+            title: page_title.clone(),
+            description: page_description.clone(),
         };
 
         match database.get_mut(&word) {