diff options
author | Baitinq <manuelpalenzuelamerino@gmail.com> | 2022-10-30 12:59:46 +0100 |
---|---|---|
committer | Baitinq <manuelpalenzuelamerino@gmail.com> | 2022-10-30 12:59:46 +0100 |
commit | 620d82cc3910d21940e9cc737900440be3b0ced4 (patch) | |
tree | 82f8b4b9e93deed5f72a61c5784d6fff5c5bc971 | |
parent | Frontend: Add props to the OSSE Component (diff) | |
download | OSSE-620d82cc3910d21940e9cc737900440be3b0ced4.tar.gz OSSE-620d82cc3910d21940e9cc737900440be3b0ced4.tar.bz2 OSSE-620d82cc3910d21940e9cc737900440be3b0ced4.zip |
Misc: Add local lib crate to share common structs
-rw-r--r-- | Cargo.lock | 10 | ||||
-rw-r--r-- | Cargo.toml | 2 | ||||
-rw-r--r-- | crawler/Cargo.toml | 1 | ||||
-rw-r--r-- | crawler/src/main.rs | 8 | ||||
-rw-r--r-- | frontend/Cargo.toml | 1 | ||||
-rw-r--r-- | frontend/src/main.rs | 45 | ||||
-rw-r--r-- | indexer/Cargo.toml | 1 | ||||
-rw-r--r-- | indexer/src/main.rs | 33 | ||||
-rw-r--r-- | lib/Cargo.toml | 9 | ||||
-rw-r--r-- | lib/src/lib.rs | 50 |
10 files changed, 77 insertions, 83 deletions
diff --git a/Cargo.lock b/Cargo.lock index fd3a99f..81bd414 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -444,6 +444,7 @@ version = "0.1.0" dependencies = [ "async-channel", "itertools", + "lib", "rand 0.7.3", "reqwest", "scraper", @@ -619,6 +620,7 @@ dependencies = [ "gloo 0.8.0", "gloo-net", "itertools", + "lib", "scraper", "serde", "wasm-bindgen", @@ -1120,6 +1122,7 @@ dependencies = [ "actix-cors", "actix-web", "html2text", + "lib", "rand 0.7.3", "scraper", "serde", @@ -1222,6 +1225,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] +name = "lib" +version = "0.1.0" +dependencies = [ + "serde", +] + +[[package]] name = "libc" version = "0.2.135" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/Cargo.toml b/Cargo.toml index 7a2596a..72f2764 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = [ "crawler", "indexer", "frontend" ] +members = [ "crawler", "indexer", "frontend", "lib" ] diff --git a/crawler/Cargo.toml b/crawler/Cargo.toml index c58b458..6badd7a 100644 --- a/crawler/Cargo.toml +++ b/crawler/Cargo.toml @@ -14,6 +14,7 @@ tokio = { version = "0.2.22", features = ["full"] } async-channel = "1.7.1" url = "2.3.1" rand = "0.7.3" +lib = { path = "../lib" } [[bin]] name = "crawler" diff --git a/crawler/src/main.rs b/crawler/src/main.rs index a831655..a3dc06b 100644 --- a/crawler/src/main.rs +++ b/crawler/src/main.rs @@ -1,8 +1,8 @@ use itertools::Itertools; use rand::seq::IteratorRandom; use reqwest::{Client, Response, StatusCode}; -use serde::Serialize; use url::Url; +use lib::lib::*; #[tokio::main] async fn main() { @@ -137,12 +137,6 @@ async fn push_crawl_entry_to_indexer( ) -> Result<Response, String> { dbg!("Pushin to indexer"); - #[derive(Serialize, Debug)] - struct CrawledResource { - url: String, - content: String, - } - let request_body = CrawledResource { url, content }; match http_client diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index 23acba2..b528d2b 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -15,3 +15,4 @@ serde = { version = "1.0", features = ["derive", "rc"] } wasm-bindgen-futures = "0.4" itertools = "0.10.5" scraper = "0.12.0" +lib = { path = "../lib" } \ No newline at end of file diff --git a/frontend/src/main.rs b/frontend/src/main.rs index 2052a89..b83ccdd 100644 --- a/frontend/src/main.rs +++ b/frontend/src/main.rs @@ -1,50 +1,9 @@ -use gloo::console::log; use gloo_net::http::Request; use itertools::Itertools; -use serde::Deserialize; -use std::cmp::Ordering; -use std::hash::{Hash, Hasher}; -use std::{ops::Deref, sync::Arc}; use wasm_bindgen::*; use web_sys::{EventTarget, HtmlInputElement}; use yew::prelude::*; - -//TODO: we should import this from the indexer -#[derive(Debug, Clone, Deserialize)] -pub struct IndexedResource { - url: String, - title: String, - description: String, - priority: u32, - word: Arc<String>, -} - -//We implement PartialEq, Eq and Hash to ignore the priority field. -impl PartialEq for IndexedResource { - fn eq(&self, other: &Self) -> bool { - self.url == other.url && self.word == other.word - } -} -impl Eq for IndexedResource {} - -impl PartialOrd for IndexedResource { - fn partial_cmp(&self, other: &Self) -> Option<Ordering> { - Some(self.cmp(other)) - } -} - -impl Ord for IndexedResource { - fn cmp(&self, other: &Self) -> Ordering { - self.priority.cmp(&other.priority).reverse() - } -} - -impl Hash for IndexedResource { - fn hash<H: Hasher>(&self, state: &mut H) { - self.url.hash(state); - self.word.hash(state); - } -} +use lib::lib::*; #[derive(Properties, Clone, PartialEq, Eq)] pub struct ResultComponentProps { @@ -65,7 +24,7 @@ pub struct OSSE { pub results: Option<Vec<IndexedResource>>, //TODO: some loading? } -#[derive(Properties, PartialEq)] +#[derive(Properties, PartialEq, Eq)] pub struct OSSEProps { pub api_endpoint: String, } diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 32355db..3627da8 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -13,6 +13,7 @@ scraper = "0.12.0" html2text = "0.4.3" rand = "0.7.3" serde_json = "1.0.87" +lib = { path = "../lib" } [[bin]] name = "indexer" diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 825fe4d..18604a6 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -1,32 +1,8 @@ use actix_cors::Cors; use actix_web::{get, post, web, App, HttpServer, Responder}; -use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; -use std::hash::{Hash, Hasher}; use std::sync::{Arc, Mutex}; - -#[derive(Debug, Clone, Serialize)] -struct IndexedResource { - url: String, - title: String, - description: String, - priority: u32, - word: Arc<String>, -} - -//We implement PartialEq, Eq and Hash to ignore the priority field. -impl PartialEq for IndexedResource { - fn eq(&self, other: &Self) -> bool { - self.url == other.url && self.word == other.word - } -} -impl Eq for IndexedResource {} -impl Hash for IndexedResource { - fn hash<H: Hasher>(&self, state: &mut H) { - self.url.hash(state); - self.word.hash(state); - } -} +use lib::lib::*; struct AppState { database: Mutex<HashMap<String, HashSet<IndexedResource>>>, @@ -58,13 +34,6 @@ async fn serve_http_endpoint(address: &str, port: u16) -> std::io::Result<()> { } //TODO: sufficiently simmilar word in search (algorithm) -//we need to rename stuff -#[derive(Deserialize, Debug)] -struct CrawledResource { - url: String, - content: String, -} - #[post("/resource")] async fn add_resource( data: web::Data<AppState>, diff --git a/lib/Cargo.toml b/lib/Cargo.toml new file mode 100644 index 0000000..a43e09e --- /dev/null +++ b/lib/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "lib" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { version = "1.0", features = ["derive"] } diff --git a/lib/src/lib.rs b/lib/src/lib.rs new file mode 100644 index 0000000..a11637f --- /dev/null +++ b/lib/src/lib.rs @@ -0,0 +1,50 @@ +pub mod lib { + use serde::{Serialize,Deserialize}; + use std::sync::Arc; + use std::hash::{Hash, Hasher}; + use std::cmp::Ordering; + + #[derive(Serialize, Deserialize, Debug)] + pub struct CrawledResource { + pub url: String, + pub content: String, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct IndexedResource { + pub url: String, + pub title: String, + pub description: String, + pub priority: u32, + pub word: Arc<String>, + } + + //We implement PartialEq, Eq and Hash to ignore the priority field. + impl PartialEq for IndexedResource { + fn eq(&self, other: &Self) -> bool { + self.url == other.url && self.word == other.word + } + } + impl Eq for IndexedResource {} + + impl PartialOrd for IndexedResource { + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + Some(self.cmp(other)) + } + } + + impl Ord for IndexedResource { + fn cmp(&self, other: &Self) -> Ordering { + self.priority.cmp(&other.priority).reverse() + } + } + + impl Hash for IndexedResource { + fn hash<H: Hasher>(&self, state: &mut H) { + self.url.hash(state); + self.word.hash(state); + } + } + + +} \ No newline at end of file |