diff options
| author | Baitinq <[email protected]> | 2022-11-05 18:41:48 +0100 |
|---|---|---|
| committer | Baitinq <[email protected]> | 2022-11-10 17:56:59 +0100 |
| commit | 93009fd53e58286c6a5e2da600d70a8ec85d9a0b (patch) | |
| tree | 74cc0c9cd1ff89dab3ec82cab7c6b56d015f609f /indexer | |
| parent | Indexer: Switch back to not serving frontend with actix (diff) | |
| download | OSSE-ngrams_indexer.tar.gz OSSE-ngrams_indexer.tar.bz2 OSSE-ngrams_indexer.zip | |
Indexer: Ngrams ngrams_indexer
Diffstat (limited to '')
| -rw-r--r-- | indexer/Cargo.toml | 1 | ||||
| -rw-r--r-- | indexer/src/indexer_implementation.rs | 5 |
2 files changed, 6 insertions, 0 deletions
diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 7b64bb3..1dfb33f 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -14,6 +14,7 @@ html2text = "0.4.3" serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0.87" kuchiki = "0.8.1" +ngrams = "1.0.1" lib = { path = "../lib" } [[bin]] diff --git a/indexer/src/indexer_implementation.rs b/indexer/src/indexer_implementation.rs index f24c2bd..6f12644 100644 --- a/indexer/src/indexer_implementation.rs +++ b/indexer/src/indexer_implementation.rs @@ -1,4 +1,5 @@ use lib::lib::*; +use ngrams::Ngram; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -36,6 +37,10 @@ impl crate::Indexer for IndexerImplementation { content: &str, ) -> Result<(), String> { for word in words { + let ngrams: Vec<_> = word.chars().ngrams(2).pad().collect(); + + println!("Ngrams for {}: {:?}", word, ngrams); + let resource_to_add = IndexedResource { url: url.to_string(), priority: Self::calculate_word_priority(word, content, words), |