diff options
| author | Baitinq <[email protected]> | 2022-10-20 13:59:03 +0200 |
|---|---|---|
| committer | Baitinq <[email protected]> | 2022-10-20 13:59:11 +0200 |
| commit | dd3f705e04f8a74c03ff7ea0f4fabdaf25c310ef (patch) | |
| tree | c11543992b607360fe5fde6fcf2cabe2bc3c9a4f /crawler/Cargo.toml | |
| parent | Crawler: Add skeleton crawler implementation (diff) | |
| download | OSSE-dd3f705e04f8a74c03ff7ea0f4fabdaf25c310ef.tar.gz OSSE-dd3f705e04f8a74c03ff7ea0f4fabdaf25c310ef.tar.bz2 OSSE-dd3f705e04f8a74c03ff7ea0f4fabdaf25c310ef.zip | |
Crawler: Add basic html parsing and link-following
Extremely basic implementation. Needs max queue size, error handling, formatting of parsed links.
Diffstat (limited to 'crawler/Cargo.toml')
| -rw-r--r-- | crawler/Cargo.toml | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/crawler/Cargo.toml b/crawler/Cargo.toml index ce2876a..3f03217 100644 --- a/crawler/Cargo.toml +++ b/crawler/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" [dependencies] blockingqueue = "0.1.1" +reqwest = {version = "0.11", features = ["blocking"]} +scraper = "0.12.0" [[bin]] name = "crawler" |