feat: tweak new readablity lib flags

This commit is contained in:
Moritz Böhme 2025-10-03 13:12:24 +02:00
parent 0424544abf
commit 63705c0745
No known key found for this signature in database
GPG key ID: 970C6E89EB0547A9

View file

@ -2,6 +2,7 @@ use std::time::Duration;
use anyhow::{Context, Result};
use itertools::Itertools;
use readability_rust::{ReadabilityFlags, ReadabilityOptions};
use reqwest::{Client, Url};
use rss::Channel;
use tokio::{task::JoinSet, time::sleep};
@ -58,8 +59,26 @@ async fn complete(mut channel: Channel, client: &Client) -> Result<Box<Channel>>
async fn get_content(link: &str, client: &Client) -> Result<String> {
let response = client.get(link).send().await?;
let mut readablity = readability_rust::Readability::new(response.text().await?.as_ref(), None)?;
let content = readablity.parse().context("readablity parse error")?.content.context("readablity no content")?;
let mut readablity = readability_rust::Readability::new(
response.text().await?.as_ref(),
Some(ReadabilityOptions {
max_elems_to_parse: 0,
nb_top_candidates: 5,
char_threshold: 500,
link_density_modifier: 0.0,
flags: ReadabilityFlags {
strip_unlikelys: false,
clean_conditionally: false,
weight_classes: false,
},
..Default::default()
}),
)?;
let content = readablity
.parse()
.context("readablity parse error")?
.content
.context("readablity no content")?;
Ok(content)
}