From 6c64ebd56b98f5616c2014e2e0567fa37791844c Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Fri, 15 Nov 2024 16:53:00 -0500 Subject: [PATCH 01/38] fix(scraper): additionally grab common words --- Cargo.lock | 46 ++++++++++++++++++++++++++ Cargo.toml | 1 + src/scraper/main.rs | 79 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4a1a05..6447c2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + [[package]] name = "arc-swap" version = "1.7.1" @@ -307,6 +313,18 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "common-words-all" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a6ff47eb813c9e315610ceca0ddd247827e22f2cdadc4189e4676a81470c77" +dependencies = [ + "anyhow", + "csv", + "glob", + "serde", +] + [[package]] name = "cookie" version = "0.18.1" @@ -370,6 +388,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.20.10" @@ -642,6 +681,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.15" @@ -1160,6 +1205,7 @@ dependencies = [ "build_html", "cached", "clap", + "common-words-all", "cookie", "dotenvy", "fastrand", diff --git a/Cargo.toml b/Cargo.toml index 2fe1456..dac3f48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ rss = "2.0.7" arc-swap = "1.7.1" serde_json_path = "0.6.7" async-recursion = "1.1.1" +common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } [dev-dependencies] diff --git a/src/scraper/main.rs b/src/scraper/main.rs index bc154ff..f2e48d6 100644 --- a/src/scraper/main.rs +++ b/src/scraper/main.rs @@ -1,6 +1,7 @@ -use std::{fmt::Display, io::Write}; +use std::{collections::HashMap, fmt::Display, io::Write}; use clap::{Parser, ValueEnum}; +use common_words_all::{get_top, Language, NgramSize}; use redlib::utils::Post; #[derive(Parser)] @@ -10,9 +11,6 @@ struct Cli { #[arg(short = 's', long = "sub")] sub: String, - #[arg(short = 'c', long = "count")] - count: usize, - #[arg(long = "sort")] sort: SortOrder, @@ -50,28 +48,85 @@ enum Format { #[tokio::main] async fn main() { + pretty_env_logger::init(); let cli = Cli::parse(); - let (sub, final_count, sort, format, output) = (cli.sub, cli.count, cli.sort, cli.format, cli.output); + let (sub, sort, format, output) = (cli.sub, cli.sort, cli.format, cli.output); let initial = format!("/r/{sub}/{sort}.json?&raw_json=1"); - let (mut posts, mut after) = Post::fetch(&initial, false).await.unwrap(); - while posts.len() < final_count { + let (posts, mut after) = Post::fetch(&initial, false).await.unwrap(); + let mut hashmap = HashMap::new(); + hashmap.extend(posts.into_iter().map(|post| (post.id.clone(), post))); + loop { print!("\r"); let path = format!("/r/{sub}/{sort}.json?sort={sort}&t=&after={after}&raw_json=1"); let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap(); - posts.extend(new_posts); + let old_len = hashmap.len(); + // convert to hashmap and extend hashmap + let new_posts = new_posts.into_iter().map(|post| (post.id.clone(), post)).collect::>(); + let len = new_posts.len(); + hashmap.extend(new_posts); + if hashmap.len() - old_len < 3 { + break; + } + + let x = hashmap.len() - old_len; after = new_after; // Print number of posts fetched - print!("Fetched {} posts", posts.len()); + print!("Fetched {len} posts (+{x})",); std::io::stdout().flush().unwrap(); } + println!("\n\n"); + // additionally search if final count not reached - posts.truncate(final_count); + for word in get_top(Language::English, 10_000, NgramSize::One) { + let mut retrieved_posts_from_search = 0; + let initial = format!("/r/{sub}/search.json?q={word}&restrict_sr=on&include_over_18=on&raw_json=1&sort={sort}"); + println!("Grabbing posts with word {word}."); + let (posts, mut after) = Post::fetch(&initial, false).await.unwrap(); + hashmap.extend(posts.into_iter().map(|post| (post.id.clone(), post))); + 'search: loop { + let path = format!("/r/{sub}/search.json?q={word}&restrict_sr=on&include_over_18=on&raw_json=1&sort={sort}&after={after}"); + let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap(); + if new_posts.is_empty() || new_after.is_empty() { + println!("No more posts for word {word}"); + break 'search; + } + retrieved_posts_from_search += new_posts.len(); + let old_len = hashmap.len(); + let new_posts = new_posts.into_iter().map(|post| (post.id.clone(), post)).collect::>(); + let len = new_posts.len(); + hashmap.extend(new_posts); + let delta = hashmap.len() - old_len; + after = new_after; + // Print number of posts fetched + println!("Fetched {len} posts (+{delta})",); + if retrieved_posts_from_search > 1000 { + println!("Reached 1000 posts from search"); + break 'search; + } + } + // Need to save incrementally. atomic save + move + let tmp_file = output.clone().unwrap_or_else(|| format!("{sub}.json.tmp")); + let perm_file = output.clone().unwrap_or_else(|| format!("{sub}.json")); + write_posts(&hashmap.values().collect(), tmp_file.clone()); + // move file + std::fs::rename(tmp_file, perm_file).unwrap(); + } + + println!("\n\n"); + + println!("Size of hashmap: {}", hashmap.len()); + + let posts: Vec<&Post> = hashmap.values().collect(); match format { Format::Json => { let filename: String = output.unwrap_or_else(|| format!("{sub}.json")); - let json = serde_json::to_string(&posts).unwrap(); - std::fs::write(filename, json).unwrap(); + write_posts(&posts, filename); } } } + +fn write_posts(posts: &Vec<&Post>, filename: String) { + let json = serde_json::to_string(&posts).unwrap(); + std::fs::write(filename, json).unwrap(); +} From a96bebb0991558ff278b4cabf3c54678d7d8b208 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:08:00 -0500 Subject: [PATCH 02/38] fix(client): switch to hyper-tls --- Cargo.lock | 209 ++++++++++++++++++++++++-------------------------- Cargo.toml | 4 +- src/client.rs | 9 +-- 3 files changed, 107 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6447c2c..6906b65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -139,12 +139,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -253,9 +247,9 @@ checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" [[package]] name = "cc" -version = "1.1.31" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "shlex", ] @@ -565,6 +559,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -818,19 +827,16 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.24.2" +name = "hyper-tls" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "futures-util", - "http", + "bytes", "hyper", - "log", - "rustls", - "rustls-native-certs", + "native-tls", "tokio", - "tokio-rustls", + "tokio-native-tls", ] [[package]] @@ -1002,6 +1008,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "never" version = "0.1.0" @@ -1057,12 +1080,50 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.1" @@ -1110,6 +1171,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1200,7 +1267,7 @@ version = "0.35.1" dependencies = [ "arc-swap", "async-recursion", - "base64 0.22.1", + "base64", "brotli", "build_html", "cached", @@ -1211,7 +1278,7 @@ dependencies = [ "fastrand", "futures-lite", "hyper", - "hyper-rustls", + "hyper-tls", "libflate", "lipsum", "log", @@ -1273,21 +1340,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rinja" version = "0.3.5" @@ -1408,49 +1460,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rusty-forkfork" version = "0.4.0" @@ -1480,9 +1489,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -1493,16 +1502,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sealed_test" version = "1.1.0" @@ -1540,9 +1539,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -1706,12 +1705,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "strsim" version = "0.11.1" @@ -1849,12 +1842,12 @@ dependencies = [ ] [[package]] -name = "tokio-rustls" -version = "0.24.1" +name = "tokio-native-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ - "rustls", + "native-tls", "tokio", ] @@ -1975,12 +1968,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - [[package]] name = "url" version = "2.5.2" @@ -2001,6 +1988,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index dac3f48..96c7952 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,8 +23,7 @@ regex = "1.10.2" serde = { version = "1.0.193", features = ["derive"] } cookie = "0.18.0" futures-lite = "2.2.0" -hyper = { version = "0.14.28", features = ["full"] } -hyper-rustls = { version = "0.24.2", features = [ "http2" ] } +hyper = { version = "0.14.31", features = ["full"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" serde_json = "1.0.108" @@ -49,6 +48,7 @@ arc-swap = "1.7.1" serde_json_path = "0.6.7" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } +hyper-tls = "=0.5.0" [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index 3d20db0..a0a01d2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,8 +4,9 @@ use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::client::HttpConnector; use hyper::header::HeaderValue; -use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri}; -use hyper_rustls::HttpsConnector; +use hyper::Client; +use hyper::{body, body::Buf, header, Body, Method, Request, Response, Uri}; +use hyper_tls::HttpsConnector; use libflate::gzip; use log::{error, trace, warn}; use once_cell::sync::Lazy; @@ -30,8 +31,7 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it"; const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com"; const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com"; -pub static HTTPS_CONNECTOR: Lazy> = - Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build()); +pub static HTTPS_CONNECTOR: Lazy> = Lazy::new(HttpsConnector::new); pub static CLIENT: Lazy>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone())); @@ -242,7 +242,6 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .header("Authorization", &format!("Bearer {token}")) .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) .header("Accept-Language", "en-US,en;q=0.5") - .header("Connection", "keep-alive") .header( "Cookie", if quarantine { From 0714d58efe0b6d1e1bb94890ea2a402ffe86e944 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:12:04 -0500 Subject: [PATCH 03/38] fix(ci): install new openssl requirements --- .github/workflows/build-artifacts.yaml | 6 +++--- .github/workflows/main-rust.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-artifacts.yaml b/.github/workflows/build-artifacts.yaml index 695d1bf..673ef54 100644 --- a/.github/workflows/build-artifacts.yaml +++ b/.github/workflows/build-artifacts.yaml @@ -38,17 +38,17 @@ jobs: - if: matrix.target == 'x86_64-unknown-linux-musl' run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends musl-tools + sudo apt-get install -y --no-install-recommends musl-tools libssl-dev - if: matrix.target == 'armv7-unknown-linux-musleabihf' run: | sudo apt update - sudo apt install -y gcc-arm-linux-gnueabihf musl-tools + sudo apt install -y gcc-arm-linux-gnueabihf musl-tools libssl-dev - if: matrix.target == 'aarch64-unknown-linux-musl' run: | sudo apt update - sudo apt install -y gcc-aarch64-linux-gnu musl-tools + sudo apt install -y gcc-aarch64-linux-gnu musl-tools libssl-dev - name: Versions id: version diff --git a/.github/workflows/main-rust.yml b/.github/workflows/main-rust.yml index f38c01d..64f30b3 100644 --- a/.github/workflows/main-rust.yml +++ b/.github/workflows/main-rust.yml @@ -30,8 +30,8 @@ jobs: with: toolchain: stable - - name: Install musl-gcc - run: sudo apt-get install musl-tools + - name: Install musl-gcc and libssl-dev + run: sudo apt-get install musl-tools libssl-dev - name: Install cargo musl target run: rustup target add x86_64-unknown-linux-musl From f7240208f1f7204a108af193f7270ad00dc8e402 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:18:20 -0500 Subject: [PATCH 04/38] fix(tls): vendor native-tls --- Cargo.lock | 10 ++++++++++ Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 6906b65..fdb2d9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1112,6 +1112,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.4.1+3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.104" @@ -1120,6 +1129,7 @@ checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index 96c7952..e130f8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ arc-swap = "1.7.1" serde_json_path = "0.6.7" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } -hyper-tls = "=0.5.0" +hyper-tls = { version = "=0.5.0", features = ["vendored"] } [dev-dependencies] From f8a9ad363de6acde0d4ffb9dcadad189fec0e8f0 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:37:30 -0500 Subject: [PATCH 05/38] chore(deps): updates --- Cargo.lock | 520 ++++++++++++++++++++++++++++++++++++++++++----------- Cargo.toml | 6 +- 2 files changed, 414 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdb2d9b..846d8dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,15 +61,15 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "atom_syndication" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3a5ed3201df5658d1aa45060c5a57dc9dba8a8ada20d696d67cb0c479ee043" +checksum = "3ee79fb83c725eae67b55218870813d2fc39fd85e4f1583848ef9f4f823cfe7c" dependencies = [ "chrono", "derive_builder", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" dependencies = [ "memchr", "serde", @@ -197,6 +197,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225eb82ce9e70dcc0cfa6e404d0f353326b6e163bf500ec4711cec317d11935c" +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" @@ -211,9 +217,9 @@ checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cached" -version = "0.51.4" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0feb64151eed3da6107fddd2d717a6ca4b9dbd74e43784c55c841d1abfe5a295" +checksum = "9718806c4a2fe9e8a56fd736f97b340dd10ed1be8ed733ed50449f351dc33cae" dependencies = [ "ahash", "async-trait", @@ -221,17 +227,17 @@ dependencies = [ "cached_proc_macro_types", "futures", "hashbrown 0.14.5", - "instant", "once_cell", "thiserror", "tokio", + "web-time", ] [[package]] name = "cached_proc_macro" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771aa57f3b17da6c8bcacb187bb9ec9bc81c8160e72342e67c329e0e1651a669" +checksum = "2f42a145ed2d10dce2191e1dcf30cfccfea9026660e143662ba5eec4017d5daa" dependencies = [ "darling", "proc-macro2", @@ -271,9 +277,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -281,9 +287,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstyle", "clap_lex", @@ -303,9 +309,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "common-words-all" @@ -356,9 +362,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -496,13 +502,24 @@ dependencies = [ [[package]] name = "diligent-date-parser" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" +checksum = "c8ede7d79366f419921e2e2f67889c12125726692a313bffb474bd5f37a581e9" dependencies = [ "chrono", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -549,9 +566,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fnv" @@ -627,9 +644,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" dependencies = [ "fastrand", "futures-core", @@ -740,9 +757,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -839,6 +856,124 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -847,12 +982,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -862,16 +1008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", + "hashbrown 0.15.1", ] [[package]] @@ -898,10 +1035,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] -name = "libc" -version = "0.2.161" +name = "js-sys" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libflate" @@ -943,6 +1089,12 @@ dependencies = [ "rand_chacha", ] +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -1229,9 +1381,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "encoding_rs", "memchr", @@ -1335,9 +1487,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1400,9 +1552,9 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "rss" -version = "2.0.9" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e92048f840d98c6d6dd870af9101610ea9ff413f11f1bcebf4f4c31d96d957" +checksum = "554a62b3dd5450fcbb0435b3db809f9dd3c6e9f5726172408f7ad3b57ed59057" dependencies = [ "atom_syndication", "derive_builder", @@ -1459,9 +1611,9 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags", "errno", @@ -1559,18 +1711,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -1579,9 +1731,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -1591,13 +1743,12 @@ dependencies = [ [[package]] name = "serde_json_path" -version = "0.6.7" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bc0207b6351893eafa1e39aa9aea452abb6425ca7b02dd64faf29109e7a33ba" +checksum = "e176fbf9bd62f75c2d8be33207fa13af2f800a506635e89759e46f934c520f4d" dependencies = [ "inventory", "nom", - "once_cell", "regex", "serde", "serde_json", @@ -1608,12 +1759,11 @@ dependencies = [ [[package]] name = "serde_json_path_core" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d64fe53ce1aaa31bea2b2b46d3b6ab6a37e61854bedcbd9f174e188f3f7d79" +checksum = "ea3bfd54a421bec8328aefede43ac9f18c8c7ded3b2afc8addd44b4813d99fd0" dependencies = [ "inventory", - "once_cell", "serde", "serde_json", "thiserror", @@ -1621,21 +1771,20 @@ dependencies = [ [[package]] name = "serde_json_path_macros" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31e8177a443fd3e94917f12946ae7891dfb656e6d4c5e79b8c5d202fbcb723" +checksum = "ee05bac728cc5232af5c23896b34fbdd17cf0bb0c113440588aeeb1b57c6ba1f" dependencies = [ "inventory", - "once_cell", "serde_json_path_core", "serde_json_path_macros_internal", ] [[package]] name = "serde_json_path_macros_internal" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75dde5a1d2ed78dfc411fc45592f72d3694436524d3353683ecb3d22009731dc" +checksum = "aafbefbe175fa9bf03ca83ef89beecff7d2a95aaacd5732325b90ac8c3bd7b90" dependencies = [ "proc-macro2", "quote", @@ -1715,6 +1864,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -1723,9 +1878,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1733,10 +1888,21 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.13.0" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -1756,18 +1922,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -1808,25 +1974,20 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.8.0" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", @@ -1951,27 +2112,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1980,15 +2126,27 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.11.0" @@ -2044,6 +2202,71 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -2144,6 +2367,42 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2164,3 +2423,46 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index e130f8c..77aec78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ default-run = "redlib" [dependencies] rinja = { version = "0.3.4", default-features = false } -cached = { version = "0.51.3", features = ["async"] } +cached = { version = "0.54.0", features = ["async"] } clap = { version = "4.4.11", default-features = false, features = [ "std", "env", @@ -26,7 +26,7 @@ futures-lite = "2.2.0" hyper = { version = "0.14.31", features = ["full"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" -serde_json = "1.0.108" +serde_json = "1.0.133" tokio = { version = "1.35.1", features = ["full"] } time = { version = "0.3.31", features = ["local-offset"] } url = "2.5.0" @@ -45,7 +45,7 @@ pretty_env_logger = "0.5.0" dotenvy = "0.15.7" rss = "2.0.7" arc-swap = "1.7.1" -serde_json_path = "0.6.7" +serde_json_path = "0.7.1" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } hyper-tls = { version = "=0.5.0", features = ["vendored"] } From 96e40e888742966e73ff1265918c6d256d8d3726 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 11:40:17 -0500 Subject: [PATCH 06/38] style(clippy): small clippy change --- src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 170f53d..1edb528 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -507,7 +507,7 @@ impl std::ops::Deref for Awards { impl std::fmt::Display for Awards { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.iter().fold(Ok(()), |result, award| result.and_then(|()| writeln!(f, "{award}"))) + self.iter().try_fold((), |_, award| writeln!(f, "{award}")) } } From 3e1718bfc971017aacee180d497f398be2d13855 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 12:44:20 -0500 Subject: [PATCH 07/38] fix(client): ??? no accept language --- src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index a0a01d2..978214e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -241,7 +241,6 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .header("Host", host) .header("Authorization", &format!("Bearer {token}")) .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) - .header("Accept-Language", "en-US,en;q=0.5") .header( "Cookie", if quarantine { From 96ebfd2d3a6d2b7c45c6cdf7dfc65140d2288747 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 12:53:36 -0500 Subject: [PATCH 08/38] fix(ci): statically build on artifacts --- .github/workflows/build-artifacts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yaml b/.github/workflows/build-artifacts.yaml index 673ef54..9dd94c9 100644 --- a/.github/workflows/build-artifacts.yaml +++ b/.github/workflows/build-artifacts.yaml @@ -55,7 +55,7 @@ jobs: run: echo "VERSION=$(cargo metadata --format-version 1 --no-deps | jq .packages[0].version -r | sed 's/^/v/')" >> "$GITHUB_OUTPUT" - name: Build - run: cargo build --release --target ${{ matrix.target }} + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target ${{ matrix.target }} - name: Package release run: tar czf redlib-${{ matrix.target }}.tar.gz -C target/${{ matrix.target }}/release/ redlib From 0bc36d529cd8208a07acfa65a071119f63cb7a9a Mon Sep 17 00:00:00 2001 From: James Musselman <134752354+musselmandev@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:19:48 -0600 Subject: [PATCH 09/38] Add Quadlet Container File (#319) * Add Quadlet Container File * Update README.md with Quadlet instructions --- README.md | 36 +++++++++++++++++++++++++++++++++++- redlib.container | 16 ++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 redlib.container diff --git a/README.md b/README.md index 16be29e..4026791 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ - [Docker](#docker) - [Docker Compose](#docker-compose) - [Docker CLI](#docker-cli) + - Podman + - Quadlets + - [Binary](#binary) - [Running as a systemd service](#running-as-a-systemd-service) - [Building from source](#building-from-source) @@ -180,7 +183,7 @@ For configuration options, see the [Configuration section](#Configuration). [Docker](https://www.docker.com) lets you run containerized applications. Containers are loosely isolated environments that are lightweight and contain everything needed to run the application, so there's no need to rely on what's installed on the host. -Docker images for Redlib are available at [quay.io](https://quay.io/repository/redlib/redlib), with support for `amd64`, `arm64`, and `armv7` platforms. +Container images for Redlib are available at [quay.io](https://quay.io/repository/redlib/redlib), with support for `amd64`, `arm64`, and `armv7` platforms. ### Docker Compose @@ -224,6 +227,37 @@ Stream logs from the Redlib container: ```bash docker logs -f redlib ``` +## Podman + +[Podman](https://podman.io/) lets you run containerized applications in a rootless fashion. Containers are loosely isolated environments that are lightweight and contain everything needed to run the application, so there's no need to rely on what's installed on the host. + +Container images for Redlib are available at [quay.io](https://quay.io/repository/redlib/redlib), with support for `amd64`, `arm64`, and `armv7` platforms. + +### Quadlets + +> [!IMPORTANT] +> These instructions assume that you are on a systemd based distro with [podman](https://podman.io/). If not, follow these [instructions on podman's website](https://podman.io/docs/installation) for how to do so. +> It also assumes you have used `loginctl enable-linger ` to enable the service to start for your user without logging in. + +Copy the `redlib.container` and `.env.example` files to `.config/containers/systemd/` and modify any relevant values (for example, the ports Redlib should listen on, renaming the .env file and editing its values, etc.). + +To start Redlib either reboot or follow the instructions below: + +Notify systemd of the new files +```bash +systemctl --user daemon-reload +``` + +Start the newly generated service file + +```bash +systemctl --user start redlib.service +``` + +You can check the status of your container by using the following command: +```bash +systemctl --user status redlib.service +``` ## Binary diff --git a/redlib.container b/redlib.container new file mode 100644 index 0000000..e66051e --- /dev/null +++ b/redlib.container @@ -0,0 +1,16 @@ +[Install] +WantedBy=default.target + +[Container] +AutoUpdate=registry +ContainerName=redlib +DropCapability=ALL +EnvironmentFile=.env +HealthCmd=["wget","--spider","-q","--tries=1","http://localhost:8080/settings"] +HealthInterval=5m +HealthTimeout=3s +Image=quay.io/redlib/redlib:latest +NoNewPrivileges=true +PublishPort=8080:8080 +ReadOnly=true +User=nobody From 18efb8c714ad10e0082059e1d47e0f95686d04a7 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 14:10:59 -0500 Subject: [PATCH 10/38] fix(client): update headers --- src/client.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/client.rs b/src/client.rs index 978214e..f13e864 100644 --- a/src/client.rs +++ b/src/client.rs @@ -229,6 +229,12 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo ) }; + let (key, value) = match fastrand::u32(0..3) { + 0 => ("X-Reddit-Width", fastrand::u32(300..500).to_string()), + 1 => ("X-Reddit-DPR", "2".to_owned()), + _ => ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), + }; + // Build request to Reddit. When making a GET, request gzip compression. // (Reddit doesn't do brotli yet.) let builder = Request::builder() @@ -241,6 +247,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .header("Host", host) .header("Authorization", &format!("Bearer {token}")) .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) + .header(key, value) .header( "Cookie", if quarantine { From 6ecdedd2ede7fa72a6651bf51f1d535c9e64ec93 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 14:54:06 -0500 Subject: [PATCH 11/38] feat(client): additionally randomize headers --- src/client.rs | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/client.rs b/src/client.rs index f13e864..98c42fb 100644 --- a/src/client.rs +++ b/src/client.rs @@ -229,34 +229,39 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo ) }; - let (key, value) = match fastrand::u32(0..3) { - 0 => ("X-Reddit-Width", fastrand::u32(300..500).to_string()), - 1 => ("X-Reddit-DPR", "2".to_owned()), - _ => ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), - }; - // Build request to Reddit. When making a GET, request gzip compression. // (Reddit doesn't do brotli yet.) - let builder = Request::builder() - .method(method) - .uri(&url) - .header("User-Agent", user_agent) - .header("Client-Vendor-Id", vendor_id) - .header("X-Reddit-Device-Id", device_id) - .header("x-reddit-loid", loid) - .header("Host", host) - .header("Authorization", &format!("Bearer {token}")) - .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) - .header(key, value) - .header( + let mut headers = vec![ + ("User-Agent", user_agent), + ("Client-Vendor-Id", vendor_id), + ("X-Reddit-Device-Id", device_id), + ("x-reddit-loid", loid), + ("Host", host.to_string()), + ("Authorization", format!("Bearer {token}")), + ("Accept-Encoding", if method == Method::GET { "gzip".into() } else { "identity".into() }), + ( "Cookie", if quarantine { - "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D" + "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D".into() } else { - "" + "".into() }, - ) - .body(Body::empty()); + ), + ("X-Reddit-Width", fastrand::u32(300..500).to_string()), + ("X-Reddit-DPR", "2".to_owned()), + ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), + ]; + + // shuffle headers: https://github.com/redlib-org/redlib/issues/324 + fastrand::shuffle(&mut headers); + + let mut builder = Request::builder().method(method).uri(&url); + + for (key, value) in headers { + builder = builder.header(key, value); + } + + let builder = builder.body(Body::empty()); async move { match builder { From cb9a2a3c391f2c148521770aa70556fa5a500623 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 15:48:42 -0500 Subject: [PATCH 12/38] fix(client): revert to hyper_rustls :P hi SWE :wave: --- .github/workflows/build-artifacts.yaml | 8 +- .github/workflows/main-rust.yml | 4 +- Cargo.lock | 205 ++++++++++++------------- Cargo.toml | 2 +- src/client.rs | 8 +- 5 files changed, 112 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build-artifacts.yaml b/.github/workflows/build-artifacts.yaml index 9dd94c9..695d1bf 100644 --- a/.github/workflows/build-artifacts.yaml +++ b/.github/workflows/build-artifacts.yaml @@ -38,24 +38,24 @@ jobs: - if: matrix.target == 'x86_64-unknown-linux-musl' run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends musl-tools libssl-dev + sudo apt-get install -y --no-install-recommends musl-tools - if: matrix.target == 'armv7-unknown-linux-musleabihf' run: | sudo apt update - sudo apt install -y gcc-arm-linux-gnueabihf musl-tools libssl-dev + sudo apt install -y gcc-arm-linux-gnueabihf musl-tools - if: matrix.target == 'aarch64-unknown-linux-musl' run: | sudo apt update - sudo apt install -y gcc-aarch64-linux-gnu musl-tools libssl-dev + sudo apt install -y gcc-aarch64-linux-gnu musl-tools - name: Versions id: version run: echo "VERSION=$(cargo metadata --format-version 1 --no-deps | jq .packages[0].version -r | sed 's/^/v/')" >> "$GITHUB_OUTPUT" - name: Build - run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target ${{ matrix.target }} + run: cargo build --release --target ${{ matrix.target }} - name: Package release run: tar czf redlib-${{ matrix.target }}.tar.gz -C target/${{ matrix.target }}/release/ redlib diff --git a/.github/workflows/main-rust.yml b/.github/workflows/main-rust.yml index 64f30b3..f38c01d 100644 --- a/.github/workflows/main-rust.yml +++ b/.github/workflows/main-rust.yml @@ -30,8 +30,8 @@ jobs: with: toolchain: stable - - name: Install musl-gcc and libssl-dev - run: sudo apt-get install musl-tools libssl-dev + - name: Install musl-gcc + run: sudo apt-get install musl-tools - name: Install cargo musl target run: rustup target add x86_64-unknown-linux-musl diff --git a/Cargo.lock b/Cargo.lock index 846d8dd..4143b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -576,21 +582,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -844,16 +835,19 @@ dependencies = [ ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "hyper-rustls" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ - "bytes", + "futures-util", + "http", "hyper", - "native-tls", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-native-tls", + "tokio-rustls", ] [[package]] @@ -1160,23 +1154,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "native-tls" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "never" version = "0.1.0" @@ -1232,60 +1209,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "openssl" -version = "0.10.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "300.4.1+3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" -dependencies = [ - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking" version = "2.2.1" @@ -1333,12 +1262,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" - [[package]] name = "powerfmt" version = "0.2.0" @@ -1429,7 +1352,7 @@ version = "0.35.1" dependencies = [ "arc-swap", "async-recursion", - "base64", + "base64 0.22.1", "brotli", "build_html", "cached", @@ -1440,7 +1363,7 @@ dependencies = [ "fastrand", "futures-lite", "hyper", - "hyper-tls", + "hyper-rustls", "libflate", "lipsum", "log", @@ -1502,6 +1425,21 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rinja" version = "0.3.5" @@ -1622,6 +1560,49 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rusty-forkfork" version = "0.4.0" @@ -1664,6 +1645,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sealed_test" version = "1.1.0" @@ -1864,6 +1855,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2013,12 +2010,12 @@ dependencies = [ ] [[package]] -name = "tokio-native-tls" -version = "0.3.1" +name = "tokio-rustls" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "native-tls", + "rustls", "tokio", ] @@ -2124,6 +2121,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.3" @@ -2156,12 +2159,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 77aec78..7bb7e93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ arc-swap = "1.7.1" serde_json_path = "0.7.1" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } -hyper-tls = { version = "=0.5.0", features = ["vendored"] } +hyper-rustls = { version = "0.24.2", features = [ "http2" ] } [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index 98c42fb..248fc88 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,9 +4,8 @@ use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::client::HttpConnector; use hyper::header::HeaderValue; -use hyper::Client; -use hyper::{body, body::Buf, header, Body, Method, Request, Response, Uri}; -use hyper_tls::HttpsConnector; +use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri}; +use hyper_rustls::HttpsConnector; use libflate::gzip; use log::{error, trace, warn}; use once_cell::sync::Lazy; @@ -31,7 +30,8 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it"; const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com"; const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com"; -pub static HTTPS_CONNECTOR: Lazy> = Lazy::new(HttpsConnector::new); +pub static HTTPS_CONNECTOR: Lazy> = + Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build()); pub static CLIENT: Lazy>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone())); From d3ba5f3efb6825f4b0454523dae472b2adda3066 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 19 Nov 2024 16:30:37 -0500 Subject: [PATCH 13/38] feat(error): add new instance buttom --- src/main.rs | 20 ++++++++++++++++++++ static/check_update.js | 18 ++++++++++++++++++ templates/error.html | 2 ++ 3 files changed, 40 insertions(+) diff --git a/src/main.rs b/src/main.rs index 4923921..7342597 100644 --- a/src/main.rs +++ b/src/main.rs @@ -223,6 +223,7 @@ async fn main() { .get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed()); app.at("/commits.atom").get(|_| async move { proxy_commit_info().await }.boxed()); + app.at("/instances.json").get(|_| async move { proxy_instances().await }.boxed()); // Proxy media through Redlib app.at("/vid/:id/:size").get(|r| proxy(r, "https://v.redd.it/{id}/DASH_{size}").boxed()); @@ -399,3 +400,22 @@ async fn fetch_commit_info() -> String { hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() } + +pub async fn proxy_instances() -> Result, String> { + Ok( + Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(fetch_instances().await)) + .unwrap_or_default(), + ) +} + +#[cached(time = 600)] +async fn fetch_instances() -> String { + let uri = Uri::from_str("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json").expect("Invalid URI"); + + let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); + + hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() +} diff --git a/static/check_update.js b/static/check_update.js index b68c508..b747203 100644 --- a/static/check_update.js +++ b/static/check_update.js @@ -37,4 +37,22 @@ async function checkInstanceUpdateStatus() { } } +async function checkOtherInstances() { + try { + const response = await fetch('/instances.json'); + const data = await response.json(); + const randomInstance = data.instances[Math.floor(Math.random() * data.instances.length)]; + const instanceUrl = randomInstance.url; + // Set the href of the tag to the instance URL with path included + document.getElementById('random-instance').href = instanceUrl + window.location.pathname; + document.getElementById('random-instance').innerText = "Visit Random Instance"; + } catch (error) { + console.error('Error fetching instances:', error); + document.getElementById('update-status').innerText = '⚠️ Error checking update status.'; + } +} + +// Set the target URL when the page loads +window.addEventListener('load', checkOtherInstances); + checkInstanceUpdateStatus(); diff --git a/templates/error.html b/templates/error.html index 8f2f44d..2a56f68 100644 --- a/templates/error.html +++ b/templates/error.html @@ -7,6 +7,8 @@

Reddit Status


+
+


From 95ab6c53856291a4d9cebb9fa5c5045031836120 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Wed, 20 Nov 2024 18:50:06 -0500 Subject: [PATCH 14/38] fix(oauth): update oauth resources and script --- scripts/update_oauth_resources.sh | 10 ++--- src/oauth_resources.rs | 70 ++++++++++++++++--------------- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/scripts/update_oauth_resources.sh b/scripts/update_oauth_resources.sh index 1d6b486..a3014ae 100755 --- a/scripts/update_oauth_resources.sh +++ b/scripts/update_oauth_resources.sh @@ -39,12 +39,12 @@ done echo "];" >> "$filename" # Fetch Android app versions -page_1=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions/" | rg "" -r "https://apkcombo.com\$1" | sort | uniq) +page_1=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions/" | rg "" -r "https://apkcombo.com\$1" | sort | uniq | sed 's/ //g') # Append with pages -page_2=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=2" | rg "" -r "https://apkcombo.com\$1" | sort | uniq) -page_3=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=3" | rg "" -r "https://apkcombo.com\$1" | sort | uniq) -page_4=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=4" | rg "" -r "https://apkcombo.com\$1" | sort | uniq) -page_5=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=5" | rg "" -r "https://apkcombo.com\$1" | sort | uniq) +page_2=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=2" | rg "" -r "https://apkcombo.com\$1" | sort | uniq | sed 's/ //g') +page_3=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=3" | rg "" -r "https://apkcombo.com\$1" | sort | uniq | sed 's/ //g') +page_4=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=4" | rg "" -r "https://apkcombo.com\$1" | sort | uniq | sed 's/ //g') +page_5=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=5" | rg "" -r "https://apkcombo.com\$1" | sort | uniq | sed 's/ //g') # Concatenate all pages versions="${page_1}" diff --git a/src/oauth_resources.rs b/src/oauth_resources.rs index 3272939..a5dc2f3 100644 --- a/src/oauth_resources.rs +++ b/src/oauth_resources.rs @@ -2,8 +2,40 @@ // Rerun scripts/update_oauth_resources.sh to update this file // Please do not edit manually // Filled in with real app versions -pub static _IOS_APP_VERSION_LIST: &[&str; 1] = &[""]; +pub static _IOS_APP_VERSION_LIST: &[&str; 1] = &[ + "", +]; pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ + "Version 2024.22.1/Build 1652272", + "Version 2024.23.1/Build 1665606", + "Version 2024.24.1/Build 1682520", + "Version 2024.25.0/Build 1693595", + "Version 2024.25.2/Build 1700401", + "Version 2024.25.3/Build 1703490", + "Version 2024.26.0/Build 1710470", + "Version 2024.26.1/Build 1717435", + "Version 2024.28.0/Build 1737665", + "Version 2024.28.1/Build 1741165", + "Version 2024.30.0/Build 1770787", + "Version 2024.31.0/Build 1786202", + "Version 2024.32.0/Build 1809095", + "Version 2024.32.1/Build 1813258", + "Version 2024.33.0/Build 1819908", + "Version 2024.34.0/Build 1837909", + "Version 2024.35.0/Build 1861437", + "Version 2024.36.0/Build 1875012", + "Version 2024.37.0/Build 1888053", + "Version 2024.38.0/Build 1902791", + "Version 2024.39.0/Build 1916713", + "Version 2024.40.0/Build 1928580", + "Version 2024.41.0/Build 1941199", + "Version 2024.41.1/Build 1947805", + "Version 2024.42.0/Build 1952440", + "Version 2024.43.0/Build 1972250", + "Version 2024.44.0/Build 1988458", + "Version 2024.45.0/Build 2001943", + "Version 2024.46.0/Build 2012731", + "Version 2024.47.0/Build 2029755", "Version 2023.48.0/Build 1319123", "Version 2023.49.0/Build 1321715", "Version 2023.49.1/Build 1322281", @@ -31,9 +63,9 @@ pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ "Version 2024.20.0/Build 1612800", "Version 2024.20.1/Build 1615586", "Version 2024.20.2/Build 1624969", + "Version 2024.20.3/Build 1624970", "Version 2024.21.0/Build 1631686", "Version 2024.22.0/Build 1645257", - "Version 2024.22.1/Build 1652272", "Version 2023.21.0/Build 956283", "Version 2023.22.0/Build 968223", "Version 2023.23.0/Build 983896", @@ -124,35 +156,7 @@ pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ "Version 2022.40.0/Build 624782", "Version 2022.41.0/Build 630468", "Version 2022.41.1/Build 634168", - "Version 2021.39.1/Build 372418", - "Version 2021.41.0/Build 376052", - "Version 2021.42.0/Build 378193", - "Version 2021.43.0/Build 382019", - "Version 2021.44.0/Build 385129", - "Version 2021.45.0/Build 387663", - "Version 2021.46.0/Build 392043", - "Version 2021.47.0/Build 394342", - "Version 2022.10.0/Build 429896", - "Version 2022.1.0/Build 402829", - "Version 2022.11.0/Build 433004", - "Version 2022.12.0/Build 436848", - "Version 2022.13.0/Build 442084", - "Version 2022.13.1/Build 444621", - "Version 2022.14.1/Build 452742", - "Version 2022.15.0/Build 455453", - "Version 2022.16.0/Build 462377", - "Version 2022.17.0/Build 468480", - "Version 2022.18.0/Build 473740", - "Version 2022.19.1/Build 482464", - "Version 2022.2.0/Build 405543", - "Version 2022.3.0/Build 408637", - "Version 2022.4.0/Build 411368", - "Version 2022.5.0/Build 414731", - "Version 2022.6.0/Build 418391", - "Version 2022.6.1/Build 419585", - "Version 2022.6.2/Build 420562", - "Version 2022.7.0/Build 420849", - "Version 2022.8.0/Build 423906", - "Version 2022.9.0/Build 426592", ]; -pub static _IOS_OS_VERSION_LIST: &[&str; 1] = &[""]; +pub static _IOS_OS_VERSION_LIST: &[&str; 1] = &[ + "", +]; From 6be6f892a4eb159f5a27ce48f0ce615298071eac Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Wed, 20 Nov 2024 19:19:29 -0500 Subject: [PATCH 15/38] feat(oauth): better oauth client matching --- Cargo.lock | 15 +++++++++++++++ Cargo.toml | 1 + src/client.rs | 36 +++++++++++++----------------------- src/oauth.rs | 20 +++++++++++++++----- src/oauth_resources.rs | 8 ++------ 5 files changed, 46 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4143b80..057234f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,6 +1327,8 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ + "libc", + "rand_chacha", "rand_core", ] @@ -1345,6 +1347,9 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] [[package]] name = "redlib" @@ -1380,6 +1385,7 @@ dependencies = [ "serde_json", "serde_json_path", "serde_yaml", + "tegen", "time", "tokio", "toml", @@ -1895,6 +1901,15 @@ dependencies = [ "syn", ] +[[package]] +name = "tegen" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a2d5a357b7c859b410139734a875136473c3b18b1bbd8d5bdc1769d9002acd" +dependencies = [ + "rand", +] + [[package]] name = "tempfile" version = "3.14.0" diff --git a/Cargo.toml b/Cargo.toml index 7bb7e93..616d8e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ serde_json_path = "0.7.1" async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } hyper-rustls = { version = "0.24.2", features = [ "http2" ] } +tegen = "0.1.4" [dev-dependencies] diff --git a/src/client.rs b/src/client.rs index 248fc88..1e1661d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -218,40 +218,30 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo // Construct the hyper client from the HTTPS connector. let client: &Lazy> = &CLIENT; - let (token, vendor_id, device_id, user_agent, loid) = { - let client = OAUTH_CLIENT.load_full(); - ( - client.token.clone(), - client.headers_map.get("Client-Vendor-Id").cloned().unwrap_or_default(), - client.headers_map.get("X-Reddit-Device-Id").cloned().unwrap_or_default(), - client.headers_map.get("User-Agent").cloned().unwrap_or_default(), - client.headers_map.get("x-reddit-loid").cloned().unwrap_or_default(), - ) - }; - // Build request to Reddit. When making a GET, request gzip compression. // (Reddit doesn't do brotli yet.) - let mut headers = vec![ - ("User-Agent", user_agent), - ("Client-Vendor-Id", vendor_id), - ("X-Reddit-Device-Id", device_id), - ("x-reddit-loid", loid), - ("Host", host.to_string()), - ("Authorization", format!("Bearer {token}")), - ("Accept-Encoding", if method == Method::GET { "gzip".into() } else { "identity".into() }), + let mut headers: Vec<(String, String)> = vec![ + ("Host".into(), host.into()), + ("Accept-Encoding".into(), if method == Method::GET { "gzip".into() } else { "identity".into() }), ( - "Cookie", + "Cookie".into(), if quarantine { "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D".into() } else { "".into() }, ), - ("X-Reddit-Width", fastrand::u32(300..500).to_string()), - ("X-Reddit-DPR", "2".to_owned()), - ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), ]; + { + let client = OAUTH_CLIENT.load_full(); + for (key, value) in client.initial_headers.clone() { + headers.push((key, value)); + } + } + + trace!("Headers: {:#?}", headers); + // shuffle headers: https://github.com/redlib-org/redlib/issues/324 fastrand::shuffle(&mut headers); diff --git a/src/oauth.rs b/src/oauth.rs index 80bf318..576b647 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -7,8 +7,8 @@ use crate::{ use base64::{engine::general_purpose, Engine as _}; use hyper::{client, Body, Method, Request}; use log::{error, info, trace}; - use serde_json::json; +use tegen::tegen::TextGenerator; use tokio::time::{error::Elapsed, timeout}; static REDDIT_ANDROID_OAUTH_CLIENT_ID: &str = "ohXpoqrZYub1kg"; @@ -84,7 +84,7 @@ impl Oauth { // Set JSON body. I couldn't tell you what this means. But that's what the client sends let json = json!({ - "scopes": ["*","email"] + "scopes": ["*","email", "pii"] }); let body = Body::from(json.to_string()); @@ -185,11 +185,21 @@ impl Device { let android_user_agent = format!("Reddit/{android_app_version}/Android {android_version}"); + let qos = fastrand::u32(1000..=100_000); + let qos: f32 = qos as f32 / 1000.0; + let qos = format!("{:.3}", qos); + + let codecs = TextGenerator::new().generate("available-codecs=video/avc, video/hevc{, video/x-vnd.on2.vp9|}"); + // Android device headers - let headers = HashMap::from([ - ("Client-Vendor-Id".into(), uuid.clone()), - ("X-Reddit-Device-Id".into(), uuid.clone()), + let headers: HashMap = HashMap::from([ ("User-Agent".into(), android_user_agent), + ("x-reddit-retry".into(), "algo=no-retries".into()), + ("x-reddit-compression".into(), "1".into()), + ("x-reddit-qos".into(), qos), + ("x-reddit-media-codecs".into(), codecs), + ("Content-Type".into(), "application/json; charset=UTF-8".into()), + ("client-vendor-id".into(), uuid.clone()), ]); info!("[🔄] Spoofing Android client with headers: {headers:?}, uuid: \"{uuid}\", and OAuth ID \"{REDDIT_ANDROID_OAUTH_CLIENT_ID}\""); diff --git a/src/oauth_resources.rs b/src/oauth_resources.rs index a5dc2f3..faf7873 100644 --- a/src/oauth_resources.rs +++ b/src/oauth_resources.rs @@ -2,9 +2,7 @@ // Rerun scripts/update_oauth_resources.sh to update this file // Please do not edit manually // Filled in with real app versions -pub static _IOS_APP_VERSION_LIST: &[&str; 1] = &[ - "", -]; +pub static _IOS_APP_VERSION_LIST: &[&str; 1] = &[""]; pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ "Version 2024.22.1/Build 1652272", "Version 2024.23.1/Build 1665606", @@ -157,6 +155,4 @@ pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ "Version 2022.41.0/Build 630468", "Version 2022.41.1/Build 634168", ]; -pub static _IOS_OS_VERSION_LIST: &[&str; 1] = &[ - "", -]; +pub static _IOS_OS_VERSION_LIST: &[&str; 1] = &[""]; From 100a7b65a6a79968cbc8548f4ce12d722dfb0cba Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 23 Nov 2024 21:17:52 -0500 Subject: [PATCH 16/38] fix(client): update headers management, add self check (fix #334, fix #318) --- src/client.rs | 41 ++++++++++++++++++++++++++++++++++++++--- src/main.rs | 12 +++++++++++- src/oauth.rs | 6 ++++-- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/client.rs b/src/client.rs index 1e1661d..0e2c301 100644 --- a/src/client.rs +++ b/src/client.rs @@ -19,6 +19,7 @@ use std::{io, result::Result}; use crate::dbg_msg; use crate::oauth::{force_refresh_token, token_daemon, Oauth}; use crate::server::RequestExt; +use crate::subreddit::community; use crate::utils::format_url; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; @@ -235,13 +236,11 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo { let client = OAUTH_CLIENT.load_full(); - for (key, value) in client.initial_headers.clone() { + for (key, value) in client.headers_map.clone() { headers.push((key, value)); } } - trace!("Headers: {:#?}", headers); - // shuffle headers: https://github.com/redlib-org/redlib/issues/324 fastrand::shuffle(&mut headers); @@ -390,6 +389,12 @@ pub async fn json(path: String, quarantine: bool) -> Result { "Ratelimit remaining: Header says {remaining}, we have {current_rate_limit}. Resets in {reset}. Rollover: {}. Ratelimit used: {used}", if is_rolling_over { "yes" } else { "no" }, ); + + // If can parse remaining as a float, round to a u16 and save + if let Ok(val) = remaining.parse::() { + OAUTH_RATELIMIT_REMAINING.store(val.round() as u16, Ordering::SeqCst); + } + Some(reset) } else { None @@ -474,6 +479,36 @@ pub async fn json(path: String, quarantine: bool) -> Result { } } +async fn self_check(sub: &str) -> Result<(), String> { + let request = Request::get(format!("/r/{sub}/")).body(Body::empty()).unwrap(); + + match community(request).await { + Ok(sub) if sub.status().is_success() => Ok(()), + Ok(sub) => Err(sub.status().to_string()), + Err(e) => Err(e), + } +} + +pub async fn rate_limit_check() -> Result<(), String> { + // First, check a subreddit. + self_check("reddit").await?; + // This will reduce the rate limit to 99. Assert this check. + if OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst) != 99 { + return Err(format!("Rate limit check failed: expected 99, got {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst))); + } + // Now, we switch out the OAuth client. + // This checks for the IP rate limit association. + force_refresh_token().await; + // Now, check a new sub to break cache. + self_check("rust").await?; + // Again, assert the rate limit check. + if OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst) != 99 { + return Err(format!("Rate limit check failed: expected 99, got {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst))); + } + + Ok(()) +} + #[cfg(test)] static POPULAR_URL: &str = "/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL"; diff --git a/src/main.rs b/src/main.rs index 7342597..abae968 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,7 @@ use hyper::Uri; use hyper::{header::HeaderValue, Body, Request, Response}; use log::info; use once_cell::sync::Lazy; -use redlib::client::{canonical_path, proxy, CLIENT}; +use redlib::client::{canonical_path, proxy, rate_limit_check, CLIENT}; use redlib::server::{self, RequestExt}; use redlib::utils::{error, redirect, ThemeAssets}; use redlib::{config, duplicates, headers, instance_info, post, search, settings, subreddit, user}; @@ -146,6 +146,16 @@ async fn main() { ) .get_matches(); + match rate_limit_check().await { + Ok(()) => { + info!("[✅] Rate limit check passed"); + }, + Err(e) => { + log::error!("[❌] Rate limit check failed: {}", e); + std::process::exit(1); + } + } + let address = matches.get_one::("address").unwrap(); let port = matches.get_one::("port").unwrap(); let hsts = matches.get_one("hsts").map(|m: &String| m.as_str()); diff --git a/src/oauth.rs b/src/oauth.rs index 576b647..12b0f37 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -38,12 +38,12 @@ impl Oauth { } Ok(None) => { error!("Failed to create OAuth client. Retrying in 5 seconds..."); - continue; } Err(duration) => { error!("Failed to create OAuth client in {duration:?}. Retrying in 5 seconds..."); } } + tokio::time::sleep(Duration::from_secs(5)).await; } } @@ -91,13 +91,14 @@ impl Oauth { // Build request let request = builder.body(body).unwrap(); - trace!("Sending token request..."); + trace!("Sending token request...\n\n{request:?}"); // Send request let client: &once_cell::sync::Lazy> = &CLIENT; let resp = client.request(request).await.ok()?; trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length")); + trace!("OAuth headers: {:#?}", resp.headers()); // Parse headers - loid header _should_ be saved sent on subsequent token refreshes. // Technically it's not needed, but it's easy for Reddit API to check for this. @@ -200,6 +201,7 @@ impl Device { ("x-reddit-media-codecs".into(), codecs), ("Content-Type".into(), "application/json; charset=UTF-8".into()), ("client-vendor-id".into(), uuid.clone()), + ("X-Reddit-Device-Id".into(), uuid.clone()), ]); info!("[🔄] Spoofing Android client with headers: {headers:?}, uuid: \"{uuid}\", and OAuth ID \"{REDDIT_ANDROID_OAUTH_CLIENT_ID}\""); From 7fe109df2267f292459c2154554c7bb40e77908d Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sat, 23 Nov 2024 21:41:30 -0500 Subject: [PATCH 17/38] style(clippy) --- src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index abae968..9c8de97 100644 --- a/src/main.rs +++ b/src/main.rs @@ -149,9 +149,10 @@ async fn main() { match rate_limit_check().await { Ok(()) => { info!("[✅] Rate limit check passed"); - }, + } Err(e) => { - log::error!("[❌] Rate limit check failed: {}", e); + log::error!(" Rate limit check failed: {}", e); + println!("[❌] Rate limit check failed: {}", e); std::process::exit(1); } } From a4f511f67e350fb4e4792416d42dbeaa9f1d544b Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sun, 24 Nov 2024 10:50:21 -0500 Subject: [PATCH 18/38] fix(client): update rate limit self-check (fix #335) --- src/client.rs | 30 ++++++++++++++++++++++++------ src/subreddit.rs | 3 +++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/client.rs b/src/client.rs index 0e2c301..ba08531 100644 --- a/src/client.rs +++ b/src/client.rs @@ -19,8 +19,7 @@ use std::{io, result::Result}; use crate::dbg_msg; use crate::oauth::{force_refresh_token, token_daemon, Oauth}; use crate::server::RequestExt; -use crate::subreddit::community; -use crate::utils::format_url; +use crate::utils::{format_url, Post}; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com"; @@ -480,11 +479,10 @@ pub async fn json(path: String, quarantine: bool) -> Result { } async fn self_check(sub: &str) -> Result<(), String> { - let request = Request::get(format!("/r/{sub}/")).body(Body::empty()).unwrap(); + let query = format!("/r/{sub}/hot.json?&raw_json=1"); - match community(request).await { - Ok(sub) if sub.status().is_success() => Ok(()), - Ok(sub) => Err(sub.status().to_string()), + match Post::fetch(&query, true).await { + Ok(_) => Ok(()), Err(e) => Err(e), } } @@ -509,6 +507,26 @@ pub async fn rate_limit_check() -> Result<(), String> { Ok(()) } +#[cfg(test)] +use {crate::config::get_setting, sealed_test::prelude::*}; + +#[tokio::test(flavor = "multi_thread")] +async fn test_rate_limit_check() { + rate_limit_check().await.unwrap(); +} + +#[test] +#[sealed_test(env = [("REDLIB_DEFAULT_SUBSCRIPTIONS", "rust")])] +fn test_default_subscriptions() { + tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap().block_on(async { + let subscriptions = get_setting("REDLIB_DEFAULT_SUBSCRIPTIONS"); + assert!(subscriptions.is_some()); + + // check rate limit + rate_limit_check().await.unwrap(); + }); +} + #[cfg(test)] static POPULAR_URL: &str = "/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL"; diff --git a/src/subreddit.rs b/src/subreddit.rs index 3a07bdc..88aa542 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -8,6 +8,7 @@ use crate::utils::{ use crate::{client::json, server::RequestExt, server::ResponseExt}; use cookie::Cookie; use hyper::{Body, Request, Response}; +use log::{debug, trace}; use rinja::Template; use once_cell::sync::Lazy; @@ -62,6 +63,7 @@ pub async fn community(req: Request) -> Result, String> { // Build Reddit API path let root = req.uri().path() == "/"; let query = req.uri().query().unwrap_or_default().to_string(); + trace!("query: {}", query); let subscribed = setting(&req, "subscriptions"); let front_page = setting(&req, "front_page"); let post_sort = req.cookie("post_sort").map_or_else(|| "hot".to_string(), |c| c.value().to_string()); @@ -123,6 +125,7 @@ pub async fn community(req: Request) -> Result, String> { } let path = format!("/r/{}/{sort}.json?{}{params}", sub_name.replace('+', "%2B"), req.uri().query().unwrap_or_default()); + debug!("Path: {}", path); let url = String::from(req.uri().path_and_query().map_or("", |val| val.as_str())); let redirect_url = url[1..].replace('?', "%3F").replace('&', "%26").replace('+', "%2B"); let filters = get_filters(&req); From 9f6b08cbb2d0f43644a34f5d0210ac32b9add30c Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Tue, 26 Nov 2024 22:55:48 -0500 Subject: [PATCH 19/38] fix(main): reduce rate limit check fail to warned error --- src/main.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9c8de97..8732d20 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ use std::str::FromStr; use futures_lite::FutureExt; use hyper::Uri; use hyper::{header::HeaderValue, Body, Request, Response}; -use log::info; +use log::{info, warn}; use once_cell::sync::Lazy; use redlib::client::{canonical_path, proxy, rate_limit_check, CLIENT}; use redlib::server::{self, RequestExt}; @@ -151,9 +151,12 @@ async fn main() { info!("[✅] Rate limit check passed"); } Err(e) => { - log::error!(" Rate limit check failed: {}", e); - println!("[❌] Rate limit check failed: {}", e); - std::process::exit(1); + let mut message = format!("Rate limit check failed: {}", e); + message += "\nThis may cause issues with the rate limit."; + message += "\nPlease report this error with the above information."; + message += "\nhttps://github.com/redlib-org/redlib/issues/new?assignees=sigaloid&labels=bug&title=%F0%9F%90%9B+Bug+Report%3A+Rate+limit+mismatch"; + warn!("{}", message); + eprintln!("{}", message); } } From e4fc22cf906b7e8213e0b96108106a9ad34c0e29 Mon Sep 17 00:00:00 2001 From: Integral Date: Tue, 3 Dec 2024 00:28:31 +0800 Subject: [PATCH 20/38] refactor: replace static with const for global constants (#340) --- scripts/update_oauth_resources.sh | 6 +++--- src/client.rs | 6 +++--- src/oauth.rs | 4 ++-- src/oauth_resources.rs | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/update_oauth_resources.sh b/scripts/update_oauth_resources.sh index a3014ae..7eeb959 100755 --- a/scripts/update_oauth_resources.sh +++ b/scripts/update_oauth_resources.sh @@ -24,7 +24,7 @@ echo "// Please do not edit manually" >> "$filename" echo "// Filled in with real app versions" >> "$filename" # Open the array in the source file -echo "pub static _IOS_APP_VERSION_LIST: &[&str; $ios_app_count] = &[" >> "$filename" +echo "pub const _IOS_APP_VERSION_LIST: &[&str; $ios_app_count] = &[" >> "$filename" num=0 @@ -63,7 +63,7 @@ android_count=$(echo "$versions" | wc -l) echo -e "Fetching \e[32m$android_count Android app versions...\e[0m" # Append to the source file -echo "pub static ANDROID_APP_VERSION_LIST: &[&str; $android_count] = &[" >> "$filename" +echo "pub const ANDROID_APP_VERSION_LIST: &[&str; $android_count] = &[" >> "$filename" num=0 @@ -89,7 +89,7 @@ ios_count=$(echo "$table" | wc -l) echo -e "Fetching \e[34m$ios_count iOS versions...\e[0m" # Append to the source file -echo "pub static _IOS_OS_VERSION_LIST: &[&str; $ios_count] = &[" >> "$filename" +echo "pub const _IOS_OS_VERSION_LIST: &[&str; $ios_count] = &[" >> "$filename" num=0 diff --git a/src/client.rs b/src/client.rs index ba08531..fa32fc0 100644 --- a/src/client.rs +++ b/src/client.rs @@ -45,7 +45,7 @@ pub static OAUTH_RATELIMIT_REMAINING: AtomicU16 = AtomicU16::new(99); pub static OAUTH_IS_ROLLING_OVER: AtomicBool = AtomicBool::new(false); -static URL_PAIRS: [(&str, &str); 2] = [ +const URL_PAIRS: [(&str, &str); 2] = [ (ALTERNATIVE_REDDIT_URL_BASE, ALTERNATIVE_REDDIT_URL_BASE_HOST), (REDDIT_SHORT_URL_BASE, REDDIT_SHORT_URL_BASE_HOST), ]; @@ -262,7 +262,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo return Ok(response); }; let location_header = response.headers().get(header::LOCATION); - if location_header == Some(&HeaderValue::from_static("https://www.reddit.com/")) { + if location_header == Some(&HeaderValue::from_static(ALTERNATIVE_REDDIT_URL_BASE)) { return Err("Reddit response was invalid".to_string()); } return request( @@ -528,7 +528,7 @@ fn test_default_subscriptions() { } #[cfg(test)] -static POPULAR_URL: &str = "/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL"; +const POPULAR_URL: &str = "/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL"; #[tokio::test(flavor = "multi_thread")] async fn test_localization_popular() { diff --git a/src/oauth.rs b/src/oauth.rs index 12b0f37..5627900 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -11,9 +11,9 @@ use serde_json::json; use tegen::tegen::TextGenerator; use tokio::time::{error::Elapsed, timeout}; -static REDDIT_ANDROID_OAUTH_CLIENT_ID: &str = "ohXpoqrZYub1kg"; +const REDDIT_ANDROID_OAUTH_CLIENT_ID: &str = "ohXpoqrZYub1kg"; -static AUTH_ENDPOINT: &str = "https://www.reddit.com"; +const AUTH_ENDPOINT: &str = "https://www.reddit.com"; // Spoofed client for Android devices #[derive(Debug, Clone, Default)] diff --git a/src/oauth_resources.rs b/src/oauth_resources.rs index faf7873..01928c3 100644 --- a/src/oauth_resources.rs +++ b/src/oauth_resources.rs @@ -2,8 +2,8 @@ // Rerun scripts/update_oauth_resources.sh to update this file // Please do not edit manually // Filled in with real app versions -pub static _IOS_APP_VERSION_LIST: &[&str; 1] = &[""]; -pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ +pub const _IOS_APP_VERSION_LIST: &[&str; 1] = &[""]; +pub const ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ "Version 2024.22.1/Build 1652272", "Version 2024.23.1/Build 1665606", "Version 2024.24.1/Build 1682520", @@ -155,4 +155,4 @@ pub static ANDROID_APP_VERSION_LIST: &[&str; 150] = &[ "Version 2022.41.0/Build 630468", "Version 2022.41.1/Build 634168", ]; -pub static _IOS_OS_VERSION_LIST: &[&str; 1] = &[""]; +pub const _IOS_OS_VERSION_LIST: &[&str; 1] = &[""]; From d7ec07cd0d713fc308e1004663b0053db8f00a0f Mon Sep 17 00:00:00 2001 From: Jeidnx Date: Mon, 2 Dec 2024 17:29:57 +0100 Subject: [PATCH 21/38] Implement a serializer for user preferences (#336) --- Cargo.lock | 13 +++++++++++ Cargo.toml | 1 + src/utils.rs | 49 ++++++++++++++++++++++++++++++++++++++--- templates/settings.html | 12 ++++++++-- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 057234f..819d4bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1384,6 +1384,7 @@ dependencies = [ "serde", "serde_json", "serde_json_path", + "serde_urlencoded", "serde_yaml", "tegen", "time", @@ -1797,6 +1798,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" diff --git a/Cargo.toml b/Cargo.toml index 616d8e9..a1d3ec0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ async-recursion = "1.1.1" common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } hyper-rustls = { version = "0.24.2", features = [ "http2" ] } tegen = "0.1.4" +serde_urlencoded = "0.7.1" [dev-dependencies] diff --git a/src/utils.rs b/src/utils.rs index 1edb528..c15dcea 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -13,7 +13,7 @@ use once_cell::sync::Lazy; use regex::Regex; use rinja::Template; use rust_embed::RustEmbed; -use serde::Serialize; +use serde::{Serialize, Serializer}; use serde_json::Value; use serde_json_path::{JsonPath, JsonPathExt}; use std::collections::{HashMap, HashSet}; @@ -601,8 +601,9 @@ pub struct Params { pub before: Option, } -#[derive(Default)] +#[derive(Default, Serialize)] pub struct Preferences { + #[serde(skip)] pub available_themes: Vec, pub theme: String, pub front_page: String, @@ -620,12 +621,21 @@ pub struct Preferences { pub disable_visit_reddit_confirmation: String, pub comment_sort: String, pub post_sort: String, + #[serde(serialize_with = "serialize_vec_with_plus")] pub subscriptions: Vec, + #[serde(serialize_with = "serialize_vec_with_plus")] pub filters: Vec, pub hide_awards: String, pub hide_score: String, } +fn serialize_vec_with_plus(vec: &Vec, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&vec.join("+")) +} + #[derive(RustEmbed)] #[folder = "static/themes/"] #[include = "*.css"] @@ -665,6 +675,10 @@ impl Preferences { hide_score: setting(req, "hide_score"), } } + + pub fn to_urlencoded(&self) -> Result { + serde_urlencoded::to_string(self).map_err(|e| e.to_string()) + } } /// Gets a `HashSet` of filters from the cookie in the given `Request`. @@ -1277,7 +1291,7 @@ pub fn get_post_url(post: &Post) -> String { #[cfg(test)] mod tests { - use super::{format_num, format_url, rewrite_urls}; + use super::{format_num, format_url, rewrite_urls, Preferences}; #[test] fn format_num_works() { @@ -1344,6 +1358,35 @@ mod tests { assert_eq!(format_url("nsfw"), ""); assert_eq!(format_url("spoiler"), ""); } + #[test] + fn serialize_prefs() { + let prefs = Preferences { + available_themes: vec![], + theme: "laserwave".to_owned(), + front_page: "default".to_owned(), + layout: "compact".to_owned(), + wide: "on".to_owned(), + blur_spoiler: "on".to_owned(), + show_nsfw: "off".to_owned(), + blur_nsfw: "on".to_owned(), + hide_hls_notification: "off".to_owned(), + video_quality: "best".to_owned(), + hide_sidebar_and_summary: "off".to_owned(), + use_hls: "on".to_owned(), + autoplay_videos: "on".to_owned(), + fixed_navbar: "on".to_owned(), + disable_visit_reddit_confirmation: "on".to_owned(), + comment_sort: "confidence".to_owned(), + post_sort: "top".to_owned(), + subscriptions: vec!["memes".to_owned(), "mildlyinteresting".to_owned()], + filters: vec![], + hide_awards: "off".to_owned(), + hide_score: "off".to_owned(), + }; + let urlencoded = serde_urlencoded::to_string(prefs).expect("Failed to serialize Prefs"); + + assert_eq!(urlencoded, "theme=laserwave&front_page=default&layout=compact&wide=on&blur_spoiler=on&show_nsfw=off&blur_nsfw=on&hide_hls_notification=off&video_quality=best&hide_sidebar_and_summary=off&use_hls=on&autoplay_videos=on&fixed_navbar=on&disable_visit_reddit_confirmation=on&comment_sort=confidence&post_sort=top&subscriptions=memes%2Bmildlyinteresting&filters=&hide_awards=off&hide_score=off") + } } #[test] diff --git a/templates/settings.html b/templates/settings.html index a7d6615..fef91cf 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -161,8 +161,16 @@ {% endif %} From 5c1e15c359e7b01316baab27d410130f1557cdf8 Mon Sep 17 00:00:00 2001 From: Butter Cat Date: Sun, 2 Feb 2025 21:48:46 -0500 Subject: [PATCH 22/38] Make subscription and filter cookies split into multiple cookies if they're too large (#288) * Split subscriptions and filters cookies into multiple cookies and make old cookies properly delete * Cleanup * Fix mispelling for removing subscription cookies * Fix many subscription misspellings * Fix subreddits and filters that were at the end and beginning of the cookies getting merged * Make join_until_size_limit take the +'s into account when calculating length * Start cookies without number to be backwards compatible * Fix old split cookies not being removed and subreddits/filters between cookies occasionally getting merged * Make updating subscription/filters cookies safer * Small cleanup * Make restore properly add new subscriptions/filters cookies and delete old unused subscriptions/filters cookies * Fix misspellings on variable name --- src/server.rs | 8 ++- src/settings.rs | 116 ++++++++++++++++++++++++++++++++++++++- src/subreddit.rs | 138 +++++++++++++++++++++++++++++++++++++++++------ src/utils.rs | 78 ++++++++++++++++++++++----- 4 files changed, 307 insertions(+), 33 deletions(-) diff --git a/src/server.rs b/src/server.rs index 15c56ad..e1f464d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -25,7 +25,7 @@ use std::{ str::{from_utf8, Split}, string::ToString, }; -use time::Duration; +use time::OffsetDateTime; use crate::dbg_msg; @@ -170,10 +170,8 @@ impl ResponseExt for Response { } fn remove_cookie(&mut self, name: String) { - let mut cookie = Cookie::from(name); - cookie.set_path("/"); - cookie.set_max_age(Duration::seconds(1)); - if let Ok(val) = header::HeaderValue::from_str(&cookie.to_string()) { + let removal_cookie = Cookie::build(name).path("/").http_only(true).expires(OffsetDateTime::now_utc()); + if let Ok(val) = header::HeaderValue::from_str(&removal_cookie.to_string()) { self.headers_mut().append("Set-Cookie", val); } } diff --git a/src/settings.rs b/src/settings.rs index 4404912..34718c2 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; // CRATES use crate::server::ResponseExt; +use crate::subreddit::join_until_size_limit; use crate::utils::{redirect, template, Preferences}; use cookie::Cookie; use futures_lite::StreamExt; @@ -119,7 +120,7 @@ fn set_cookies_method(req: Request, remove_cookies: bool) -> Response response.insert_cookie( Cookie::build((name.to_owned(), value.clone())) @@ -136,6 +137,119 @@ fn set_cookies_method(req: Request, remove_cookies: bool) -> Response = subscriptions.expect("Subscriptions").split('+').map(str::to_string).collect(); + + // Start at 0 to keep track of what number we need to start deleting old subscription cookies from + let mut subscriptions_number_to_delete_from = 0; + + // Starting at 0 so we handle the subscription cookie without a number first + for (subscriptions_number, list) in join_until_size_limit(&sub_list).into_iter().enumerate() { + let subscriptions_cookie = if subscriptions_number == 0 { + "subscriptions".to_string() + } else { + format!("subscriptions{}", subscriptions_number) + }; + + response.insert_cookie( + Cookie::build((subscriptions_cookie, list)) + .path("/") + .http_only(true) + .expires(OffsetDateTime::now_utc() + Duration::weeks(52)) + .into(), + ); + + subscriptions_number_to_delete_from += 1; + } + + // While subscriptionsNUMBER= is in the string of cookies add a response removing that cookie + while cookies_string.contains(&format!("subscriptions{subscriptions_number_to_delete_from}=")) { + // Remove that subscriptions cookie + response.remove_cookie(format!("subscriptions{subscriptions_number_to_delete_from}")); + + // Increment subscriptions cookie number + subscriptions_number_to_delete_from += 1; + } + } else { + // Remove unnumbered subscriptions cookie + response.remove_cookie("subscriptions".to_string()); + + // Starts at one to deal with the first numbered subscription cookie and onwards + let mut subscriptions_number_to_delete_from = 1; + + // While subscriptionsNUMBER= is in the string of cookies add a response removing that cookie + while cookies_string.contains(&format!("subscriptions{subscriptions_number_to_delete_from}=")) { + // Remove that subscriptions cookie + response.remove_cookie(format!("subscriptions{subscriptions_number_to_delete_from}")); + + // Increment subscriptions cookie number + subscriptions_number_to_delete_from += 1; + } + } + + // If there are filters to restore set them and delete any old filters cookies, otherwise delete them all + if filters.is_some() { + let filters_list: Vec = filters.expect("Filters").split('+').map(str::to_string).collect(); + + // Start at 0 to keep track of what number we need to start deleting old subscription cookies from + let mut filters_number_to_delete_from = 0; + + // Starting at 0 so we handle the subscription cookie without a number first + for (filters_number, list) in join_until_size_limit(&filters_list).into_iter().enumerate() { + let filters_cookie = if filters_number == 0 { + "filters".to_string() + } else { + format!("filters{}", filters_number) + }; + + response.insert_cookie( + Cookie::build((filters_cookie, list)) + .path("/") + .http_only(true) + .expires(OffsetDateTime::now_utc() + Duration::weeks(52)) + .into(), + ); + + filters_number_to_delete_from += 1; + } + + // While filtersNUMBER= is in the string of cookies add a response removing that cookie + while cookies_string.contains(&format!("filters{filters_number_to_delete_from}=")) { + // Remove that filters cookie + response.remove_cookie(format!("filters{filters_number_to_delete_from}")); + + // Increment filters cookie number + filters_number_to_delete_from += 1; + } + } else { + // Remove unnumbered filters cookie + response.remove_cookie("filters".to_string()); + + // Starts at one to deal with the first numbered subscription cookie and onwards + let mut filters_number_to_delete_from = 1; + + // While filtersNUMBER= is in the string of cookies add a response removing that cookie + while cookies_string.contains(&format!("filters{filters_number_to_delete_from}=")) { + // Remove that sfilters cookie + response.remove_cookie(format!("filters{filters_number_to_delete_from}")); + + // Increment filters cookie number + filters_number_to_delete_from += 1; + } + } + response } diff --git a/src/subreddit.rs b/src/subreddit.rs index 88aa542..2362a12 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -214,6 +214,41 @@ pub fn can_access_quarantine(req: &Request, sub: &str) -> bool { setting(req, &format!("allow_quaran_{}", sub.to_lowercase())).parse().unwrap_or_default() } +// Join items in chunks of 4000 bytes in length for cookies +pub fn join_until_size_limit(vec: &[T]) -> Vec { + let mut result = Vec::new(); + let mut list = String::new(); + let mut current_size = 0; + + for item in vec { + // Size in bytes + let item_size = item.to_string().len(); + // Use 4000 bytes to leave us some headroom because the name and options of the cookie count towards the 4096 byte cap + if current_size + item_size > 4000 { + // If last item add a seperator on the end of the list so it's interpreted properly in tanden with the next cookie + list.push('+'); + + // Push current list to result vector + result.push(list); + + // Reset the list variable so we can continue with only new items + list = String::new(); + } + // Add separator if not the first item + if !list.is_empty() { + list.push('+'); + } + // Add current item to list + list.push_str(&item.to_string()); + current_size = list.len() + item_size; + } + // Make sure to push whatever the remaining subreddits are there into the result vector + result.push(list); + + // Return resulting vector + result +} + // Sub, filter, unfilter, or unsub by setting subscription cookie using response "Set-Cookie" header pub async fn subscriptions_filters(req: Request) -> Result, String> { let sub = req.param("sub").unwrap_or_default(); @@ -306,28 +341,101 @@ pub async fn subscriptions_filters(req: Request) -> Result, let mut response = redirect(&path); - // Delete cookie if empty, else set + // If sub_list is empty remove all subscriptions cookies, otherwise update them and remove old ones if sub_list.is_empty() { + // Remove subscriptions cookie response.remove_cookie("subscriptions".to_string()); + + // Start with first numbered subscriptions cookie + let mut subscriptions_number = 1; + + // While whatever subscriptionsNUMBER cookie we're looking at has a value + while req.cookie(&format!("subscriptions{}", subscriptions_number)).is_some() { + // Remove that subscriptions cookie + response.remove_cookie(format!("subscriptions{}", subscriptions_number)); + + // Increment subscriptions cookie number + subscriptions_number += 1; + } } else { - response.insert_cookie( - Cookie::build(("subscriptions", sub_list.join("+"))) - .path("/") - .http_only(true) - .expires(OffsetDateTime::now_utc() + Duration::weeks(52)) - .into(), - ); + // Start at 0 to keep track of what number we need to start deleting old subscription cookies from + let mut subscriptions_number_to_delete_from = 0; + + // Starting at 0 so we handle the subscription cookie without a number first + for (subscriptions_number, list) in join_until_size_limit(&sub_list).into_iter().enumerate() { + let subscriptions_cookie = if subscriptions_number == 0 { + "subscriptions".to_string() + } else { + format!("subscriptions{}", subscriptions_number) + }; + + response.insert_cookie( + Cookie::build((subscriptions_cookie, list)) + .path("/") + .http_only(true) + .expires(OffsetDateTime::now_utc() + Duration::weeks(52)) + .into(), + ); + + subscriptions_number_to_delete_from += 1; + } + + // While whatever subscriptionsNUMBER cookie we're looking at has a value + while req.cookie(&format!("subscriptions{}", subscriptions_number_to_delete_from)).is_some() { + // Remove that subscriptions cookie + response.remove_cookie(format!("subscriptions{}", subscriptions_number_to_delete_from)); + + // Increment subscriptions cookie number + subscriptions_number_to_delete_from += 1; + } } + + // If filters is empty remove all filters cookies, otherwise update them and remove old ones if filters.is_empty() { + // Remove filters cookie response.remove_cookie("filters".to_string()); + + // Start with first numbered filters cookie + let mut filters_number = 1; + + // While whatever filtersNUMBER cookie we're looking at has a value + while req.cookie(&format!("filters{}", filters_number)).is_some() { + // Remove that filters cookie + response.remove_cookie(format!("filters{}", filters_number)); + + // Increment filters cookie number + filters_number += 1; + } } else { - response.insert_cookie( - Cookie::build(("filters", filters.join("+"))) - .path("/") - .http_only(true) - .expires(OffsetDateTime::now_utc() + Duration::weeks(52)) - .into(), - ); + // Start at 0 to keep track of what number we need to start deleting old filters cookies from + let mut filters_number_to_delete_from = 0; + + for (filters_number, list) in join_until_size_limit(&filters).into_iter().enumerate() { + let filters_cookie = if filters_number == 0 { + "filters".to_string() + } else { + format!("filters{}", filters_number) + }; + + response.insert_cookie( + Cookie::build((filters_cookie, list)) + .path("/") + .http_only(true) + .expires(OffsetDateTime::now_utc() + Duration::weeks(52)) + .into(), + ); + + filters_number_to_delete_from += 1; + } + + // While whatever filtersNUMBER cookie we're looking at has a value + while req.cookie(&format!("filters{}", filters_number_to_delete_from)).is_some() { + // Remove that filters cookie + response.remove_cookie(format!("filters{}", filters_number_to_delete_from)); + + // Increment filters cookie number + filters_number_to_delete_from += 1; + } } Ok(response) diff --git a/src/utils.rs b/src/utils.rs index c15dcea..e2cefd1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -825,18 +825,72 @@ pub fn param(path: &str, value: &str) -> Option { // Retrieve the value of a setting by name pub fn setting(req: &Request, name: &str) -> String { // Parse a cookie value from request - req - .cookie(name) - .unwrap_or_else(|| { - // If there is no cookie for this setting, try receiving a default from the config - if let Some(default) = get_setting(&format!("REDLIB_DEFAULT_{}", name.to_uppercase())) { - Cookie::new(name, default) - } else { - Cookie::from(name) - } - }) - .value() - .to_string() + + // If this was called with "subscriptions" and the "subscriptions" cookie has a value + if name == "subscriptions" && req.cookie("subscriptions").is_some() { + // Create subscriptions string + let mut subscriptions = String::new(); + + // Default subscriptions cookie + if req.cookie("subscriptions").is_some() { + subscriptions.push_str(req.cookie("subscriptions").unwrap().value()); + } + + // Start with first numbered subscription cookie + let mut subscriptions_number = 1; + + // While whatever subscriptionsNUMBER cookie we're looking at has a value + while req.cookie(&format!("subscriptions{}", subscriptions_number)).is_some() { + // Push whatever subscriptionsNUMBER cookie we're looking at into the subscriptions string + subscriptions.push_str(req.cookie(&format!("subscriptions{}", subscriptions_number)).unwrap().value()); + + // Increment subscription cookie number + subscriptions_number += 1; + } + + // Return the subscriptions cookies as one large string + subscriptions + } + // If this was called with "filters" and the "filters" cookie has a value + else if name == "filters" && req.cookie("filters").is_some() { + // Create filters string + let mut filters = String::new(); + + // Default filters cookie + if req.cookie("filters").is_some() { + filters.push_str(req.cookie("filters").unwrap().value()); + } + + // Start with first numbered filters cookie + let mut filters_number = 1; + + // While whatever filtersNUMBER cookie we're looking at has a value + while req.cookie(&format!("filters{}", filters_number)).is_some() { + // Push whatever filtersNUMBER cookie we're looking at into the filters string + filters.push_str(req.cookie(&format!("filters{}", filters_number)).unwrap().value()); + + // Increment filters cookie number + filters_number += 1; + } + + // Return the filters cookies as one large string + filters + } + // The above two still come to this if there was no existing value + else { + req + .cookie(name) + .unwrap_or_else(|| { + // If there is no cookie for this setting, try receiving a default from the config + if let Some(default) = get_setting(&format!("REDLIB_DEFAULT_{}", name.to_uppercase())) { + Cookie::new(name, default) + } else { + Cookie::from(name) + } + }) + .value() + .to_string() + } } // Retrieve the value of a setting by name or the default value From 9e47bc37c7e3d1b5b929926d84459d5ca4a244a9 Mon Sep 17 00:00:00 2001 From: Kot C Date: Sun, 2 Feb 2025 20:49:46 -0600 Subject: [PATCH 23/38] Support HEAD requests (resolves #292) (#363) * Support HEAD requests * Remove body from error responses too --- src/server.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/server.rs b/src/server.rs index e1f464d..5297c22 100644 --- a/src/server.rs +++ b/src/server.rs @@ -238,8 +238,14 @@ impl Server { path.pop(); } + // Replace HEAD with GET for routing + let (method, is_head) = match req.method() { + &Method::HEAD => (&Method::GET, true), + method => (method, false), + }; + // Match the visited path with an added route - match router.recognize(&format!("/{}{}", req.method().as_str(), path)) { + match router.recognize(&format!("/{}{}", method.as_str(), path)) { // If a route was configured for this path Ok(found) => { let mut parammed = req; @@ -251,17 +257,21 @@ impl Server { match func.await { Ok(mut res) => { res.headers_mut().extend(def_headers); - let _ = compress_response(&req_headers, &mut res).await; + if is_head { + *res.body_mut() = Body::empty(); + } else { + let _ = compress_response(&req_headers, &mut res).await; + } Ok(res) } - Err(msg) => new_boilerplate(def_headers, req_headers, 500, Body::from(msg)).await, + Err(msg) => new_boilerplate(def_headers, req_headers, 500, if is_head { Body::empty() } else { Body::from(msg) }).await, } } .boxed() } // If there was a routing error - Err(e) => new_boilerplate(def_headers, req_headers, 404, e.into()).boxed(), + Err(e) => new_boilerplate(def_headers, req_headers, 404, if is_head { Body::empty() } else { e.into() }).boxed(), } })) } From adf25cb15b61984581422ac798cc7c1364ad8e75 Mon Sep 17 00:00:00 2001 From: Martin Lindhe Date: Mon, 3 Feb 2025 03:56:47 +0100 Subject: [PATCH 24/38] unescape selftext_html from json api, fixes #354 (#357) * unescape selftext_html from json api, fixes #354 * fix(fmt) --------- Co-authored-by: Matthew Esposito --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/utils.rs | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 819d4bc..20d528b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,6 +770,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.12" @@ -1367,6 +1373,7 @@ dependencies = [ "dotenvy", "fastrand", "futures-lite", + "htmlescape", "hyper", "hyper-rustls", "libflate", diff --git a/Cargo.toml b/Cargo.toml index a1d3ec0..a4d0170 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ common-words-all = { version = "0.0.2", default-features = false, features = ["e hyper-rustls = { version = "0.24.2", features = [ "http2" ] } tegen = "0.1.4" serde_urlencoded = "0.7.1" +htmlescape = "0.3.1" [dev-dependencies] diff --git a/src/utils.rs b/src/utils.rs index e2cefd1..ea14dac 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -7,6 +7,7 @@ use crate::config::{self, get_setting}; // use crate::{client::json, server::RequestExt}; use cookie::Cookie; +use htmlescape::decode_html; use hyper::{Body, Request, Response}; use log::error; use once_cell::sync::Lazy; @@ -376,7 +377,7 @@ impl Post { let awards = Awards::parse(&data["all_awardings"]); // selftext_html is set for text posts when browsing. - let mut body = rewrite_urls(&val(post, "selftext_html")); + let mut body = rewrite_urls(&decode_html(&val(post, "selftext_html")).unwrap()); if body.is_empty() { body = rewrite_urls(&val(post, "body_html")); } From fd1c32f5552cc116e1cb4c95fcd7cc7a7b069335 Mon Sep 17 00:00:00 2001 From: Martin Lindhe Date: Mon, 3 Feb 2025 04:00:44 +0100 Subject: [PATCH 25/38] rss: add field, fixes #356 (#358) * rss: add field, fixes #356 * rss: also add pub_date on user feed * fix(fmt) --------- Co-authored-by: Matthew Esposito --- Cargo.lock | 5 +++-- Cargo.toml | 1 + src/subreddit.rs | 2 ++ src/user.rs | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20d528b..24791b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,9 +274,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "num-traits", ] @@ -1367,6 +1367,7 @@ dependencies = [ "brotli", "build_html", "cached", + "chrono", "clap", "common-words-all", "cookie", diff --git a/Cargo.toml b/Cargo.toml index a4d0170..843b9c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ common-words-all = { version = "0.0.2", default-features = false, features = ["e hyper-rustls = { version = "0.24.2", features = [ "http2" ] } tegen = "0.1.4" serde_urlencoded = "0.7.1" +chrono = { version = "0.4.39", default-features = false, features = [ "std" ] } htmlescape = "0.3.1" diff --git a/src/subreddit.rs b/src/subreddit.rs index 2362a12..d5d5196 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -11,6 +11,7 @@ use hyper::{Body, Request, Response}; use log::{debug, trace}; use rinja::Template; +use chrono::DateTime; use once_cell::sync::Lazy; use regex::Regex; use time::{Duration, OffsetDateTime}; @@ -607,6 +608,7 @@ pub async fn rss(req: Request) -> Result, String> { link: Some(utils::get_post_url(&post)), author: Some(post.author.name), content: Some(rewrite_urls(&post.body)), + pub_date: Some(DateTime::from_timestamp(post.created_ts as i64, 0).unwrap_or_default().to_rfc2822()), description: Some(format!( "Comments", config::get_setting("REDLIB_FULL_URL").unwrap_or_default(), diff --git a/src/user.rs b/src/user.rs index 50a4daa..2fb8b0d 100644 --- a/src/user.rs +++ b/src/user.rs @@ -5,6 +5,7 @@ use crate::client::json; use crate::server::RequestExt; use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, param, setting, template, Post, Preferences, User}; use crate::{config, utils}; +use chrono::DateTime; use hyper::{Body, Request, Response}; use rinja::Template; use time::{macros::format_description, OffsetDateTime}; @@ -165,6 +166,7 @@ pub async fn rss(req: Request) -> Result, String> { title: Some(post.title.to_string()), link: Some(utils::get_post_url(&post)), author: Some(post.author.name), + pub_date: Some(DateTime::from_timestamp(post.created_ts as i64, 0).unwrap_or_default().to_rfc2822()), content: Some(rewrite_urls(&post.body)), ..Default::default() }) From cb659cc8a3fa8e85e7fcd6c5e96d67fa83081c7b Mon Sep 17 00:00:00 2001 From: Martin Lindhe Date: Mon, 3 Feb 2025 04:00:58 +0100 Subject: [PATCH 26/38] rss: proxy links in users and subreddit feeds, fixes #359 (#361) --- src/subreddit.rs | 2 +- src/user.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/subreddit.rs b/src/subreddit.rs index d5d5196..0db4f77 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -605,7 +605,7 @@ pub async fn rss(req: Request) -> Result, String> { .into_iter() .map(|post| Item { title: Some(post.title.to_string()), - link: Some(utils::get_post_url(&post)), + link: Some(format_url(&utils::get_post_url(&post))), author: Some(post.author.name), content: Some(rewrite_urls(&post.body)), pub_date: Some(DateTime::from_timestamp(post.created_ts as i64, 0).unwrap_or_default().to_rfc2822()), diff --git a/src/user.rs b/src/user.rs index 2fb8b0d..818f368 100644 --- a/src/user.rs +++ b/src/user.rs @@ -164,7 +164,7 @@ pub async fn rss(req: Request) -> Result, String> { .into_iter() .map(|post| Item { title: Some(post.title.to_string()), - link: Some(utils::get_post_url(&post)), + link: Some(format_url(&utils::get_post_url(&post))), author: Some(post.author.name), pub_date: Some(DateTime::from_timestamp(post.created_ts as i64, 0).unwrap_or_default().to_rfc2822()), content: Some(rewrite_urls(&post.body)), From 0703fa103611786637328bf569928c0619aff759 Mon Sep 17 00:00:00 2001 From: Vivek Date: Sun, 2 Feb 2025 19:10:12 -0800 Subject: [PATCH 27/38] [build] add new dockerfiles for building from source (#244) * add new dockerfiles * update default ubuntu base images * updates * update comment * update cargo command Co-authored-by: Pim * update cargo command Co-authored-by: Pim * specify binary * use label instead of maintainer --------- Co-authored-by: Pim --- Dockerfile.alpine | 45 +++++++++++++++++++++++++++++++++++++++++ Dockerfile.ubuntu | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 Dockerfile.alpine create mode 100644 Dockerfile.ubuntu diff --git a/Dockerfile.alpine b/Dockerfile.alpine new file mode 100644 index 0000000..051476a --- /dev/null +++ b/Dockerfile.alpine @@ -0,0 +1,45 @@ +# supported versions here: https://hub.docker.com/_/rust +ARG ALPINE_VERSION=3.20 + +######################## +## builder image +######################## +FROM rust:alpine${ALPINE_VERSION} AS builder + +RUN apk add --no-cache musl-dev + +WORKDIR /redlib + +# download (most) dependencies in their own layer +COPY Cargo.lock Cargo.toml ./ +RUN mkdir src && echo "fn main() { panic!(\"why am i running?\") }" > src/main.rs +RUN cargo build --release --locked --bin redlib +RUN rm ./src/main.rs && rmdir ./src + +# copy the source and build the redlib binary +COPY . ./ +RUN cargo build --release --locked --bin redlib +RUN echo "finished building redlib!" + +######################## +## release image +######################## +FROM alpine:${ALPINE_VERSION} AS release + +# Import redlib binary from builder +COPY --from=builder /redlib/target/release/redlib /usr/local/bin/redlib + +# Add non-root user for running redlib +RUN adduser --home /nonexistent --no-create-home --disabled-password redlib +USER redlib + +# Document that we intend to expose port 8080 to whoever runs the container +EXPOSE 8080 + +# Run a healthcheck every minute to make sure redlib is functional +HEALTHCHECK --interval=1m --timeout=3s CMD wget --spider --q http://localhost:8080/settings || exit 1 + +# Add container metadata +LABEL org.opencontainers.image.authors="sigaloid" + +CMD ["redlib"] diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu new file mode 100644 index 0000000..2e277c5 --- /dev/null +++ b/Dockerfile.ubuntu @@ -0,0 +1,51 @@ +# supported versions here: https://hub.docker.com/_/rust +ARG RUST_BUILDER_VERSION=slim-bookworm +ARG UBUNTU_RELEASE_VERSION=noble + +######################## +## builder image +######################## +FROM rust:${RUST_BUILDER_VERSION} AS builder + +WORKDIR /redlib + +# download (most) dependencies in their own layer +COPY Cargo.lock Cargo.toml ./ +RUN mkdir src && echo "fn main() { panic!(\"why am i running?\") }" > src/main.rs +RUN cargo build --release --locked --bin redlib +RUN rm ./src/main.rs && rmdir ./src + +# copy the source and build the redlib binary +COPY . ./ +RUN cargo build --release --locked --bin redlib +RUN echo "finished building redlib!" + +######################## +## release image +######################## +FROM ubuntu:${UBUNTU_RELEASE_VERSION} AS release + +# Install ca-certificates +RUN apt-get update && apt-get install -y ca-certificates + +# Import redlib binary from builder +COPY --from=builder /redlib/target/release/redlib /usr/local/bin/redlib + +# Add non-root user for running redlib +RUN useradd \ + --no-create-home \ + --password "!" \ + --comment "user for running redlib" \ + redlib +USER redlib + +# Document that we intend to expose port 8080 to whoever runs the container +EXPOSE 8080 + +# Run a healthcheck every minute to make sure redlib is functional +HEALTHCHECK --interval=1m --timeout=3s CMD wget --spider --q http://localhost:8080/settings || exit 1 + +# Add container metadata +LABEL org.opencontainers.image.authors="sigaloid" + +CMD ["redlib"] From 9e39a75e82cbf0c83b09e051c13073fa4a5e3f5a Mon Sep 17 00:00:00 2001 From: Joel Koen Date: Mon, 3 Feb 2025 14:16:59 +1000 Subject: [PATCH 28/38] build(nix): update deps (#331) --- flake.lock | 32 ++++++++++++-------------------- flake.nix | 10 ++-------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/flake.lock b/flake.lock index 4569244..2b0b585 100644 --- a/flake.lock +++ b/flake.lock @@ -1,17 +1,12 @@ { "nodes": { "crane": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, "locked": { - "lastModified": 1717025063, - "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=", + "lastModified": 1731974733, + "narHash": "sha256-enYSSZVVl15FI5p+0Y5/Ckf5DZAvXe6fBrHxyhA/njc=", "owner": "ipetkov", "repo": "crane", - "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e", + "rev": "3cb338ce81076ce5e461cf77f7824476addb0e1c", "type": "github" }, "original": { @@ -25,11 +20,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -40,11 +35,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1717112898, - "narHash": "sha256-7R2ZvOnvd9h8fDd65p0JnB7wXfUvreox3xFdYWd1BnY=", + "lastModified": 1731890469, + "narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6132b0f6e344ce2fe34fc051b72fb46e34f668e0", + "rev": "5083ec887760adfe12af64830a66807423a859a7", "type": "github" }, "original": { @@ -64,19 +59,16 @@ }, "rust-overlay": { "inputs": { - "flake-utils": [ - "flake-utils" - ], "nixpkgs": [ "nixpkgs" ] }, "locked": { - "lastModified": 1717121863, - "narHash": "sha256-/3sxIe7MZqF/jw1RTQCSmgTjwVod43mmrk84m50MJQ4=", + "lastModified": 1732069891, + "narHash": "sha256-moKx8AVJrViCSdA0e0nSsG8b1dAsObI4sRAtbqbvBY8=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "2a7b53172ed08f856b8382d7dcfd36a4e0cbd866", + "rev": "8509a51241c407d583b1963d5079585a992506e8", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 8bcacf6..0180c8d 100644 --- a/flake.nix +++ b/flake.nix @@ -4,19 +4,13 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - crane = { - url = "github:ipetkov/crane"; - inputs.nixpkgs.follows = "nixpkgs"; - }; + crane.url = "github:ipetkov/crane"; flake-utils.url = "github:numtide/flake-utils"; rust-overlay = { url = "github:oxalica/rust-overlay"; - inputs = { - nixpkgs.follows = "nixpkgs"; - flake-utils.follows = "flake-utils"; - }; + inputs.nixpkgs.follows = "nixpkgs"; }; }; From 96ad7bf1632ef4bafeba148288422ffe4cf9bfb2 Mon Sep 17 00:00:00 2001 From: mooons <10822203+mooons@users.noreply.github.com> Date: Sun, 2 Feb 2025 20:26:36 -0800 Subject: [PATCH 29/38] feat: render bullet lists (#321) * feat: render bullet lists * tests: add tests --------- Co-authored-by: Matthew Esposito --- src/utils.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/utils.rs b/src/utils.rs index ea14dac..1bc70b9 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -989,6 +989,17 @@ pub fn format_url(url: &str) -> String { } } +static REGEX_BULLET: Lazy = Lazy::new(|| Regex::new(r"(?m)^- (.*)$").unwrap()); +static REGEX_BULLET_CONSECUTIVE_LINES: Lazy = Lazy::new(|| Regex::new(r"\n
    ").unwrap()); + +pub fn render_bullet_lists(input_text: &str) -> String { + // ref: https://stackoverflow.com/a/4902622 + // First enclose each bullet with
    • tags + let text1 = REGEX_BULLET.replace_all(&input_text, "
      • $1
      ").to_string(); + // Then remove any consecutive
      tags + REGEX_BULLET_CONSECUTIVE_LINES.replace_all(&text1, "").to_string() +} + // These are links we want to replace in-body static REDDIT_REGEX: Lazy = Lazy::new(|| Regex::new(r#"href="(https|http|)://(www\.|old\.|np\.|amp\.|new\.|)(reddit\.com|redd\.it)/"#).unwrap()); static REDDIT_PREVIEW_REGEX: Lazy = Lazy::new(|| Regex::new(r"https?://(external-preview|preview|i)\.redd\.it(.*)[^?]").unwrap()); @@ -1143,6 +1154,10 @@ pub fn rewrite_emotes(media_metadata: &Value, comment: String) -> String { } } } + + // render bullet (unordered) lists + comment = render_bullet_lists(&comment); + // Call rewrite_urls() to transform any other Reddit links rewrite_urls(&comment) } @@ -1505,3 +1520,28 @@ fn test_rewriting_emotes() { let output = r#"

      "#; assert_eq!(rewrite_emotes(&json_input, comment_input.to_string()), output); } + +#[test] +fn test_rewriting_bullet_list() { + let input = r#"

      Hi, I've bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I've installed its driver from the LG website and it works ok. I also used http://www.lagom.nl/lcd-test/ to calibrate it. After some good tinkering I've found the following settings + the color profile from the driver gets me past all the tests perfectly: +- Brightness 50 (still have to settle on this one, it's personal preference, it controls the backlight, not the colors) +- Contrast 70 (which for me was the default one) +- Picture mode Custom +- Super resolution + Off (it looks horrible anyway) +- Sharpness 50 (default one I think) +- Black level High (low messes up gray colors) +- DFC Off +- Response Time Middle (personal preference, https://www.blurbusters.com/ show horrible overdrive with it on high) +- Freesync doesn't matter +- Black stabilizer 50 +- Gamma setting on 0 +- Color Temp Medium +How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge's icon for example, the blue background is just blocky, don't know why.

      +
      "#; +let output = r#"

      Hi, I've bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I've installed its driver from the LG website and it works ok. I also used http://www.lagom.nl/lcd-test/ to calibrate it. After some good tinkering I've found the following settings + the color profile from the driver gets me past all the tests perfectly: +

      • Brightness 50 (still have to settle on this one, it's personal preference, it controls the backlight, not the colors)
      • Contrast 70 (which for me was the default one)
      • Picture mode Custom
      • Super resolution + Off (it looks horrible anyway)
      • Sharpness 50 (default one I think)
      • Black level High (low messes up gray colors)
      • DFC Off
      • Response Time Middle (personal preference, https://www.blurbusters.com/ show horrible overdrive with it on high)
      • Freesync doesn't matter
      • Black stabilizer 50
      • Gamma setting on 0
      • Color Temp Medium
      +How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge's icon for example, the blue background is just blocky, don't know why.

      +
      "#; + + assert_eq!(render_bullet_lists(input), output); +} \ No newline at end of file From 23cda23d013e4a2866afd1cad0649cb40bb612d0 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Sun, 2 Feb 2025 23:30:33 -0500 Subject: [PATCH 30/38] feat: add environment variables and dedicated flags for ipv4/6 only (#307) * feat: add environment variables and dedicated flags for ipv4/6 only * fix(readme): mention all flags on README --- README.md | 11 +++++++++++ src/main.rs | 25 ++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4026791..6fabfcc 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,17 @@ REDLIB_DEFAULT_USE_HLS = "on" > > If using Docker Compose, no changes are needed as the `.env` file is already referenced in `compose.yaml` via the `env_file: .env` line. +## Command Line Flags + +Redlib supports the following command line flags: + +- `-4`, `--ipv4-only`: Listen on IPv4 only. +- `-6`, `--ipv6-only`: Listen on IPv6 only. +- `-r`, `--redirect-https`: Redirect all HTTP requests to HTTPS (no longer functional). +- `-a`, `--address
      `: Sets address to listen on. Default is `[::]`. +- `-p`, `--port `: Port to listen on. Default is `8080`. +- `-H`, `--hsts `: HSTS header to tell browsers that this site should only be accessed over HTTPS. Default is `604800`. + ## Instance settings Assign a default value for each instance-specific setting by passing environment variables to Redlib in the format `REDLIB_{X}`. Replace `{X}` with the setting name (see list below) in capital letters. diff --git a/src/main.rs b/src/main.rs index 8732d20..a109560 100644 --- a/src/main.rs +++ b/src/main.rs @@ -108,6 +108,20 @@ async fn main() { let matches = Command::new("Redlib") .version(env!("CARGO_PKG_VERSION")) .about("Private front-end for Reddit written in Rust ") + .arg( + Arg::new("ipv4-only") + .short('4') + .long("ipv4-only") + .help("Listen on IPv4 only") + .num_args(0), + ) + .arg( + Arg::new("ipv6-only") + .short('6') + .long("ipv6-only") + .help("Listen on IPv6 only") + .num_args(0), + ) .arg( Arg::new("redirect-https") .short('r') @@ -164,7 +178,16 @@ async fn main() { let port = matches.get_one::("port").unwrap(); let hsts = matches.get_one("hsts").map(|m: &String| m.as_str()); - let listener = [address, ":", port].concat(); + let ipv4_only = std::env::var("IPV4_ONLY").is_ok() || matches.get_flag("ipv4-only"); + let ipv6_only = std::env::var("IPV6_ONLY").is_ok() || matches.get_flag("ipv6-only"); + + let listener = if ipv4_only { + format!("0.0.0.0:{}", port) + } else if ipv6_only { + format!("[::]:{}", port) + } else { + [address, ":", port].concat() + }; println!("Starting Redlib..."); From 2e0e1a1aaab5e020365e06343e5d9def30a46466 Mon Sep 17 00:00:00 2001 From: Butter Cat Date: Sun, 2 Feb 2025 23:31:37 -0500 Subject: [PATCH 31/38] Fix crossposted galleries not working (#293) --- src/utils.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/utils.rs b/src/utils.rs index 1bc70b9..c4991d8 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -234,6 +234,14 @@ impl Media { // If this post contains a gallery of images gallery = GalleryMedia::parse(&data["gallery_data"]["items"], &data["media_metadata"]); + ("gallery", &data["url"], None) + } else if data["crosspost_parent_list"][0]["is_gallery"].as_bool().unwrap_or_default() { + // If this post contains a gallery of images + gallery = GalleryMedia::parse( + &data["crosspost_parent_list"][0]["gallery_data"]["items"], + &data["crosspost_parent_list"][0]["media_metadata"], + ); + ("gallery", &data["url"], None) } else if data["is_reddit_media_domain"].as_bool().unwrap_or_default() && data["domain"] == "i.redd.it" { // If this post contains a reddit media (image) URL. From 68a0517115b31d5188c01d0c5c30575bf877a6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Val=C3=A9rio?= Date: Mon, 3 Feb 2025 04:32:23 +0000 Subject: [PATCH 32/38] update devcontainer image, that includes a more recent version of rust (#294) --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3a941de..3ec1ead 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Rust", - "image": "mcr.microsoft.com/devcontainers/rust:0-1-bullseye", + "image": "mcr.microsoft.com/devcontainers/rust:1.0.9-bookworm", "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, From 51386671d3474b5020cd43db8d7536ba1ac7979d Mon Sep 17 00:00:00 2001 From: Butter Cat Date: Sun, 2 Feb 2025 23:38:52 -0500 Subject: [PATCH 33/38] Fix embedded images sometimes having gaps around them (#295) * Fix images embedded by rewrite_urls() having an empty

      above and below them that caused weird gaps in some scenarios * Fix test for new embedding behavior * fix: remove println --------- Co-authored-by: Matthew Esposito --- src/utils.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index c4991d8..4ed7664 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1010,7 +1010,7 @@ pub fn render_bullet_lists(input_text: &str) -> String { // These are links we want to replace in-body static REDDIT_REGEX: Lazy = Lazy::new(|| Regex::new(r#"href="(https|http|)://(www\.|old\.|np\.|amp\.|new\.|)(reddit\.com|redd\.it)/"#).unwrap()); -static REDDIT_PREVIEW_REGEX: Lazy = Lazy::new(|| Regex::new(r"https?://(external-preview|preview|i)\.redd\.it(.*)[^?]").unwrap()); +static REDDIT_PREVIEW_REGEX: Lazy = Lazy::new(|| Regex::new(r"https?://(external-preview|preview|i)\.redd\.it(.*)").unwrap()); static REDDIT_EMOJI_REGEX: Lazy = Lazy::new(|| Regex::new(r"https?://(www|).redditstatic\.com/(.*)").unwrap()); static REDLIB_PREVIEW_LINK_REGEX: Lazy = Lazy::new(|| Regex::new(r#"/(img|preview/)(pre|external-pre)?/(.*?)>"#).unwrap()); static REDLIB_PREVIEW_TEXT_REGEX: Lazy = Lazy::new(|| Regex::new(r">(.*?)").unwrap()); @@ -1052,7 +1052,7 @@ pub fn rewrite_urls(input_text: &str) -> String { } // image_url contains > at the end of it, and right above this we remove image_text's front >, leaving us with just a single > between them - let image_to_replace = format!(""); + let image_to_replace = format!("

      "); // _image_replacement needs to be in scope for the replacement at the bottom of the loop let mut _image_replacement = String::new(); @@ -1501,7 +1501,7 @@ async fn test_fetching_ws() { fn test_rewriting_image_links() { let input = r#"

      caption 1

      "#; - let output = r#"

      caption 1
      caption 1
      "#; assert_eq!(rewrite_urls(input), output); } From bbe5f8191475ea20e41e5a4eb1572fbd2a45d8b3 Mon Sep 17 00:00:00 2001 From: internationalcrisis <97415475+internationalcrisis@users.noreply.github.com> Date: Sun, 2 Feb 2025 22:40:19 -0600 Subject: [PATCH 34/38] fix: gracefully shutdown on CTRL+C and SIGTERM (#273) Fixes #205 --- src/server.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/server.rs b/src/server.rs index 5297c22..a287de2 100644 --- a/src/server.rs +++ b/src/server.rs @@ -282,8 +282,19 @@ impl Server { // Bind server to address specified above. Gracefully shut down if CTRL+C is pressed let server = HyperServer::bind(address).serve(make_svc).with_graceful_shutdown(async { + #[cfg(windows)] // Wait for the CTRL+C signal tokio::signal::ctrl_c().await.expect("Failed to install CTRL+C signal handler"); + + #[cfg(unix)] + { + // Wait for CTRL+C or SIGTERM signals + let mut signal_terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("Failed to install SIGTERM signal handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => (), + _ = signal_terminate.recv() => () + } + } }); server.boxed() From 257871b56ca1d5612482df05fed14b1c45e04de9 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Mon, 3 Feb 2025 00:30:48 -0500 Subject: [PATCH 35/38] fix(tests) --- src/utils.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 4ed7664..cf226b9 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1483,7 +1483,10 @@ async fn test_fetching_subreddit_quarantined() { #[tokio::test(flavor = "multi_thread")] async fn test_fetching_nsfw_subreddit() { - let subreddit = Post::fetch("/r/randnsfw", false).await; + // Gonwild is a place for closed, Euclidean Geometric shapes to exchange their nth terms for karma; showing off their edges in a comfortable environment without pressure. + // Find a good sub that is tagged NSFW but that actually isn't in case my future employers are watching (they probably are) + // switched from randnsfw as it is no longer functional. + let subreddit = Post::fetch("/r/gonwild", false).await; assert!(subreddit.is_ok()); assert!(!subreddit.unwrap().0.is_empty()); } From 7930b19809f00c99705949027550add74068da3b Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Mon, 3 Feb 2025 00:47:25 -0500 Subject: [PATCH 36/38] fix: fix clippy + tests --- src/client.rs | 6 ------ src/main.rs | 18 +++--------------- src/utils.rs | 4 ++-- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/client.rs b/src/client.rs index fa32fc0..76369ca 100644 --- a/src/client.rs +++ b/src/client.rs @@ -544,12 +544,6 @@ async fn test_obfuscated_share_link() { assert_eq!(canonical_path(share_link, 3).await, Ok(Some(canonical_link))); } -#[tokio::test(flavor = "multi_thread")] -async fn test_share_link_strip_json() { - let link = "/17krzvz".into(); - let canonical_link = "/comments/17krzvz".into(); - assert_eq!(canonical_path(link, 3).await, Ok(Some(canonical_link))); -} #[tokio::test(flavor = "multi_thread")] async fn test_private_sub() { let link = json("/r/suicide/about.json?raw_json=1".into(), true).await; diff --git a/src/main.rs b/src/main.rs index a109560..165f0cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -108,20 +108,8 @@ async fn main() { let matches = Command::new("Redlib") .version(env!("CARGO_PKG_VERSION")) .about("Private front-end for Reddit written in Rust ") - .arg( - Arg::new("ipv4-only") - .short('4') - .long("ipv4-only") - .help("Listen on IPv4 only") - .num_args(0), - ) - .arg( - Arg::new("ipv6-only") - .short('6') - .long("ipv6-only") - .help("Listen on IPv6 only") - .num_args(0), - ) + .arg(Arg::new("ipv4-only").short('4').long("ipv4-only").help("Listen on IPv4 only").num_args(0)) + .arg(Arg::new("ipv6-only").short('6').long("ipv6-only").help("Listen on IPv6 only").num_args(0)) .arg( Arg::new("redirect-https") .short('r') @@ -392,7 +380,7 @@ async fn main() { Some("best" | "hot" | "new" | "top" | "rising" | "controversial") => subreddit::community(req).await, // Short link for post - Some(id) if (5..8).contains(&id.len()) => match canonical_path(format!("/{id}"), 3).await { + Some(id) if (5..8).contains(&id.len()) => match canonical_path(format!("/comments/{id}"), 3).await { Ok(path_opt) => match path_opt { Some(path) => Ok(redirect(&path)), None => error(req, "Post ID is invalid. It may point to a post on a community that has been banned.").await, diff --git a/src/utils.rs b/src/utils.rs index cf226b9..867f73f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1549,10 +1549,10 @@ fn test_rewriting_bullet_list() { - Color Temp Medium How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge's icon for example, the blue background is just blocky, don't know why.

      "#; -let output = r#"

      Hi, I've bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I've installed its driver from the LG website and it works ok. I also used http://www.lagom.nl/lcd-test/ to calibrate it. After some good tinkering I've found the following settings + the color profile from the driver gets me past all the tests perfectly: + let output = r#"

      Hi, I've bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I've installed its driver from the LG website and it works ok. I also used http://www.lagom.nl/lcd-test/ to calibrate it. After some good tinkering I've found the following settings + the color profile from the driver gets me past all the tests perfectly:

      • Brightness 50 (still have to settle on this one, it's personal preference, it controls the backlight, not the colors)
      • Contrast 70 (which for me was the default one)
      • Picture mode Custom
      • Super resolution + Off (it looks horrible anyway)
      • Sharpness 50 (default one I think)
      • Black level High (low messes up gray colors)
      • DFC Off
      • Response Time Middle (personal preference, https://www.blurbusters.com/ show horrible overdrive with it on high)
      • Freesync doesn't matter
      • Black stabilizer 50
      • Gamma setting on 0
      • Color Temp Medium
      How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge's icon for example, the blue background is just blocky, don't know why.

      "#; assert_eq!(render_bullet_lists(input), output); -} \ No newline at end of file +} From ef2cc01bf7f43018dddf60d9b575b1b66934c2c1 Mon Sep 17 00:00:00 2001 From: Integral Date: Mon, 3 Feb 2025 13:51:54 +0800 Subject: [PATCH 37/38] refactor(utils): avoid redundant String conversions & use match (#347) * refactor(utils): avoid redundant String conversions & use match * ci: fix clippy lint --- src/utils.rs | 57 ++++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 867f73f..21461a7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -638,7 +638,7 @@ pub struct Preferences { pub hide_score: String, } -fn serialize_vec_with_plus(vec: &Vec, serializer: S) -> Result +fn serialize_vec_with_plus(vec: &[String], serializer: S) -> Result where S: Serializer, { @@ -915,11 +915,12 @@ pub fn setting_or_default(req: &Request, name: &str, default: String) -> S // Detect and redirect in the event of a random subreddit pub async fn catch_random(sub: &str, additional: &str) -> Result, String> { if sub == "random" || sub == "randnsfw" { - let new_sub = json(format!("/r/{sub}/about.json?raw_json=1"), false).await?["data"]["display_name"] - .as_str() - .unwrap_or_default() - .to_string(); - Ok(redirect(&format!("/r/{new_sub}{additional}"))) + Ok(redirect(&format!( + "/r/{}{additional}", + json(format!("/r/{sub}/about.json?raw_json=1"), false).await?["data"]["display_name"] + .as_str() + .unwrap_or_default() + ))) } else { Err("No redirect needed".to_string()) } @@ -1019,8 +1020,7 @@ static REDLIB_PREVIEW_TEXT_REGEX: Lazy = Lazy::new(|| Regex::new(r">(.*?) pub fn rewrite_urls(input_text: &str) -> String { let mut text1 = // Rewrite Reddit links to Redlib - REDDIT_REGEX.replace_all(input_text, r#"href="/"#) - .to_string(); + REDDIT_REGEX.replace_all(input_text, r#"href="/"#).to_string(); loop { if REDDIT_EMOJI_REGEX.find(&text1).is_none() { @@ -1042,49 +1042,44 @@ pub fn rewrite_urls(input_text: &str) -> String { } else { let formatted_url = format_url(REDDIT_PREVIEW_REGEX.find(&text1).map(|x| x.as_str()).unwrap_or_default()); - let image_url = REDLIB_PREVIEW_LINK_REGEX.find(&formatted_url).map_or("", |m| m.as_str()).to_string(); - let mut image_caption = REDLIB_PREVIEW_TEXT_REGEX.find(&formatted_url).map_or("", |m| m.as_str()).to_string(); + let image_url = REDLIB_PREVIEW_LINK_REGEX.find(&formatted_url).map_or("", |m| m.as_str()); + let mut image_caption = REDLIB_PREVIEW_TEXT_REGEX.find(&formatted_url).map_or("", |m| m.as_str()); /* As long as image_caption isn't empty remove first and last four characters of image_text to leave us with just the text in the caption without any HTML. This makes it possible to enclose it in a
      later on without having stray HTML breaking it */ if !image_caption.is_empty() { - image_caption = image_caption[1..image_caption.len() - 4].to_string(); + image_caption = &image_caption[1..image_caption.len() - 4]; } // image_url contains > at the end of it, and right above this we remove image_text's front >, leaving us with just a single > between them let image_to_replace = format!("

      "); - // _image_replacement needs to be in scope for the replacement at the bottom of the loop - let mut _image_replacement = String::new(); - /* We don't want to show a caption that's just the image's link, so we check if we find a Reddit preview link within the image's caption. If we don't find one we must have actual text, so we include a
      block that contains it. Otherwise we don't include the
      block as we don't need it. */ - if REDDIT_PREVIEW_REGEX.find(&image_caption).is_none() { + let _image_replacement = if REDDIT_PREVIEW_REGEX.find(image_caption).is_none() { // Without this " would show as \" instead. "\"" is how the quotes are formatted within image_text beforehand - image_caption = image_caption.replace("\\"", "\""); - - _image_replacement = format!("
      {image_caption}
      "); + format!( + "
      {}
      ", + image_caption.replace("\\"", "\"") + ) } else { - _image_replacement = format!("
      "); - } + format!("
      ") + }; /* In order to know if we're dealing with a normal or external preview we need to take a look at the first capture group of REDDIT_PREVIEW_REGEX if it's preview we're dealing with something that needs /preview/pre, external-preview is /preview/external-pre, and i is /img */ - let reddit_preview_regex_capture = REDDIT_PREVIEW_REGEX.captures(&text1).unwrap().get(1).map_or("", |m| m.as_str()).to_string(); - let mut _preview_type = String::new(); - if reddit_preview_regex_capture == "preview" { - _preview_type = "/preview/pre".to_string(); - } else if reddit_preview_regex_capture == "external-preview" { - _preview_type = "/preview/external-pre".to_string(); - } else { - _preview_type = "/img".to_string(); - } + let reddit_preview_regex_capture = REDDIT_PREVIEW_REGEX.captures(&text1).unwrap().get(1).map_or("", |m| m.as_str()); + + let _preview_type = match reddit_preview_regex_capture { + "preview" => "/preview/pre", + "external-preview" => "/preview/external-pre", + _ => "/img", + }; text1 = REDDIT_PREVIEW_REGEX .replace(&text1, format!("{_preview_type}$2")) .replace(&image_to_replace, &_image_replacement) - .to_string() } } } @@ -1158,7 +1153,7 @@ pub fn rewrite_emotes(media_metadata: &Value, comment: String) -> String { ); // Inside the comment replace the ID we found with the string that will embed the image - comment = comment.replace(&id, &to_replace_with).to_string(); + comment = comment.replace(&id, &to_replace_with); } } } From c7f55c146a641540c293967e53ab6ee55eac2516 Mon Sep 17 00:00:00 2001 From: Matthew Esposito Date: Mon, 3 Feb 2025 00:53:13 -0500 Subject: [PATCH 38/38] fix(clippy): minor clippy changes --- src/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 21461a7..23db3d0 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1004,7 +1004,7 @@ static REGEX_BULLET_CONSECUTIVE_LINES: Lazy = Lazy::new(|| Regex::new(r"< pub fn render_bullet_lists(input_text: &str) -> String { // ref: https://stackoverflow.com/a/4902622 // First enclose each bullet with
      • tags - let text1 = REGEX_BULLET.replace_all(&input_text, "
        • $1
        ").to_string(); + let text1 = REGEX_BULLET.replace_all(input_text, "
        • $1
        ").to_string(); // Then remove any consecutive
        tags REGEX_BULLET_CONSECUTIVE_LINES.replace_all(&text1, "").to_string() } @@ -1344,7 +1344,7 @@ pub fn url_path_basename(path: &str) -> String { let mut url = url_result.unwrap(); url.path_segments_mut().unwrap().pop_if_empty(); - url.path_segments().unwrap().last().unwrap().to_string() + url.path_segments().unwrap().next_back().unwrap().to_string() } }
-

Note: settings and subscriptions are saved in browser cookies. Clearing your cookies will reset them.


-

You can restore your current settings and subscriptions after clearing your cookies using this link.

+

Note: settings and subscriptions are saved in browser cookies. Clearing your cookies will reset them.

+
+ {% match prefs.to_urlencoded() %} + {% when Ok with (encoded_prefs) %} +

You can restore your current settings and subscriptions after clearing your cookies using this link.

+ {% when Err with (err) %} +

There was an error creating your restore link: {{ err }}

+

Please report this issue

+ {% endmatch %}