Skip to content

Commit

Permalink
refactor: Genericize creation of checksum hashmaps
Browse files Browse the repository at this point in the history
  • Loading branch information
@lj3954
lj3954 committed Jul 3, 2024
1 parent 7092da4 commit 1a546a4
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 54 deletions.
22 changes: 4 additions & 18 deletions src/bsd/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use crate::store_data::{ArchiveFormat, Config, Disk, Distro, Source, WebSource};
use crate::store_data::{ArchiveFormat, ChecksumSeparation, Config, Disk, Distro, Source, WebSource};
use crate::utils::capture_page;
use quickemu::config::{Arch, GuestOS};
use regex::Regex;
use std::collections::HashMap;
use std::sync::Arc;

const FREEBSD_X86_64_RELEASES: &str = "https://download.freebsd.org/ftp/releases/amd64/amd64/";
Expand All @@ -18,36 +17,22 @@ impl Distro for FreeBSD {
const DESCRIPTION: Option<&'static str> = Some("Operating system used to power modern servers, desktops, and embedded platforms.");
async fn generate_configs() -> Option<Vec<Config>> {
let freebsd_regex = Arc::new(Regex::new(r#"href="([0-9\.]+)-RELEASE"#).unwrap());
let checksum_regex = Arc::new(Regex::new(r#"SHA256 \(([^)]+)\) = ([0-9a-f]+)"#).unwrap());
let futures = [
(FREEBSD_X86_64_RELEASES, "amd64", Arch::x86_64),
(FREEBSD_AARCH64_RELEASES, "arm64-aarch64", Arch::aarch64),
(FREEBSD_RISCV64_RELEASES, "riscv-riscv64", Arch::riscv64),
]
.iter()
.map(|(mirror, denom, arch)| {
let checksum_regex = checksum_regex.clone();
let freebsd_regex = freebsd_regex.clone();

let build_checksums = |cs_url: String, cs_regex: Arc<Regex>| async move {
let checksum_page = capture_page(&cs_url).await;
checksum_page.map(|cs| {
cs_regex
.captures_iter(&cs)
.map(|c| (c[1].to_string(), c[2].to_string()))
.collect::<HashMap<String, String>>()
})
};

async move {
if let Some(page) = capture_page(mirror).await {
let futures = freebsd_regex
.captures_iter(&page)
.flat_map(|c| {
let release = c[1].to_string();
let vm_image_release = release.clone();
let normal_checksum_regex = checksum_regex.clone();
let vm_checksum_regex = checksum_regex.clone();

let vm_image_mirror = {
let arch = if *arch == Arch::x86_64 { "amd64" } else { &arch.to_string() };
Expand All @@ -56,7 +41,7 @@ impl Distro for FreeBSD {

let normal_editions = tokio::spawn(async move {
let checksum_url = format!("{mirror}ISO-IMAGES/{release}/CHECKSUM.SHA256-FreeBSD-{release}-RELEASE-{denom}");
let mut checksums = build_checksums(checksum_url, normal_checksum_regex).await;
let mut checksums = ChecksumSeparation::Sha256Regex.build(&checksum_url).await;
FREEBSD_EDITIONS
.iter()
.map(|edition| {
Expand All @@ -78,7 +63,8 @@ impl Distro for FreeBSD {
let vm_image = tokio::spawn(async move {
let iso = format!("FreeBSD-{vm_image_release}-RELEASE-{denom}.qcow2.xz");
let checksum_url = format!("{vm_image_mirror}CHECKSUM.SHA256");
let checksum = build_checksums(checksum_url, vm_checksum_regex)
let checksum = ChecksumSeparation::Sha256Regex
.build(&checksum_url)
.await
.and_then(|mut cs| cs.remove(&iso));
let url = vm_image_mirror + &iso;
Expand Down
17 changes: 5 additions & 12 deletions src/linux/arch.rs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
pub mod manjaro;

use crate::{
store_data::{Config, Distro, Source, WebSource},
store_data::{ChecksumSeparation, Config, Distro, Source, WebSource},
utils::{capture_page, GatherData, GithubAPI},
};
use regex::Regex;
use serde::Deserialize;
use std::{collections::HashMap, sync::Arc};
use std::sync::Arc;

const ARCHCRAFT_MIRROR: &str = "https://sourceforge.net/projects/archcraft/files/";

Expand Down Expand Up @@ -117,13 +117,10 @@ impl Distro for ArcoLinux {
let release = c[1].to_string();
let mirror = format!("{ARCOLINUX_MIRROR}{release}/");
let iso_regex = iso_regex.clone();
let checksum_regex = checksum_regex.clone();
let checksums = ChecksumSeparation::CustomRegex(checksum_regex.clone(), 2, 1);
async move {
let page = capture_page(&mirror).await?;
let checksums = checksum_regex
.captures_iter(&page)
.map(|c| (c[2].to_string(), c[1].to_string()))
.collect::<HashMap<String, String>>();
let checksums = checksums.build_with_data(&page).await;

let futures = iso_regex
.captures_iter(&page)
Expand Down Expand Up @@ -177,11 +174,7 @@ impl Distro for ArtixLinux {
let page = capture_page(ARTIX_MIRROR).await?;
let iso_regex = Regex::new(r#"href="(artix-(.*?)-([^-]+-[0-9]+)-x86_64.iso)""#).unwrap();

let checksums = capture_page(&format!("{ARTIX_MIRROR}sha256sums")).await.map(|c| {
c.lines()
.filter_map(|l| l.split_once(" ").map(|(hash, file)| (file.to_string(), hash.to_string())))
.collect::<HashMap<String, String>>()
});
let checksums = ChecksumSeparation::Whitespace.build(&format!("{ARTIX_MIRROR}sha256sums")).await;

iso_regex
.captures_iter(&page)
Expand Down
37 changes: 24 additions & 13 deletions src/linux/debian.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
use crate::{
store_data::{Config, Distro, Source, WebSource},
store_data::{ChecksumSeparation, Config, Distro, Source, WebSource},
utils::capture_page,
};
use regex::Regex;
use std::collections::HashMap;
use std::sync::Arc;

const ANTIX_MIRROR: &str = "https://sourceforge.net/projects/antix-linux/files/Final/";
Expand All @@ -20,6 +19,14 @@ impl Distro for Antix {
let releases_regex = Regex::new(r#""name":"antiX-([0-9.]+)""#).unwrap();
let iso_regex = Arc::new(Regex::new(r#""name":"(antiX-[0-9.]+(?:-runit)?(?:-[^_]+)?_x64-([^.]+).iso)".*?"download_url":"(.*?)""#).unwrap());

let skip_until_sha256 = |cs_data: String| {
cs_data
.lines()
.skip_while(|l| !l.starts_with("sha256"))
.collect::<Vec<_>>()
.join("\n")
};

let futures = releases_regex.captures_iter(&releases).take(3).map(|r| {
let release = r[1].to_string();
let mirror = format!("{ANTIX_MIRROR}antiX-{release}/");
Expand All @@ -29,17 +36,10 @@ impl Distro for Antix {
let iso_regex = iso_regex.clone();

async move {
let main_checksums = capture_page(&checksum_mirror).await;
let runit_checksums = capture_page(&runit_checksum_mirror).await;
let mut checksums = main_checksums
.iter()
.chain(runit_checksums.iter())
.flat_map(|cs| {
cs.lines()
.skip_while(|l| !l.starts_with("sha256"))
.filter_map(|l| l.split_once(" ").map(|(a, b)| (b.to_string(), a.to_string())))
})
.collect::<HashMap<String, String>>();
let main_checksums = capture_page(&checksum_mirror).await.map(skip_until_sha256).unwrap_or_default();
let runit_checksums = capture_page(&runit_checksum_mirror).await.map(skip_until_sha256);
let checksums = main_checksums + "\n" + &runit_checksums.unwrap_or_default();
let mut checksums = ChecksumSeparation::Whitespace.build_with_data(&checksums).await;

let page = capture_page(&mirror).await?;
let iso_regex = iso_regex.clone();
Expand Down Expand Up @@ -75,3 +75,14 @@ impl Distro for Antix {
.into()
}
}

pub struct BunsenLabs;
impl Distro for BunsenLabs {
const NAME: &'static str = "bunsenlabs";
const PRETTY_NAME: &'static str = "BunsenLabs";
const HOMEPAGE: Option<&'static str> = Some("https://www.bunsenlabs.org/");
const DESCRIPTION: Option<&'static str> = Some("Light-weight and easily customizable Openbox desktop. The project is a community continuation of CrunchBang Linux.");
async fn generate_configs() -> Option<Vec<Config>> {
todo!()
}
}
13 changes: 2 additions & 11 deletions src/linux/fedora_redhat.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
use crate::{
store_data::{Arch, Config, Distro, Source, WebSource},
store_data::{Arch, ChecksumSeparation, Config, Distro, Source, WebSource},
utils::capture_page,
};
use regex::Regex;
use std::collections::HashMap;
use std::sync::Arc;

const ALMA_MIRROR: &str = "https://repo.almalinux.org/almalinux/";
Expand All @@ -19,7 +18,6 @@ impl Distro for Alma {

let releases_regex = Regex::new(r#"<a href="([0-9]+)/""#).unwrap();
let iso_regex = Arc::new(Regex::new(r#"<a href="(AlmaLinux-[0-9]+-latest-(?:x86_64|aarch64)-([^-]+).iso)">"#).unwrap());
let checksum_regex = Arc::new(Regex::new(r#"SHA256 \(([^)]+)\) = ([0-9a-f]+)"#).unwrap());

let futures = releases_regex.captures_iter(&releases).flat_map(|r| {
let release = r[1].to_string();
Expand All @@ -28,18 +26,11 @@ impl Distro for Alma {
.map(|arch| {
let release = release.clone();
let iso_regex = iso_regex.clone();
let checksum_regex = checksum_regex.clone();
let mirror = format!("{ALMA_MIRROR}{release}/isos/{arch}/");

async move {
let page = capture_page(&mirror).await?;
let checksum_page = capture_page(&format!("{mirror}CHECKSUM")).await;
let checksums = checksum_page.map(|cs| {
checksum_regex
.captures_iter(&cs)
.map(|c| (c[1].to_string(), c[2].to_string()))
.collect::<HashMap<String, String>>()
});
let checksums = ChecksumSeparation::Sha256Regex.build(&format!("{mirror}CHECKSUM")).await;

Some(
iso_regex
Expand Down
37 changes: 37 additions & 0 deletions src/store_data.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use crate::utils::all_valid;
use once_cell::sync::Lazy;
pub use quickemu::config::Arch;
pub use quickget::data_structures::{ArchiveFormat, Config, Disk, Source, WebSource, OS};
use regex::Regex;
use std::{collections::HashMap, sync::Arc};

pub trait Distro {
const NAME: &'static str;
Expand Down Expand Up @@ -89,3 +92,37 @@ pub fn extract_disk_urls(sources: Option<&[Disk]>) -> Vec<String> {
})
.collect()
}

pub static DEFAULT_SHA256_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r#"SHA256 \(([^)]+)\) = ([0-9a-f]+)"#).unwrap());

pub enum ChecksumSeparation {
Whitespace,
Sha256Regex,
CustomRegex(Arc<Regex>, usize, usize),
}

impl ChecksumSeparation {
pub async fn build(self, url: &str) -> Option<HashMap<String, String>> {
let data = crate::utils::capture_page(url).await?;
Some(self.build_with_data(&data).await)
}
pub async fn build_with_data(self, data: &str) -> HashMap<String, String> {
match self {
Self::Whitespace => data
.lines()
.filter_map(|l| {
l.split_once(' ')
.map(|(hash, file)| (file.trim().to_string(), hash.trim().to_string()))
})
.collect(),
Self::Sha256Regex => DEFAULT_SHA256_REGEX
.captures_iter(data)
.map(|c| (c[1].to_string(), c[2].to_string()))
.collect(),
Self::CustomRegex(regex, keyindex, valueindex) => regex
.captures_iter(data)
.map(|c| (c[keyindex].to_string(), c[valueindex].to_string()))
.collect(),
}
}
}

0 comments on commit 1a546a4

Please sign in to comment.