Add the rest of the distros beginning in E & F (#8)

* feat: Add EndeavourOS

* feat: Add EndlessOS

* feat: Add Fedora (all editions & architectures)

* refactor: Allow trailing commas in macro call

* feat: Add FreeDOS

---------

Co-authored-by: Martin Wimpress <304639+flexiondotorg@users.noreply.github.com>

src/linux.rs

@@ -4,8 +4,8 @@ mod fedora_redhat;
 mod independent;
 mod ubuntu;
 
-pub(crate) use arch::{manjaro::BigLinux, ArchLinux, Archcraft, ArcoLinux, ArtixLinux, AthenaOS, BlendOS, CachyOS};
-pub(crate) use debian::{Antix, BunsenLabs, CrunchbangPlusPlus, Debian, Devuan, EasyOS};
-pub(crate) use fedora_redhat::{Alma, Bazzite, CentOSStream};
+pub(crate) use arch::{manjaro::BigLinux, ArchLinux, Archcraft, ArcoLinux, ArtixLinux, AthenaOS, BlendOS, CachyOS, EndeavourOS};
+pub(crate) use debian::{Antix, BunsenLabs, CrunchbangPlusPlus, Debian, Devuan, EasyOS, EndlessOS};
+pub(crate) use fedora_redhat::{Alma, Bazzite, CentOSStream, Fedora};
 pub(crate) use independent::{Alpine, Batocera, ChimeraLinux, NixOS};
 pub(crate) use ubuntu::{Bodhi, Edubuntu, Elementary, Kubuntu, Lubuntu, Ubuntu, UbuntuBudgie, UbuntuCinnamon, UbuntuKylin, UbuntuMATE, UbuntuServer, UbuntuStudio, UbuntuUnity, Xubuntu};

src/linux/arch.rs

@@ -315,3 +315,32 @@ impl Distro for CachyOS {
             .into()
     }
 }
+
+const ENDEAVOUROS_MIRROR: &str = "https://mirror.alpix.eu/endeavouros/iso/";
+
+pub struct EndeavourOS;
+impl Distro for EndeavourOS {
+    const NAME: &'static str = "endeavouros";
+    const PRETTY_NAME: &'static str = "EndeavourOS";
+    const HOMEPAGE: Option<&'static str> = Some("https://endeavouros.com/");
+    const DESCRIPTION: Option<&'static str> = Some("Provides an Arch experience without the hassle of installing it manually for both x86_64 and ARM systems.");
+    async fn generate_configs() -> Option<Vec<Config>> {
+        let iso_regex = Regex::new(r#"href="(EndeavourOS_[^\d]+(\d{4}.\d{2}.\d{2}).iso)""#).unwrap();
+        let page = capture_page(ENDEAVOUROS_MIRROR).await?;
+        let futures = iso_regex.captures_iter(&page).map(|c| c.extract()).map(|(_, [iso, release])| {
+            let url = ENDEAVOUROS_MIRROR.to_string() + iso;
+            let checksum_url = url.clone() + ".sha512sum";
+            async move {
+                let checksum = capture_page(&checksum_url)
+                    .await
+                    .and_then(|c| c.split_whitespace().next().map(ToString::to_string));
+                Config {
+                    release: Some(release.to_string()),
+                    iso: Some(vec![Source::Web(WebSource::new(url, checksum, None, None))]),
+                    ..Default::default()
+                }
+            }
+        });
+        futures::future::join_all(futures).await.into()
+    }
+}

src/linux/debian.rs

@@ -433,3 +433,62 @@ impl Distro for EasyOS {
             .into()
     }
 }
+
+const ENDLESS_DL_MIRROR: &str = "https://images-dl.endlessm.com/release/";
+const ENDLESS_DATA_MIRROR: &str = "https://mirror.leitecastro.com/endless/release/";
+
+pub struct EndlessOS;
+impl Distro for EndlessOS {
+    const NAME: &'static str = "endless";
+    const PRETTY_NAME: &'static str = "Endless OS";
+    const HOMEPAGE: Option<&'static str> = Some("https://endlessos.org/");
+    const DESCRIPTION: Option<&'static str> = Some("Completely Free, User-Friendly Operating System Packed with Educational Tools, Games, and More.");
+    async fn generate_configs() -> Option<Vec<Config>> {
+        let release_html = capture_page(ENDLESS_DATA_MIRROR).await?;
+        let release_regex = Regex::new(r#"href="(\d+(?:.\d+){2})\/""#).unwrap();
+        let edition_regex = Arc::new(Regex::new(r#"href="([^./]+)"#).unwrap());
+        let iso_regex = Arc::new(Regex::new(r#"href="(eos-eos[\d.]+-amd64-amd64.[-\d]+.[^.]+.iso)""#).unwrap());
+
+        let futures = release_regex.captures_iter(&release_html).map(|c| {
+            let release = c[1].to_string();
+            let mirror = ENDLESS_DATA_MIRROR.to_string() + &release + "/eos-amd64-amd64/";
+            let edition_regex = edition_regex.clone();
+            let iso_regex = iso_regex.clone();
+            async move {
+                let edition_html = capture_page(&mirror).await?;
+                let futures = edition_regex.captures_iter(&edition_html).map(|c| {
+                    let edition = c[1].to_string();
+                    let mirror = mirror.clone() + &edition + "/";
+                    let iso_regex = iso_regex.clone();
+                    let release = release.clone();
+                    async move {
+                        let page = capture_page(&mirror).await?;
+                        let iso = &iso_regex.captures(&page)?[1];
+                        let url = format!("{ENDLESS_DL_MIRROR}{release}/eos-amd64-amd64/{edition}/{iso}");
+
+                        let checksum_url = url.clone() + ".sha256";
+                        let checksum = capture_page(&checksum_url)
+                            .await
+                            .and_then(|cs| cs.split_whitespace().next().map(ToString::to_string));
+                        Some(Config {
+                            release: Some(release),
+                            edition: Some(edition),
+                            iso: Some(vec![Source::Web(WebSource::new(url, checksum, None, None))]),
+                            ..Default::default()
+                        })
+                    }
+                });
+                Some(futures::future::join_all(futures).await)
+            }
+        });
+
+        futures::future::join_all(futures)
+            .await
+            .into_iter()
+            .flatten()
+            .flatten()
+            .flatten()
+            .collect::<Vec<_>>()
+            .into()
+    }
+}

src/linux/fedora_redhat.rs

@@ -1,7 +1,9 @@
 use crate::{
     store_data::{Arch, ChecksumSeparation, Config, Distro, Source, WebSource},
-    utils::capture_page,
+    utils::{arch_from_str, capture_page, FedoraRelease, GatherData},
 };
+use quickemu::config::DiskFormat;
+use quickget::data_structures::{ArchiveFormat, Disk};
 use regex::Regex;
 use std::sync::Arc;
 
@@ -182,3 +184,65 @@ impl Distro for CentOSStream {
             .into()
     }
 }
+
+const FEDORA_RELEASE_URL: &str = "https://fedoraproject.org/releases.json";
+const VALID_FEDORA_FILETYPES: [&str; 2] = ["raw.xz", "iso"];
+const BLACKLISTED_EDITIONS: [&str; 2] = ["Server", "Cloud_Base"];
+
+pub struct Fedora;
+impl Distro for Fedora {
+    const NAME: &'static str = "fedora";
+    const PRETTY_NAME: &'static str = "Fedora";
+    const HOMEPAGE: Option<&'static str> = Some("https://fedoraproject.org/");
+    const DESCRIPTION: Option<&'static str> = Some("Innovative platform for hardware, clouds, and containers, built with love by you.");
+    async fn generate_configs() -> Option<Vec<Config>> {
+        let mut releases = FedoraRelease::gather_data(FEDORA_RELEASE_URL).await?;
+        // Filter out unwanted filetypes and editions
+        releases.retain(|FedoraRelease { link, edition, .. }| VALID_FEDORA_FILETYPES.iter().any(|ext| link.ends_with(ext)) && !BLACKLISTED_EDITIONS.iter().any(|e| edition == e));
+
+        releases
+            .iter_mut()
+            .for_each(|FedoraRelease { link, edition, archive_format, .. }| {
+                if link.ends_with("raw.xz") {
+                    *edition += "_preinstalled";
+                    *archive_format = Some(ArchiveFormat::Xz);
+                }
+            });
+        releases.dedup_by(|a, b| a.release == b.release && a.edition == b.edition);
+
+        releases
+            .into_iter()
+            .filter_map(
+                |FedoraRelease {
+                     release,
+                     edition,
+                     arch,
+                     link,
+                     archive_format,
+                     sha256,
+                 }| {
+                    let is_disk_image = archive_format.is_some();
+                    let source = Source::Web(WebSource::new(link, sha256, archive_format, None));
+                    let arch = arch_from_str(&arch)?;
+                    let mut config = Config {
+                        release: Some(release),
+                        edition: Some(edition),
+                        arch,
+                        ..Default::default()
+                    };
+                    if is_disk_image {
+                        config.disk_images = Some(vec![Disk {
+                            source,
+                            format: DiskFormat::Raw,
+                            ..Default::default()
+                        }])
+                    } else {
+                        config.iso = Some(vec![source]);
+                    }
+                    Some(config)
+                },
+            )
+            .collect::<Vec<Config>>()
+            .into()
+    }
+}

src/main.rs

@@ -1,5 +1,6 @@
 mod bsd;
 mod linux;
+mod other;
 mod store_data;
 mod utils;
 
@@ -48,7 +49,11 @@ async fn main() {
         linux::Debian,
         linux::Devuan,
         bsd::DragonFlyBSD,
-        linux::EasyOS
+        linux::EasyOS,
+        linux::EndeavourOS,
+        linux::EndlessOS,
+        linux::Fedora,
+        other::FreeDOS,
     );
 
     let distros = futures::future::join_all(futures)

src/other/mod.rs

@@ -0,0 +1,70 @@
+use crate::store_data::{ArchiveFormat, ChecksumSeparation, Config, Distro, Source, WebSource};
+use crate::utils::capture_page;
+use quickemu::config::GuestOS;
+use regex::Regex;
+use std::sync::Arc;
+
+const FREEDOS_MIRROR: &str = "https://www.ibiblio.org/pub/micro/pc-stuff/freedos/files/distributions/";
+
+pub struct FreeDOS;
+impl Distro for FreeDOS {
+    const NAME: &'static str = "freedos";
+    const PRETTY_NAME: &'static str = "FreeDOS";
+    const HOMEPAGE: Option<&'static str> = Some("https://www.freedos.org/");
+    const DESCRIPTION: Option<&'static str> = Some("DOS-compatible operating system that you can use to play classic DOS games, run legacy business software, or develop embedded systems.");
+    async fn generate_configs() -> Option<Vec<Config>> {
+        let release_html = capture_page(FREEDOS_MIRROR).await?;
+        let release_regex = Regex::new(r#"href="(\d+\.\d+)/""#).unwrap();
+        let iso_regex = Arc::new(Regex::new(r#"href="(FD\d+-?(.*?CD)\.(iso|zip))""#).unwrap());
+        let checksum_regex = Arc::new(Regex::new(r#"FD\d+.sha|verify.txt"#).unwrap());
+
+        let futures = release_regex.captures_iter(&release_html).map(|c| {
+            let release = c[1].to_string();
+            let mirror = format!("{FREEDOS_MIRROR}{release}/official/");
+            let iso_regex = iso_regex.clone();
+            let checksum_regex = checksum_regex.clone();
+            async move {
+                let page = capture_page(&mirror).await?;
+
+                let mut checksums = match checksum_regex.find(&page) {
+                    Some(cs_match) => {
+                        let checksum_url = format!("{mirror}{}", cs_match.as_str());
+                        ChecksumSeparation::Whitespace.build(&checksum_url).await
+                    }
+                    None => None,
+                };
+
+                Some(
+                    iso_regex
+                        .captures_iter(&page)
+                        .map(|c| c.extract())
+                        .map(|(_, [iso, edition, filetype])| {
+                            let url = format!("{mirror}{iso}");
+                            let checksum = checksums.as_mut().and_then(|cs| cs.remove(iso));
+                            let archive_format = match filetype {
+                                "iso" => None,
+                                "zip" => Some(ArchiveFormat::Zip),
+                                _ => panic!("FreeDOS: Regex allowed an invalid filetype"),
+                            };
+                            Config {
+                                guest_os: GuestOS::FreeDOS,
+                                release: Some(release.clone()),
+                                edition: Some(edition.to_string()),
+                                iso: Some(vec![Source::Web(WebSource::new(url, checksum, archive_format, None))]),
+                                ..Default::default()
+                            }
+                        })
+                        .collect::<Vec<Config>>(),
+                )
+            }
+        });
+
+        futures::future::join_all(futures)
+            .await
+            .into_iter()
+            .flatten()
+            .flatten()
+            .collect::<Vec<Config>>()
+            .into()
+    }
+}

src/utils.rs

@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 use once_cell::sync::Lazy;
+use quickget::data_structures::ArchiveFormat;
 use quickemu::config::Arch;
 use reqwest::{StatusCode, Url};
 use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
@@ -118,9 +119,30 @@ pub struct GithubAsset {
     pub browser_download_url: String,
 }
 
+impl GatherData for FedoraRelease {
+    type Output = Vec<FedoraRelease>;
+    async fn gather_data(url: &str) -> Option<Self::Output> {
+        let data = capture_page(url).await?;
+        serde_json::from_str(&data).ok()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct FedoraRelease {
+    #[serde(rename = "version")]
+    pub release: String,
+    pub arch: String,
+    pub link: String,
+    #[serde(rename = "subvariant")]
+    pub edition: String,
+    pub sha256: Option<String>,
+    // This is not contained within Fedora's data, we'll add it ourselves based on the file extension
+    pub archive_format: Option<ArchiveFormat>,
+}
+
 #[macro_export]
 macro_rules! spawn_distros {
-    ($( $distro:ty ),* ) => {{
+    ($( $distro:ty ),* $(,)? ) => {{
         let mut handles = Vec::new();
         $(
             let handle = spawn(<$distro>::to_os());