add outline of gao mateer - ppp: milestone 2

Cargo.toml

@@ -11,6 +11,7 @@ debug = true
 members = [
      "./reed-solomon-tester",
      "./reed-solomon-novelpoly",
+     "./reed-solomon-gao-mateer",
      "./reed-solomon-novelpoly-fuzzit",
      "./reed-solomon-benches",
 ]

reed-solomon-gao-mateer/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "reed-solomon-gao-mateer"
+version = "1.0.1-alpha.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+repository = "https://github.com/paritytech/reed-solomon-novelpoly"
+edition = "2018"
+license = "Apache-2.0 AND MIT"
+description = "An implementation of a reed solomon code / encoder / decoder with complexity `O(n lg(n))`"
+keywords = ["reed-solomon", "erasure", "encoding", "algorithm"]
+readme = "../README.md"
+
+[build-dependencies]
+derive_more = { version = "0.99.0", default-features = false, features = ["add_assign", "add"] }
+fs-err = "2.5.0"
+
+[dependencies]
+reed-solomon-erasure = { version = "4.0", features = ["simd-accel"], optional = true }
+static_init = "0.5.2"
+
+thiserror = "1.0.23"
+derive_more = { version = "0.99.0", default-features = false, features = ["add_assign", "add"] }
+itertools = "0.10.0"
+
+[dev-dependencies]
+reed-solomon-tester = { path = "../reed-solomon-tester" }
+rand = { version = "0.8.3", features = ["alloc", "small_rng"] }
+assert_matches = "1.5.0"

reed-solomon-gao-mateer/inc_gen_field_tables.rs

@@ -0,0 +1,89 @@
+use std::io;
+use std::env;
+use std::fmt;
+use std::path::PathBuf;
+use fs_err as fs;
+
+/// Write Rust `const` declaration
+pub fn write_const<W, T>(mut w: W, name: &str, value: &T, type_name: &str) -> io::Result<()>
+where
+	W: io::Write,
+	T: fmt::Debug,
+{
+	writeln!(w, r###"#[allow(unused)]
+pub(crate) static {}: {} = {:#?};"###, name, type_name, value)
+}
+
+/// Compute tables determined solely by the field, which never depend
+/// upon the FFT domain or erasure coding paramaters.
+///
+/// We compute `LOG_TABLE` and `EXP_TABLE` here of course.  We compute
+/// the Walsh transform table `LOG_WALSH` here too because we never figured
+/// out how to shrink `LOG_WALSH` below the size of the full field (TODO).
+/// We thus assume it depends only upon the field for now.
+#[allow(unused)]
+fn write_field_tables<W: io::Write>(mut w: W) -> io::Result<()> {
+	let mut log_table: [Elt; FIELD_SIZE] = [0; FIELD_SIZE];
+	let mut exp_table: [Elt; FIELD_SIZE] = [0; FIELD_SIZE];
+
+	let mas: Elt = (1 << FIELD_BITS - 1) - 1;
+	let mut state: usize = 1;
+	for i in 0_usize..(ONEMASK as usize) {
+		exp_table[state] = i as Elt;
+		if (state >> FIELD_BITS - 1) != 0 {
+			state &= mas as usize;
+			state = state << 1_usize ^ GENERATOR as usize;
+		} else {
+			state <<= 1;
+		}
+	}
+	exp_table[0] = ONEMASK;
+
+	log_table[0] = 0;
+	for i in 0..FIELD_BITS {
+		for j in 0..(1 << i) {
+			log_table[j + (1 << i)] = log_table[j] ^ BASE[i];
+		}
+	}
+	for i in 0..FIELD_SIZE {
+		log_table[i] = exp_table[log_table[i] as usize];
+	}
+
+	for i in 0..FIELD_SIZE {
+		exp_table[log_table[i] as usize] = i as Elt;
+	}
+	exp_table[ONEMASK as usize] = exp_table[0];
+
+	write_const(&mut w, "LOG_TABLE", &log_table, "[Elt; FIELD_SIZE]")?;
+	write_const(&mut w, "EXP_TABLE", &exp_table, "[Elt; FIELD_SIZE]")?;
+
+	// mem_cpy(&mut log_walsh[..], &log_table[..]);
+	let log_walsh = log_table.clone();
+	let mut log_walsh = unsafe { core::mem::transmute::<_, [Multiplier; FIELD_SIZE]>(log_walsh) };
+	log_walsh[0] = Multiplier(0);
+	walsh(&mut log_walsh[..], FIELD_SIZE);
+
+	write_const(w, "LOG_WALSH", &log_walsh, "[Multiplier; FIELD_SIZE]")?;
+	Ok(())
+}
+
+/// Create tables file
+///
+/// We'll eventually need a seperate tables.rs build target because cargo
+/// dislikes build artifacts appearing outside env!("OUT_DIR") and we
+/// require tables to build other tables.
+/// ref.  https://doc.rust-lang.org/cargo/reference/build-scripts.html#outputs-of-the-build-script
+pub fn gen_field_tables() -> io::Result<()> {
+	// to avoid a circular loop, we need to import a dummy
+	// table, such that we do not depend on the thing we are
+	// about to spawn
+	println!("cargo:rustc-cfg=table_bootstrap_complete");
+
+	let out = env::var("OUT_DIR").expect("OUT_DIR is set by cargo after process launch. qed");
+
+	let path = PathBuf::from(out).join(format!("table_{}.rs", FIELD_NAME));
+	let f = fs::OpenOptions::new().create(true).truncate(true).write(true).open(path)?;
+	write_field_tables(f)?;
+
+	Ok(())
+}

reed-solomon-gao-mateer/src/errors.rs

@@ -0,0 +1,28 @@
+/// Error type for interfacing with the novel poly basis
+#[non_exhaustive]
+#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]
+pub enum Error {
+	#[error("Number of wanted shards {0} exceeds max of 2^16")]
+	WantedShardCountTooHigh(usize),
+
+	#[error("Number of wanted shards must be at least 2, but is {0}")]
+	WantedShardCountTooLow(usize),
+
+	#[error("Number of wanted payload shards must be at least 1, but is {0}")]
+	WantedPayloadShardCountTooLow(usize),
+
+	#[error("Size of the payload is zero")]
+	PayloadSizeIsZero,
+
+	#[error("Needs at least {min} shards of {all} to recover, have {have}")]
+	NeedMoreShards { have: usize, min: usize, all: usize },
+
+	#[error("Parameters: n (= {n}) and k (= {k}) both must be a power of 2")]
+	ParamterMustBePowerOf2 { n: usize, k: usize },
+
+	#[error("Shards do have inconsistent lengths: first = {first}, other = {other})")]
+	InconsistentShardLengths { first: usize, other: usize },
+}
+
+/// Result alias to simplify API.
+pub type Result<T> = std::result::Result<T, Error>;

reed-solomon-gao-mateer/src/gao_mateer.rs

@@ -0,0 +1,191 @@
+// let evaluated = fft::<Polynom, Field>(values: &[Field::Element]);
+
+
+pub type Ele = u16;
+
+trait ExtensionField {
+	const BITS: usize = 16;
+	type Element = Ele;
+}
+
+pub struct Polynom<F: ExtensionField> {
+	pub polynom: u16,
+}
+
+impl<F: ExtensionField> Polynom<F> {
+	pub const fn deg(&self) -> usize {
+		(u16::BITS - self.polynom.leading_ones()) as usize
+	}
+
+}
+
+impl<F: ExtensionField> std::ops::Add<Polynom<F>> for  Polynom<F> {
+	fn add(&self, other: Polynom) -> Self {
+		self.polynom ^= other.polynom;
+		self.clone()
+	}
+}
+
+impl<F: ExtensionField> std::ops::Mul<Polynom<F>> for Polynom<F> {
+	fn mul(&self, other: Polynom) -> Self {
+		// let mut acc = 0;
+		// for (idx, bit) in other.bits().enumerate() {
+		//     if bit == 1 { acc ^= (self.bits() << idx) }
+		// }
+		// self.polynom *= other.polynom;
+		self.clone()
+	}
+}
+
+
+type K = usize;
+
+type T = usize;
+const T: T = 2;
+
+#[inline(always)]
+fn lower(k: K, t: T) -> usize {
+	t * (1 << k)
+}
+
+#[inline(always)]
+fn upper(k: K, t: T) -> usize {
+	t * (1 << (k+1))
+}
+
+#[inline(always)]
+fn find_k_between(t: T, n: usize) -> K {
+	let mut k: K = log2(n) as K;
+	let mut next = k;
+	while lower(k, t) < n {
+		k = next;
+		next += 1;
+	}
+	while n <= upper(k, t) {
+		k = next;
+		next += 1;
+	}
+	k
+}
+
+// values is `B` in the paper
+fn fft_inner<F: ExtensionField>(f: Polynom<F>, m: usize, B: &[Ele]) -> Vec<Ele> {
+	let n  = 1_usize << m;
+	assert!(f.deg() < n);
+	if m == 1 {
+		return vec![
+			f.eval(0 as Ele),
+			f.eval(B[0]),
+		]
+	}
+
+	// TODO linearly independent vs what?
+	let betas = B;
+	let beta_m = betas[m];
+
+	// linear mapping of the polynomial
+	let g = f * beta_m;
+
+	let taylor = taylor_expansion(f, n, T);
+
+	// G
+	let gammas = Vec::from_iter(betas.into_iter().map(|beta| beta / beta_m));
+	// D
+	let sigmas = Vec::from_iter(gammas.into_iter().map(|gamma| gamma * gamma  - gamma));
+
+	let us = fft_inner(taylor.g0, m-1, sigmas);
+	let vs = fft_inner(taylor.g1, m-1, sigmas);
+
+	// FIXME is this `Ele` or just a bit?
+	const N_HALF: usize = n/2;
+	let mut ws  = [0 as Ele; N_HALF];
+	assert!(us.len() == k);
+	assert!(2 * k == ws.len());
+	for (i,(u,v)) in us.into_iter().zip(vs.into_iter).enumerate() {
+		ws[i] = u[i] + v[i] * betas[i];
+		ws[i + k] = ws[i] + v[i];
+	}
+	ws
+}
+
+
+/// Create a taylor expansion consisting of `f0` and `f1` at
+/// the point `x^t - x` with `t > 1`.
+///
+/// `n` is the number of ...
+pub fn taylor_expansion<F: FieldExtension>(f: Polynom<F>, n: usize, t: T) -> Polynom<F> {
+	assert!(f.deg() < n);
+
+	let k = find_k_between(t, n);
+
+	let [f0, f1, f2] = split_polynom(f, t, k);
+
+	// polynom domain ops
+	let h = f0 + f1;
+	let g0 = f0 + Polynom::x_at(1 << k) * h;
+	let g1 = h + Polynom::x_at((t-1)*(1 << k)) * h;
+
+	// recurse
+	let v1 = taylor_expansion(g0, t * (1<<k), t);
+	let v2 = taylor_expansion(g1, n - t * (1<<k), t);
+
+	// assumes a problem of for `n = 2^(k+1)`
+	v1 | (v2 << n/2)
+}
+
+
+
+struct PolySplit<F> {
+	f0: Polynom<F>,
+	f1: Polynom<F>,
+	f2: Polynom<F>,
+}
+
+fn split_polynom<F>(polynom: Polynom<F>, k: K, t: T) -> PolySplit<F> {
+	// assert!(T must be of 2^x!);
+
+	let mask0 = 1 << k; // deg f0 < 0..2^k
+	let mask2 = T * (1 << k); // deg f2 < t * 2^k
+	let mask1 = !mask0 & !mask2; // deg f1 < t * 2^k - 2^k
+
+	let mut f0 = polynom.clone();
+	f0.polynom &= mask0;
+
+	let mut f1 = polynom.clone();
+	f1.polynom &= mask1;
+
+	let mut f2 = polynom.clone();
+	f2.polynom &= mask2;
+
+	PolySplit { f0
+	, f1: (), f2: () }
+}
+
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn bounds_work() {
+		for k in 0..128 {
+			for t in 2..n {
+				assert!(lower(k,t) < upper(k,t));
+			}
+		}
+
+		assert_eq!(lower(16,2), 4);
+		assert_eq!(upper(16,2), 8);
+	}
+	#[test]
+	fn select_k() {
+		for z in 0..13 {
+			let n = 1 << z;
+			for t in 2..n {
+				let k = find_k_between(t, n);
+				assert!(n <= upper(k,t));
+				assert!(lower(k,t) < n);
+			}
+		}
+	}
+}