From fd2007ca041ad8833df5f3dac59d6a5bf9ade98d Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 11:07:32 +0100 Subject: [PATCH 01/43] Sketch of Store trait --- data-model/src/lib.rs | 2 + data-model/src/store.rs | 104 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 data-model/src/store.rs diff --git a/data-model/src/lib.rs b/data-model/src/lib.rs index e7db738..190d9d0 100644 --- a/data-model/src/lib.rs +++ b/data-model/src/lib.rs @@ -33,3 +33,5 @@ pub use parameters::*; mod path; pub use path::*; mod relative_encodings; +mod store; +pub use store::*; diff --git a/data-model/src/store.rs b/data-model/src/store.rs new file mode 100644 index 0000000..e8b8472 --- /dev/null +++ b/data-model/src/store.rs @@ -0,0 +1,104 @@ +use crate::{ + entry::{AuthorisedEntry, Entry}, + grouping::{area::Area, area_of_interest::AreaOfInterest, range_3d::Range3d}, + parameters::{NamespaceId, PayloadDigest, SubspaceId}, + path::Path, +}; + +// eheheheh +type Payload = String; + +/// Returned when an entry is successfully ingested into a [`Store`]. +pub enum EntryIngestionSuccess< + N: NamespaceId, + S: SubspaceId, + P: Path, + PD: PayloadDigest, + AT, + AEI: Iterator>, +> { + /// The entry was successfully ingested. + Success(AuthorisedEntry), + /// The entry was successfully ingested and prefix pruned some entries. + SuccessAndPruned(AEI), +} + +pub enum EntryIngestionError { + NewerPrefix { + /// The obsolete entry which is pruned by the newer entry. + pruned: AuthorisedEntry, + /// The newer entry whose path prefixes the obsolete entry's. + newer: AuthorisedEntry, + }, + Obsolete { + /// The obsolete entry which was not ingested. + obsolete: AuthorisedEntry, + /// The newer entry which was not overwritten. + newer: AuthorisedEntry, + }, + /// The entry belonged to another namespace. + WrongNamespace(AuthorisedEntry), +} + +/// Returned when a [`Payload`] fails to be ingested into the [`Store`the ] +pub enum PayloadIngestionError { + /// None of the entries in the store reference this payload. + NotEntryReference, + /// The payload is already held in storage. + AlreadyHaveIt, + /// The received's payload digest is not what was expected. + DigestMismatch, + /// Try deleting some files!!! + SomethingElseWentWrong, +} + +pub enum QueryOrder { + Subspace, + Path, + Timestamp, +} + +/// A [`Store`] is a set of [`AuthorisedEntry`]. +pub trait Store< + N: NamespaceId, + S: SubspaceId, + P: Path, + PD: PayloadDigest, + AT, + AEI: Iterator>, + // Do we need a trait for fingerprints? + FP, +> +{ + /// Attempt to store an [`AuthorisedEntry`] in the [`Store`]. + fn ingest_entry( + &self, + expected_digest: PD, + authorised_entry: AuthorisedEntry, + ) -> Result, EntryIngestionError>; + // I'm told this is too complex, but I'm also told trait bounds can't be enforced on type aliases. + + /// Attempt to ingest a payload for a given [`AuthorisedEntry`]. + fn ingest_payload(&self, payload: Payload) -> Result<(), PayloadIngestionError>; + + /// Query the store's set of [`AuthorisedEntry`] using an [`Area`]. + fn query_area(&self, area: Area) -> AEI; + + /// Query the store's set of [`AuthorisedEntry`] using a [`Range3d`]. + fn query_range(&self, area: Range3d, order: QueryOrder, reverse: bool) -> AEI; + + /// Try to get the payload for a given [`Entry`]. + fn getPayload(&self, entry: &Entry) -> Option; + + /// Summarise a given range into a fingerprint. + fn summarise(&self, range: Range3d) -> FP; + + /// Split a range into two approximately equally sized ranges, or return the original range if it can't be split. + fn split_range( + &self, + range: Range3d, + ) -> Result<(Range3d, Range3d), Range3d>; + + /// Transform an [`AreaOfInterest`] into a concrete range by using its `max_count` and `max_size` properties as clamps. + fn aoi_to_range(&self, aoi: AreaOfInterest) -> Range3d; +} From 54968da4b5818b1a16eb3edf203881d493c60d73 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 26 Jun 2024 14:38:41 +0100 Subject: [PATCH 02/43] ok --- data-model/src/store.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index e8b8472..527eaba 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -18,24 +18,19 @@ pub enum EntryIngestionSuccess< AEI: Iterator>, > { /// The entry was successfully ingested. - Success(AuthorisedEntry), + Success, /// The entry was successfully ingested and prefix pruned some entries. SuccessAndPruned(AEI), -} - -pub enum EntryIngestionError { - NewerPrefix { - /// The obsolete entry which is pruned by the newer entry. - pruned: AuthorisedEntry, - /// The newer entry whose path prefixes the obsolete entry's. - newer: AuthorisedEntry, - }, + /// The entry was not ingested because a newer entry with same prfieiuhnsuthaeusntaheouonsth Obsolete { /// The obsolete entry which was not ingested. obsolete: AuthorisedEntry, /// The newer entry which was not overwritten. newer: AuthorisedEntry, }, +} + +pub enum EntryIngestionError { /// The entry belonged to another namespace. WrongNamespace(AuthorisedEntry), } @@ -70,19 +65,27 @@ pub trait Store< FP, > { + // associtade type named AEI, which must implement the ufotofu producer trait / or std Iterator + // associated type of something went wrong error. + /// Attempt to store an [`AuthorisedEntry`] in the [`Store`]. fn ingest_entry( + // this will end up not working but in theory it's right. ask aljoscha later. + // later came. &self, - expected_digest: PD, authorised_entry: AuthorisedEntry, ) -> Result, EntryIngestionError>; // I'm told this is too complex, but I'm also told trait bounds can't be enforced on type aliases. /// Attempt to ingest a payload for a given [`AuthorisedEntry`]. - fn ingest_payload(&self, payload: Payload) -> Result<(), PayloadIngestionError>; + fn ingest_payload( + &self, + expected_digest: PD, + payload: Payload, + ) -> Result<(), PayloadIngestionError>; /// Query the store's set of [`AuthorisedEntry`] using an [`Area`]. - fn query_area(&self, area: Area) -> AEI; + fn query_area(&self, area: Area, order: QueryOrder, reverse: bool) -> AEI; /// Query the store's set of [`AuthorisedEntry`] using a [`Range3d`]. fn query_range(&self, area: Range3d, order: QueryOrder, reverse: bool) -> AEI; From b7ad9b0e1b8e6a3d33bd3412c6316aa8f3840df9 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 27 Jun 2024 09:54:43 +0100 Subject: [PATCH 03/43] good comments --- data-model/src/store.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 527eaba..d992de4 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -75,7 +75,12 @@ pub trait Store< &self, authorised_entry: AuthorisedEntry, ) -> Result, EntryIngestionError>; - // I'm told this is too complex, but I'm also told trait bounds can't be enforced on type aliases. + + // commit / barf - ingest becomes queue_ingestion_entry? + // we want bulk ingestion of entries too + // forget_entry + // forget_range + // forget_area /// Attempt to ingest a payload for a given [`AuthorisedEntry`]. fn ingest_payload( From f2b490d68f5c1ffcbdf785420d115bd9c18a20ec Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 11:08:20 +0100 Subject: [PATCH 04/43] wip who cares --- data-model/src/store.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index d992de4..cb04a68 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -1,8 +1,9 @@ use crate::{ entry::{AuthorisedEntry, Entry}, - grouping::{area::Area, area_of_interest::AreaOfInterest, range_3d::Range3d}, + grouping::{Area, AreaOfInterest, Range3d}, parameters::{NamespaceId, PayloadDigest, SubspaceId}, path::Path, + AuthorisationToken, }; // eheheheh @@ -10,12 +11,13 @@ type Payload = String; /// Returned when an entry is successfully ingested into a [`Store`]. pub enum EntryIngestionSuccess< + const MCL: usize, + const MCC: usize, + const MPL: usize, N: NamespaceId, S: SubspaceId, - P: Path, PD: PayloadDigest, - AT, - AEI: Iterator>, + AT: AuthorisationToken, > { /// The entry was successfully ingested. Success, @@ -24,9 +26,9 @@ pub enum EntryIngestionSuccess< /// The entry was not ingested because a newer entry with same prfieiuhnsuthaeusntaheouonsth Obsolete { /// The obsolete entry which was not ingested. - obsolete: AuthorisedEntry, + obsolete: AuthorisedEntry, /// The newer entry which was not overwritten. - newer: AuthorisedEntry, + newer: AuthorisedEntry, }, } @@ -57,10 +59,8 @@ pub enum QueryOrder { pub trait Store< N: NamespaceId, S: SubspaceId, - P: Path, PD: PayloadDigest, AT, - AEI: Iterator>, // Do we need a trait for fingerprints? FP, > From c6bb42ed0c1220ed3bf07d096e62113f1f6a61c0 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 13:31:57 +0100 Subject: [PATCH 05/43] Begin rework: ingest_entry, append_bytes, LengthyEntry --- data-model/src/lengthy_entry.rs | 39 ++++++++++ data-model/src/lib.rs | 2 + data-model/src/store.rs | 121 +++++++++++++++++--------------- 3 files changed, 104 insertions(+), 58 deletions(-) create mode 100644 data-model/src/lengthy_entry.rs diff --git a/data-model/src/lengthy_entry.rs b/data-model/src/lengthy_entry.rs new file mode 100644 index 0000000..e3b3965 --- /dev/null +++ b/data-model/src/lengthy_entry.rs @@ -0,0 +1,39 @@ +use crate::{Entry, NamespaceId, PayloadDigest, SubspaceId}; + +/// An [`Entry`] together with information about how much of its payload a given [`Store`] holds. +/// +/// [Definition](https://willowprotocol.org/specs/3d-range-based-set-reconciliation/index.html#LengthyEntry) +pub struct LengthyEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, +{ + /// The Entry in question. + entry: Entry, + /// The number of consecutive bytes from the start of the entry’s payload that the peer holds. + available: u64, +} + +impl + LengthyEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, +{ + /// Create a new lengthy entry from a given [`Entry`] and the number of consecutive bytes from the start of the entry’s payload that are held. + pub fn new(entry: Entry, available: u64) -> Self { + Self { entry, available } + } + + /// The entry in question. + pub fn entry(&self) -> &Entry { + &self.entry + } + + /// The number of consecutive bytes from the start of the entry’s Payload that the peer holds. + pub fn available(&self) -> u64 { + self.available + } +} diff --git a/data-model/src/lib.rs b/data-model/src/lib.rs index 190d9d0..bd4dba5 100644 --- a/data-model/src/lib.rs +++ b/data-model/src/lib.rs @@ -27,6 +27,8 @@ mod entry; pub use entry::*; +mod lengthy_entry; +pub use lengthy_entry::*; pub mod grouping; mod parameters; pub use parameters::*; diff --git a/data-model/src/store.rs b/data-model/src/store.rs index cb04a68..e147ecd 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -1,14 +1,11 @@ +use ufotofu::nb::BulkProducer; + use crate::{ - entry::{AuthorisedEntry, Entry}, - grouping::{Area, AreaOfInterest, Range3d}, - parameters::{NamespaceId, PayloadDigest, SubspaceId}, - path::Path, - AuthorisationToken, + entry::AuthorisedEntry, + parameters::{AuthorisationToken, NamespaceId, PayloadDigest, SubspaceId}, + LengthyEntry, }; -// eheheheh -type Payload = String; - /// Returned when an entry is successfully ingested into a [`Store`]. pub enum EntryIngestionSuccess< const MCL: usize, @@ -22,8 +19,8 @@ pub enum EntryIngestionSuccess< /// The entry was successfully ingested. Success, /// The entry was successfully ingested and prefix pruned some entries. - SuccessAndPruned(AEI), - /// The entry was not ingested because a newer entry with same prfieiuhnsuthaeusntaheouonsth + SuccessAndPruned(Vec>), + /// The entry was not ingested because a newer entry with same Obsolete { /// The obsolete entry which was not ingested. obsolete: AuthorisedEntry, @@ -32,18 +29,41 @@ pub enum EntryIngestionSuccess< }, } -pub enum EntryIngestionError { +pub enum EntryIngestionError< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT, +> { /// The entry belonged to another namespace. - WrongNamespace(AuthorisedEntry), + WrongNamespace(AuthorisedEntry), +} + +/// Return when a payload is successfully appended to the [`Store`]. +pub enum PayloadAppendSuccess +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, +{ + /// The payload was appended to but not completed. + Appended(Vec>), + /// The payload was completed by the appendment. + Completed(Vec>), } -/// Returned when a [`Payload`] fails to be ingested into the [`Store`the ] -pub enum PayloadIngestionError { +/// Returned when a payload fails to be appended into the [`Store`]. +pub enum PayloadAppendError { /// None of the entries in the store reference this payload. NotEntryReference, /// The payload is already held in storage. AlreadyHaveIt, - /// The received's payload digest is not what was expected. + /// The received payload is larger than was expected. + PayloadTooLarge, + /// The completed payload's digest is not what was expected. DigestMismatch, /// Try deleting some files!!! SomethingElseWentWrong, @@ -55,58 +75,43 @@ pub enum QueryOrder { Timestamp, } -/// A [`Store`] is a set of [`AuthorisedEntry`]. -pub trait Store< +/// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. +pub trait Store +where N: NamespaceId, S: SubspaceId, PD: PayloadDigest, - AT, - // Do we need a trait for fingerprints? - FP, -> + AT: AuthorisationToken, { - // associtade type named AEI, which must implement the ufotofu producer trait / or std Iterator - // associated type of something went wrong error. + /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. + fn namespace_id() -> N; /// Attempt to store an [`AuthorisedEntry`] in the [`Store`]. + /// Will fail if the entry belonged to a different namespace than the store's. fn ingest_entry( - // this will end up not working but in theory it's right. ask aljoscha later. - // later came. &self, - authorised_entry: AuthorisedEntry, - ) -> Result, EntryIngestionError>; - - // commit / barf - ingest becomes queue_ingestion_entry? - // we want bulk ingestion of entries too - // forget_entry - // forget_range - // forget_area + authorised_entry: AuthorisedEntry, + ) -> Result< + EntryIngestionSuccess, + EntryIngestionError, + >; - /// Attempt to ingest a payload for a given [`AuthorisedEntry`]. - fn ingest_payload( + /// Attempt to append part of a payload for a given [`AuthorisedEntry`]. + /// + /// Will fail if: + /// - The payload digest is not referred to by any of the store's entries. + /// - A complete payload with the same digest is already held in storage. + /// - The payload exceeded the expected size + /// - The final payload's digest did not match the expected digest + /// - Something else went wrong, e.g. there was no space for the payload on disk. + /// + /// This method **cannot** verify the integrity of partial payload. This means that arbitrary (and possibly malicious) payloads smaller than the expected size will be stored unless partial verification is implemented upstream (e.g. during [the Willow General Sync Protocol's payload transformation](https://willowprotocol.org/specs/sync/index.html#sync_payloads_transform)). + fn append_payload( &self, expected_digest: PD, - payload: Payload, - ) -> Result<(), PayloadIngestionError>; - - /// Query the store's set of [`AuthorisedEntry`] using an [`Area`]. - fn query_area(&self, area: Area, order: QueryOrder, reverse: bool) -> AEI; - - /// Query the store's set of [`AuthorisedEntry`] using a [`Range3d`]. - fn query_range(&self, area: Range3d, order: QueryOrder, reverse: bool) -> AEI; - - /// Try to get the payload for a given [`Entry`]. - fn getPayload(&self, entry: &Entry) -> Option; - - /// Summarise a given range into a fingerprint. - fn summarise(&self, range: Range3d) -> FP; - - /// Split a range into two approximately equally sized ranges, or return the original range if it can't be split. - fn split_range( - &self, - range: Range3d, - ) -> Result<(Range3d, Range3d), Range3d>; - - /// Transform an [`AreaOfInterest`] into a concrete range by using its `max_count` and `max_size` properties as clamps. - fn aoi_to_range(&self, aoi: AreaOfInterest) -> Range3d; + expected_size: u64, + producer: &mut Producer, + ) -> Result, PayloadAppendError> + where + Producer: BulkProducer; } From 0885197a99918a317bd64608e1a0e8008bc66b32 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 13:50:18 +0100 Subject: [PATCH 06/43] Make ingest / append async, add TODO on bulk entry ingestion --- data-model/src/store.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index e147ecd..c4e5811 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -1,3 +1,5 @@ +use std::future::Future; + use ufotofu::nb::BulkProducer; use crate::{ @@ -91,11 +93,15 @@ where fn ingest_entry( &self, authorised_entry: AuthorisedEntry, - ) -> Result< - EntryIngestionSuccess, - EntryIngestionError, + ) -> impl Future< + Output = Result< + EntryIngestionSuccess, + EntryIngestionError, + >, >; + // TODO: Bulk ingestion entry. The annoying there is it needs its own success / error types. We could get around this by exposing a BulkConsumer from the Store, but [then we need to support multi-writer consumption](https://github.com/earthstar-project/willow-rs/pull/21#issuecomment-2192393204). + /// Attempt to append part of a payload for a given [`AuthorisedEntry`]. /// /// Will fail if: @@ -111,7 +117,7 @@ where expected_digest: PD, expected_size: u64, producer: &mut Producer, - ) -> Result, PayloadAppendError> + ) -> impl Future, PayloadAppendError>> where Producer: BulkProducer; } From f999e19400869ca728852da9316f482bdc7cd968 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 14:10:31 +0100 Subject: [PATCH 07/43] Store.forgetEntry --- data-model/src/store.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index c4e5811..71cf272 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -5,7 +5,7 @@ use ufotofu::nb::BulkProducer; use crate::{ entry::AuthorisedEntry, parameters::{AuthorisationToken, NamespaceId, PayloadDigest, SubspaceId}, - LengthyEntry, + LengthyEntry, Path, }; /// Returned when an entry is successfully ingested into a [`Store`]. @@ -71,11 +71,8 @@ pub enum PayloadAppendError { SomethingElseWentWrong, } -pub enum QueryOrder { - Subspace, - Path, - Timestamp, -} +/// Returned when no entry was found for some criteria. +pub struct NoSuchEntryError(); /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store @@ -120,4 +117,15 @@ where ) -> impl Future, PayloadAppendError>> where Producer: BulkProducer; + + /// Locally forget an entry with a given [path] and [subspace] id, returning the forgotten entry, or an error if no entry with that path and subspace ID are held by this store. + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the data. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// Forgetting is not the same as deleting. + fn forget_entry( + path: Path, + subspace_id: S, + traceless: bool, + ) -> Result, NoSuchEntryError>; } From fe8702cd95a55b0463d3520cdebb647cb2bd9afe Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 14:26:36 +0100 Subject: [PATCH 08/43] Add forgetting by area methods --- data-model/src/store.rs | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 71cf272..e8422ef 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -4,6 +4,7 @@ use ufotofu::nb::BulkProducer; use crate::{ entry::AuthorisedEntry, + grouping::AreaOfInterest, parameters::{AuthorisationToken, NamespaceId, PayloadDigest, SubspaceId}, LengthyEntry, Path, }; @@ -120,12 +121,32 @@ where /// Locally forget an entry with a given [path] and [subspace] id, returning the forgotten entry, or an error if no entry with that path and subspace ID are held by this store. /// - /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the data. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting. + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entry back. fn forget_entry( - path: Path, + path: &Path, subspace_id: S, traceless: bool, - ) -> Result, NoSuchEntryError>; + ) -> impl Future, NoSuchEntryError>>; + + /// Locally forget all [`AuthorisedEntry`] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. + fn forget_area( + area: &AreaOfInterest, + traceless: bool, + ) -> impl Future>>; + + /// Locally forget all [`AuthorisedEntry`] **not** [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. + fn forget_everything_but_area( + area: &AreaOfInterest, + traceless: bool, + ) -> impl Future>>; } From a23e76d495a262ebcb1e0626bb714e118811d419 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 16:30:06 +0100 Subject: [PATCH 09/43] Add payload forgetting methods --- data-model/src/store.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index e8422ef..7496798 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -149,4 +149,34 @@ where area: &AreaOfInterest, traceless: bool, ) -> impl Future>>; + + /// Locally forget a payload with a given [`PayloadDigest`], or an error if no payload with that digest is held by this store. + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. + fn forget_payload( + digest: PD, + traceless: bool, + ) -> impl Future>; + + /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. + fn forget_area_payloads( + area: &AreaOfInterest, + traceless: bool, + ) -> impl Future>; + + /// Locally forget all payloads with corresponding [`AuthorisedEntry`] **not** [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. + fn forget_everything_but_area_payloads( + area: &AreaOfInterest, + traceless: bool, + ) -> impl Future>; } From f01dcff91c44acbcc1720293bfbedaa3ca2257c5 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 20 Aug 2024 17:02:17 +0100 Subject: [PATCH 10/43] Add flush --- data-model/src/store.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 7496798..b04abc6 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -179,4 +179,7 @@ where area: &AreaOfInterest, traceless: bool, ) -> impl Future>; + + /// Force persistence of all previous mutations + fn flush() -> impl Future; } From 38ecd75e717634eb5330330650368dd375ee2e01 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 21 Aug 2024 12:00:33 +0100 Subject: [PATCH 11/43] Add bulk_ingest_entry, Store::FlushError --- data-model/src/store.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index b04abc6..da7effb 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -9,7 +9,7 @@ use crate::{ LengthyEntry, Path, }; -/// Returned when an entry is successfully ingested into a [`Store`]. +/// Returned when an entry could be ingested into a [`Store`]. pub enum EntryIngestionSuccess< const MCL: usize, const MCC: usize, @@ -32,6 +32,7 @@ pub enum EntryIngestionSuccess< }, } +/// Returned when an entry cannot be ingested into a [`Store`]. pub enum EntryIngestionError< const MCL: usize, const MCC: usize, @@ -45,6 +46,31 @@ pub enum EntryIngestionError< WrongNamespace(AuthorisedEntry), } +/// Returned when an entry failed to be ingested during a bulk ingestion. +pub enum BulkIngestionError< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT, +> { + /// The entry belonged to another namespace. + WrongNamespace(AuthorisedEntry), + /// The consumer failed for some reason. + ConsumerFailure, +} + +/// A tuple of an [`AuthorisedEntry`] and how a [`Store`] responded to its ingestion. +pub type BulkIngestionResult = ( + AuthorisedEntry, + Result< + EntryIngestionSuccess, + BulkIngestionError, + >, +); + /// Return when a payload is successfully appended to the [`Store`]. pub enum PayloadAppendSuccess where @@ -83,6 +109,8 @@ where PD: PayloadDigest, AT: AuthorisationToken, { + type FlushError; + /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -99,6 +127,12 @@ where >; // TODO: Bulk ingestion entry. The annoying there is it needs its own success / error types. We could get around this by exposing a BulkConsumer from the Store, but [then we need to support multi-writer consumption](https://github.com/earthstar-project/willow-rs/pull/21#issuecomment-2192393204). + /// Attempt to store an [`AuthorisedEntry`] in the [`Store`]. + /// Will fail if the entry belonged to a different namespace than the store's. + fn bulk_ingest_entry( + &self, + authorised_entries: &[AuthorisedEntry], + ) -> impl Future>>; /// Attempt to append part of a payload for a given [`AuthorisedEntry`]. /// @@ -181,5 +215,5 @@ where ) -> impl Future>; /// Force persistence of all previous mutations - fn flush() -> impl Future; + fn flush() -> impl Future>; } From 98a9b73a57d6f341d273394f5f17a722078796a0 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 21 Aug 2024 12:33:58 +0100 Subject: [PATCH 12/43] prevent_pruning param, revisit bulk ingestion result --- data-model/src/store.rs | 51 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index da7effb..ad9f90c 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -44,33 +44,34 @@ pub enum EntryIngestionError< > { /// The entry belonged to another namespace. WrongNamespace(AuthorisedEntry), + /// The ingestion would have triggered prefix pruning when that was not desired. + PruningPrevented, } -/// Returned when an entry failed to be ingested during a bulk ingestion. -pub enum BulkIngestionError< +/// A tuple of an [`AuthorisedEntry`] and how a [`Store`] responded to its ingestion. +pub type BulkIngestionResult = ( + AuthorisedEntry, + Result< + EntryIngestionSuccess, + EntryIngestionError, + >, +); + +/// Returned when a bulk ingestion failed due to a consumer error. +pub struct BulkIngestionError< const MCL: usize, const MCC: usize, const MPL: usize, N: NamespaceId, S: SubspaceId, PD: PayloadDigest, - AT, + AT: AuthorisationToken, + ConsumerError, > { - /// The entry belonged to another namespace. - WrongNamespace(AuthorisedEntry), - /// The consumer failed for some reason. - ConsumerFailure, + pub ingested: Vec>, + pub error: ConsumerError, } -/// A tuple of an [`AuthorisedEntry`] and how a [`Store`] responded to its ingestion. -pub type BulkIngestionResult = ( - AuthorisedEntry, - Result< - EntryIngestionSuccess, - BulkIngestionError, - >, -); - /// Return when a payload is successfully appended to the [`Store`]. pub enum PayloadAppendSuccess where @@ -110,15 +111,17 @@ where AT: AuthorisationToken, { type FlushError; + type ConsumerError; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; - /// Attempt to store an [`AuthorisedEntry`] in the [`Store`]. - /// Will fail if the entry belonged to a different namespace than the store's. + /// Attempt to ingest an [`AuthorisedEntry`] into the [`Store`]. + /// Will fail if the entry belonged to a different namespace than the store's, or if the `prevent_pruning` param is `true` and an ingestion would have triggered [prefix pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning). fn ingest_entry( &self, authorised_entry: AuthorisedEntry, + prevent_pruning: bool, ) -> impl Future< Output = Result< EntryIngestionSuccess, @@ -126,13 +129,17 @@ where >, >; - // TODO: Bulk ingestion entry. The annoying there is it needs its own success / error types. We could get around this by exposing a BulkConsumer from the Store, but [then we need to support multi-writer consumption](https://github.com/earthstar-project/willow-rs/pull/21#issuecomment-2192393204). - /// Attempt to store an [`AuthorisedEntry`] in the [`Store`]. - /// Will fail if the entry belonged to a different namespace than the store's. + /// Attempt to ingest many [`AuthorisedEntry`] in the [`Store`]. fn bulk_ingest_entry( &self, authorised_entries: &[AuthorisedEntry], - ) -> impl Future>>; + prevent_pruning: bool, + ) -> impl Future< + Output = Result< + Vec>, + BulkIngestionError, + >, + >; /// Attempt to append part of a payload for a given [`AuthorisedEntry`]. /// From fe7b7b11a4d34d17ed63f888560176f6f8ed7179 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 21 Aug 2024 12:51:58 +0100 Subject: [PATCH 13/43] Better docs for bulk ingestion --- data-model/src/store.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index ad9f90c..2388f3c 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -66,10 +66,10 @@ pub struct BulkIngestionError< S: SubspaceId, PD: PayloadDigest, AT: AuthorisationToken, - ConsumerError, + IngestionError, > { pub ingested: Vec>, - pub error: ConsumerError, + pub error: IngestionError, } /// Return when a payload is successfully appended to the [`Store`]. @@ -111,7 +111,7 @@ where AT: AuthorisationToken, { type FlushError; - type ConsumerError; + type BulkIngestionError; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -130,6 +130,8 @@ where >; /// Attempt to ingest many [`AuthorisedEntry`] in the [`Store`]. + /// + /// The result being `Ok` does **not** indicate that all entry ingestions were successful, only that each entry had an ingestion attempt, some of which *may* have returned [`EntryIngestionError`]. The `Err` type of this result is only returned if there was some internal error. fn bulk_ingest_entry( &self, authorised_entries: &[AuthorisedEntry], @@ -137,7 +139,7 @@ where ) -> impl Future< Output = Result< Vec>, - BulkIngestionError, + BulkIngestionError, >, >; From f4f72bc55aebc6ea56d57b5369c0d949c3af9de9 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 21 Aug 2024 19:31:55 +0100 Subject: [PATCH 14/43] add entry method --- data-model/src/lengthy_entry.rs | 47 ++++++++++++++++++++++++++++++++- data-model/src/store.rs | 15 +++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/data-model/src/lengthy_entry.rs b/data-model/src/lengthy_entry.rs index e3b3965..609b4ba 100644 --- a/data-model/src/lengthy_entry.rs +++ b/data-model/src/lengthy_entry.rs @@ -1,4 +1,4 @@ -use crate::{Entry, NamespaceId, PayloadDigest, SubspaceId}; +use crate::{AuthorisationToken, AuthorisedEntry, Entry, NamespaceId, PayloadDigest, SubspaceId}; /// An [`Entry`] together with information about how much of its payload a given [`Store`] holds. /// @@ -37,3 +37,48 @@ where self.available } } + +/// An [`AuthorisedEntry`] together with information about how much of its payload a given [`Store`] holds. +pub struct LengthyAuthorisedEntry< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N, + S, + PD, + AT, +> where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + /// The Entry in question. + entry: AuthorisedEntry, + /// The number of consecutive bytes from the start of the entry’s payload that the peer holds. + available: u64, +} + +impl + LengthyAuthorisedEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + /// Create a new lengthy entry from a given [`AuthorisedEntry`] and the number of consecutive bytes from the start of the entry’s payload that are held. + pub fn new(entry: AuthorisedEntry, available: u64) -> Self { + Self { entry, available } + } + + /// The entry in question. + pub fn entry(&self) -> &AuthorisedEntry { + &self.entry + } + + /// The number of consecutive bytes from the start of the entry’s Payload that the peer holds. + pub fn available(&self) -> u64 { + self.available + } +} diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 2388f3c..a8fc033 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -6,7 +6,7 @@ use crate::{ entry::AuthorisedEntry, grouping::AreaOfInterest, parameters::{AuthorisationToken, NamespaceId, PayloadDigest, SubspaceId}, - LengthyEntry, Path, + LengthyAuthorisedEntry, LengthyEntry, Path, }; /// Returned when an entry could be ingested into a [`Store`]. @@ -162,7 +162,7 @@ where where Producer: BulkProducer; - /// Locally forget an entry with a given [path] and [subspace] id, returning the forgotten entry, or an error if no entry with that path and subspace ID are held by this store. + /// Locally forget an entry with a given [`Path`] and [subspace](https://willowprotocol.org/specs/data-model/index.html#subspace) id, returning the forgotten entry, or an error if no entry with that path and subspace ID are held by this store. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// @@ -225,4 +225,15 @@ where /// Force persistence of all previous mutations fn flush() -> impl Future>; + + /// Return a [`LengthyAuthorisedEntry`] with the given [`Path`] and [subspace](https://willowprotocol.org/specs/data-model/index.html#subspace) ID, if present. + /// + /// If `ignore_incomplete_payloads` is `true`, will return `None` if the entry's corresponding payload is incomplete, even if there is an entry present. + /// If `ignore_empty_payloads` is `true`, will return `None` if the entry's payload length is `0`, even if there is an entry present. + fn entry( + path: Path, + subspace_id: S, + ignore_incomplete_payloads: bool, + ignore_empty_payloads: bool, + ) -> Option>; } From 9305acae9bcdff6cd0b8fa849f67d69f6fb78f5d Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 22 Aug 2024 11:54:50 +0100 Subject: [PATCH 15/43] Add query_area method --- data-model/src/store.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index a8fc033..258196d 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -1,6 +1,6 @@ use std::future::Future; -use ufotofu::nb::BulkProducer; +use ufotofu::{local_nb::Producer, nb::BulkProducer}; use crate::{ entry::AuthorisedEntry, @@ -102,6 +102,18 @@ pub enum PayloadAppendError { /// Returned when no entry was found for some criteria. pub struct NoSuchEntryError(); +/// Orderings for a +pub enum QueryOrder { + /// Ordered by subspace, then path, then timestamp + Subspace, + /// Ordered by path, them timestamp, then subspace + Path, + /// Ordered by timestamp, then subspace, then path + Timestamp, + /// Whichever order is most efficient. + Efficient, +} + /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store where @@ -112,6 +124,7 @@ where { type FlushError; type BulkIngestionError; + type EntryProducer: Producer>; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -236,4 +249,16 @@ where ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, ) -> Option>; + + /// Query which entries are [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`]. + /// + /// If `ignore_incomplete_payloads` is `true`, entries with incomplete corresponding payloads will be excluded from results. + /// If `ignore_empty_payloads` is `true`, entries with a `payload_length` of `0` will be excluded from results. + fn query_area( + area: AreaOfInterest, + order: QueryOrder, + reverse: bool, + ignore_incomplete_payloads: bool, + ignore_empty_payloads: bool, + ) -> Self::EntryProducer; } From bbf09c0c81972b160672c57eb224c426fad23996 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 22 Aug 2024 12:53:34 +0100 Subject: [PATCH 16/43] Add subscription fns --- data-model/src/store.rs | 51 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 258196d..c0d445a 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -114,6 +114,32 @@ pub enum QueryOrder { Efficient, } +/// An event which took place within a [`Store`]. +/// Each event includes a *progress ID* which can be used to *resume* a subscription at any point in the future. +pub enum StoreEvent +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + /// A new entry was ingested. + Ingested(u64, AuthorisedEntry), + /// An existing entry received a portion of its corresponding payload. + Appended(u64, LengthyAuthorisedEntry), + /// An entry was forgotten. + EntryForgotten(u64, AuthorisedEntry), + /// A payload was forgotten. + PayloadForgotten(u64, PD), + /// An entry was overwritten. + Overwritten(u64, AuthorisedEntry), + /// An entry was pruned via prefix pruning. + Pruned(u64, AuthorisedEntry), +} + +/// Returned when the store could not resume a subscription because the provided ID was too outdated or not present. +pub struct ResumptionFailedError(pub u64); + /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store where @@ -125,6 +151,7 @@ where type FlushError; type BulkIngestionError; type EntryProducer: Producer>; + type EventProducer: Producer>; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -250,10 +277,10 @@ where ignore_empty_payloads: bool, ) -> Option>; - /// Query which entries are [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`]. + /// Query which entries are [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of [`LengthyAuthorisedEntry`]. /// - /// If `ignore_incomplete_payloads` is `true`, entries with incomplete corresponding payloads will be excluded from results. - /// If `ignore_empty_payloads` is `true`, entries with a `payload_length` of `0` will be excluded from results. + /// If `ignore_incomplete_payloads` is `true`, the producer will not produce entries with incomplete corresponding payloads. + /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. fn query_area( area: AreaOfInterest, order: QueryOrder, @@ -261,4 +288,22 @@ where ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, ) -> Self::EntryProducer; + + /// Subscribe to events concerning entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of `StoreEvent`s which occurred since the moment of calling this function. + /// + /// If `ignore_incomplete_payloads` is `true`, the producer will not produce entries with incomplete corresponding payloads. + /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. + fn subscribe_area( + area: AreaOfInterest, + ignore_incomplete_payloads: bool, + ignore_empty_payloads: bool, + ) -> Self::EventProducer; + + /// Attempt to resume a subscription using a *progress ID* obtained from a previous subscription, or return an error if this store implementation is unable to resume the subscription. + fn resume_subscription( + area: AreaOfInterest, + ignore_incomplete_payloads: bool, + ignore_empty_payloads: bool, + progress_id: Option, + ) -> Result; } From f1cf5e1b88780e8b9069208ee73953be8e090a1b Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 22 Aug 2024 14:15:05 +0100 Subject: [PATCH 17/43] fix Store.resume_subscription params --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index c0d445a..2b3b066 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -301,9 +301,9 @@ where /// Attempt to resume a subscription using a *progress ID* obtained from a previous subscription, or return an error if this store implementation is unable to resume the subscription. fn resume_subscription( + progress_id: u64, area: AreaOfInterest, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, - progress_id: Option, ) -> Result; } From 348347840f21d4984f2edd11e648b3a001710c8f Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 16:49:26 +0100 Subject: [PATCH 18/43] Add into_entry, AsRef impls for LengthEntry --- data-model/src/lengthy_entry.rs | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/data-model/src/lengthy_entry.rs b/data-model/src/lengthy_entry.rs index 609b4ba..1941062 100644 --- a/data-model/src/lengthy_entry.rs +++ b/data-model/src/lengthy_entry.rs @@ -36,6 +36,23 @@ where pub fn available(&self) -> u64 { self.available } + + /// Turn this into a regular [`Entry`]. + pub fn into_entry(self) -> Entry { + self.entry + } +} + +impl + AsRef> for LengthyEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, +{ + fn as_ref(&self) -> &Entry { + &self.entry + } } /// An [`AuthorisedEntry`] together with information about how much of its payload a given [`Store`] holds. @@ -81,4 +98,23 @@ where pub fn available(&self) -> u64 { self.available } + + /// Turn this into a [`AuthorisedEntry`]. + pub fn into_authorised_entry(self) -> AuthorisedEntry { + self.entry + } +} + +impl + AsRef> + for LengthyAuthorisedEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + fn as_ref(&self) -> &AuthorisedEntry { + &self.entry + } } From 108d765f62055bd8c7ce4e664ac8cc2818fa4f5a Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 16:58:06 +0100 Subject: [PATCH 19/43] Refactor how pruned entries are reported --- data-model/src/store.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 2b3b066..912b141 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -21,8 +21,6 @@ pub enum EntryIngestionSuccess< > { /// The entry was successfully ingested. Success, - /// The entry was successfully ingested and prefix pruned some entries. - SuccessAndPruned(Vec>), /// The entry was not ingested because a newer entry with same Obsolete { /// The obsolete entry which was not ingested. @@ -114,6 +112,20 @@ pub enum QueryOrder { Efficient, } +/// Describes an [`AuthorisedEntry`] which was pruned and the [`AuthorisedEntry`] which triggered the pruning. +pub struct PruneEvent +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + /// The entry which was pruned. + pub pruned: AuthorisedEntry, + /// The entry which triggered the pruning. + pub by: AuthorisedEntry, +} + /// An event which took place within a [`Store`]. /// Each event includes a *progress ID* which can be used to *resume* a subscription at any point in the future. pub enum StoreEvent @@ -131,10 +143,8 @@ where EntryForgotten(u64, AuthorisedEntry), /// A payload was forgotten. PayloadForgotten(u64, PD), - /// An entry was overwritten. - Overwritten(u64, AuthorisedEntry), /// An entry was pruned via prefix pruning. - Pruned(u64, AuthorisedEntry), + Pruned(u64, PruneEvent), } /// Returned when the store could not resume a subscription because the provided ID was too outdated or not present. From 851b8c9bc903f526a26ce1d8b59344d1a0086e4c Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 16:59:37 +0100 Subject: [PATCH 20/43] Store.entry should return future --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 912b141..3459f69 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -285,7 +285,7 @@ where subspace_id: S, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, - ) -> Option>; + ) -> impl Future>; /// Query which entries are [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of [`LengthyAuthorisedEntry`]. /// From c6bdc09dca47d01457ddea5775dfaf6be7ea5bb9 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 17:00:02 +0100 Subject: [PATCH 21/43] NoSuchEntryError does not need parentheses --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 3459f69..6c4c3d7 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -98,7 +98,7 @@ pub enum PayloadAppendError { } /// Returned when no entry was found for some criteria. -pub struct NoSuchEntryError(); +pub struct NoSuchEntryError; /// Orderings for a pub enum QueryOrder { From 5079c761d67eba83fd801da890c97f0e301fb685 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 17:02:50 +0100 Subject: [PATCH 22/43] Use impl return types for producers instead of associated type --- data-model/src/store.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 6c4c3d7..72f411c 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -160,8 +160,6 @@ where { type FlushError; type BulkIngestionError; - type EntryProducer: Producer>; - type EventProducer: Producer>; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -297,7 +295,7 @@ where reverse: bool, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, - ) -> Self::EntryProducer; + ) -> impl Producer>; /// Subscribe to events concerning entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of `StoreEvent`s which occurred since the moment of calling this function. /// @@ -307,7 +305,7 @@ where area: AreaOfInterest, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, - ) -> Self::EventProducer; + ) -> impl Producer>; /// Attempt to resume a subscription using a *progress ID* obtained from a previous subscription, or return an error if this store implementation is unable to resume the subscription. fn resume_subscription( @@ -315,5 +313,5 @@ where area: AreaOfInterest, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, - ) -> Result; + ) -> impl Producer>; } From 9161dcbf72e37e6e3b745e49591adb2f86555548 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 17:04:01 +0100 Subject: [PATCH 23/43] Update data-model/src/store.rs Co-authored-by: Aljoscha Meyer --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 72f411c..c57c714 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -147,7 +147,7 @@ where Pruned(u64, PruneEvent), } -/// Returned when the store could not resume a subscription because the provided ID was too outdated or not present. +/// Returned when the store chooses to not resume a subscription. pub struct ResumptionFailedError(pub u64); /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. From e780240ea83214badb4758ac434b75c5e73a3f5e Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 17:29:17 +0100 Subject: [PATCH 24/43] Add Store::OperationsError associated type --- data-model/src/store.rs | 49 ++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 2388f3c..daf4a9e 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -41,19 +41,31 @@ pub enum EntryIngestionError< S: SubspaceId, PD: PayloadDigest, AT, + OE, > { /// The entry belonged to another namespace. WrongNamespace(AuthorisedEntry), /// The ingestion would have triggered prefix pruning when that was not desired. PruningPrevented, + /// Something specific to this store implementation went wrong. + OperationsError(OE), } /// A tuple of an [`AuthorisedEntry`] and how a [`Store`] responded to its ingestion. -pub type BulkIngestionResult = ( +pub type BulkIngestionResult< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N, + S, + PD, + AT, + OE, +> = ( AuthorisedEntry, Result< EntryIngestionSuccess, - EntryIngestionError, + EntryIngestionError, >, ); @@ -66,9 +78,10 @@ pub struct BulkIngestionError< S: SubspaceId, PD: PayloadDigest, AT: AuthorisationToken, + OE, IngestionError, > { - pub ingested: Vec>, + pub ingested: Vec>, pub error: IngestionError, } @@ -86,7 +99,7 @@ where } /// Returned when a payload fails to be appended into the [`Store`]. -pub enum PayloadAppendError { +pub enum PayloadAppendError { /// None of the entries in the store reference this payload. NotEntryReference, /// The payload is already held in storage. @@ -95,8 +108,8 @@ pub enum PayloadAppendError { PayloadTooLarge, /// The completed payload's digest is not what was expected. DigestMismatch, - /// Try deleting some files!!! - SomethingElseWentWrong, + /// Something specific to this store implementation went wrong. + OperationError(OE), } /// Returned when no entry was found for some criteria. @@ -112,6 +125,7 @@ where { type FlushError; type BulkIngestionError; + type OperationsError; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -125,7 +139,7 @@ where ) -> impl Future< Output = Result< EntryIngestionSuccess, - EntryIngestionError, + EntryIngestionError, >, >; @@ -138,8 +152,18 @@ where prevent_pruning: bool, ) -> impl Future< Output = Result< - Vec>, - BulkIngestionError, + Vec>, + BulkIngestionError< + MCL, + MCC, + MPL, + N, + S, + PD, + AT, + Self::BulkIngestionError, + Self::OperationsError, + >, >, >; @@ -158,7 +182,12 @@ where expected_digest: PD, expected_size: u64, producer: &mut Producer, - ) -> impl Future, PayloadAppendError>> + ) -> impl Future< + Output = Result< + PayloadAppendSuccess, + PayloadAppendError, + >, + > where Producer: BulkProducer; From aec0d59b52a89e566c256878a6da087c1cde221e Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 18:06:29 +0100 Subject: [PATCH 25/43] Add forget_payload_unchecked --- data-model/src/store.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index daf4a9e..9f135de 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -115,6 +115,12 @@ pub enum PayloadAppendError { /// Returned when no entry was found for some criteria. pub struct NoSuchEntryError(); +/// Returned when a payload could not be forgotten. +pub enum ForgetPayloadError { + NoSuchEntry, + ReferredToByOtherEntries, +} + /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store where @@ -222,13 +228,27 @@ where traceless: bool, ) -> impl Future>>; - /// Locally forget a payload with a given [`PayloadDigest`], or an error if no payload with that digest is held by this store. + /// Locally forget the corresponding payload of the entry with a given path and subspace, or an error if no entry with that path and subspace ID is held by this store or if the entry's payload corresponds to other entries. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. fn forget_payload( - digest: PD, + path: &Path, + subspace_id: S, + traceless: bool, + ) -> impl Future>; + + /// Locally forget the corresponding payload of the entry with a given path and subspace, or an error if no entry with that path and subspace ID is held by this store. **The payload will be forgotten even if it corresponds to other entries**. + /// + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// + /// If the `even_if_referred_to_by_other_entries` parameter is `true`, the payload will be forgotten even if other entries with a different path and/or subspace ID refer to this payload. + /// + /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. + fn forget_payload_unchecked( + path: &Path, + subspace_id: S, traceless: bool, ) -> impl Future>; From 288fe8c2aab49a1e709e7650871d00ab30a9271c Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 18:10:44 +0100 Subject: [PATCH 26/43] Improve producer param name --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 9f135de..0d01b4e 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -187,7 +187,7 @@ where &self, expected_digest: PD, expected_size: u64, - producer: &mut Producer, + payload_source: &mut Producer, ) -> impl Future< Output = Result< PayloadAppendSuccess, From 89d7d19853e4a9f2d0af3dbab90c317aa6f4aece Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 18:13:47 +0100 Subject: [PATCH 27/43] Different return type for forget_entry --- data-model/src/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 0d01b4e..5d31cda 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -197,7 +197,7 @@ where where Producer: BulkProducer; - /// Locally forget an entry with a given [path] and [subspace] id, returning the forgotten entry, or an error if no entry with that path and subspace ID are held by this store. + /// Locally forget an entry with a given [path] and [subspace] id, succeeding if there is no entry at that path and subspace ID, or returning an implementation-specific error if something went wrong. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// @@ -206,7 +206,7 @@ where path: &Path, subspace_id: S, traceless: bool, - ) -> impl Future, NoSuchEntryError>>; + ) -> impl Future>; /// Locally forget all [`AuthorisedEntry`] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries /// From 9277addc4f31db07963085a9229eb6585d9ab162 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 18:17:03 +0100 Subject: [PATCH 28/43] no references to deleting --- data-model/src/store.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 5d31cda..592d2ef 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -201,7 +201,7 @@ where /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entry back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entry back. fn forget_entry( path: &Path, subspace_id: S, @@ -212,7 +212,7 @@ where /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entries back. fn forget_area( area: &AreaOfInterest, traceless: bool, @@ -222,7 +222,7 @@ where /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entries back. fn forget_everything_but_area( area: &AreaOfInterest, traceless: bool, @@ -232,7 +232,7 @@ where /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payload back. fn forget_payload( path: &Path, subspace_id: S, @@ -245,7 +245,7 @@ where /// /// If the `even_if_referred_to_by_other_entries` parameter is `true`, the payload will be forgotten even if other entries with a different path and/or subspace ID refer to this payload. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payload back. fn forget_payload_unchecked( path: &Path, subspace_id: S, @@ -256,7 +256,7 @@ where /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. fn forget_area_payloads( area: &AreaOfInterest, traceless: bool, @@ -266,7 +266,7 @@ where /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. fn forget_everything_but_area_payloads( area: &AreaOfInterest, traceless: bool, From 32c931812dd1fc5b33ca630b3c374b60c1eb9782 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 18:21:46 +0100 Subject: [PATCH 29/43] Update data-model/src/store.rs Co-authored-by: Aljoscha Meyer --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 592d2ef..4c6f0ab 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -182,7 +182,7 @@ where /// - The final payload's digest did not match the expected digest /// - Something else went wrong, e.g. there was no space for the payload on disk. /// - /// This method **cannot** verify the integrity of partial payload. This means that arbitrary (and possibly malicious) payloads smaller than the expected size will be stored unless partial verification is implemented upstream (e.g. during [the Willow General Sync Protocol's payload transformation](https://willowprotocol.org/specs/sync/index.html#sync_payloads_transform)). + /// This method **cannot** verify the integrity of partial payloads. This means that arbitrary (and possibly malicious) payloads smaller than the expected size will be stored unless partial verification is implemented upstream (e.g. during [the Willow General Sync Protocol's payload transformation](https://willowprotocol.org/specs/sync/index.html#sync_payloads_transform)). fn append_payload( &self, expected_digest: PD, From a130c474af02c78e0bf7397e33af0a11ef501474 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Mon, 26 Aug 2024 18:22:06 +0100 Subject: [PATCH 30/43] Update data-model/src/store.rs Co-authored-by: Aljoscha Meyer --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 4c6f0ab..6070f73 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -199,7 +199,7 @@ where /// Locally forget an entry with a given [path] and [subspace] id, succeeding if there is no entry at that path and subspace ID, or returning an implementation-specific error if something went wrong. /// - /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgotten for an arbitrary amount of time. /// /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entry back. fn forget_entry( From 09b70160d253ef5c4b232b7cafc9737f932e5968 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 27 Aug 2024 11:03:29 +0100 Subject: [PATCH 31/43] =?UTF-8?q?Clarify=20=E2=80=9Cpayload=20too=20long?= =?UTF-8?q?=E2=80=9D=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data-model/src/store.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 6070f73..956afa2 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -104,8 +104,8 @@ pub enum PayloadAppendError { NotEntryReference, /// The payload is already held in storage. AlreadyHaveIt, - /// The received payload is larger than was expected. - PayloadTooLarge, + /// The payload source produced more bytes than were expected for this payload. + TooManyBytes, /// The completed payload's digest is not what was expected. DigestMismatch, /// Something specific to this store implementation went wrong. @@ -178,7 +178,7 @@ where /// Will fail if: /// - The payload digest is not referred to by any of the store's entries. /// - A complete payload with the same digest is already held in storage. - /// - The payload exceeded the expected size + /// - The payload source produced more bytes than were expected for this payload. /// - The final payload's digest did not match the expected digest /// - Something else went wrong, e.g. there was no space for the payload on disk. /// From cc2a67ef5e3d816f6558eea2e4d40dcd5426c68b Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 27 Aug 2024 11:13:45 +0100 Subject: [PATCH 32/43] Single forget_area fn with protected param --- data-model/src/store.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 956afa2..36212a7 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -4,7 +4,7 @@ use ufotofu::nb::BulkProducer; use crate::{ entry::AuthorisedEntry, - grouping::AreaOfInterest, + grouping::{Area, AreaOfInterest}, parameters::{AuthorisationToken, NamespaceId, PayloadDigest, SubspaceId}, LengthyEntry, Path, }; @@ -210,21 +210,14 @@ where /// Locally forget all [`AuthorisedEntry`] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries /// - /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. - /// - /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entries back. - fn forget_area( - area: &AreaOfInterest, - traceless: bool, - ) -> impl Future>>; - - /// Locally forget all [`AuthorisedEntry`] **not** [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries + /// If `protected` is `Some`, then all entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by that [`Area`] will be prevented from being forgotten, even though they are included by `area`. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entries back. - fn forget_everything_but_area( + fn forget_area( area: &AreaOfInterest, + protected: Option>, traceless: bool, ) -> impl Future>>; From e1330971f4495ad851f15a9b109e42121c5ad6bb Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 27 Aug 2024 12:59:13 +0100 Subject: [PATCH 33/43] Add missing &self param / make many params references --- data-model/src/store.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index c57c714..3e5adc3 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -203,7 +203,7 @@ where /// This method **cannot** verify the integrity of partial payload. This means that arbitrary (and possibly malicious) payloads smaller than the expected size will be stored unless partial verification is implemented upstream (e.g. during [the Willow General Sync Protocol's payload transformation](https://willowprotocol.org/specs/sync/index.html#sync_payloads_transform)). fn append_payload( &self, - expected_digest: PD, + expected_digest: &PD, expected_size: u64, producer: &mut Producer, ) -> impl Future, PayloadAppendError>> @@ -216,8 +216,9 @@ where /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entry back. fn forget_entry( + &self, path: &Path, - subspace_id: S, + subspace_id: &S, traceless: bool, ) -> impl Future, NoSuchEntryError>>; @@ -227,6 +228,7 @@ where /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. fn forget_area( + &self, area: &AreaOfInterest, traceless: bool, ) -> impl Future>>; @@ -237,6 +239,7 @@ where /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. fn forget_everything_but_area( + &self, area: &AreaOfInterest, traceless: bool, ) -> impl Future>>; @@ -247,6 +250,7 @@ where /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. fn forget_payload( + &self, digest: PD, traceless: bool, ) -> impl Future>; @@ -257,6 +261,7 @@ where /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. fn forget_area_payloads( + &self, area: &AreaOfInterest, traceless: bool, ) -> impl Future>; @@ -267,6 +272,7 @@ where /// /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. fn forget_everything_but_area_payloads( + &self, area: &AreaOfInterest, traceless: bool, ) -> impl Future>; @@ -279,8 +285,9 @@ where /// If `ignore_incomplete_payloads` is `true`, will return `None` if the entry's corresponding payload is incomplete, even if there is an entry present. /// If `ignore_empty_payloads` is `true`, will return `None` if the entry's payload length is `0`, even if there is an entry present. fn entry( - path: Path, - subspace_id: S, + &self, + path: &Path, + subspace_id: &S, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, ) -> impl Future>; @@ -290,8 +297,9 @@ where /// If `ignore_incomplete_payloads` is `true`, the producer will not produce entries with incomplete corresponding payloads. /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. fn query_area( - area: AreaOfInterest, - order: QueryOrder, + &self, + area: &AreaOfInterest, + order: &QueryOrder, reverse: bool, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, @@ -302,15 +310,17 @@ where /// If `ignore_incomplete_payloads` is `true`, the producer will not produce entries with incomplete corresponding payloads. /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. fn subscribe_area( - area: AreaOfInterest, + &self, + area: &AreaOfInterest, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, ) -> impl Producer>; /// Attempt to resume a subscription using a *progress ID* obtained from a previous subscription, or return an error if this store implementation is unable to resume the subscription. fn resume_subscription( + &self, progress_id: u64, - area: AreaOfInterest, + area: &AreaOfInterest, ignore_incomplete_payloads: bool, ignore_empty_payloads: bool, ) -> impl Producer>; From f824d43997bd6d66893fbb8afe14f3e1763fdbbd Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 27 Aug 2024 14:11:43 +0100 Subject: [PATCH 34/43] Add QueryIgnoreParams --- data-model/src/store.rs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 3e5adc3..4071a9d 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -150,6 +150,25 @@ where /// Returned when the store chooses to not resume a subscription. pub struct ResumptionFailedError(pub u64); +/// Describes which entries to ignore during a query. +#[derive(Default)] +pub struct QueryIgnoreParams { + /// Omit entries with locally incomplete corresponding payloads. + pub ignore_incomplete_payloads: bool, + /// Omit entries with an empty payload. + pub ignore_empty_payloads: bool, +} + +impl QueryIgnoreParams { + pub fn ignore_incomplete_payloads(&mut self) { + self.ignore_incomplete_payloads = true; + } + + pub fn ignore_empty_payloads(&mut self) { + self.ignore_empty_payloads = true; + } +} + /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store where @@ -288,8 +307,7 @@ where &self, path: &Path, subspace_id: &S, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, + ignore: Option, ) -> impl Future>; /// Query which entries are [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of [`LengthyAuthorisedEntry`]. @@ -301,8 +319,7 @@ where area: &AreaOfInterest, order: &QueryOrder, reverse: bool, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, + ignore: Option, ) -> impl Producer>; /// Subscribe to events concerning entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of `StoreEvent`s which occurred since the moment of calling this function. @@ -312,8 +329,7 @@ where fn subscribe_area( &self, area: &AreaOfInterest, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, + ignore: Option, ) -> impl Producer>; /// Attempt to resume a subscription using a *progress ID* obtained from a previous subscription, or return an error if this store implementation is unable to resume the subscription. @@ -321,7 +337,6 @@ where &self, progress_id: u64, area: &AreaOfInterest, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, + ignore: Option, ) -> impl Producer>; } From 4f3fcd4018dcde201f2b45b540fc67cc0d7fb71c Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 27 Aug 2024 14:16:15 +0100 Subject: [PATCH 35/43] Update QueryOrder --- data-model/src/store.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 4071a9d..f0f4ab6 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -100,16 +100,16 @@ pub enum PayloadAppendError { /// Returned when no entry was found for some criteria. pub struct NoSuchEntryError; -/// Orderings for a +/// The order by which entries should be returned for a given query. pub enum QueryOrder { - /// Ordered by subspace, then path, then timestamp + /// Ordered by subspace, then path, then timestamp. Subspace, - /// Ordered by path, them timestamp, then subspace + /// Ordered by path, then by an arbitrary order determined by the implementation. Path, - /// Ordered by timestamp, then subspace, then path + /// Ordered by timestamp, then by an arbitrary order determined by the implementation. Timestamp, - /// Whichever order is most efficient. - Efficient, + /// An arbitrary order chosen by the implementation, hopefully the most efficient one. + Arbitrary, } /// Describes an [`AuthorisedEntry`] which was pruned and the [`AuthorisedEntry`] which triggered the pruning. From a5782295e149701990dadcdf0cb85305367d4886 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Tue, 27 Aug 2024 14:49:30 +0100 Subject: [PATCH 36/43] Update data-model/src/store.rs Co-authored-by: Aljoscha Meyer --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index f0f4ab6..31a5edb 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -155,7 +155,7 @@ pub struct ResumptionFailedError(pub u64); pub struct QueryIgnoreParams { /// Omit entries with locally incomplete corresponding payloads. pub ignore_incomplete_payloads: bool, - /// Omit entries with an empty payload. + /// Omit entries whose payload is the empty string. pub ignore_empty_payloads: bool, } From 4c6cfc7eef772e2ff1e59e740076099c411133c0 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 28 Aug 2024 12:01:47 +0100 Subject: [PATCH 37/43] Rename forget_payload_unchecked --- data-model/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index d9d4064..50454f0 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -306,7 +306,7 @@ where /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payload back. - fn forget_payload_unchecked( + fn force_forget_payload( path: &Path, subspace_id: S, traceless: bool, From b8a87ad2973e87812de9d01811fd9eaa5473f50d Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Wed, 28 Aug 2024 12:15:12 +0100 Subject: [PATCH 38/43] Make payload forgetting APIs more consistent --- data-model/src/store.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 50454f0..e7b9790 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -133,8 +133,8 @@ where PD: PayloadDigest, AT: AuthorisationToken, { - /// The entry which was pruned. - pub pruned: AuthorisedEntry, + /// The subspace ID and path of the entry which was pruned. + pub pruned: (S, Path), /// The entry which triggered the pruning. pub by: AuthorisedEntry, } @@ -153,7 +153,7 @@ where /// An existing entry received a portion of its corresponding payload. Appended(u64, LengthyAuthorisedEntry), /// An entry was forgotten. - EntryForgotten(u64, AuthorisedEntry), + EntryForgotten(u64, (S, Path)), /// A payload was forgotten. PayloadForgotten(u64, PD), /// An entry was pruned via prefix pruning. @@ -312,7 +312,9 @@ where traceless: bool, ) -> impl Future>; - /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. + /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. Payloads corresponding to entries *outside* of the given `area` param will be be prevented from being forgotten. + /// + /// If `protected` is `Some`, then all payloads corresponding to entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by that [`Area`] will be prevented from being forgotten, even though they are included by `area`. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// @@ -320,17 +322,21 @@ where fn forget_area_payloads( &self, area: &AreaOfInterest, + protected: Option>, traceless: bool, ) -> impl Future>; - /// Locally forget all payloads with corresponding [`AuthorisedEntry`] **not** [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. + /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. **Payloads will be forgotten even if it corresponds to other entries outside the given area**. + /// + /// If `protected` is `Some`, then all payloads corresponding to entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by that [`Area`] will be prevented from being forgotten, even though they are included by `area`. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. - fn forget_everything_but_area_payloads( + fn force_forget_area_payloads( &self, area: &AreaOfInterest, + protected: Option>, traceless: bool, ) -> impl Future>; From b8de760a7a1b8ee88dc88d43e8a1859bbe27f9da Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 29 Aug 2024 12:16:27 +0100 Subject: [PATCH 39/43] Add EntryOrigin to ingestion --- data-model/src/store.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index e7b9790..9231355 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -149,7 +149,11 @@ where AT: AuthorisationToken, { /// A new entry was ingested. - Ingested(u64, AuthorisedEntry), + Ingested( + u64, + AuthorisedEntry, + EntryOrigin, + ), /// An existing entry received a portion of its corresponding payload. Appended(u64, LengthyAuthorisedEntry), /// An entry was forgotten. @@ -188,6 +192,14 @@ pub enum ForgetPayloadError { ReferredToByOtherEntries, } +/// The origin of an entry ingestion event. +pub enum EntryOrigin { + /// The entry was probably created on this machine. + Local, + /// The entry was sourced from another device, e.g. a networked sync session. + Remote(u64), +} + /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store where From de0f72fc972de30c9590bc8d7b2a255e8666ce49 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 29 Aug 2024 12:31:46 +0100 Subject: [PATCH 40/43] Use Areas for subscriptions --- data-model/src/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 9231355..578924d 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -384,7 +384,7 @@ where /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. fn subscribe_area( &self, - area: &AreaOfInterest, + area: &Area, ignore: Option, ) -> impl Producer>; @@ -392,7 +392,7 @@ where fn resume_subscription( &self, progress_id: u64, - area: &AreaOfInterest, + area: &Area, ignore: Option, ) -> impl Producer>; } From d39f7e04e2f1b774f9b6b5d4aa9aab49c07de791 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 29 Aug 2024 14:13:25 +0100 Subject: [PATCH 41/43] Add standard derives --- data-model/src/lengthy_entry.rs | 2 ++ data-model/src/store.rs | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/data-model/src/lengthy_entry.rs b/data-model/src/lengthy_entry.rs index 1941062..06fa898 100644 --- a/data-model/src/lengthy_entry.rs +++ b/data-model/src/lengthy_entry.rs @@ -3,6 +3,7 @@ use crate::{AuthorisationToken, AuthorisedEntry, Entry, NamespaceId, PayloadDige /// An [`Entry`] together with information about how much of its payload a given [`Store`] holds. /// /// [Definition](https://willowprotocol.org/specs/3d-range-based-set-reconciliation/index.html#LengthyEntry) +#[derive(Debug, Clone, PartialEq, Eq)] pub struct LengthyEntry where N: NamespaceId, @@ -56,6 +57,7 @@ where } /// An [`AuthorisedEntry`] together with information about how much of its payload a given [`Store`] holds. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct LengthyAuthorisedEntry< const MCL: usize, const MCC: usize, diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 578924d..e09a754 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -10,6 +10,7 @@ use crate::{ }; /// Returned when an entry could be ingested into a [`Store`]. +#[derive(Debug, Clone)] pub enum EntryIngestionSuccess< const MCL: usize, const MCC: usize, @@ -31,6 +32,7 @@ pub enum EntryIngestionSuccess< } /// Returned when an entry cannot be ingested into a [`Store`]. +#[derive(Debug, Clone)] pub enum EntryIngestionError< const MCL: usize, const MCC: usize, @@ -68,6 +70,7 @@ pub type BulkIngestionResult< ); /// Returned when a bulk ingestion failed due to a consumer error. +#[derive(Debug, Clone)] pub struct BulkIngestionError< const MCL: usize, const MCC: usize, @@ -84,6 +87,7 @@ pub struct BulkIngestionError< } /// Return when a payload is successfully appended to the [`Store`]. +#[derive(Debug, Clone)] pub enum PayloadAppendSuccess where N: NamespaceId, @@ -97,6 +101,7 @@ where } /// Returned when a payload fails to be appended into the [`Store`]. +#[derive(Debug, Clone)] pub enum PayloadAppendError { /// None of the entries in the store reference this payload. NotEntryReference, @@ -111,9 +116,11 @@ pub enum PayloadAppendError { } /// Returned when no entry was found for some criteria. +#[derive(Debug, Clone)] pub struct NoSuchEntryError; /// The order by which entries should be returned for a given query. +#[derive(Debug, Clone)] pub enum QueryOrder { /// Ordered by subspace, then path, then timestamp. Subspace, @@ -126,6 +133,7 @@ pub enum QueryOrder { } /// Describes an [`AuthorisedEntry`] which was pruned and the [`AuthorisedEntry`] which triggered the pruning. +#[derive(Debug, Clone)] pub struct PruneEvent where N: NamespaceId, @@ -141,6 +149,7 @@ where /// An event which took place within a [`Store`]. /// Each event includes a *progress ID* which can be used to *resume* a subscription at any point in the future. +#[derive(Debug, Clone)] pub enum StoreEvent where N: NamespaceId, @@ -165,10 +174,11 @@ where } /// Returned when the store chooses to not resume a subscription. +#[derive(Debug, Clone)] pub struct ResumptionFailedError(pub u64); /// Describes which entries to ignore during a query. -#[derive(Default)] +#[derive(Default, Clone)] pub struct QueryIgnoreParams { /// Omit entries with locally incomplete corresponding payloads. pub ignore_incomplete_payloads: bool, @@ -187,12 +197,14 @@ impl QueryIgnoreParams { } /// Returned when a payload could not be forgotten. +#[derive(Debug, Clone)] pub enum ForgetPayloadError { NoSuchEntry, ReferredToByOtherEntries, } /// The origin of an entry ingestion event. +#[derive(Debug, Clone, PartialEq, Eq)] pub enum EntryOrigin { /// The entry was probably created on this machine. Local, From 51ab2c4882d943236444b69a7adc5ccafe8a8066 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Thu, 29 Aug 2024 15:44:19 +0100 Subject: [PATCH 42/43] tedious implementation of Error --- data-model/src/store.rs | 151 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 6 deletions(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index e09a754..04c62ae 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -1,4 +1,8 @@ -use std::future::Future; +use std::{ + error::Error, + fmt::{Debug, Display}, + future::Future, +}; use ufotofu::{local_nb::Producer, nb::BulkProducer}; @@ -41,7 +45,7 @@ pub enum EntryIngestionError< S: SubspaceId, PD: PayloadDigest, AT, - OE, + OE: Display + Error, > { /// The entry belonged to another namespace. WrongNamespace(AuthorisedEntry), @@ -51,6 +55,43 @@ pub enum EntryIngestionError< OperationsError(OE), } +impl< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT, + OE: Display + Error, + > Display for EntryIngestionError +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EntryIngestionError::WrongNamespace(_) => { + write!(f, "Tried to ingest an entry from a different namespace.") + } + EntryIngestionError::PruningPrevented => { + write!(f, "Entry ingestion would have triggered undesired pruning.") + } + EntryIngestionError::OperationsError(err) => Display::fmt(err, f), + } + } +} + +impl< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N: NamespaceId + Debug, + S: SubspaceId + Debug, + PD: PayloadDigest + Debug, + AT: Debug, + OE: Display + Error, + > Error for EntryIngestionError +{ +} + /// A tuple of an [`AuthorisedEntry`] and how a [`Store`] responded to its ingestion. pub type BulkIngestionResult< const MCL: usize, @@ -79,13 +120,48 @@ pub struct BulkIngestionError< S: SubspaceId, PD: PayloadDigest, AT: AuthorisationToken, - OE, + OE: Error, IngestionError, > { pub ingested: Vec>, pub error: IngestionError, } +impl< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, + OE: Display + Error, + IngestionError: Error, + > std::fmt::Display for BulkIngestionError +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "An error stopped bulk ingestion after successfully ingesting {:?} entries", + self.ingested.len() + ) + } +} + +impl< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N: NamespaceId + Debug, + S: SubspaceId + Debug, + PD: PayloadDigest + Debug, + AT: AuthorisationToken + Debug, + OE: Display + Error, + IngestionError: Error, + > Error for BulkIngestionError +{ +} + /// Return when a payload is successfully appended to the [`Store`]. #[derive(Debug, Clone)] pub enum PayloadAppendSuccess @@ -115,10 +191,42 @@ pub enum PayloadAppendError { OperationError(OE), } +impl Display for PayloadAppendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PayloadAppendError::NotEntryReference => write!( + f, + "None of the entries in the soter reference this payload." + ), + PayloadAppendError::AlreadyHaveIt => { + write!(f, "The payload is already held in storage.") + } + PayloadAppendError::TooManyBytes => write!( + f, + "The payload source produced more bytes than were expected for this payload." + ), + PayloadAppendError::DigestMismatch => { + write!(f, "The complete payload's digest is not what was expected.") + } + PayloadAppendError::OperationError(err) => std::fmt::Display::fmt(err, f), + } + } +} + +impl Error for PayloadAppendError {} + /// Returned when no entry was found for some criteria. #[derive(Debug, Clone)] pub struct NoSuchEntryError; +impl Display for NoSuchEntryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "No entry was found for the given criteria.") + } +} + +impl Error for NoSuchEntryError {} + /// The order by which entries should be returned for a given query. #[derive(Debug, Clone)] pub enum QueryOrder { @@ -177,6 +285,18 @@ where #[derive(Debug, Clone)] pub struct ResumptionFailedError(pub u64); +impl Display for ResumptionFailedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "The subscription with ID {:?} could not be resumed.", + self.0 + ) + } +} + +impl Error for ResumptionFailedError {} + /// Describes which entries to ignore during a query. #[derive(Default, Clone)] pub struct QueryIgnoreParams { @@ -203,6 +323,25 @@ pub enum ForgetPayloadError { ReferredToByOtherEntries, } +impl Display for ForgetPayloadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ForgetPayloadError::NoSuchEntry => { + write!( + f, + "No entry for the given criteria could be found in this store." + ) + } + ForgetPayloadError::ReferredToByOtherEntries => write!( + f, + "The payload could not be forgotten because it is referred to by other entries." + ), + } + } +} + +impl Error for ForgetPayloadError {} + /// The origin of an entry ingestion event. #[derive(Debug, Clone, PartialEq, Eq)] pub enum EntryOrigin { @@ -220,9 +359,9 @@ where PD: PayloadDigest, AT: AuthorisationToken, { - type FlushError; - type BulkIngestionError; - type OperationsError; + type FlushError: Display + Error; + type BulkIngestionError: Display + Error; + type OperationsError: Display + Error; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; From 914bc631e7d4e35bcb916da255c05a62de38cc09 Mon Sep 17 00:00:00 2001 From: Sam Gwilym Date: Fri, 30 Aug 2024 11:26:54 +0100 Subject: [PATCH 43/43] clearer docs for EntryOrigin --- data-model/src/store.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-model/src/store.rs b/data-model/src/store.rs index 04c62ae..fe4386f 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -347,7 +347,8 @@ impl Error for ForgetPayloadError {} pub enum EntryOrigin { /// The entry was probably created on this machine. Local, - /// The entry was sourced from another device, e.g. a networked sync session. + /// The entry was sourced from another source with an ID assigned by us. + /// This is useful if you want to suppress the forwarding of entries to the peers from which the entry was originally sourced. Remote(u64), }