Skip to content

Commit

Permalink
Standardize Variable and Factor Levels (#1109)
Browse files Browse the repository at this point in the history 
* better progress in loadDFG!

* rn SkeletonDFGVariable -> VariableSkeleton

* rn DFGVariableSummary -> VariableSummary

* rn DFGVariable -> VariableCompute

* rn PackedVariable/Variable -> VariableDFG

* rn SkeletonDFGFactor -> FactorSkeleton

* rn DFGFactorSummary -> FactorSummary

* rn DFGFactor -> FactorCompute

* rn PackedFactor/Factor -> FactorDFG
  • Loading branch information
@Affie
Affie authored Nov 28, 2024
1 parent b0ae00c commit e2e6151
Show file tree
Hide file tree
Showing 27 changed files with 468 additions and 404 deletions.
14 changes: 14 additions & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@ Listing news on any major breaking changes in DFG. For regular changes, see int
UserLabel -> AgentLabel
RobotLabel -> AgentLabel
SessionLabel -> GraphLabel

Variables and Factors are renamed and aliased to the old names, see #1109.
- Factor-level noun-adjectives
SkeletonDFGFactor -> FactorSkeleton
DFGFactorSummary -> FactorSummary
DFGFactor -> FactorCompute
PackedFactor/Factor -> FactorDFG

- Variable-level noun-adjectives
SkeletonDFGVariable -> VariableSkeleton
DFGVariableSummary -> VariableSummary
DFGVariable -> VariableCompute
PackedVariable/Variable -> VariableDFG

- v0.24 FileDFGs can be loaded with v0.25 with the exception of the User[Label/Data/BlobEntries]

# v0.24
Expand Down
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ Graphs = "1.4"
InteractiveUtils = "1.10"
JSON3 = "1"
LinearAlgebra = "1.10"
Manifolds = "0.9"
Manifolds = "0.9, 0.10"
ManifoldsBase = "0.14, 0.15"
OrderedCollections = "1.4"
Pkg = "1.4, 1.5"
Expand Down
20 changes: 10 additions & 10 deletions attic/SerializationOld.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ end
##==============================================================================
## Variable Packing and unpacking
##==============================================================================
function packVariable(v::DFGVariable)
function packVariable(v::VariableCompute)
props = Dict{String, Any}()
props["label"] = string(v.label)
props["timestamp"] = v.timestamp
Expand Down Expand Up @@ -286,7 +286,7 @@ end

"""
$SIGNATURES
Returns a DFGVariable.
Returns a VariableCompute.
DevNotes
- v0.19 packVariable fixed nested JSON bug on these fields, see #867:
Expand Down Expand Up @@ -404,11 +404,11 @@ function unpackVariable(
else
Dict{Symbol, VariableNodeData{variableType, pointType}}()
end
# Rebuild DFGVariable using the first solver variableType in solverData
# Rebuild VariableCompute using the first solver variableType in solverData
# @info "dbg Serialization 171" variableType Symbol(packedProps["label"]) timestamp nstime ppeDict solverData smallData Dict{Symbol,AbstractBlobEntry}() Ref(packedProps["solvable"])
# variable = DFGVariable{variableType}(Symbol(packedProps["label"]), timestamp, nstime, Set(tags), ppeDict, solverData, smallData, Dict{Symbol,AbstractBlobEntry}(), Ref(packedProps["solvable"]))
# variable = VariableCompute{variableType}(Symbol(packedProps["label"]), timestamp, nstime, Set(tags), ppeDict, solverData, smallData, Dict{Symbol,AbstractBlobEntry}(), Ref(packedProps["solvable"]))

variable = DFGVariable{variableType}(;
variable = VariableCompute{variableType}(;
id,
label = Symbol(packedProps["label"]),
# variableType = variableType,
Expand Down Expand Up @@ -562,7 +562,7 @@ end
## Factor Packing and unpacking
##==============================================================================

function _packSolverData(f::DFGFactor, fnctype::AbstractFactor)
function _packSolverData(f::FactorCompute, fnctype::AbstractFactor)
#
packtype = convertPackedType(fnctype)
try
Expand All @@ -579,7 +579,7 @@ function _packSolverData(f::DFGFactor, fnctype::AbstractFactor)
end

# returns ::Dict{String, <:Any}
function packFactor(dfg::AbstractDFG, f::DFGFactor)
function packFactor(dfg::AbstractDFG, f::FactorCompute)
# Construct the properties to save
props = Dict{String, Any}()
props["id"] = f.id !== nothing ? string(f.id) : nothing
Expand Down Expand Up @@ -692,7 +692,7 @@ function fncStringToData(fncType::String, data::Union{String, <:NamedTuple})
return fncStringToData(packtype, data)
end

# Returns `::DFGFactor`
# Returns `::FactorCompute`
function unpackFactor(
dfg::G,
packedProps::Dict{String, Any};
Expand Down Expand Up @@ -753,9 +753,9 @@ function unpackFactor(
Dict{Symbol, SmallDataTypes}()
end

# Rebuild DFGFactor
# Rebuild FactorCompute
#TODO use constuctor to create factor
factor = DFGFactor(
factor = FactorCompute(
Symbol(label),
timestamp,
nstime,
Expand Down
28 changes: 14 additions & 14 deletions docs/src/DataStructure.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,37 +15,37 @@ Accessible properties for each of the variable structures:

| | Label | Timestamp | Tags | Estimates | Soft Type | Solvable | Solver Data | Metadata | Blob Entries |
|---------------------|-------|-----------|------|-----------|-----------|----------|-------------|----------|--------------|
| SkeletonDFGVariable | X | | X | | | | | | |
| DFGVariableSummary | X | X | X | X | Symbol | | | | X |
| DFGVariable | X | X | X | X | X | X | X | X | X |
| VariableSkeleton | X | | X | | | | | | |
| VariableSummary | X | X | X | X | Symbol | | | | X |
| VariableCompute | X | X | X | X | X | X | X | X | X |

Accessible properties for each of the factor structures:

| | Label | Timestamp | Tags | Factor Type | Solvable | Solver Data |
|-------------------|-------|-----------|------|-------------|----------|-------------|
| SkeletonDFGFactor | X | | X | | | |
| DFGFactorSummary | X | X | X | | | |
| DFGFactor | X | X | X | X | X | X |
| FactorSkeleton | X | | X | | | |
| FactorSummary | X | X | X | | | |
| FactorCompute | X | X | X | X | X | X |

## DFG Skeleton types

- [`SkeletonDFGVariable`](@ref)
- [`SkeletonDFGFactor`](@ref)
- [`VariableSkeleton`](@ref)
- [`FactorSkeleton`](@ref)

## DFG Summary types

- [`DFGVariableSummary`](@ref)
- [`DFGFactorSummary`](@ref)
- [`VariableSummary`](@ref)
- [`FactorSummary`](@ref)

## DFG Portable and Storeable types

- [`Variable`](@ref)
- [`PackedFactor`](@ref)
- [`VariableDFG`](@ref)
- [`FactorDFG`](@ref)

## DFG Full solvable types

- [`DFGVariable`](@ref)
- [`DFGFactor`](@ref)
- [`VariableCompute`](@ref)
- [`FactorCompute`](@ref)

## Additional Offloaded Data

Expand Down
16 changes: 8 additions & 8 deletions src/DataBlobs/services/BlobEntry.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ function getBlobEntry(var::AbstractDFGVariable, key::Symbol)
return var.dataDict[key]
end

function getBlobEntry(var::PackedVariable, key::Symbol)
function getBlobEntry(var::VariableDFG, key::Symbol)
if !hasBlobEntry(var, key)
throw(
KeyError(
Expand Down Expand Up @@ -116,7 +116,7 @@ function getBlobEntryFirst(var::AbstractDFGVariable, key::Regex)
)
end

function getBlobEntryFirst(var::Variable, key::Regex)
function getBlobEntryFirst(var::VariableDFG, key::Regex)
firstIdx = findfirst(x -> contains(string(x.label), key), var.blobEntries)
if isnothing(firstIdx)
throw(KeyError("$key"))
Expand All @@ -129,7 +129,7 @@ function getBlobEntryFirst(dfg::AbstractDFG, label::Symbol, key::Regex)
end

# TODO Consider autogenerating all methods of the form:
# verbNoun(dfg::DFGVariable, label::Symbol, args...; kwargs...) = verbNoun(getVariable(dfg, label), args...; kwargs...)
# verbNoun(dfg::VariableCompute, label::Symbol, args...; kwargs...) = verbNoun(getVariable(dfg, label), args...; kwargs...)
# with something like:
# getvariablemethod = [
# :getBlobEntryFirst,
Expand Down Expand Up @@ -160,7 +160,7 @@ function addBlobEntry!(var::AbstractDFGVariable, entry::BlobEntry;)
return entry
end

function addBlobEntry!(var::PackedVariable, entry::BlobEntry)
function addBlobEntry!(var::VariableDFG, entry::BlobEntry)
entry.label in getproperty.(var.blobEntries, :label) &&
error("blobEntry $(entry.label) already exists on variable $(getLabel(var))")
push!(var.blobEntries, entry)
Expand Down Expand Up @@ -205,7 +205,7 @@ function deleteBlobEntry!(var::AbstractDFGVariable, key::Symbol)
return pop!(var.dataDict, key)
end

function deleteBlobEntry!(var::PackedVariable, key::Symbol)
function deleteBlobEntry!(var::VariableDFG, key::Symbol)
if !hasBlobEntry(var, key)
throw(
KeyError(
Expand Down Expand Up @@ -238,7 +238,7 @@ Does a blob entry (element) exist with `blobLabel`.
"""
hasBlobEntry(var::AbstractDFGVariable, blobLabel::Symbol) = haskey(var.dataDict, blobLabel)

function hasBlobEntry(var::PackedVariable, label::Symbol)
function hasBlobEntry(var::VariableDFG, label::Symbol)
return label in getproperty.(var.blobEntries, :label)
end

Expand All @@ -252,7 +252,7 @@ function getBlobEntries(var::AbstractDFGVariable)
return collect(values(var.dataDict))
end

function getBlobEntries(var::PackedVariable)
function getBlobEntries(var::VariableDFG)
return var.blobEntries
end

Expand Down Expand Up @@ -311,7 +311,7 @@ function listBlobEntries(var::AbstractDFGVariable)
return collect(keys(var.dataDict))
end

function listBlobEntries(var::PackedVariable)
function listBlobEntries(var::VariableDFG)
return getproperty.(var.blobEntries, :label)
end

Expand Down
36 changes: 34 additions & 2 deletions src/DistributedFactorGraphs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,10 @@ export getSummaryGraph
export DFGNode, AbstractDFGVariable, AbstractDFGFactor

# Variables
export DFGVariable, DFGVariableSummary, SkeletonDFGVariable, PackedVariable
export VariableCompute, VariableSummary, VariableSkeleton, VariableDFG

# Factors
export DFGFactor, DFGFactorSummary, SkeletonDFGFactor, PackedFactor, Factor
export FactorCompute, FactorSummary, FactorSkeleton, FactorDFG

# Common
export getSolvable, setSolvable!, isSolvable
Expand Down Expand Up @@ -330,6 +330,7 @@ export plotDFG

## TODO maybe move to DFG
# addAgent!
# deleteAgent!
# listAgents
# addGraph!
# deleteGraph!
Expand Down Expand Up @@ -393,4 +394,35 @@ include("Common.jl")

include("weakdeps_prototypes.jl")

#TODO start off as just an alias before deprecating
# Starting Variable-level nouns-adjective standardisation
export SkeletonDFGVariable
const SkeletonDFGVariable = VariableSkeleton

export DFGVariableSummary
const DFGVariableSummary = VariableSummary

export DFGVariable
const DFGVariable = VariableCompute

export PackedVariable
const PackedVariable = VariableDFG
export Variable
const Variable = VariableDFG

# Starting Factor-level noun-adjective standardisation
export SkeletonDFGFactor
const SkeletonDFGFactor = FactorSkeleton

export DFGFactorSummary
const DFGFactorSummary = FactorSummary

export DFGFactor
const DFGFactor = FactorCompute

export PackedFactor
const PackedFactor = FactorDFG
export Factor
const Factor = FactorDFG

end
52 changes: 30 additions & 22 deletions src/FileDFG/services/FileDFG.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ function loadDFG!(
end

# extract the factor graph from fileDFG folder
factors = DFGFactor[]
factors = FactorCompute[]
varFolder = "$folder/variables"
factorFolder = "$folder/factors"
# Folder preparations
Expand All @@ -155,45 +155,53 @@ function loadDFG!(
!isdir(factorFolder) &&
error("Can't load DFG graph - folder '$factorFolder' doesn't exist")

varFiles = sort(readdir(varFolder; sort = false); lt = natural_lt)
factorFiles = sort(readdir(factorFolder; sort = false); lt = natural_lt)
# varFiles = sort(readdir(varFolder; sort = false); lt = natural_lt)
# factorFiles = sort(readdir(factorFolder; sort = false); lt = natural_lt)
varFiles = readdir(varFolder; sort = false)
factorFiles = readdir(factorFolder; sort = false)

packedvars = @showprogress 1 "loading variables" asyncmap(varFiles) do varFile
jstr = read("$varFolder/$varFile", String)
return JSON3.read(jstr, PackedVariable)
end
# FIXME, why is this treated different from VariableSkeleton, VariableSummary?
# FIXME, still creates type instability on `variables` as either `::Variable` or `::DFGVariable`
if isa(dfgLoadInto, GraphsDFG) && getTypeDFGVariables(dfgLoadInto) == Variable
variables = packedvars
else
variables = unpackVariable.(packedvars)

usePackedVariable =
isa(dfgLoadInto, GraphsDFG) && getTypeDFGVariables(dfgLoadInto) == VariableDFG
# type instability on `variables` as either `::Vector{Variable}` or `::Vector{VariableCompute{<:}}` (vector of abstract)
variables = @showprogress 1 "loading variables" asyncmap(varFiles) do varFile
jstr = read("$varFolder/$varFile", String)
packedvar = JSON3.read(jstr, VariableDFG)
if usePackedVariable
return packedvar
else
return unpackVariable(packedvar)
end
end

@info "Loaded $(length(variables)) variables"#- $(map(v->v.label, variables))"
@info "Inserting variables into graph..."
# Adding variables
map(v -> addVariable!(dfgLoadInto, v), variables)

packedfacts = @showprogress 1 "loading factors" asyncmap(factorFiles) do factorFile
usePackedFactor =
isa(dfgLoadInto, GraphsDFG) && getTypeDFGFactors(dfgLoadInto) == FactorDFG

# `factors` is not type stable `::Vector{Factor}` or `::Vector{FactorCompute{<:}}` (vector of abstract)
factors = @showprogress 1 "loading factors" asyncmap(factorFiles) do factorFile
jstr = read("$factorFolder/$factorFile", String)
return JSON3.read(jstr, PackedFactor)
end
# FIXME, still creates type instability on `variables` as either `::Factor` or `::DFGFactor{<:}`
if isa(dfgLoadInto, GraphsDFG) && getTypeDFGFactors(dfgLoadInto) == PackedFactor
factors = packedfacts
else
factors = unpackFactor.(dfgLoadInto, packedfacts)
packedfact = JSON3.read(jstr, FactorDFG)
if usePackedFactor
return packedfact
else
return unpackFactor(dfgLoadInto, packedfact)
end
end

@info "Loaded $(length(factors)) factors"# - $(map(f->f.label, factors))"
@info "Inserting factors into graph..."
# # Adding factors
map(f -> addFactor!(dfgLoadInto, f), factors)

if isa(dfgLoadInto, GraphsDFG) && getTypeDFGFactors(dfgLoadInto) != PackedFactor
if isa(dfgLoadInto, GraphsDFG) && getTypeDFGFactors(dfgLoadInto) != FactorDFG
# Finally, rebuild the CCW's for the factors to completely reinflate them
# NOTE CREATES A NEW DFGFactor IF CCW TYPE CHANGES
# NOTE CREATES A NEW FactorCompute IF CCW TYPE CHANGES
@info "Rebuilding CCW's for the factors..."
@showprogress 1 "build factor operational memory" for factor in factors
rebuildFactorMetadata!(dfgLoadInto, factor)
Expand Down
Loading

2 comments on commit e2e6151

@Affie
Copy link
Member Author

@Affie Affie commented on e2e6151 Dec 2, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register branch=release/v0.24

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/120512

Tip: Release Notes

Did you know you can add release notes too? Just add markdown formatted text underneath the comment after the text
"Release notes:" and it will be added to the registry PR, and if TagBot is installed it will also be added to the
release that TagBot creates. i.e.

@JuliaRegistrator register

Release notes:

## Breaking changes

- blah

To add them here just re-invoke and the PR will be updated.

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.25.2 -m "<description of version>" e2e6151df9ea25a5e4465acc90fe37a7e2de507d
git push origin v0.25.2

Please sign in to comment.