Skip to content

Commit

Permalink
fix motion photo files with MP~2 extension marked unsupported and ski…
Browse files Browse the repository at this point in the history 
…pped #405 (#419)

* Improve get debug data

* adapt the fakefs reader

* fix duplicate counts when same path in the archive

* fix Duplicated assets already uploaded are not counted as Duplicates #414

* Merge branch 'main' into fix-0.21

* fix Movie part of live pictures already present on the server aren't counted, leading to a no 100% progression #414

* fix upload counters for live photos

* add upload  of .MP~* motion pictures

* add support of .MP~* in google photos

* fix motion photo files with MP~2 extension marked unsupported and skipped #405

* linter messages
  • Loading branch information
@simulot
simulot authored Aug 1, 2024
1 parent ff81fd0 commit 1d3016e
Show file tree
Hide file tree
Showing 11 changed files with 234 additions and 132 deletions.
146 changes: 101 additions & 45 deletions browser/files/localassets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ type fileLinks struct {
type LocalAssetBrowser struct {
fsyss []fs.FS
albums map[string]string
catalogs map[fs.FS]map[string]map[string]fileLinks // per FS, DIR and base name
catalogs map[fs.FS]map[string][]string
log *fileevent.Recorder
sm immich.SupportedMedia
bannedFiles namematcher.List // list of file pattern to be exclude
Expand All @@ -38,9 +38,10 @@ func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, fsyss ...fs.FS) (
return &LocalAssetBrowser{
fsyss: fsyss,
albums: map[string]string{},
catalogs: map[fs.FS]map[string]map[string]fileLinks{},
catalogs: map[fs.FS]map[string][]string{},
log: l,
whenNoDate: "FILE",
sm: immich.DefaultSupportedMedia,
}, nil
}

Expand Down Expand Up @@ -71,17 +72,16 @@ func (la *LocalAssetBrowser) Prepare(ctx context.Context) error {
}

func (la *LocalAssetBrowser) passOneFsWalk(ctx context.Context, fsys fs.FS) error {
fsCatalog := map[string]map[string]fileLinks{}
la.catalogs[fsys] = map[string][]string{}
err := fs.WalkDir(fsys, ".",
func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

if d.IsDir() {
fsCatalog[name] = map[string]fileLinks{}
return nil
la.catalogs[fsys][name] = []string{}
}

select {
case <-ctx.Done():
// If the context has been cancelled, return immediately
Expand All @@ -100,40 +100,25 @@ func (la *LocalAssetBrowser) passOneFsWalk(ctx context.Context, fsys fs.FS) erro
return nil
}

linkBase := strings.TrimSuffix(base, ext)
for {
e := path.Ext(linkBase)
if la.sm.IsMedia(e) {
linkBase = strings.TrimSuffix(linkBase, e)
continue
}
break
}
dirLinks := fsCatalog[dir]
links := dirLinks[linkBase]
cat := la.catalogs[fsys][dir]

switch mediaType {
case immich.TypeImage:
links.image = name
la.log.Record(ctx, fileevent.DiscoveredImage, nil, name)
case immich.TypeVideo:
links.video = name
la.log.Record(ctx, fileevent.DiscoveredVideo, nil, name)
case immich.TypeSidecar:
links.sidecar = name
la.log.Record(ctx, fileevent.DiscoveredSidecar, nil, name)
}

if la.bannedFiles.Match(name) {
la.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, name, "reason", "banned file")
return nil
}
dirLinks[linkBase] = links
fsCatalog[dir] = dirLinks
la.catalogs[fsys][dir] = append(cat, name)
}
return nil
})
la.catalogs[fsys] = fsCatalog
return err
}

Expand All @@ -150,46 +135,117 @@ func (la *LocalAssetBrowser) Browse(ctx context.Context) chan *browser.LocalAsse
}
}
for _, fsys := range la.fsyss {
dirLinks := la.catalogs[fsys]
dirKeys := gen.MapKeys(dirLinks)
sort.Strings(dirKeys)
for _, d := range dirKeys {
linksList := la.catalogs[fsys][d]
linksKeys := gen.MapKeys(linksList)
sort.Strings(linksKeys)
for _, l := range linksKeys {
dirs := gen.MapKeys(la.catalogs[fsys])
sort.Strings(dirs)
for _, dir := range dirs {
links := map[string]fileLinks{}
files := la.catalogs[fsys][dir]

if len(files) == 0 {
continue
}

// Scan images first
for _, file := range files {
ext := path.Ext(file)
if la.sm.TypeFromExt(ext) == immich.TypeImage {
linked := links[file]
linked.image = file
links[file] = linked
}
}

next:
for _, file := range files {
ext := path.Ext(file)
t := la.sm.TypeFromExt(ext)
if t == immich.TypeImage {
continue next
}

base := strings.TrimSuffix(file, ext)
switch t {
case immich.TypeSidecar:
if image, ok := links[base]; ok {
// file.ext.XMP -> file.ext
image.sidecar = file
links[base] = image
continue next
}
for f := range links {
if strings.TrimSuffix(f, path.Ext(f)) == base {
if image, ok := links[f]; ok {
// base.XMP -> base.ext
image.sidecar = file
links[f] = image
continue next
}
}
}
case immich.TypeVideo:
if image, ok := links[base]; ok {
// file.MP.ext -> file.ext
image.sidecar = file
links[base] = image
continue next
}
for f := range links {
if strings.TrimSuffix(f, path.Ext(f)) == base {
if image, ok := links[f]; ok {
// base.MP4 -> base.ext
image.video = file
links[f] = image
continue next
}
}
if strings.TrimSuffix(f, path.Ext(f)) == file {
if image, ok := links[f]; ok {
// base.MP4 -> base.ext
image.video = file
links[f] = image
continue next
}
}
}
// Unlinked video
links[file] = fileLinks{video: file}
}
}

files = gen.MapKeys(links)
sort.Strings(files)
for _, file := range files {
var a *browser.LocalAssetFile
links := linksList[l]
linked := links[file]

if links.image != "" {
a, err = la.assetFromFile(fsys, links.image)
if linked.image != "" {
a, err = la.assetFromFile(fsys, linked.image)
if err != nil {
errFn(links.image, err)
errFn(linked.image, err)
return
}
if links.video != "" {
a.LivePhoto, err = la.assetFromFile(fsys, links.video)
if linked.video != "" {
a.LivePhoto, err = la.assetFromFile(fsys, linked.video)
if err != nil {
errFn(links.video, err)
errFn(linked.video, err)
return
}
}
} else if links.video != "" {
a, err = la.assetFromFile(fsys, links.video)
} else if linked.video != "" {
a, err = la.assetFromFile(fsys, linked.video)
if err != nil {
errFn(links.video, err)
errFn(linked.video, err)
return
}
}

if a != nil && links.sidecar != "" {
if a != nil && linked.sidecar != "" {
a.SideCar = metadata.SideCarFile{
FSys: fsys,
FileName: links.sidecar,
FileName: linked.sidecar,
}
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, links.sidecar, "main", a.FileName)
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, linked.sidecar, "main", a.FileName)
}

select {
case <-ctx.Done():
return
Expand Down
112 changes: 75 additions & 37 deletions browser/files/localassets_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
package files_test
package files

import (
"context"
"errors"
"io/fs"
"path"
"reflect"
"sort"
"testing"

"github.com/kr/pretty"
"github.com/psanford/memfs"
"github.com/simulot/immich-go/browser/files"
"github.com/simulot/immich-go/helpers/fileevent"
"github.com/simulot/immich-go/helpers/namematcher"
"github.com/simulot/immich-go/immich"
Expand All @@ -37,49 +36,77 @@ func (mfs *inMemFS) addFile(name string) *inMemFS {
return mfs
}

func generateFS() *inMemFS {
return newInMemFS().
addFile("root_01.jpg").
addFile("photos/photo_01.jpg").
addFile("photos/photo_02.cr3").
addFile("photos/photo_03.jpg").
addFile("photos/summer 2023/20230801-001.jpg").
addFile("photos/summer 2023/20230801-002.jpg").
addFile("photos/summer 2023/20230801-003.cr3").
addFile("@eaDir/thb1.jpg").
addFile("photos/SYNOFILE_THUMB_0001.jpg").
addFile("photos/summer 2023/.@__thumb/thb2.jpg")
}

func TestLocalAssets(t *testing.T) {
tc := []struct {
name string
expected []string
fsys fs.FS
expected map[string]fileLinks
}{
{
name: "all",
expected: []string{
"root_01.jpg",
"photos/photo_01.jpg",
"photos/photo_02.cr3",
"photos/photo_03.jpg",
"photos/summer 2023/20230801-001.jpg",
"photos/summer 2023/20230801-002.jpg",
"photos/summer 2023/20230801-003.cr3",
name: "simple",
fsys: newInMemFS().
addFile("root_01.jpg").
addFile("photos/photo_01.jpg").
addFile("photos/photo_02.cr3").
addFile("photos/photo_03.jpg").
addFile("photos/summer 2023/20230801-001.jpg").
addFile("photos/summer 2023/20230801-002.jpg").
addFile("photos/summer 2023/20230801-003.cr3").
addFile("@eaDir/thb1.jpg").
addFile("photos/SYNOFILE_THUMB_0001.jpg").
addFile("photos/summer 2023/.@__thumb/thb2.jpg"),
expected: map[string]fileLinks{
"root_01.jpg": {image: "root_01.jpg"},
"photos/photo_01.jpg": {image: "photos/photo_01.jpg"},
"photos/photo_02.cr3": {image: "photos/photo_02.cr3"},
"photos/photo_03.jpg": {image: "photos/photo_03.jpg"},
"photos/summer 2023/20230801-001.jpg": {image: "photos/summer 2023/20230801-001.jpg"},
"photos/summer 2023/20230801-002.jpg": {image: "photos/summer 2023/20230801-002.jpg"},
"photos/summer 2023/20230801-003.cr3": {image: "photos/summer 2023/20230801-003.cr3"},
},
},
{
name: "motion picture",
fsys: newInMemFS().
addFile("motion/PXL_20210102_221126856.MP~2").
addFile("motion/PXL_20210102_221126856.MP~2.jpg").
addFile("motion/PXL_20210102_221126856.MP.jpg").
addFile("motion/PXL_20210102_221126856.MP").
addFile("motion/20231227_152817.jpg").
addFile("motion/20231227_152817.MP4"),
expected: map[string]fileLinks{
"motion/PXL_20210102_221126856.MP.jpg": {image: "motion/PXL_20210102_221126856.MP.jpg", video: "motion/PXL_20210102_221126856.MP"},
"motion/PXL_20210102_221126856.MP~2.jpg": {image: "motion/PXL_20210102_221126856.MP~2.jpg", video: "motion/PXL_20210102_221126856.MP~2"},
"motion/20231227_152817.jpg": {image: "motion/20231227_152817.jpg", video: "motion/20231227_152817.MP4"},
},
},
{
name: "sidecar",
fsys: newInMemFS().
addFile("root_01.jpg").
addFile("root_01.XMP").
addFile("root_02.jpg").
addFile("root_02.jpg.XMP").
addFile("video_01.mp4").
addFile("video_01.mp4.XMP").
addFile("root_03.MP.jpg").
addFile("root_03.MP.jpg.XMP").
addFile("root_03.MP"),
expected: map[string]fileLinks{
"root_01.jpg": {image: "root_01.jpg", sidecar: "root_01.XMP"},
"root_02.jpg": {image: "root_02.jpg", sidecar: "root_02.jpg.XMP"},
"root_03.MP.jpg": {image: "root_03.MP.jpg", sidecar: "root_03.MP.jpg.XMP", video: "root_03.MP"},
"video_01.mp4": {video: "video_01.mp4", sidecar: "video_01.mp4.XMP"},
},
},
}

for _, c := range tc {
t.Run(c.name, func(t *testing.T) {
fsys := generateFS()
if fsys.err != nil {
t.Error(fsys.err)
return
}
fsys := c.fsys
ctx := context.Background()

b, err := files.NewLocalFiles(ctx, fileevent.NewRecorder(nil, false), fsys)
b, err := NewLocalFiles(ctx, fileevent.NewRecorder(nil, false), fsys)
if err != nil {
t.Error(err)
}
Expand All @@ -96,12 +123,23 @@ func TestLocalAssets(t *testing.T) {
t.Error(err)
}

results := []string{}
results := map[string]fileLinks{}
for a := range b.Browse(ctx) {
results = append(results, a.FileName)
links := fileLinks{}
ext := path.Ext(a.FileName)
if b.sm.TypeFromExt(ext) == immich.TypeImage {
links.image = a.FileName
if a.LivePhoto != nil {
links.video = a.LivePhoto.FileName
}
} else {
links.video = a.FileName
}
if a.SideCar.FileName != "" {
links.sidecar = a.SideCar.FileName
}
results[a.FileName] = links
}
sort.Strings(c.expected)
sort.Strings(results)

if !reflect.DeepEqual(results, c.expected) {
t.Errorf("difference\n")
Expand Down
Loading

0 comments on commit 1d3016e

Please sign in to comment.