Skip to content

Commit

Permalink
Merge pull request #1090 from maiko3tattun/0329_FixMODP
Browse files Browse the repository at this point in the history 
Cache frq for MOD+
  • Loading branch information
@stakira
stakira authored Apr 20, 2024
2 parents d62403d + 5fc1701 commit 73b5a2a
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 130 deletions.
1 change: 1 addition & 0 deletions OpenUtau.Core/Classic/ClassicSinger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public class ClassicSinger : USinger {
OtoWatcher otoWatcher;

public bool? UseFilenameAsAlias { get => voicebank.UseFilenameAsAlias; set => voicebank.UseFilenameAsAlias = value; }
public Dictionary<string, Frq> Frqs { get; set; } = new Dictionary<string, Frq>();

public ClassicSinger(Voicebank voicebank) {
this.voicebank = voicebank;
Expand Down
90 changes: 90 additions & 0 deletions OpenUtau.Core/Classic/Frq.cs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,107 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using NAudio.Wave;
using OpenUtau.Core;
using OpenUtau.Core.Ustx;

namespace OpenUtau.Classic {
public class OtoFrq {
public double[] toneDiffFix = new double[0];
public double[] toneDiffStretch = new double[0];
public int hopSize;
public bool loaded = false;

public OtoFrq(UOto oto, Dictionary<string, Frq> dict) {
if (!dict.TryGetValue(oto.File, out var frq)) {
frq = new Frq();
if (frq.Load(oto.File)){
dict.Add(oto.File, frq);
} else {
frq = null;
}
}
if(frq != null && frq.wavSampleLength != - 1) {
this.hopSize = frq.hopSize;

if (frq.wavSampleLength == 0) {
try {
using (var waveStream = Core.Format.Wave.OpenFile(oto.File)) {
var sampleProvider = waveStream.ToSampleProvider();
if (sampleProvider.WaveFormat.SampleRate == 44100) {
frq.wavSampleLength = Core.Format.Wave.GetSamples(sampleProvider).Length;
} else {
frq.wavSampleLength = -1;
}
}
} catch {
frq.wavSampleLength = - 1;
}
}

if (frq.wavSampleLength > 0) {
int offset = (int)Math.Floor(oto.Offset * 44100 / 1000 / frq.hopSize); // frq samples
int consonant = (int)Math.Floor((oto.Offset + oto.Consonant) * 44100 / 1000 / frq.hopSize);
int cutoff = oto.Cutoff < 0 ?
(int)Math.Floor((oto.Offset - oto.Cutoff) * 44100 / 1000 / frq.hopSize)
: frq.wavSampleLength - (int)Math.Floor(oto.Cutoff * 44100 / 1000 / frq.hopSize);
var completionF0 = Completion(frq.f0);
var averageTone = MusicMath.FreqToTone(frq.averageF0);
toneDiffFix = completionF0.Skip(offset).Take(consonant - offset).Select(f => MusicMath.FreqToTone(f) - averageTone).ToArray();
toneDiffStretch = completionF0.Skip(consonant).Take(cutoff - consonant).Select(f => MusicMath.FreqToTone(f) - averageTone).ToArray();

loaded = true;
}
}
}

private double[] Completion(double[] frqs) {
var list = new List<double>();
for (int i = 0; i < frqs.Length; i++) {
if (frqs[i] <= 60) {
int min = i - 1;
double minFrq = 0;
while (min >= 0) {
if (frqs[min] > 60) {
minFrq = frqs[min];
break;
}
min--;
}
int max = i + 1;
double maxFrq = 0;
while (max < frqs.Length) {
if (frqs[max] > 60) {
maxFrq = frqs[max];
break;
}
max++;
}
if (minFrq <= 60) {
list.Add(maxFrq);
} else if (maxFrq <= 60) {
list.Add(minFrq);
} else {
list.Add(MusicMath.Linear(min, max, minFrq, maxFrq, i));
}
} else {
list.Add(frqs[i]);
}
}
return list.ToArray();
}
}

public class Frq {
public const int kHopSize = 256;

public int hopSize;
public double averageF0;
public double[] f0 = new double[0];
public double[] amp = new double[0];
public int wavSampleLength = 0;

/// <summary>
/// If the wav path is null (machine learning voicebank), return false.
Expand Down
116 changes: 60 additions & 56 deletions OpenUtau.Core/Render/RenderPhrase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Numerics;
using K4os.Hash.xxHash;
using OpenUtau.Classic;
using OpenUtau.Core.Ustx;
using Serilog;

Expand Down Expand Up @@ -299,69 +300,72 @@ internal RenderPhrase(UProject project, UTrack track, UVoicePart part, IEnumerab
}
}
// Mod plus
if (track.TryGetExpDescriptor(project, Format.Ustx.MODP, out var modp) && renderer.SupportsExpression(modp)) {
if (track.TryGetExpDescriptor(project, Format.Ustx.MODP, out var modp) && renderer.SupportsExpression(modp) && singer is ClassicSinger cSinger) {
foreach (var phoneme in phonemes) {
var mod = phoneme.GetExpression(project, track, Format.Ustx.MODP).Item1;
if (mod == 0) {
var phonemeModp = phoneme.GetExpression(project, track, Format.Ustx.MODP).Item1;
if (phonemeModp == 0) {
continue;
}

try {
if (phoneme.TryGetFrq(out var frqFix, out var frqStretch, out double average, out int hopSize)) {
UTempo[] noteTempos = project.timeAxis.TemposBetweenTicks(part.position + phoneme.position, part.position + phoneme.End);
var tempo = noteTempos[0].bpm; // compromise 妥協！
var frqIntervalTick = MusicMath.TempoMsToTick(tempo, (double)1 * 1000 / 44100 * hopSize);
double consonantStretch = Math.Pow(2f, 1.0f - phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 / 100f);

var preutter = MusicMath.TempoMsToTick(tempo, Math.Min(phoneme.preutter, phoneme.oto.Preutter * consonantStretch));
int startIndex = Math.Max(0, (int)Math.Floor((phoneme.position - pitchStart - preutter) / pitchInterval));
int position = (int)Math.Round((double)((phoneme.position - pitchStart) / pitchInterval));
int startStretch = position + (int)Math.Round(MusicMath.TempoMsToTick(tempo, (phoneme.oto.Consonant - phoneme.oto.Preutter) * consonantStretch) / pitchInterval);
int endIndex = Math.Min(pitches.Length, (int)Math.Ceiling(phoneme.End - pitchStart - MusicMath.TempoMsToTick(tempo, phoneme.tailIntrude - phoneme.tailOverlap)) / pitchInterval);

frqFix = frqFix.Select(f => f - average).ToArray();
frqStretch = frqStretch.Select(f => f - average).ToArray();
double stretch = 1;
if (frqStretch.Length * frqIntervalTick < ((double)endIndex - startStretch) * pitchInterval) {
stretch = ((double)endIndex - startStretch) * pitchInterval / (frqStretch.Length * frqIntervalTick);
}
var env0 = new Vector2(0, 0);
var env1 = new Vector2((phoneme.envelope.data[1].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env3 = new Vector2((phoneme.envelope.data[3].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env4 = new Vector2(1, 0);

for (int i = 0; startStretch + i <= endIndex; i++) {
var pit = startStretch + i;
if (pit >= pitches.Length) break;
var frq = i * (pitchInterval / frqIntervalTick) / stretch;
var frqMin = Math.Clamp((int)Math.Floor(frq), 0, frqStretch.Length - 1);
var frqMax = Math.Clamp((int)Math.Ceiling(frq), 0, frqStretch.Length - 1);
var diff = MusicMath.Linear(frqMin, frqMax, frqStretch[frqMin], frqStretch[frqMax], frq);
diff = diff * mod / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
}
for (int i = 0; startStretch + i - 1 >= startIndex; i--) {
var pit = startStretch + i - 1;
if (pit > endIndex || pit >= pitches.Length) continue;
var frq = frqFix.Length + (i * (pitchInterval / frqIntervalTick) / consonantStretch);
var frqMin = Math.Clamp((int)Math.Floor(frq), 0, frqFix.Length - 1);
var frqMax = Math.Clamp((int)Math.Ceiling(frq), 0, frqFix.Length - 1);
var diff = MusicMath.Linear(frqMin, frqMax, frqFix[frqMin], frqFix[frqMax], frq);
diff = diff * mod / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
if (phoneme.oto.Frq == null) {
phoneme.oto.Frq = new OtoFrq(phoneme.oto, cSinger.Frqs);
}
if (phoneme.oto.Frq.loaded == false) {
continue;
}
var frq = phoneme.oto.Frq;
UTempo[] noteTempos = project.timeAxis.TemposBetweenTicks(part.position + phoneme.position, part.position + phoneme.End);
var tempo = noteTempos[0].bpm; // compromise 妥協！
var frqIntervalTick = MusicMath.TempoMsToTick(tempo, (double)1 * 1000 / 44100 * frq.hopSize);
double consonantStretch = Math.Pow(2f, 1.0f - phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 / 100f);

var preutter = MusicMath.TempoMsToTick(tempo, Math.Min(phoneme.preutter, phoneme.oto.Preutter * consonantStretch));
int startIndex = Math.Max(0, (int)Math.Floor((phoneme.position - pitchStart - preutter) / pitchInterval));
int position = (int)Math.Round((double)((phoneme.position - pitchStart) / pitchInterval));
int startStretch = position + (int)Math.Round(MusicMath.TempoMsToTick(tempo, (phoneme.oto.Consonant - phoneme.oto.Preutter) * consonantStretch) / pitchInterval);
int endIndex = Math.Min(pitches.Length, (int)Math.Ceiling(phoneme.End - pitchStart - MusicMath.TempoMsToTick(tempo, phoneme.tailIntrude - phoneme.tailOverlap)) / pitchInterval);

double stretch = 1;
if (frq.toneDiffStretch.Length * frqIntervalTick < ((double)endIndex - startStretch) * pitchInterval) {
stretch = ((double)endIndex - startStretch) * pitchInterval / (frq.toneDiffStretch.Length * frqIntervalTick);
}
var env0 = new Vector2(0, 0);
var env1 = new Vector2((phoneme.envelope.data[1].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env3 = new Vector2((phoneme.envelope.data[3].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env4 = new Vector2(1, 0);

for (int i = 0; startStretch + i <= endIndex; i++) {
var pit = startStretch + i;
if (pit >= pitches.Length) break;
var frqPoint = i * (pitchInterval / frqIntervalTick) / stretch;
var frqPointMin = Math.Clamp((int)Math.Floor(frqPoint), 0, frq.toneDiffStretch.Length - 1);
var frqPointMax = Math.Clamp((int)Math.Ceiling(frqPoint), 0, frq.toneDiffStretch.Length - 1);
var diff = MusicMath.Linear(frqPointMin, frqPointMax, frq.toneDiffStretch[frqPointMin], frq.toneDiffStretch[frqPointMax], frqPoint);
diff = diff * phonemeModp / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
}
for (int i = 0; startStretch + i - 1 >= startIndex; i--) {
var pit = startStretch + i - 1;
if (pit > endIndex || pit >= pitches.Length) continue;
var frqPoint = frq.toneDiffFix.Length + (i * (pitchInterval / frqIntervalTick) / consonantStretch);
var frqPointMin = Math.Clamp((int)Math.Floor(frqPoint), 0, frq.toneDiffFix.Length - 1);
var frqPointMax = Math.Clamp((int)Math.Ceiling(frqPoint), 0, frq.toneDiffFix.Length - 1);
var diff = MusicMath.Linear(frqPointMin, frqPointMax, frq.toneDiffFix[frqPointMin], frq.toneDiffFix[frqPointMax], frqPoint);
diff = diff * phonemeModp / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
}
double Fade(double diff, int pit) {
var percentage = (double)(pit - startIndex) / (endIndex - startIndex);
if (phoneme.Next != null && phoneme.End == phoneme.Next.position && percentage > env3.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env3.X, env4.X, env3.Y, env4.Y, percentage), 0, 100) / 100;
}
double Fade(double diff, int pit) {
var percentage = (double)(pit - startIndex) / (endIndex - startIndex);
if (phoneme.Next != null && phoneme.End == phoneme.Next.position && percentage > env3.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env3.X, env4.X, env3.Y, env4.Y, percentage), 0, 100) / 100;
}
if (phoneme.Prev != null && phoneme.Prev.End == phoneme.position && percentage < env1.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env0.X, env1.X, env0.Y, env1.Y, percentage), 0, 100) / 100;
}
return diff;
if (phoneme.Prev != null && phoneme.Prev.End == phoneme.position && percentage < env1.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env0.X, env1.X, env0.Y, env1.Y, percentage), 0, 100) / 100;
}
return diff;
}
} catch(Exception e) {
Log.Error(e, "Failed to compute mod plus.");
Expand Down
74 changes: 0 additions & 74 deletions OpenUtau.Core/Ustx/UPhoneme.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using NAudio.Wave;
using OpenUtau.Classic;
using SharpCompress;
using YamlDotNet.Serialization;

namespace OpenUtau.Core.Ustx {
Expand Down Expand Up @@ -237,77 +234,6 @@ public string GetVoiceColor(UProject project, UTrack track) {
}
return track.VoiceColorExp.options[index];
}

public bool TryGetFrq(out double[] frqFix, out double[] frqStretch, out double average, out int hopSize) {
frqFix = new double[0];
frqStretch = new double[0];
average = 0;
hopSize = 0;

var frq = new Frq();
if (frq.Load(oto.File)) {
average = MusicMath.FreqToTone(frq.averageF0); // 1 = 1tone
hopSize = frq.hopSize;

int wavLength;
using (var waveStream = Format.Wave.OpenFile(oto.File)) {
var sampleProvider = waveStream.ToSampleProvider();
if (sampleProvider.WaveFormat.SampleRate != 44100) {
return false;
}
wavLength = Format.Wave.GetSamples(sampleProvider).Length;
}

int offset = (int)Math.Floor(oto.Offset * 44100 / 1000 / frq.hopSize); // frq samples
int consonant = (int)Math.Floor((oto.Offset + oto.Consonant) * 44100 / 1000 / frq.hopSize);
int cutoff = oto.Cutoff < 0 ?
(int)Math.Floor((oto.Offset - oto.Cutoff) * 44100 / 1000 / frq.hopSize)
: wavLength - (int)Math.Floor(oto.Cutoff * 44100 / 1000 / frq.hopSize);
var avr = average;
var f0 = Completion(frq.f0);
frqFix = f0.Skip(offset).Take(consonant - offset).Select(f => MusicMath.FreqToTone(f)).ToArray();
frqStretch = f0.Skip(consonant).Take(cutoff - consonant).Select(f => MusicMath.FreqToTone(f)).ToArray();

double[] Completion(double[] frqs) {
var list = new List<double>();
for (int i = 0; i < frqs.Length; i++) {
if (frqs[i] <= 0) {
int min = i - 1;
double minFrq = 0;
while (min >= 0) {
if (frqs[min] > 0) {
minFrq = frqs[min];
break;
}
min--;
}
int max = i + 1;
double maxFrq = 0;
while (max < frqs.Length) {
if (frqs[max] > 0) {
maxFrq = frqs[max];
break;
}
max++;
}
if(minFrq <= 0) {
list.Add(maxFrq);
} else if (maxFrq <= 0) {
list.Add(minFrq);
} else {
list.Add(MusicMath.Linear(min, max, minFrq, maxFrq, i));
}
} else {
list.Add(frqs[i]);
}
}
return list.ToArray();
}
return true;
} else {
return false;
}
}
}

public class UEnvelope {
Expand Down
1 change: 1 addition & 0 deletions OpenUtau.Core/Ustx/USinger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ public double Overlap {
NotifyPropertyChanged(nameof(Overlap));
}
}
public OtoFrq Frq { get;set; }
public List<string> SearchTerms { get; private set; }

public event PropertyChangedEventHandler PropertyChanged;
Expand Down

0 comments on commit 73b5a2a

Please sign in to comment.