From b1bae9230815a7b0cf7a4b5a0a0e845a8800c540 Mon Sep 17 00:00:00 2001 From: AnimeDL Date: Wed, 10 Apr 2024 22:03:17 -0700 Subject: [PATCH] Allow for byterange mpd/dash downloads Prep for new service --- @types/mpd-parser.d.ts | 30 +++++++++++++ crunchy.ts | 2 +- hidive.ts | 4 +- modules/module.transform-mpd.ts | 79 ++++++++++++++++++++++++++++++--- 4 files changed, 107 insertions(+), 8 deletions(-) diff --git a/@types/mpd-parser.d.ts b/@types/mpd-parser.d.ts index 516cf2a1..a45968f7 100644 --- a/@types/mpd-parser.d.ts +++ b/@types/mpd-parser.d.ts @@ -7,11 +7,40 @@ declare module 'mpd-parser' { map: { uri: string, resolvedUri: string, + byterange?: { + length: number, + offset: number + } + }, + byterange?: { + length: number, + offset: number }, number: number, presentationTime: number } + export type Sidx = { + uri: string, + resolvedUri: string, + byterange: { + length: number, + offset: number + }, + map: { + uri: string, + resolvedUri: string, + byterange: { + length: number, + offset: number + } + }, + duration: number, + timeline: number, + presentationTime: number, + number: number + } + export type Playlist = { attributes: { NAME: string, @@ -45,6 +74,7 @@ declare module 'mpd-parser' { } } segments: Segment[] + sidx?: Sidx } export type Manifest = { diff --git a/crunchy.ts b/crunchy.ts index 967bf359..4f3e1e93 100644 --- a/crunchy.ts +++ b/crunchy.ts @@ -1563,7 +1563,7 @@ export default class Crunchy implements ServiceClass { const streamPlaylistBody = await streamPlaylistsReq.res.text(); if (streamPlaylistBody.match('MPD')) { //Parse MPD Playlists - const streamPlaylists = parse(streamPlaylistBody, langsData.findLang(langsData.fixLanguageTag(pbData.meta.audio_locale as string) || ''), curStream.url.match(/.*\.urlset\//)[0]); + const streamPlaylists = await parse(streamPlaylistBody, langsData.findLang(langsData.fixLanguageTag(pbData.meta.audio_locale as string) || ''), curStream.url.match(/.*\.urlset\//)[0]); //Get name of CDNs/Servers const streamServers = Object.keys(streamPlaylists); diff --git a/hidive.ts b/hidive.ts index fcbe1770..ce58d902 100644 --- a/hidive.ts +++ b/hidive.ts @@ -1014,7 +1014,7 @@ export default class Hidive implements ServiceClass { console.info('\tSubs : ' + availableSubs.map(a => langsData.languages.find(b => b.new_hd_locale == a.language)?.name).join('\n\t\t')); console.info(`[INFO] Selected dub(s): ${options.dubLang.join(', ')}`); const baseUrl = playbackData.dash[0].url.split('master')[0]; - const parsedmpd = parse(mpd, undefined, baseUrl); + const parsedmpd = await parse(mpd, undefined, baseUrl); const res = await this.downloadMPD(parsedmpd, availableSubs, selectedEpisode, options); if (res === undefined || res.error) { console.error('Failed to download media list'); @@ -1103,7 +1103,7 @@ export default class Hidive implements ServiceClass { console.info('\tSubs : ' + availableSubs.map(a => langsData.languages.find(b => b.new_hd_locale == a.language)?.name).join('\n\t\t')); console.info(`[INFO] Selected dub(s): ${options.dubLang.join(', ')}`); const baseUrl = playbackData.dash[0].url.split('master')[0]; - const parsedmpd = parse(mpd, undefined, baseUrl); + const parsedmpd = await parse(mpd, undefined, baseUrl); const res = await this.downloadMPD(parsedmpd, availableSubs, selectedEpisode, options); if (res === undefined || res.error) { console.error('Failed to download media list'); diff --git a/modules/module.transform-mpd.ts b/modules/module.transform-mpd.ts index a56a42c7..11d6a558 100644 --- a/modules/module.transform-mpd.ts +++ b/modules/module.transform-mpd.ts @@ -7,9 +7,17 @@ type Segment = { duration: number; map: { uri: string; + byterange?: { + length: number, + offset: number + }; + }; + byterange?: { + length: number, + offset: number }; - number: number; - presentationTime: number; + number?: number; + presentationTime?: number; } export type PlaylistItem = { @@ -38,19 +46,49 @@ export type MPDParsed = { } } -export function parse(manifest: string, language?: LanguageItem, url?: string) { +export async function parse(manifest: string, language?: LanguageItem, url?: string) { if (!manifest.includes('BaseURL') && url) { manifest = manifest.replace(/(]*>)/gm, `$1${url}`); } const parsed = mpdParse(manifest); const ret: MPDParsed = {}; + // Audio Loop for (const item of Object.values(parsed.mediaGroups.AUDIO.audio)){ for (const playlist of item.playlists) { const host = new URL(playlist.resolvedUri).hostname; if (!Object.prototype.hasOwnProperty.call(ret, host)) ret[host] = { audio: [], video: [] }; + + if (playlist.sidx) { + const item = await fetch(playlist.sidx.uri, { + 'method': 'head' + }); + const byteLength = parseInt(item.headers.get('content-length') as string); + let currentByte = playlist.sidx.map.byterange.length; + while (currentByte <= byteLength) { + playlist.segments.push({ + 'duration': 0, + 'map': { + 'uri': playlist.resolvedUri, + 'resolvedUri': playlist.resolvedUri, + 'byterange': playlist.sidx.map.byterange + }, + 'uri': playlist.resolvedUri, + 'resolvedUri': playlist.resolvedUri, + 'byterange': { + 'length': 500000, + 'offset': currentByte + }, + timeline: 0, + number: 0, + presentationTime: 0 + }); + currentByte = currentByte + 500000; + } + } + //Find and add audio language if it is found in the MPD let audiolang: LanguageItem; const foundlanguage = findLang(languages.find(a => a.code === item.language)?.cr_locale ?? 'unknown'); @@ -68,10 +106,11 @@ export function parse(manifest: string, language?: LanguageItem, url?: string) { const map_uri = segment.map.resolvedUri; return { duration: segment.duration, - map: { uri: map_uri }, + map: { uri: map_uri, byterange: segment.map.byterange }, number: segment.number, presentationTime: segment.presentationTime, timeline: segment.timeline, + byterange: segment.byterange, uri }; }) @@ -85,11 +124,40 @@ export function parse(manifest: string, language?: LanguageItem, url?: string) { } } + // Video Loop for (const playlist of parsed.playlists) { const host = new URL(playlist.resolvedUri).hostname; if (!Object.prototype.hasOwnProperty.call(ret, host)) ret[host] = { audio: [], video: [] }; + if (playlist.sidx) { + const item = await fetch(playlist.sidx.uri, { + 'method': 'head' + }); + const byteLength = parseInt(item.headers.get('content-length') as string); + let currentByte = playlist.sidx.map.byterange.length; + while (currentByte <= byteLength) { + playlist.segments.push({ + 'duration': 0, + 'map': { + 'uri': playlist.resolvedUri, + 'resolvedUri': playlist.resolvedUri, + 'byterange': playlist.sidx.map.byterange + }, + 'uri': playlist.resolvedUri, + 'resolvedUri': playlist.resolvedUri, + 'byterange': { + 'length': 2000000, + 'offset': currentByte + }, + timeline: 0, + number: 0, + presentationTime: 0 + }); + currentByte = currentByte + 2000000; + } + } + const pItem: VideoPlayList = { bandwidth: playlist.attributes.BANDWIDTH, quality: playlist.attributes.RESOLUTION!, @@ -98,10 +166,11 @@ export function parse(manifest: string, language?: LanguageItem, url?: string) { const map_uri = segment.map.resolvedUri; return { duration: segment.duration, - map: { uri: map_uri }, + map: { uri: map_uri, byterange: segment.map.byterange }, number: segment.number, presentationTime: segment.presentationTime, timeline: segment.timeline, + byterange: segment.byterange, uri }; })