Fork for YouTube downloader
This commit is contained in:
parent
518a01052c
commit
51d3008094
|
@ -3,12 +3,12 @@ requires = ["hatchling"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "pix360_krpano"
|
name = "pix360_youtube"
|
||||||
version = "0.0.1"
|
version = "0.0.1"
|
||||||
authors = [
|
authors = [
|
||||||
{ name="Kumi Systems e.U.", email="office@kumi.systems" },
|
{ name="Kumi Systems e.U.", email="office@kumi.systems" },
|
||||||
]
|
]
|
||||||
description = "krpano downloader module for PIX360"
|
description = "YouTube downloader module for PIX360"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.8"
|
requires-python = ">=3.8"
|
||||||
classifiers = [
|
classifiers = [
|
||||||
|
@ -16,10 +16,13 @@ classifiers = [
|
||||||
"License :: OSI Approved :: MIT License",
|
"License :: OSI Approved :: MIT License",
|
||||||
"Operating System :: OS Independent",
|
"Operating System :: OS Independent",
|
||||||
]
|
]
|
||||||
|
dependencies = [
|
||||||
|
"yt-dlp",
|
||||||
|
]
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
"Homepage" = "https://kumig.it/kumisystems/pix360-eyerevolution"
|
"Homepage" = "https://kumig.it/kumisystems/pix360-youtube"
|
||||||
"Bug Tracker" = "https://kumig.it/kumisystems/pix360-eyerevolution/issues"
|
"Bug Tracker" = "https://kumig.it/kumisystems/pix360-youtube/issues"
|
||||||
|
|
||||||
[project.entry-points."pix360downloader"]
|
[project.entry-points."pix360downloader"]
|
||||||
"pix360-krpano" = "pix360_krpano.modules:KRPanoDownloader"
|
"pix360-youtube" = "pix360_youtube.modules:YouTubeDownloader"
|
|
@ -1,243 +0,0 @@
|
||||||
from pix360core.classes import DownloaderModule, HTTPRequest, DownloadError, DEFAULT_CUBEMAP_TO_EQUIRECTANGULAR_STITCHER, DEFAULT_STITCHER
|
|
||||||
from pix360core.models import Conversion, File
|
|
||||||
|
|
||||||
from django.core.files.base import ContentFile
|
|
||||||
|
|
||||||
from typing import List, Tuple, Dict
|
|
||||||
|
|
||||||
import re
|
|
||||||
import logging
|
|
||||||
import uuid
|
|
||||||
|
|
||||||
class KRPanoDownloader(DownloaderModule):
|
|
||||||
name: str = "KRPano Downloader"
|
|
||||||
identifier: str = "systems.kumi.pix360.krpano"
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.logger = logging.getLogger("pix360")
|
|
||||||
|
|
||||||
REGEX_FULL: List[Tuple[str, int, Dict[str, str]]] = [
|
|
||||||
(r"\d+/\d+/\d+_\d+\.jpg", DownloaderModule.CERTAINTY_PROBABLE, {}),
|
|
||||||
]
|
|
||||||
|
|
||||||
REGEX_SIMPLE: List[Tuple[str, int, Dict[str, str]]] = [
|
|
||||||
(r"\_[frblud].jpg", DownloaderModule.CERTAINTY_PROBABLE, {}),
|
|
||||||
(r"^\d.jpg", DownloaderModule.CERTAINTY_POSSIBLE, {"tiles": "012345"}),
|
|
||||||
]
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def test_url(cls, url: str) -> int:
|
|
||||||
"""Test if URL looks like this module can handle it
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url (str): URL to test
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
int: Certainty level of the URL being supported by this module
|
|
||||||
CERTAINTY_UNSUPPORTED if the URL is not supported at all
|
|
||||||
CERTAINTY_POSSIBLE if the URL may be supported
|
|
||||||
CERTAINTY_PROBABLE if the URL is probably supported
|
|
||||||
"""
|
|
||||||
|
|
||||||
for regex, certainty, kwargs in cls.REGEX_FULL:
|
|
||||||
if bool(re.search(regex, url)):
|
|
||||||
return certainty
|
|
||||||
|
|
||||||
for regex, certainty, kwargs in cls.REGEX_SIMPLE:
|
|
||||||
if bool(re.search(regex, url)):
|
|
||||||
return certainty
|
|
||||||
|
|
||||||
return DownloaderModule.CERTAINTY_UNSUPPORTED
|
|
||||||
|
|
||||||
def process_conversion(self, conversion: Conversion) -> File:
|
|
||||||
"""Download content from the given URL
|
|
||||||
|
|
||||||
Args:
|
|
||||||
conversion (Conversion): Conversion object to process
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
DownloadError: If an error occurred while downloading content
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
File: File object containing the downloaded file
|
|
||||||
"""
|
|
||||||
self.logger.debug(f"Processing conversion {conversion.id} with URL {conversion.url}")
|
|
||||||
converter = KRPanoConverter(conversion)
|
|
||||||
result = converter.to_equirectangular()
|
|
||||||
self.logger.debug(f"Finished processing conversion {conversion.id} with URL {conversion.url}. Result: {result.id}")
|
|
||||||
return result
|
|
||||||
|
|
||||||
class KRPanoConverter:
|
|
||||||
def __init__(self, conversion):
|
|
||||||
self.conversion = conversion
|
|
||||||
|
|
||||||
self.logger = logging.getLogger("pix360")
|
|
||||||
|
|
||||||
self.cubemap_stitcher = DEFAULT_CUBEMAP_TO_EQUIRECTANGULAR_STITCHER()
|
|
||||||
self.stitcher = DEFAULT_STITCHER()
|
|
||||||
|
|
||||||
def url_normalize(self, url):
|
|
||||||
'''
|
|
||||||
Takes the URL of any image in a krpano panorama and returns a string
|
|
||||||
with substitutable variables for image IDs.
|
|
||||||
|
|
||||||
:param url: URL of an image contained in a krpano panorama
|
|
||||||
:return: string with substitutable variables or False if URL invalid
|
|
||||||
'''
|
|
||||||
|
|
||||||
try:
|
|
||||||
with HTTPRequest(url).open() as res:
|
|
||||||
assert res.getcode() == 200
|
|
||||||
|
|
||||||
parts = url.split("/")
|
|
||||||
|
|
||||||
assert "_" in parts[-1]
|
|
||||||
parts[-1] = "%i_%i.jpg"
|
|
||||||
parts[-2] = "%i"
|
|
||||||
parts[-3] = parts[-3].rstrip("0123456789") + "%i"
|
|
||||||
|
|
||||||
return "/".join(parts)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def get_max_zoom(self, schema):
|
|
||||||
'''
|
|
||||||
Takes a normalized string from krpano_normalize() and returns the maximum
|
|
||||||
zoom level available.
|
|
||||||
|
|
||||||
:param schema: normalized URL format output by krpano_normalize()
|
|
||||||
:return: int value of largest available zoom level
|
|
||||||
'''
|
|
||||||
|
|
||||||
self.logger.debug(f"Entering get_max_zoom for {schema}")
|
|
||||||
|
|
||||||
l = 0
|
|
||||||
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
url = schema % (0, l+1, 0, 0)
|
|
||||||
with HTTPRequest(url).open() as res:
|
|
||||||
assert res.getcode() == 200
|
|
||||||
l += 1
|
|
||||||
except:
|
|
||||||
self.logger.debug(f"Max zoom is {l}")
|
|
||||||
return l
|
|
||||||
|
|
||||||
def export(self, schema):
|
|
||||||
'''
|
|
||||||
Takes a normalized string from krpano_normalize() and returns a list of
|
|
||||||
lists of lists containing all images fit for passing into stitch().
|
|
||||||
|
|
||||||
:param schema: normalized URL format output by krpano_normalize()
|
|
||||||
:return: list of lists of lists of PIL.Image() objects for multistitch()
|
|
||||||
'''
|
|
||||||
|
|
||||||
self.logger.debug(f"Entering export for {schema}")
|
|
||||||
|
|
||||||
maxzoom = self.get_max_zoom(schema)
|
|
||||||
output = []
|
|
||||||
|
|
||||||
for tile in range(6):
|
|
||||||
t_array = []
|
|
||||||
y = 0
|
|
||||||
|
|
||||||
while True:
|
|
||||||
r_array = []
|
|
||||||
x = 0
|
|
||||||
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
res = HTTPRequest(schema % (tile, maxzoom, y, x)).open()
|
|
||||||
assert res.getcode() == 200
|
|
||||||
content = res.read()
|
|
||||||
fo = ContentFile(content, name=f"{tile}_{maxzoom}_{y}_{x}.jpg")
|
|
||||||
file = File.objects.create(conversion=self.conversion, file=fo, mime_type="image/jpeg")
|
|
||||||
r_array.append(file)
|
|
||||||
x += 1
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.debug(f"Error: {e}")
|
|
||||||
break
|
|
||||||
|
|
||||||
if not r_array:
|
|
||||||
break
|
|
||||||
|
|
||||||
t_array.append(r_array)
|
|
||||||
y += 1
|
|
||||||
|
|
||||||
output.append(t_array)
|
|
||||||
|
|
||||||
return output
|
|
||||||
|
|
||||||
def export_simple(self, url, tiles="frblud"):
|
|
||||||
'''
|
|
||||||
Exports krpano panoramas which only consist of six complete tiles. Takes
|
|
||||||
the URL of one of these images and returns a list of PIL.Image objects
|
|
||||||
|
|
||||||
:param url: URL of one of the images
|
|
||||||
:return: list of PIL.Image objects
|
|
||||||
'''
|
|
||||||
|
|
||||||
self.logger.debug(f"Entering export_simple for {url}")
|
|
||||||
|
|
||||||
output = []
|
|
||||||
|
|
||||||
for i in tiles:
|
|
||||||
cur = url[:-5] + i + url[-4:]
|
|
||||||
res = HTTPRequest(cur).open()
|
|
||||||
assert res.getcode() == 200
|
|
||||||
fo = ContentFile(res.read())
|
|
||||||
file = File.objects.create(conversion=self.conversion, file=fo, mime_type="image/jpeg")
|
|
||||||
output += [file]
|
|
||||||
|
|
||||||
return output
|
|
||||||
|
|
||||||
def export_full(self, url: str) -> File:
|
|
||||||
self.logger.debug(f"Entering export_full for {url}")
|
|
||||||
|
|
||||||
schema = self.url_normalize(url)
|
|
||||||
images = self.export(schema)
|
|
||||||
return self.stitcher.multistitch(images)
|
|
||||||
|
|
||||||
def make_tiles(self, url):
|
|
||||||
'''
|
|
||||||
Determines the type of processing needed to build the six tiles, then
|
|
||||||
creates and returns them.
|
|
||||||
|
|
||||||
:param url: URL of any image in a krpano panorama
|
|
||||||
:return: list of stitched PIL.Image objects (back, right, front, left, top,
|
|
||||||
bottom)
|
|
||||||
'''
|
|
||||||
|
|
||||||
self.logger.debug(f"Entering make_tiles for {url}")
|
|
||||||
|
|
||||||
for regex, certainty, kwargs in KRPanoDownloader.REGEX_FULL:
|
|
||||||
if bool(re.search(regex, url)):
|
|
||||||
return self.export_full(url, **kwargs)
|
|
||||||
|
|
||||||
for regex, certainty, kwargs in KRPanoDownloader.REGEX_SIMPLE:
|
|
||||||
if bool(re.search(regex, url)):
|
|
||||||
return self.export_simple(url, **kwargs)
|
|
||||||
|
|
||||||
raise ValueError("%s does not seem to be a valid krpano URL." % url)
|
|
||||||
|
|
||||||
def to_equirectangular(self):
|
|
||||||
'''
|
|
||||||
Takes the URL of any image in a krpano panorama and returns a finished
|
|
||||||
stitched image.
|
|
||||||
|
|
||||||
:param url: Image URL
|
|
||||||
:return: PIL.Image object containing the final image
|
|
||||||
'''
|
|
||||||
|
|
||||||
self.logger.debug(f"Entering to_equirectangular for {self.conversion.url}")
|
|
||||||
stitched = self.make_tiles(self.conversion.url)
|
|
||||||
self.logger.debug(f"Calling cubemap_to_equirectangular for {self.conversion.url}")
|
|
||||||
|
|
||||||
if self.conversion.properties:
|
|
||||||
rotation = self.conversion.properties.get("rotation", (0,0,0))
|
|
||||||
else:
|
|
||||||
rotation = (0,0,0)
|
|
||||||
|
|
||||||
function = self.cubemap_stitcher.cubemap_to_equirectangular
|
|
||||||
return function(stitched, rotation)
|
|
111
src/pix360_youtube/modules.py
Normal file
111
src/pix360_youtube/modules.py
Normal file
|
@ -0,0 +1,111 @@
|
||||||
|
from pix360core.classes import DownloaderModule, HTTPRequest, DownloadError, DEFAULT_CUBEMAP_TO_EQUIRECTANGULAR_STITCHER, DEFAULT_STITCHER
|
||||||
|
from pix360core.models import Conversion, File
|
||||||
|
|
||||||
|
from django.core.files.base import ContentFile
|
||||||
|
|
||||||
|
from typing import List, Tuple, Dict
|
||||||
|
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
import tempfile
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
import yt_dlp
|
||||||
|
|
||||||
|
class YouTubeDownloader(DownloaderModule):
|
||||||
|
name: str = "YouTube Downloader"
|
||||||
|
identifier: str = "systems.kumi.pix360.youtube"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.logger = logging.getLogger("pix360")
|
||||||
|
|
||||||
|
REGEX: List[Tuple[str, int, Dict[str, str]]] = [
|
||||||
|
(r"^https://(www\.)?youtube.com", DownloaderModule.CERTAINTY_PROBABLE),
|
||||||
|
]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def test_url(cls, url: str) -> int:
|
||||||
|
"""Test if URL looks like this module can handle it
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url (str): URL to test
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: Certainty level of the URL being supported by this module
|
||||||
|
CERTAINTY_UNSUPPORTED if the URL is not supported at all
|
||||||
|
CERTAINTY_POSSIBLE if the URL may be supported
|
||||||
|
CERTAINTY_PROBABLE if the URL is probably supported
|
||||||
|
"""
|
||||||
|
|
||||||
|
for regex, certainty in cls.REGEX:
|
||||||
|
if bool(re.search(regex, url)):
|
||||||
|
return certainty
|
||||||
|
|
||||||
|
return DownloaderModule.CERTAINTY_UNSUPPORTED
|
||||||
|
|
||||||
|
def process_conversion(self, conversion: Conversion) -> File:
|
||||||
|
"""Download content from the given URL
|
||||||
|
|
||||||
|
Args:
|
||||||
|
conversion (Conversion): Conversion object to process
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
DownloadError: If an error occurred while downloading content
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
File: File object containing the downloaded file
|
||||||
|
"""
|
||||||
|
self.logger.debug(f"Processing conversion {conversion.id} with URL {conversion.url}")
|
||||||
|
converter = YouTubeConverter(conversion)
|
||||||
|
result = converter.convert()
|
||||||
|
self.logger.debug(f"Finished processing conversion {conversion.id} with URL {conversion.url}. Result: {result.id}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
class YouTubeConverter:
|
||||||
|
def __init__(self, conversion):
|
||||||
|
self.conversion = conversion
|
||||||
|
self.logger = logging.getLogger("pix360")
|
||||||
|
|
||||||
|
def convert(self):
|
||||||
|
self.logger.debug(f"Entering convert for {self.conversion.url}")
|
||||||
|
file = self.download()
|
||||||
|
file.is_result = True
|
||||||
|
file.save()
|
||||||
|
return file
|
||||||
|
|
||||||
|
def hook(self, d):
|
||||||
|
if d["status"] == "finished":
|
||||||
|
self.logger.debug(f"Finished downloading {self.conversion.url}")
|
||||||
|
elif d["status"] == "downloading":
|
||||||
|
self.logger.debug(f"Downloading {self.conversion.url}: {d['filename']} ({d['_percent_str']})")
|
||||||
|
|
||||||
|
def download(self):
|
||||||
|
self.logger.debug(f"Entering download for {self.conversion.url}")
|
||||||
|
yt_dlp.utils.std_headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " \
|
||||||
|
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as outdir:
|
||||||
|
ydl_opts = {
|
||||||
|
"format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
|
||||||
|
"outtmpl": f"{outdir}/%(id)s.%(ext)s",
|
||||||
|
"logger": self.logger,
|
||||||
|
"progress_hooks": [self.hook],
|
||||||
|
"merge_output_format": "mp4",
|
||||||
|
}
|
||||||
|
|
||||||
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||||
|
ydl.download([self.conversion.url])
|
||||||
|
|
||||||
|
# Read the file
|
||||||
|
file = pathlib.Path(outdir).glob("*.mp4")
|
||||||
|
file = next(file)
|
||||||
|
|
||||||
|
self.logger.debug(f"Finished downloading {self.conversion.url}. File: {file}")
|
||||||
|
|
||||||
|
# Create a File object
|
||||||
|
with open(file, "rb") as f:
|
||||||
|
fo = ContentFile(f.read(), name="result.mp4")
|
||||||
|
file = File.objects.create(conversion=self.conversion, file=fo, mime_type="video/mp4")
|
||||||
|
|
||||||
|
return file
|
Loading…
Reference in a new issue