From 0f91c3e941ea341cebccf23397840c7a9b608fdf Mon Sep 17 00:00:00 2001 From: har0ke Date: Mon, 2 Sep 2024 20:45:46 +0200 Subject: [PATCH] Initial commit --- .gitignore | 2 + backup.sh | 27 +++ check_videos.py | 187 +++++++++++++++ copy_videos.py | 133 +++++++++++ dt_auto_group.py | 241 +++++++++++++++++++ dt_cleanup.py | 51 +++++ dt_fix_file_structure.py | 100 ++++++++ dt_remove_secondary copy.py | 114 +++++++++ dt_render.py | 34 +++ export.py | 52 +++++ pytable/database.py | 17 ++ pytable/fields.py | 69 ++++++ pytable/models.py | 304 ++++++++++++++++++++++++ pytable/modules.py | 420 ++++++++++++++++++++++++++++++++++ pytable/types.py | 202 ++++++++++++++++ synchronise.py | 297 ++++++++++++++++++++++++ vmgr/actions.py | 278 ++++++++++++++++++++++ vmgr/main.py | 445 ++++++++++++++++++++++++++++++++++++ vmgr/thumbnails.py | 74 ++++++ vmgr/video.py | 252 ++++++++++++++++++++ 20 files changed, 3299 insertions(+) create mode 100644 .gitignore create mode 100644 backup.sh create mode 100644 check_videos.py create mode 100644 copy_videos.py create mode 100644 dt_auto_group.py create mode 100644 dt_cleanup.py create mode 100644 dt_fix_file_structure.py create mode 100644 dt_remove_secondary copy.py create mode 100644 dt_render.py create mode 100644 export.py create mode 100644 pytable/database.py create mode 100644 pytable/fields.py create mode 100644 pytable/models.py create mode 100644 pytable/modules.py create mode 100644 pytable/types.py create mode 100644 synchronise.py create mode 100644 vmgr/actions.py create mode 100644 vmgr/main.py create mode 100644 vmgr/thumbnails.py create mode 100644 vmgr/video.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96553cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +**/__pycache__ diff --git a/backup.sh b/backup.sh new file mode 100644 index 0000000..75d75c2 --- /dev/null +++ b/backup.sh @@ -0,0 +1,27 @@ +#!/usr/bin/bash +shopt -s nocaseglob +shopt -s dotglob + +trap "exit" INT + +read -r -s -p "Password: " PWD +echo "" + +host="192.168.20.2" +user="oke" +extra_args=("--info=progress2" "-h" "${@:1}") +do_backup() ( + set -e + echo "Waiting for '$host' to be up ..." + until ping -c1 "$host" >/dev/null 2>&1; do :; done + echo "... done waiting" + + sshpass -p "$PWD" \ + rsync -av "${extra_args[@]}" --relative "$HOME/Pictures/DarktableLocal" "$user@$host":lenovo-darktable + + sshpass -p "$PWD" \ + rsync -av "${extra_args[@]}" --relative "$HOME/.config/darktable" "$user@$host":lenovo-darktable + + ) + +do_backup diff --git a/check_videos.py b/check_videos.py new file mode 100644 index 0000000..f639e76 --- /dev/null +++ b/check_videos.py @@ -0,0 +1,187 @@ +import subprocess +import json +import os +import datetime +import glob +from PIL import Image +import shutil +import re + +def get_time(fn): + try: + output = subprocess.check_output([ + "ffprobe", "-v", "quiet", fn, "-print_format", "json", + "-show_entries", + "stream=index,codec_type:stream_tags=creation_time:format_tags=creation_time" + ]) + except subprocess.CalledProcessError as e: + print("FFPROBE failed for ", fn) + return ValueError() + fmt = "%Y-%m-%dT%H:%M:%S.%fZ" + data = json.loads(output) + if "streams" in data: + for d in data["streams"]: + if "tags" in d and "creation_time" in d["tags"]: + return datetime.datetime.strptime(d["tags"]["creation_time"], fmt) + if "format" in data and "tags" in data["format"] and "creation_time" in data["format"]["tags"]: + return datetime.datetime.strptime(data["format"]["tags"]["creation_time"], fmt) + # import sys + # import zoneinfo + # dt = datetime.datetime.fromtimestamp( + # os.stat(fn).st_ctime, zoneinfo.ZoneInfo("UTC")) + # dt = dt.replace(tzinfo=None) + # return dt + return ValueError() +def get_time_image(fn): + fmt = "%Y:%m:%d %H:%M:%S" + return datetime.datetime.strptime(Image.open(fn)._getexif()[36867], fmt) + +def get_new_fn(fn): + base, ext = os.path.splitext(fn) + directory, base_fn = os.path.split(base) + ext = ext.lower() + + image_files = list( + filter(lambda fn: os.path.splitext(fn)[1].lower() in [".jpeg", ".jpg", ".png"], + glob.iglob(glob.escape(base) + "*"))) + assert len(image_files) < 2 + fmt_folder = "%Y%m%d" + fmt_base = "%Y%m%d_%H%M" + if image_files: + f = image_files[0] + image_ext = os.path.splitext(f)[-1] + image_dt = get_time_image(f) + image_size = os.path.getsize(f) + new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), image_dt.strftime(fmt_folder), image_dt.strftime(fmt_base)) + + g = glob.escape(new_base) + "_*" + image_ext + matches = [g for g in glob.glob(g) if image_size == os.path.getsize(g)] + if len(matches) != 1: + print() + print("NO MATCH") + print(image_files) + print(matches) + print(g) + print(image_dt) + + return None + assert len(matches) == 1, fn + nfn = os.path.splitext(matches[0])[0] + ext + nfn = list(os.path.split(nfn)) + nfn[-1] = "." + nfn[-1] + return os.path.join(*nfn) + + time = get_time(fn) + if isinstance(time, ValueError): + print("NO TIME") + return None + new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), time.strftime(fmt_folder), time.strftime(fmt_base)) + g = glob.glob(glob.escape(new_base) + "_*" + ext) + if len(g) == 0: + return new_base + "_0000" + ext + n = 0 + s = os.path.getsize(fn) + for f in g: + if os.path.getsize(f) == s: + return f + p = r".*_(\d+)" + ext + nn = re.match(p, f) + n = max(n, int(nn.group(1))) + n = new_base + "_%04d" % (n + 1) + ext + return n + +def main(dir, dry): + from datetime import timedelta, datetime + print(dir) + offsets = {} + for date in sorted(os.listdir(dir)): + path = os.path.join(dir, date) + if not os.path.isdir(path): + continue + print(date) + date = datetime.strptime(date, '%Y%m%d') + + offset = None + for fn in sorted(os.listdir(path)): + + fn = os.path.join(path, fn) + _, ext = os.path.splitext(fn) + if ext.lower() not in ['.jpeg', '.jpg']: + continue + + import pprint + from PIL import ExifTags + import exifread + import re + + with open(fn, 'rb') as f: + + tags = exifread.process_file(f) + + try: + model = str(tags['Image Model']) + except KeyError: + continue + + if str(model) != 'Pixel 6' or str(model).startswith("iPhone"): + continue + + offset_tags = ['EXIF OffsetTime', 'EXIF OffsetTimeOriginal', 'EXIF OffsetTimeDigitized', 'EXIF TimeZoneOffset'] + + for tag in offset_tags: + try: + offset_str = str(tags[tag]) + except KeyError: + continue + assert re.match(r"(\+|-)\d\d:\d\d", offset_str) + offset = timedelta(hours=int(offset_str[1:3]), minutes=int(offset_str[4:6])) + if offset_str[0] == '-': + offset = -offset + + break + + break + import pprint + + if offset: + offsets[date] = offset + + for p, folders, files in os.walk(os.path.expanduser(dir)): + for f in sorted(files): + fn = os.path.join(p, f) + + base, ext = os.path.splitext(f) + if ext.lower() not in [".mp4", ".mov", ".mts"]: + continue + + time = get_time(fn) + if isinstance(time, ValueError): + continue + + diff = sorted(offsets.keys(), key=lambda d: abs((d - time).total_seconds())) + + # print(diff) + # print(diff[0]) + # print(offsets[diff[0]]) + + print(time) + print(time + offsets[diff[0]]) + print() + continue + nfn = get_new_fn(fn) + if not nfn: + print("SKIP: ", fn) + continue + assert nfn + + print(os.path.basename(fn)) + print(os.path.basename(nfn)) + print() + + + +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("--dry", action="store_true") +options = parser.parse_args() +main("/home/oke/Pictures/DarktableLocal", options.dry) diff --git a/copy_videos.py b/copy_videos.py new file mode 100644 index 0000000..730681f --- /dev/null +++ b/copy_videos.py @@ -0,0 +1,133 @@ +import subprocess +import json +import os +import datetime +import glob +from PIL import Image +import shutil +import re + +def get_time(fn): + try: + output = subprocess.check_output([ + "ffprobe", "-v", "quiet", fn, "-print_format", "json", + "-show_entries", + "stream=index,codec_type:stream_tags=creation_time:format_tags=creation_time" + ]) + except subprocess.CalledProcessError as e: + print("FFPROBE failed for ", fn) + return ValueError() + fmt = "%Y-%m-%dT%H:%M:%S.%fZ" + data = json.loads(output) + if "streams" in data: + for d in data["streams"]: + if "tags" in d and "creation_time" in d["tags"]: + return datetime.datetime.strptime(d["tags"]["creation_time"], fmt) + if "format" in data and "tags" in data["format"] and "creation_time" in data["format"]["tags"]: + return datetime.datetime.strptime(data["format"]["tags"]["creation_time"], fmt) + #import sys + #import zoneinfo + #dt = datetime.datetime.fromtimestamp( + # os.stat(fn).st_ctime, zoneinfo.ZoneInfo("UTC")) + #dt = dt.replace(tzinfo=None) + #return dt + return ValueError() +def get_time_image(fn): + fmt = "%Y:%m:%d %H:%M:%S" + return datetime.datetime.strptime(Image.open(fn)._getexif()[36867], fmt) + +def get_new_fn(fn): + base, ext = os.path.splitext(fn) + directory, base_fn = os.path.split(base) + ext = ext.lower() + + image_files = list( + filter(lambda fn: os.path.splitext(fn)[1].lower() in [".jpeg", ".jpg", ".png"], + glob.iglob(glob.escape(base) + "*"))) + assert len(image_files) < 2 + fmt_folder = "%Y%m%d" + fmt_base = "%Y%m%d_%H%M" + if image_files: + f = image_files[0] + image_ext = os.path.splitext(f)[-1] + image_dt = get_time_image(f) + image_size = os.path.getsize(f) + new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), image_dt.strftime(fmt_folder), image_dt.strftime(fmt_base)) + + g = glob.escape(new_base) + "_*" + image_ext + matches = [g for g in glob.glob(g) if image_size == os.path.getsize(g)] + if len(matches) != 1: + print() + print("NO MATCH") + print(image_files) + print(matches) + print(g) + print(image_dt) + + return None + assert len(matches) == 1, fn + nfn = os.path.splitext(matches[0])[0] + ext + nfn = list(os.path.split(nfn)) + nfn[-1] = "." + nfn[-1] + return os.path.join(*nfn) + + time = get_time(fn) + if isinstance(time, ValueError): + print("NO TIME") + return None + new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), time.strftime(fmt_folder), time.strftime(fmt_base)) + g = glob.glob(glob.escape(new_base) + "_*" + ext) + if len(g) == 0: + return new_base + "_0000" + ext + n = 0 + s = os.path.getsize(fn) + for f in g: + if os.path.getsize(f) == s: + return f + p = r".*_(\d+)" + ext + nn = re.match(p, f) + n = max(n, int(nn.group(1))) + n = new_base + "_%04d" % (n + 1) + ext + return n + +def main(dir, dry): + print(dir) + for p, folders, files in os.walk(os.path.expanduser(dir)): + for f in sorted(files): + base, ext = os.path.splitext(f) + + ext = ext.lower() + if ext not in [".mp4", ".mov", ".mts"]: + if ext not in [".jpg", ".jpeg", ".png"]: + pass # print("Not handled: ", os.path.join(p, f)) + continue + fn = os.path.join(p, f) + nfn = get_new_fn(fn) + if not nfn: + print("SKIP: ", fn) + continue + assert nfn + + if not os.path.exists(nfn): + if not os.path.exists(os.path.dirname(nfn)): + os.makedirs(os.path.dirname(nfn)) + print(fn, nfn) + if not dry: + shutil.copy(fn, nfn) + else: + print(fn, nfn, "EXIST") + + + + +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("--dry", action="store_true") +options = parser.parse_args() +main("/run/media/oke/3634-3063", options.dry) +main("/run/media/oke/3138-3162", options.dry) +main("/run/media/oke/01D5-9878/DCIM/", options.dry) +main("/run/media/oke/E5B5-DBF0/DCIM", options.dry) +main("/run/media/oke/disk/", options.dry) +main("~/Nextcloud/InstantUpload", options.dry) +main("/home/oke/Desktop/360/", options.dry) diff --git a/dt_auto_group.py b/dt_auto_group.py new file mode 100644 index 0000000..0d63151 --- /dev/null +++ b/dt_auto_group.py @@ -0,0 +1,241 @@ +from pytable.models import Image, ImageFlags, FilmRoll +from datetime import datetime, timedelta +import os + +def is_raw(image): + return image.filename.split(".")[-1].lower() not in ["jpeg", "jpg", "png"] + +def consume_group_pixel_6(images): + # print("NEXT: ", images[0]) + i = 1 + for i in range(1, len(images)): + # print(images[i], images[i].datetime_taken, images[0].datetime_taken, images[i].datetime_taken - images[0].datetime_taken) + if images[i].datetime_taken - images[0].datetime_taken > timedelta(milliseconds=600): + break + else: + i = len(images) + # print(i) + if i == 1: + # print(images[:1]) + return images[:1], images[1:] + + def priority(image): + return - ( + image.iso - images[0].iso + + image.aperture - images[0].aperture + + image.exposure - images[0].exposure + ) + + sorted_images = sorted(filter(lambda image: is_raw(image) != is_raw(images[0]), images[1:i]), key=priority) + + if len(sorted_images) == 0: + # print(images[:1]) + return images[:1], images[1:] + + group = [images[0], sorted_images[0]] + + images = images[1:] + images.remove(sorted_images[0]) + # print(group) + return group, images + + +def consume_group_burst(images, delta=None): + if delta is None: + delta = timedelta(seconds=1) + i = -1 + for i in range(len(images)): + # print(i, images[0], images[0].datetime_taken, images[i], images[i].datetime_taken, (images[i].datetime_taken - images[0].datetime_taken)) + if (images[i].datetime_taken - images[0].datetime_taken) > delta: + break + else: + i += 1 + + return images[:i], images[i:] + +def consume_group_exposure_bracketing(images): + seen_exposures = set([]) + seen_raw_exposures = set([]) + + def trace_exposure(image): + bias = int(image.exposure_bias * 100) + info((image, image.datetime_taken, bias)) + if is_raw(image): + is_new = bias not in seen_raw_exposures + if not is_new: + info(seen_raw_exposures, seen_exposures, bias) + seen_raw_exposures.add(bias) + else: + is_new = bias not in seen_exposures + if not is_new: + info(seen_raw_exposures, seen_exposures, bias) + seen_exposures.add(bias) + return is_new + + info(images[0]) + trace_exposure(images[0]) + for i in range(1, len(images)): + if images[i].datetime_taken - images[i - 1].datetime_taken > timedelta(seconds=1): + info("::: Out of time") + break + + if not trace_exposure(images[i]): + info("::: Exposure seen") + break + info(images[i]) + + else: + i = len(images) + + return images[:i], images[i:] + + +def select_leader(images, prefer_raw): + + def leader_preference(image): + return ( + image.flag(ImageFlags.REJECTED), + -image.stars, + -votes[image], + abs(image.exposure_bias), + (-int(is_raw(image))) if prefer_raw else int(is_raw(image)), + image.datetime_taken + ) + + votes = {} + for image in images: + if image not in votes: + votes[image] = 0 + if image.group in images: + if image.group not in votes: + votes[image.group] = 0 + votes[image.group] += 1 + + leader = min(images, key=leader_preference) + + return leader + +def info(*args): + pass + # print(*args) + +def validate_group(images, prefer_raw): + will_degrade = False + has_changes = False + + leader = select_leader(images, prefer_raw) + + for image in images: + if image.group != leader: + if image.group == image and not image.flag(ImageFlags.REJECTED): + # if image was group leader and was not rejected, changing the + # group leader would degrate image + will_degrade = True + print("Below change would degrate dataset: MANUAL ATTENTION NEEDED.") + print("Change group leader of {}: {} => {}".format(image, image.group, leader)) + image.group = leader + has_changes = True + + if has_changes: + stars = max(map(lambda image: image.stars, images)) + rejected = min(map(lambda image: image.flag(ImageFlags.REJECTED), images)) + for image in images: + if image.flag(ImageFlags.REJECTED) != rejected: + print("Change rejected status of {}: {} => {}".format(image, image.flag(ImageFlags.REJECTED), rejected)) + image.set_flag(ImageFlags.REJECTED, rejected) + if image.stars != stars: + print("Change stars of {}: {} => {}".format(image, image.stars, stars)) + image.stars = stars + print() + return has_changes, will_degrade + + +def main(date_from, force, dry_run): + + images = set([]) + for r in FilmRoll.filter(): + try: + if datetime.strptime(os.path.split(r.folder)[-1], "%Y%m%d").date() >= date_from: + # print(r) + for image in Image.filter(film=r): + images.add(image) + else: + pass + # print("not", r) + except ValueError as e: + print(e) + print("Ignoring", r) + + images = sorted(filter(lambda image: image.datetime_taken, images), key=lambda image: (image.datetime_taken, -is_raw(image), image.filename)) + + by_model = {} + for image in images: + key = (image.maker, image.model) + if key not in by_model: + by_model[key] = [] + by_model[key].append(image) + + n = 0 + for model, images in by_model.items(): + + images = images[:] + prefer_raw = True + while len(images) > 0: + if model[1].name == "DSC-RX100": + group, images = consume_group_exposure_bracketing(images) + elif model[1].name == "ZV-1" or model[1].name == "ILCE-6700": + original_images = images + group, images = consume_group_exposure_bracketing(original_images) + group_burst, images_burst = consume_group_burst(original_images) + if len(group_burst) > len(group): + group, images = group_burst, images_burst + elif model[1].name == "Pixel 6": + group, images = consume_group_pixel_6(images) + prefer_raw = False + + elif ( + model[1].name == "HERO9 Black" or + model[1].name == "HERO11 Black" + ): + group, images = consume_group_burst(images, timedelta(seconds=3)) + prefer_raw = False + elif ( + model[1].name == "HERO3+ Black Edition" or + model[1].name == "iPhone 12 mini" or + model[1].name == "iPhone 15 Pro Max" + ): + group, images = images[:1], images[1:] + else: + if model[1].name != '': + for image in images: + print(image) + print("Unkown model {} - {}".format(*model)) + break + + has_changes, will_degrade = validate_group(group, prefer_raw) + if has_changes: + n += 1 + + if not dry_run and (not will_degrade or force): + for image in group: + print("Saving {}".format(image)) + image.save() + + print("incorrect", n) + +import argparse +if __name__ == '__main__': + + + parser = argparse.ArgumentParser() + parser.add_argument("--force", action="store_true") + parser.add_argument("--save", action="store_true") + parser.add_argument("--from", dest="from_date", default=None) + + options = parser.parse_args() + + if options.from_date is None: + from_date = datetime.now().date() + else: + from_date = datetime.strptime(options.from_date, "%Y%m%d").date() + main(from_date, options.force, not options.save) diff --git a/dt_cleanup.py b/dt_cleanup.py new file mode 100644 index 0000000..c6a70fb --- /dev/null +++ b/dt_cleanup.py @@ -0,0 +1,51 @@ +from pytable.models import Image, ImageFlags, FilmRoll + +import peewee as pw +import os +import subprocess + +base_path = "/home/oke/Pictures/Darktable" + +subprocess.check_call(["mountpoint", "/home/oke/Pictures/DarktableRemote"]) +real_folders = [ + "/home/oke/Pictures/DarktableLocal", + "/home/oke/Pictures/DarktableRemote", +] + +query: list[Image] = Image.filter() +pw.prefetch(query, FilmRoll) +images = list(query) +import itertools + + +required_files = list(itertools.chain(*[ + [os.path.join(image.film.folder, image.filename), + os.path.join(image.film.folder, + os.path.splitext(image.filename)[0] + + (("_%02d" % image.version) if image.version != 0 else "") + + os.path.splitext(image.filename)[1]) + ".xmp" + ] + for image in images])) + +allowed_files = list(itertools.chain(*[ + [os.path.join(image.film.folder, "." + os.path.splitext(image.filename)[0] + ".mov")] + for image in images])) + required_files + + + +for fn in allowed_files: + assert fn.startswith(base_path) + +for folder in real_folders: + real_files = set([os.path.join(path, file) for path, folders, files in os.walk(folder) for file in files if file[-3:] not in ["mov", "mp4", "mts", "ata"]]) + mapped_allowed = [fn.replace(base_path, folder) for fn in allowed_files] + + print(":::: Files to remove:") + for fn in sorted(real_files.difference(set(mapped_allowed))): + print(fn) + + if folder == "/home/oke/Pictures/DarktableRemote": + print(":::: Missing files:") + mapped_required = [fn.replace(base_path, folder) for fn in required_files] + for fn in sorted(set(mapped_required).difference(set(real_files))): + print(fn) diff --git a/dt_fix_file_structure.py b/dt_fix_file_structure.py new file mode 100644 index 0000000..1419506 --- /dev/null +++ b/dt_fix_file_structure.py @@ -0,0 +1,100 @@ +from pytable.models import Image, ImageFlags, FilmRoll + +from datetime import datetime +import peewee as pw +import os +from pprint import pprint +import subprocess + +base_path = "/home/oke/Pictures/Darktable" +real_folders = [ + "/home/oke/Pictures/DarktableRemote", + "/home/oke/Pictures/DarktableLocal", +] + +subprocess.check_call(["mountpoint", "/home/oke/Pictures/DarktableRemote"]) + +query = Image.filter() +pw.prefetch(query, FilmRoll) +dry = True + +move_files = [] + + +d_root = "/home/oke/Pictures/Darktable" +d_local = "/home/oke/Pictures/DarktableLocal" +d_remote = "/home/oke/Pictures/DarktableRemote" + + +def me_and_related_dt(fn, new_folder, new_basename): + directory, filename = os.path.split(fn) + basename, ext = os.path.splitext(filename) + yield fn, os.path.join(new_folder, new_basename + ext) + yield os.path.join(directory, "." + basename + ".mov"), os.path.join(new_folder, "." + new_basename + ".mov"), + yield fn + ".xmp", os.path.join(new_folder, new_basename + ext + ".xmp") + + +def me_and_related_potential(fn, new_folder, new_basename): + for local_fn, local_fn_related in me_and_related_dt(fn, new_folder, new_basename): + assert local_fn.startswith(d_root) + yield local_fn.replace(d_root, d_local), local_fn_related.replace(d_root, d_local) + yield local_fn.replace(d_root, d_remote), local_fn_related.replace(d_root, d_remote) + +def me_and_related(fn, new_folder, new_basename): + for fn, fn_related in me_and_related_potential(fn, new_folder, new_basename): + if os.path.exists(fn): + assert not os.path.exists(fn_related), fn + "=>" + fn_related + yield fn, fn_related + +changes = [] + +for image in query: + try: + n = 0 + dt_fn = datetime.strptime(image.filename[:13],"%Y%m%d_%H%M") + if image.datetime_taken: + dt_internal = datetime(image.datetime_taken.year, image.datetime_taken.month, image.datetime_taken.day, image.datetime_taken.hour, image.datetime_taken.minute) + expected_folder = os.path.join(base_path, dt_internal.strftime("%Y%m%d")) + if dt_fn != dt_internal: + d = ((dt_internal - dt_fn).total_seconds() / 3600) + dt_formatted = image.datetime_taken.strftime("%Y%m%d_%H%M") + new_film_roll = image.film + if expected_folder != image.film.folder: + new_film_roll = FilmRoll.get(folder=expected_folder) + assert new_film_roll + + new_fn = dt_formatted + image.filename[13:] + new_basename, _ = os.path.splitext(new_fn) + + changes.append(( + image, new_film_roll, new_fn, + list(me_and_related(os.path.join(image.film.folder, image.filename), new_film_roll.folder, new_basename)) + )) + + print("%s: %.2fH" % (image.filename, d)) + for src, dst in changes[-1][-1]: + print("\t%s => %s" % (src, dst)) + except Exception as e: + print("ERROR:", e) + + + +reply = input("Type YES to remove all these files. This is not reversible... : ") + +if reply != 'YES': + print("You did not type 'YES'. Not doing anything!") + exit(-1) + +for image, film_roll, new_fn, move in changes: + print(image.filename, "=>", new_fn) + + for src, dst in move: + assert not os.path.exists(dst) + + for src, dst in move: + print("Moving %s => %s" % (src, dst)) + os.rename(src, dst) + print("Saving image into sql") + image.film = film_roll + image.filename = new_fn + image.save() diff --git a/dt_remove_secondary copy.py b/dt_remove_secondary copy.py new file mode 100644 index 0000000..cd73765 --- /dev/null +++ b/dt_remove_secondary copy.py @@ -0,0 +1,114 @@ +from pytable.models import Image, ImageFlags, FilmRoll + +from datetime import datetime +import peewee as pw +import os +import shutil + +remove_three_stars_before = datetime(2024, 4, 1) +remove_secondary_before = datetime(2024, 6, 15) +remove_rejected_before = datetime(2024, 6, 15) + +query = Image.filter() +pw.prefetch(query, FilmRoll) + +def is_older_than(image, cmp): + return (image.datetime_taken or image.datetime_imported) < cmp + +def should_remove(image): + return ( + (is_older_than(image, remove_three_stars_before) and image.stars < 3) or + (image.flag(ImageFlags.REJECTED) and is_older_than(image, remove_rejected_before)) or + (image.group_id != image.id and is_older_than(image, remove_secondary_before)) + ) + +file_usages = {} +for x in query: + path = os.path.join(x.film.folder, x.filename) + if path not in file_usages: + file_usages[path] = 0 + if not should_remove(x): + file_usages[path] += 1 + +n_total = 0 +n_not_needed = 0 +needs_removal = [] +extra_files_to_remove = [] +bytes_to_remove = 0 + +d_root = "/home/oke/Pictures/Darktable" +d_local = "/home/oke/Pictures/DarktableLocal" +d_remote = "/home/oke/Pictures/DarktableRemote" + +def potential_extras(fn): + directory, filename = os.path.split(fn) + basename, _ = os.path.splitext(filename) + + iphone_movie = os.path.join(directory, "." + basename + ".mov") + yield iphone_movie + +needs_fetching = [] + +for file, usages in file_usages.items(): + if not file.startswith(d_root): + print("SKIPPNG", file) + continue + assert file.startswith(d_root) + remote_file = file.replace(d_root, d_remote) + local_file = file.replace(d_root, d_local) + + if usages > 0: + if not os.path.exists(local_file): + needs_fetching.append((remote_file, local_file)) + else: + n_not_needed += 1 + if os.path.exists(local_file): + needs_removal.append(local_file) + bytes_to_remove += os.stat(local_file).st_size + + for extra_fn in potential_extras(local_file): + if os.path.exists(extra_fn): + extra_files_to_remove.append(extra_fn) + bytes_to_remove += os.stat(extra_fn).st_size + +files_to_remove = list(sorted(needs_removal + extra_files_to_remove)) + +needs_fetching = sorted(needs_fetching) +import pprint +print("Files to fetch: ") +pprint.pprint(needs_fetching) +print("Extra files to remove:") +pprint.pprint(extra_files_to_remove) +print("First files to remove:") +pprint.pprint(needs_removal[:10]) +print("Last files to remove:") +pprint.pprint(needs_removal[-10:]) + +with open("remove.txt", "w") as f: + pprint.pprint(sorted(set(needs_removal).union(extra_files_to_remove)), f) + +print("Not needed: %6d / %6d (%3d%%)" % (n_not_needed, len(file_usages), 100 * n_not_needed / len(file_usages))) +print() +print("Not yet deleted: %6d / %6d (%3d%%)" % (len(needs_removal), n_not_needed, 100 * len(needs_removal) / n_not_needed)) +print("Bytes to free: %5.2fGB" % (bytes_to_remove / 1024. / 1024 / 1024)) +print("Total files to remove: %d" % len(files_to_remove)) +print() +print("Total files to fetch: %3d" % len(needs_fetching)) +print() + +reply = input("Type YES to remove all these files. This is not reversible... : ") + +if reply == 'YES': + + for fn in files_to_remove: + assert fn.startswith(d_local) + os.remove(fn) + + for f, t in needs_fetching: + assert os.path.exists(f), "Backup corruped, file not found: " + f + assert not os.path.exists(t) + print(f, t) + shutil.copy(f, t) + +else: + print("You did not type 'YES'. Not doing anything!") diff --git a/dt_render.py b/dt_render.py new file mode 100644 index 0000000..ac39117 --- /dev/null +++ b/dt_render.py @@ -0,0 +1,34 @@ +from pytable.models import Image, ImageFlags, FilmRoll + +from datetime import datetime +import peewee as pw +import os +import shutil + + +query = Image.filter() + +output_folder = "output" +for image in query: + assert isinstance(image, Image) + if image.stars >= 3 and (not image.group or image.group == image): + + print(image) + + image_path = os.path.join(image.film.folder, image.filename) + + output_path = os.path.join(output_folder, os.path.splitext(image.filename)[0] + ".jpg") + + + if os.path.exists(output_path): + continue + command = [ + 'darktable-cli', + '--width', '1920', + '--hq', 'true', + image_path, + output_path + ] + + import subprocess + subprocess.call(command) diff --git a/export.py b/export.py new file mode 100644 index 0000000..093fa84 --- /dev/null +++ b/export.py @@ -0,0 +1,52 @@ +from pytable.models import Color, Image, ImageFlags, FilmRoll, ColorLabel, TaggedImages + +from datetime import datetime +import peewee as pw +import os +import shutil +import subprocess +from dataclasses import dataclass +from typing import Tuple, Optional, Generic, TypeVar, Iterable + + +export_base = "/home/oke/Nextcloud/Blog/" +query = ( + Image.filter() + .join(ColorLabel) + .where(ColorLabel.color == Color.BLUE) + .where(Image.datetime_taken > datetime(2024, 6, 14, 22)) +) +pw.prefetch(query, FilmRoll) + +def is_blog(image: Image): + return ( + (image.group == image or image.group == None) and + not image.flag(ImageFlags.REJECTED) + ) + +images = list(filter(is_blog, query)) +for image in sorted(images, key=lambda i: i.datetime_taken or i.datetime_imported): + assert isinstance(image, Image) + fn = os.path.join(image.film.folder, image.filename) + version = "" if image.version == 0 else ("_%02d" % image.version) + base, ext = os.path.splitext(image.filename) + xmp = os.path.join(image.film.folder, base + version + ext + ".xmp") + out_fn = os.path.join(export_base, base + version + ".jpg") + + if os.path.exists(out_fn): + if not image.datetime_changed or datetime.fromtimestamp(os.stat(out_fn).st_ctime) > image.datetime_changed: + continue + os.remove(out_fn) + subprocess.call([ + 'darktable-cli', + fn, + xmp, + '--width', '1920', + '--out-ext', 'jpg', + '--upscale', 'false', + '--hq', 'true', + out_fn + ]) + +print(len(images)) + diff --git a/pytable/database.py b/pytable/database.py new file mode 100644 index 0000000..fb49d3e --- /dev/null +++ b/pytable/database.py @@ -0,0 +1,17 @@ +import peewee as pw +import sys +import os + + +if sys.platform == "linux": + db_library = pw.SqliteDatabase( + os.path.expanduser('~/.config/darktable/library.db')) + db_data = pw.SqliteDatabase( + os.path.expanduser('~/.config/darktable/data.db')) +else: + db = pw.SqliteDatabase(None) + db_data = pw.SqliteDatabase(None) + +def open_sqlite_db(sqlite_fn): + global db + db = pw.init(sqlite_fn) diff --git a/pytable/fields.py b/pytable/fields.py new file mode 100644 index 0000000..9bd6c0e --- /dev/null +++ b/pytable/fields.py @@ -0,0 +1,69 @@ +import peewee as pw + +import datetime +import itertools + +class DarktableTimestampField(pw.TimestampField): + """ + Darktable not always stores timestamps as UNIX timestamps with origin of + 1970-01-01, but sometimes 0001-01-01. This timestamp field lets the user + specifiy what origin to use for the timestamp. + """ + + def __init__(self, *args, origin=datetime.datetime(1970, 1, 1), **kwargs) -> None: + kwargs['resolution'] = 10**6 + self.epoch_diff = (datetime.datetime(1970, 1, 1) - origin).total_seconds() + super().__init__(*args, **kwargs) + + def python_value(self, value): + if value is None or value == -1: + return None + return super().python_value(value - self.epoch_diff * self.resolution) + + def db_value(self, value): + if value is None or value == -1: + return -1 + + if isinstance(value, datetime.datetime): + pass + elif isinstance(value, datetime.date): + value = datetime.datetime(value.year, value.month, value.day) + else: + raise ValueError() + + return super().db_value(value) + self.epoch_diff * self.resolution + +class ModuleOrderListField(pw.CharField): + + def db_value(self, value): + print("db_value") + value = super().db_value(value) + if hasattr(value, "__item__"): + new_value = ",".join([str(v) for v in itertools.chain(*value)]) + print(value, new_value) + return new_value + print(value) + return value + + def python_value(self, value): + value = super().python_value(value) + if isinstance(value, str): + values = value.split(",") + assert len(values) % 2 == 0 + new_value = [ + (values[i * 2], int(values[i * 2 + 1])) + for i in range(int(len(values) / 2))] + return new_value + return value + +class EnumField(pw.IntegerField): + + def __init__(self, enum_type, *args, **kwargs): + super().__init__(*args, **kwargs) + self.enum_type = enum_type + + def db_value(self, value): + return value.value + + def python_value(self, value): + return self.enum_type(super().python_value(value)) diff --git a/pytable/models.py b/pytable/models.py new file mode 100644 index 0000000..8b67812 --- /dev/null +++ b/pytable/models.py @@ -0,0 +1,304 @@ +from .database import db_library, db_data +from .fields import DarktableTimestampField, ModuleOrderListField, EnumField +from .types import ImageFlags, v30_jpg_order, v30_order, IOPOrderType, legacy_order, Color +from .modules import DT_MODULES, Module + +from typing import List +import peewee as pw + +import datetime + +class FilmRoll(pw.Model): + """ + A darktable filmroll. + """ + access_timestamp: datetime.datetime = DarktableTimestampField() # type: ignore + folder: str = pw.CharField(1024) # type: ignore + + def __str__(self): + return self.folder + + class Meta: + db_table = 'film_rolls' + database = db_library + +class Maker(pw.Model): + + name: str = pw.CharField() # type: ignore + + def __str__(self) -> str: + return self.name + + class Meta: + db_table = 'makers' + database = db_library + +class Model(pw.Model): + + name: str = pw.CharField() # type: ignore + + def __str__(self) -> str: + return self.name + + class Meta: + db_table = 'models' + database = db_library + +class Lens(pw.Model): + + name: str = pw.CharField() # type: ignore + + def __str__(self) -> str: + return self.name + + class Meta: + db_table = 'lens' + database = db_library + +class Camera(pw.Model): + + name: str = pw.CharField() # type: ignore + + class Meta: + db_table = 'cameras' + database = db_library + +class Image(pw.Model): + """ + A darktable image. + """ + id: int = pw.IntegerField(primary_key=True) # type: ignore + group = pw.ForeignKeyField('self') + film = pw.ForeignKeyField(FilmRoll) + + # Image width in pixels + width: int = pw.IntegerField() # type: ignore + # Image height in pixels + height: int = pw.IntegerField() # type: ignore + + # The filename of the image + filename: str = pw.CharField() # type: ignore + + # The maker of the camera used to capture this picture + maker = pw.ForeignKeyField(Maker) + # The model of the camera used to capture this picture + model = pw.ForeignKeyField(Model) + # The lens used to capture this picture + lens = pw.ForeignKeyField(Lens) + # The camera used to capture this picture + camera = pw.ForeignKeyField(Camera) + + # The exposure used to capture this picture + exposure = pw.FloatField() + # The aperture used to capture this picture + aperture = pw.FloatField() + # The iso used to capture this picture + iso = pw.FloatField() + # The focal length used to capture this picture + focal_length = pw.FloatField() + # The focus distance used to capture this picture + focus_distance = pw.FloatField() + + # The date this picture was taken + datetime_taken: datetime.datetime = DarktableTimestampField(utc=True, origin=datetime.datetime(1, 1, 1)) # type: ignore + + # The flags attached to this picture. To read individual flags, see 'flag()' function + flags: int = pw.IntegerField() # type: ignore + + output_width: int = pw.IntegerField() # type: ignore + output_height: int = pw.IntegerField() # type: ignore + crop = pw.DoubleField() + + raw_parameters: int = pw.IntegerField() # type: ignore + # raw_denoise_threshold = pw.DoubleField() + # raw_auto_bright_threshold = pw.DoubleField() + raw_black: int = pw.IntegerField() # type: ignore + raw_maximum: int = pw.IntegerField() # type: ignore + + # license: str = pw.CharField() # type: ignore + + # sha1sum = pw.FixedCharField(40) + + orientation: int = pw.IntegerField() # type: ignore + + # histogram = pw.BlobField() + # lightmap = pw.BlobField() + + longitude = pw.DoubleField() + latitude = pw.DoubleField() + altitude = pw.DoubleField() + + color_matrix = pw.BlobField() + + colorspace: int = pw.IntegerField() # type: ignore + version: int = pw.IntegerField() # type: ignore + max_version: int = pw.IntegerField() # type: ignore + write_timestamp: datetime.datetime = DarktableTimestampField() # type: ignore + history_end: int = pw.IntegerField() # type: ignore + position: int = pw.IntegerField() # type: ignore + + aspect_ratio = pw.DoubleField() + exposure_bias = pw.DoubleField() + + datetime_imported: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="import_timestamp") # type: ignore + datetime_changed: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="change_timestamp") # type: ignore + datetime_exported: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="export_timestamp") # type: ignore + datetime_printed: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="print_timestamp") # type: ignore + datetime_thumb: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="thumb_timestamp") # type: ignore + thumb_maxmip: int = pw.IntegerField() # type: ignore + + def flag(self, flag): + return bool(self.flags & flag.value) + + def set_flag(self, flag, value): + b = -1 + assert flag + flag = flag.value + while flag: + b += 1 + flag = flag >> 1 + self.flags = (self.flags & ~flag) | (value & flag) + + + + @property + def stars(self): + return self.flags & 0x7 + + @stars.setter + def stars(self, value): + self.flags = (self.flags & ~0x7) | (value & 0x7) + + def __str__(self): + return self.filename + + history: pw.BackrefAccessor + module_order: pw.BackrefAccessor + + def get_ordered_active_modules(self): + + # nextline: type: ignore + active_history = (self.history + .order_by(HistoryEntry.num.asc()) # type: ignore + .where(HistoryEntry.num < self.history_end)) + + if len(active_history) == 0: + return [] + + module_orders = self.module_order.select() + module_order = module_orders[0] + iop_list = None + if module_order.version == IOPOrderType.CUSTOM: + iop_list = module_order.iop_list + elif module_order.version == IOPOrderType.V30: + iop_list = [(v, 0) for v in v30_order] + elif module_order.version == IOPOrderType.V30_JPG: + iop_list = [(v, 0) for v in v30_jpg_order] + elif module_order.version == IOPOrderType.LEGACY: + iop_list = [(v, 0) for v in legacy_order] + else: + raise NotImplementedError() + + # Override if iop_list exists for the case that + # 'module_order.version != CUSTOM'... Is this a bug in dt? + if module_order.iop_list: + iop_list = module_order.iop_list + + iop_list = {iop_list[i]: i for i in range(len(iop_list))} + + active_modules = {} + for he in active_history: + if he.enabled: + active_modules[(he.module_name, he.instance)] = he.module + elif (he.module_name, he.instance) in active_modules: + del active_modules[(he.module_name, he.instance)] + + active_modules = sorted(active_modules.items(), key=lambda kv: iop_list[kv[0]]) + return list(map(lambda kv: kv[1], active_modules)) + + class Meta: + db_table = 'images' + database = db_library # This model uses the "people.db" database`` + +class ColorLabel(pw.Model): + + image: Image = pw.ForeignKeyField(Image, column_name='imgid') # type: ignore + color: Color = EnumField(Color) # type: ignore + + class Meta: + primary_key = False + db_table = 'color_labels' + database = db_library + +class HistoryEntry(pw.Model): + """ + A darktable filmroll. + """ + image: Image = pw.ForeignKeyField(Image, column_name="imgid", backref="history") # type: ignore + num: int = pw.IntegerField() # type: ignore + + module_name: str = pw.CharField(256, column_name="operation") # type: ignore + version: int = pw.IntegerField(column_name="module") # type: ignore + params: bytes = pw.BlobField(column_name="op_params") # type: ignore + + enabled: bool = pw.BooleanField() # type: ignore + + blendop_version: int = pw.IntegerField() # type: ignore + blendop_params: bytes = pw.BlobField() # type: ignore + + instance: int = pw.IntegerField(column_name="multi_priority") # type: ignore + name: str = pw.CharField(256, column_name="multi_name") # type: ignore + name_hand_edited: int = pw.IntegerField(column_name="multi_name_hand_edited") # type: ignore + + @property + def module(self): + for cls in DT_MODULES: + if cls.NAME == self.module_name and cls.VERSION == self.version: + return cls(self.instance, self.params) + return Module(self.instance, self.params, module_name=self.module_name) + + def __str__(self): + return "%3d - %sv%d-%s" % ( + self.num, self.module_name, self.version, self.instance) + + class Meta: + primary_key = False + db_table = 'history' + database = db_library + + +class ModuleOrderEntry(pw.Model): + """ + A darktable filmroll. + """ + image: Image = pw.ForeignKeyField(Image, column_name="imgid", backref="module_order") # type: ignore + version: IOPOrderType = EnumField(IOPOrderType) # type: ignore + iop_list = ModuleOrderListField() + + def __str__(self): + return str((self.version, self.iop_list)) + + class Meta: + primary_key = False + db_table = 'module_order' + database = db_library + +class Tag(pw.Model): + + name: str = pw.CharField() # type: ignore + synonyms: str = pw.CharField() # type: ignore + flags: int = pw.IntegerField() # type: ignore + + class Meta: + db_table = 'tags' + database = db_data + +class TaggedImages(pw.Model): + + image: Image = pw.ForeignKeyField(Image, backref='tags', column_name='imgid') # type: ignore + tag: Tag = pw.ForeignKeyField(Tag, backref='images', column_name='tagid') # type: ignore + + class Meta: + primary_key = False + db_table = 'tagged_images' + database = db_library diff --git a/pytable/modules.py b/pytable/modules.py new file mode 100644 index 0000000..61f696b --- /dev/null +++ b/pytable/modules.py @@ -0,0 +1,420 @@ +from .types import Adaptation, ColorIntent, ColorSpacesColorProfile, \ + Illuminant, IlluminantLED, ImageOrientation + +import struct +import enum + +DT_MODULES = [] + + +def decode_bytes(bytes): + return bytes.decode().rstrip("\0") + + +class ModuleMeta(type): + + def __new__(cls, name, bases, dct): + new_cls = super().__new__(cls, name, bases, dct) + if name != "Module": + DT_MODULES.append(new_cls) + return new_cls + + +class Module(metaclass=ModuleMeta): + + NAME = "" + VERSION = -1 + PARAMS_NAMES = () + PARAMS_FORMAT = "" + PARAMS_TYPES = () + + def __init__(self, instance, raw_params, module_name=None) -> None: + self.module_name = module_name or self.__class__.__name__ + self.instance = instance + self.params = self.parse_params(raw_params) + + def parse_params(self, raw_params): + if not self.PARAMS_FORMAT: + return {} + values = struct.unpack(self.PARAMS_FORMAT, raw_params) + params = {} + pts = list(self.PARAMS_TYPES) + [None] * (len(self.PARAMS_NAMES) - len(self.PARAMS_TYPES)) + for name, type in zip(self.PARAMS_NAMES, pts): + n = 1 + if "*" in name: + name, n = name.split("*") + n = int(n) + params[name] = values[:n] + assert len(params[name]) == n + if type: + params[name] = [type(v) for v in params[name]] + if n == 1: + params[name] = params[name][0] + values = values[n:] + assert not values, values + return params + + def __str__(self): + import pprint + return "%s-%d: %s" % (self.module_name, self.instance, pprint.pformat( + self.params, indent=0, compact=True, width=10000, depth=1 + )) + + def __repr__(self) -> str: + return "%s-%d" % (self.module_name, self.instance) + +class ColorBalanceRGBSaturation(enum.Enum): + JZAZBZ = 0 + DTUCS = 1 + + +class ColorBalanceRGB(Module): + NAME = "colorbalancergb" + VERSION = 5 + PARAMS_NAMES = ("shadows_Y", "shadows_C", "shadows_H", "midtones_Y", + "midtones_C", "midtones_H", "highlights_Y", "highlights_C", + "highlights_H", "global_Y", "global_C", "global_H", + "shadows_weight", "white_fulcrum", "highlights_weight", + "chroma_shadows", "chroma_highlights", "chroma_global", + "chroma_midtones", "saturation_global", + "saturation_highlights", "saturation_midtones", + "saturation_shadows", "hue_angle", "brilliance_global", + "brilliance_highlights", "brilliance_midtones", + "brilliance_shadows", "mask_grey_fulcrum", "vibrance", + "grey_fulcrum", "contrast", "saturation_formula") + + PARAMS_FORMAT = "32fi" + PARAMS_TYPES = [None] * 32 + [ColorBalanceRGBSaturation] + + +class ColorCalibrationV2(Module): + + NAME = "channelmixerrgb" + VERSION = 3 + PARAMS_NAMES = ("red*4", "green*4", "blue*4", "saturation*4", + "lightness*4", "grey*4", "normalize_red", + "normalize_green", "normalize_blue", + "normalize_saturation", "normalize_lightness", + "normalize_grey", "illuminant", "illum_fluo", + "illum_led", "adaptation", "x", "y", "temperature", + "gamut", "clip", "version") + PARAMS_FORMAT = "24f6i4iffffii" + PARAMS_TYPES = ([None] * 6 + [bool] * 6 + + [Illuminant, None, IlluminantLED, Adaptation, + None, None, None, None, bool]) + + +class ColorInputProfileNormalize(enum.Enum): + OFF = 0 + SRGB = 1 + ADOBE_RGB = 2 + LINEAR_REC709_RGB = 3 + LINEAR_REC2020_RGB = 4 + + +class ColorInputProfileV7(Module): + + NAME = "colorin" + VERSION = 7 + PARAMS_NAMES = ( + "type", "filename", "intent", "normalize", "blue_mapping", "type_work", + "filename_work") + PARAMS_FORMAT = "i512siiii512s" + PARAMS_TYPES = (ColorSpacesColorProfile, decode_bytes, ColorIntent, + ColorInputProfileNormalize, bool, ColorSpacesColorProfile, + decode_bytes) + + +class ColorOutputProfileV5(Module): + + NAME = "colorout" + VERSION = 5 + PARAMS_NAMES = ( + "type", "filename", "intent") + PARAMS_FORMAT = "i512si" + PARAMS_TYPES = (ColorSpacesColorProfile, decode_bytes, ColorIntent) + + +class ColorZonesChannel(enum.Enum): + lightness = 0 + chroma = 1 + hue = 2 + +class ColorZonesV5(Module): + NAME = "colorzones" + VERSION = 5 + PARAMS_NAMES = ("channel", "curve*120", "curve_num_nodes*3", + "curve_type*3", "strength", "mode", "spline_version") + + PARAMS_FORMAT = "i120f6ifii" + + def parse_params(self, raw_params): + params = super().parse_params(raw_params) + curve = params["curve"] + curve = [ + [(curve[channel * 20 * 2 + node * 2], + curve[channel * 20 * 2 + node * 2 + 1]) for node in range(20)] + for channel in range(3) + ] + curve[0] = curve[0][:params["curve_num_nodes"][0]] + curve[1] = curve[1][:params["curve_num_nodes"][1]] + curve[2] = curve[2][:params["curve_num_nodes"][2]] + params["curve"] = curve + return params + + +class DemosaicMethod(enum.Enum): + PPG = 0 + AMAZE = 1 + VNG4 = 2 + RCD = 5 + LMMSE = 6 + RCD_VNG = 2048 | 5 + AMAZE_VNG = 2048 | 6 + PASSTHROUGH_MONOCHROME = 3 + PASSTHROUGH_COLOR = 4 + VNG = 1024 | 0 + MARKESTEIJN = 1024 | 1 + MARKESTEIJN_3 = 1024 | 2 + FCD = 1024 | 4 + MARKEST3_VNG = 2048 | 1024 | 2 + PASSTHR_MONOX = 1024 | 3 + PASSTHR_COLORX = 1024 | 5 + + +class DemosaicGreenEQ(enum.Enum): + NO = 0 + LOCAL = 1 + FULL = 2 + BOTH = 3 + + +class DemosaicQualFlags(enum.Enum): + DEFAULT = 0 + FULL_SCALE = 1 + ONLY_VNG_LINEAR = 2 + + +class DemosaicSmooth(enum.Enum): + OFF = 0 + ONCE = 1 + TWICE = 2 + THREE_TIMES = 3 + FOUR_TIMES = 4 + FIVE_TIMES = 5 + + +class DemosaicLMMSE(enum.Enum): + BASIC = 0 + MEDIAN = 1 + MEDIANX3 = 2 + REFINE_AND_MEDIANS = 3 + REFINEx2_AND_MEDIANS = 4 + + +class DemosaicV4(Module): + + NAME = "demosaic" + VERSION = 4 + PARAMS_NAMES = ( + "green_eq", "median_thrs", "color_smoothing", "demosaicing_method", + "lmmse_refine", "dual_thrs") + PARAMS_FORMAT = "ifiiif" + PARAMS_TYPES = (DemosaicGreenEQ, None, DemosaicSmooth, DemosaicMethod, DemosaicLMMSE) + + +class DenoiseProfiledWaveletMode(enum.Enum): + RGB = 0 + Y0U0V0 = 1 + +class DenoiseProfiled(Module): + NAME = "denoiseprofile" + VERSION = 11 + PARAMS_NAMES = ("radius", "nbhood", "strength", "shadows", + "bias", "scattering", "central_pixel_weight", + "overshooting", "a*3", "b*3", "mode", "x*42", "y*42", + "wb_adaptive_anscombe", "fix_anscombe_and_nlmeans_norm", + "use_new_vst", "wavelet_color_mode") + + PARAMS_FORMAT = "8f6fi42f42fiiii" + PARAMS_TYPES = [None] * 16 + [DenoiseProfiledWaveletMode] + + def parse_params(self, raw_params): + params = super().parse_params(raw_params) + params["x"] = [[params["x"][profile * 7 + band] for band in range(7)] for profile in range(6)] + params["y"] = [[params["y"][profile * 7 + band] for band in range(7)] for profile in range(6)] + return params + + +class DiffuseOrSharpenV2(Module): + + NAME = "diffuse" + VERSION = 2 + PARAMS_NAMES = ( + "iterations", "sharpness", "radius", "regularization", + "variance_threshold", "anisotropy_first", "anisotropy_second", + "anisotropy_third", "anisotropy_fourth", "threshold", + "first", "second", "third", "fourth", "radius_center") + PARAMS_FORMAT = "ififffffffffffi" + + +class DisplayEncodingV1(Module): + + NAME = "gamma" + VERSION = 1 + PARAMS_NAMES = ("gamma", "linear") + PARAMS_FORMAT = "ff" + + +class ExposureMode(enum.Enum): + MANUAL = 0 + DEFLICKER = 1 + + +class ExposureV6(Module): + + NAME = "exposure" + VERSION = 6 + PARAMS_NAMES = ("mode", "black", "exposure", "deflicker_percentile", + "deflicker_target_level", "compensate_exposure_bias") + PARAMS_FORMAT = "iffffi" + PARAMS_TYPES = (ExposureMode, None, None, None, None, bool) + + +class FilmicRGBMethod(enum.Enum): + NONE = 0 + MAX_RGB = 1 + LUMINANCE = 2 + POWER_NORM = 3 + EUCLIDEAN_NORM_V1 = 4 + EUCLIDEAN_NORM_V2 = 5 + + +class FilmicRGBCurve(enum.Enum): + POLY_4 = 0 + POLY_3 = 1 + RATIONAL = 2 + + +class FilmicRGBReconstruction(enum.Enum): + RECONSTRUCT_RGB = 0 + RECONSTRUCT_RATIOS = 1 + + +class FilmicRGBNoiseDistribution(enum.Enum): + UNIFORM = 0 + GAUSSIAN = 1 + POISSONIAN = 2 + + +class FilmicRGB(Module): + + NAME = "filmicrgb" + VERSION = 6 + PARAMS_NAMES = ("grey_point_source", "black_point_source", + "white_point_source", "reconstruct_threshold", + "reconstruct_feather", "reconstruct_bloom_vs_details", + "reconstruct_grey_vs_color", + "reconstruct_structure_vs_texture", "security_factor", + "grey_point_target", "black_point_target", + "white_point_target", "output_power", "latitude", + "contrast", "saturation", "balance", "noise_level", + "preserve_color", "version", "auto_hardness", "custom_grey", + "high_quality_reconstruction", "noise_distribution", + "shadows", "highlights", "compensate_icc_black", + "spline_version", "enable_highlight_reconstruction") + PARAMS_FORMAT = "18f11i" + PARAMS_TYPES = [None] * 18 + [ + FilmicRGBMethod, None, bool, bool, None, FilmicRGBNoiseDistribution, + FilmicRGBCurve, FilmicRGBCurve, bool, None, bool] + + +class HighlightsMode(enum.Enum): + OPPOSED = 5 + LCH = 1 + CLIP = 0 + SEGMENTS = 4 + LAPLACIAN = 3 + INPAINT = 2 + +class HighlightsAtrousWaveletsScales(enum.Enum): + PX2 = 0 + PX4 = 1 + PX8 = 2 + PX16 = 3 + PX32 = 4 + PX64 = 5 + PX128 = 6 + PX256 = 7 + PX512 = 8 + PX1024 = 9 + PX2048 = 10 + PX4096 = 11 + +class HighlightsRecoveryMode(enum.Enum): + MODE_OFF = 0 + MODE_ADAPT = 5 + MODE_ADAPTF = 6 + MODE_SMALL = 1 + MODE_LARGE = 2 + MODE_SMALLF = 3 + MODE_LARGEF = 4 + + +class HighlightsV4(Module): + + NAME = "highlights" + VERSION = 4 + PARAMS_NAMES = ("mode", "blendL", "blendC", "strength", "clip", + "noise_level", "iterations", "scales", "candidating", + "combine", "recovery", "solid_color") + PARAMS_FORMAT = "ifffffiiffif" + PARAMS_TYPES = (HighlightsMode, None, None, None, None, None, None, + HighlightsAtrousWaveletsScales, None, None, + HighlightsRecoveryMode) + + +class LocalContrastMode(enum.Enum): + BILATERAL = 0 + LOCAL_LAPLACIAN = 1 + +class LocalContrastV3(Module): + NAME = "bilat" + VERSION = 3 + PARAMS_NAMES = ("mode", "sigma_r", "sigma_s", "detail", "midtone") + PARAMS_FORMAT = "i4f" + PARAMS_TYPES = (LocalContrastMode,) + + +class OrientationV2(Module): + NAME = "flip" + VERSION = 2 + PARAMS_NAMES = ("orientation",) + PARAMS_FORMAT = "i" + PARAMS_TYPES = (ImageOrientation,) + + +class SharpenV1(Module): + NAME = "sharpen" + VERSION = 1 + PARAMS_NAMES = ("radius", "amount", "threshold") + PARAMS_FORMAT = "3f" + + +class RawBlackWhitePointV2(Module): + + NAME = "rawprepare" + VERSION = 2 + PARAMS_NAMES = ( + "left", "top", "right", "bottom", "raw_black_level_separate0", + "raw_black_level_separate1", "raw_black_level_separate2", + "raw_black_level_separate3", "raw_white_point", "flat_field") + PARAMS_FORMAT = "iiiiHHHHHi" + + +class WhiteBalance(Module): + + NAME = "temperature" + VERSION = 3 + PARAMS_NAMES = ("red", "green", "blue", "g2") + PARAMS_FORMAT = "ffff" diff --git a/pytable/types.py b/pytable/types.py new file mode 100644 index 0000000..d54456f --- /dev/null +++ b/pytable/types.py @@ -0,0 +1,202 @@ +import enum + + +class Adaptation(enum.Enum): + LINEAR_BRADFORD = 0 + CAT16 = 1 + FULL_BRADFORD = 2 + XYZ = 3 + RGB = 4 + + +class ColorSpacesColorProfile(enum.Enum): + NONE = -1 + FILE = 0 + SRGB = 1 + ADOBERGB = 2 + LIN_REC709 = 3 + LIN_REC2020 = 4 + XYZ = 5 + LAB = 6 + INFRARED = 7 + DISPLAY = 8 + EMBEDDED_ICC = 9 + EMBEDDED_MATRIX = 10 + STANDARD_MATRIX = 11 + ENHANCED_MATRIX = 12 + VENDOR_MATRIX = 13 + ALTERNATE_MATRIX = 14 + BRG = 15 + EXPORT = 16 + SOFTPROOF = 17 + WORK = 18 + DISPLAY2 = 19 + REC709 = 20 + PROPHOTO_RGB = 21 + PQ_REC2020 = 22 + HLG_REC2020 = 23 + PQ_P3 = 24 + HLG_P3 = 25 + LAST = 26 + + +class ColorIntent(enum.Enum): + PERCEPTUAL = 0 + RELATIVE_COLORIMETRIC = 1 + SATURATION = 2 + ABSOLUTE_COLORIMETRIC = 3 + + +class Illuminant(enum.Enum): + PIPE = 0 + A = 1 + D = 2 + E = 3 + F = 4 + LED = 5 + BB = 6 + CUSTOM = 7 + DETECT_SURFACES = 8 + DETECT_EDGES = 9 + CAMERA = 10 + + +class IlluminantLED(enum.Enum): + LED_B1 = 0 + LED_B2 = 1 + LED_B3 = 2 + LED_B4 = 3 + LED_B5 = 4 + LED_BH1 = 5 + LED_RGB1= 6 + LED_V1 = 7 + LED_V2 = 8 + + +class IOPOrderType(enum.Enum): + CUSTOM = 0 + LEGACY = 1 + V30 = 2 + V30_JPG = 3 + +class Color(enum.Enum): + RED = 0 + YELLOW = 1 + GREEN = 2 + BLUE = 3 + PURPLE = 4 + +class ImageFlags(enum.Enum): + """ + Flags that are stored in Image::flags. Extract flags with 'image.flag(flag)' + + Extracted from: + https: """ + # Image is rejected + REJECTED = 8 + THUMBNAIL_DEPRECATED = 16 + # set during import if the image is low-dynamic range, i.e. doesn't + # need demosaic, wb, highlight clipping etc. + LDR = 32, + # set during import if the image is raw data, i.e. it needs demosaicing. + RAW = 64 + # set during import if images is a high-dynamic range image.. + HDR = 128 + # set when marked for deletion + REMOVE = 256 + # set when auto-applying presets have been applied to this image. + AUTO_PRESETS_APPLIED = 512 + # legacy flag. is set for all new images. i hate to waste a bit on this :( + NO_LEGACY_PRESETS = 1024 + # local copy status + LOCAL_COPY = 2048 + # image has an associated .txt file for overlay + HAS_TXT = 4096 + # image has an associated wav file + HAS_WAV = 8192 + # image is a bayer pattern with 4 colors (e.g., CYGM or RGBE) + BAYER = 16384 + # image was detected as monochrome + MONOCHROME = 32768 + # DNG image has exif tags which are not cached in the database but + # must be read and stored in t when the image is loaded. + HAS_ADDITIONAL_EXIF_TAGS = 65536 + # image is an sraw + S_RAW = 1 << 17 + # image has a monochrome preview tested + MONOCHROME_PREVIEW = 1 << 18 + # image has been set to monochrome via demosaic module + MONOCHROME_BAYER = 1 << 19 + # image has a flag set to use the monochrome workflow in the modules supporting it + MONOCHROME_WORKFLOW = 1 << 20 + +class ImageOrientation(enum.Enum): + NULL = -1 + NONE = 0 + FLIP_Y = 1 << 0 + FLIP_X = 1 << 1 + SWAP_XY = 1 << 2 + FLIP_HORIZONTALLY = FLIP_X + FLIP_VERTICALLY = FLIP_Y + ROTATE_180_DEG = FLIP_Y | FLIP_X + TRANSPOSE = SWAP_XY + ROTATE_CCW_90_DEG = FLIP_X | SWAP_XY + ROTATE_CW_90_DEG = FLIP_Y | SWAP_XY + TRANSVERSE = FLIP_Y | FLIP_X | SWAP_XY + + +legacy_order = ( + "rawprepare", "invert", "temperature", "highlights", "cacorrect", + "hotpixels", "rawdenoise", "demosaic", "mask_manager", "denoiseprofile", + "tonemap", "exposure", "spots", "retouch", "lens", "cacorrectrgb", "ashift", + "liquify", "rotatepixels", "scalepixels", "flip", "clipping", "toneequal", + "crop", "graduatednd", "basecurve", "bilateral", "profile_gamma", + "hazeremoval", "colorin", "channelmixerrgb", "diffuse", "censorize", + "negadoctor", "blurs", "basicadj", "colorreconstruct", "colorchecker", + "defringe", "equalizer", "vibrance", "colorbalance", "colorbalancergb", + "colorize", "colortransfer", "colormapping", "bloom", "nlmeans", + "globaltonemap", "shadhi", "atrous", "bilat", "colorzones", "lowlight", + "monochrome", "sigmoid", "filmic", "filmicrgb", "colisa", "zonesystem", + "tonecurve", "levels", "rgblevels", "rgbcurve", "relight", "colorcorrection", + "sharpen", "lowpass", "highpass", "grain", "lut3d", "colorcontrast", + "colorout", "channelmixer", "soften", "vignette", "splittoning", "velvia", + "clahe", "finalscale", "overexposed", "rawoverexposed", "dither", "borders", + "watermark", "gamma" +) + +v30_order = ( + "rawprepare", "invert", "temperature", "highlights", "cacorrect", "hotpixels", + "rawdenoise", "demosaic", "denoiseprofile", "bilateral", "rotatepixels", + "scalepixels", "lens", "cacorrectrgb","hazeremoval", "ashift", "flip", + "clipping", "liquify", "spots", "retouch", "exposure", "mask_manager", + "tonemap", "toneequal","crop","graduatednd", "profile_gamma", "equalizer", + "colorin", "channelmixerrgb", "diffuse", "censorize", "negadoctor","blurs", + "nlmeans","colorchecker","defringe","atrous", "lowpass", "highpass", "sharpen", + "colortransfer","colormapping", "channelmixer","basicadj", "colorbalance", + "colorbalancergb", "rgbcurve", "rgblevels", "basecurve","filmic", "sigmoid", + "filmicrgb", "lut3d", "colisa", "tonecurve", "levels", "shadhi", + "zonesystem", "globaltonemap", "relight", "bilat","colorcorrection", + "colorcontrast", "velvia", "vibrance", "colorzones", "bloom", "colorize", + "lowlight", "monochrome", "grain", "soften", "splittoning", "vignette", + "colorreconstruct", "colorout", "clahe", "finalscale", "overexposed", + "rawoverexposed", "dither", "borders", "watermark", "gamma" +) + +v30_jpg_order = ( + "rawprepare", "invert", "temperature", "highlights", "cacorrect", "hotpixels", + "rawdenoise", "demosaic", "colorin", "denoiseprofile", "bilateral", + "rotatepixels", "scalepixels", "lens", "cacorrectrgb", "hazeremoval", + "ashift", "flip", "clipping", "liquify", "spots", "retouch", "exposure", + "mask_manager", "tonemap", "toneequal", "crop", "graduatednd", + "profile_gamma", "equalizer", "channelmixerrgb", "diffuse", "censorize", + "negadoctor", "blurs", "nlmeans", "colorchecker","defringe","atrous", + "lowpass", "highpass", "sharpen", "colortransfer","colormapping", + "channelmixer","basicadj", "colorbalance", "colorbalancergb", "rgbcurve", + "rgblevels", "basecurve","filmic", "sigmoid", "filmicrgb", "lut3d", "colisa", + "tonecurve", "levels", "shadhi", "zonesystem", "globaltonemap", "relight", + "bilat","colorcorrection","colorcontrast", "velvia", "vibrance", + "colorzones", "bloom", "colorize", "lowlight", "monochrome", "grain", + "soften", "splittoning", "vignette", "colorreconstruct","colorout", "clahe", + "finalscale", "overexposed", "rawoverexposed", "dither", "borders", + "watermark", "gamma" +) diff --git a/synchronise.py b/synchronise.py new file mode 100644 index 0000000..8fcbe68 --- /dev/null +++ b/synchronise.py @@ -0,0 +1,297 @@ +from pytable.models import Image, ImageFlags, FilmRoll + +from datetime import datetime +import peewee as pw +import os +import shutil +from dataclasses import dataclass +from typing import Tuple, Optional, Generic, TypeVar, Iterable + +T = TypeVar('T') + +Range = Tuple[Optional[T], Optional[T]] + +class Filter: + + + def __call__(self, image: Image): + return self.matches(image) + + def matches(self, image: Image) -> bool: + ... + +@dataclass +class LocalRegion(Filter): + + date_range: Range[datetime] = (None, None) + stars_range: Range[int] = (None, None) + + def matches(self, image: Image) -> bool: + + stars = image.stars if not image.flag(ImageFlags.REJECTED) else -1 + if self.stars_range[0] is not None and stars < self.stars_range[0]: + return False + if self.stars_range[1] is not None and stars > self.stars_range[1]: + return False + + dt = (image.datetime_taken or image.datetime_imported) + if self.date_range[0] is not None and dt < self.date_range[0]: + return False + if self.date_range[1] is not None and dt > self.date_range[1]: + return False + return True + +@dataclass +class Or(Filter): + + filters: Iterable[Filter] + + def matches(self, image: Image) -> bool: + return any(f(image) for f in self.filters) + +@dataclass +class And(Filter): + + filters: Iterable[Filter] + + def matches(self, image: Image) -> bool: + return all(f(image) for f in self.filters) + + +class Config: + + def __init__(self) -> None: + self.filter = Or([ + LocalRegion((datetime(2024, 6, 14), None), (0, None)), + LocalRegion((datetime(2024, 6, 22, 14), datetime(2024, 6, 22, 14, 24))) + ]) + + self.d_root = "/home/oke/Pictures/Darktable" + self.d_local = "/home/oke/Pictures/DarktableLocal" + self.d_remote = "/home/oke/Pictures/DarktableRemote" + + + @staticmethod + def _is_older_than(image: Image, date: datetime): + return (image.datetime_taken or image.datetime_imported) < date + + def should_not_be_local(self, image: Image): + return not self.filter(image) + + def should_be_local(self, image: Image): + return self.filter(image) + + def image_local_folder(self, image: Image): + folder = image.film.folder + if not folder.startswith(self.d_root): + raise NotImplementedError() + assert folder.startswith(self.d_root) + return folder.replace(self.d_root, self.d_local) + + def image_local_image(self, image: Image) -> str: + return os.path.join(self.image_local_folder(image), image.filename) + + def image_local_image_xmp(self, image: Image) -> str: + base, ext = os.path.splitext(image.filename) + version = ("_%02d" % image.version) if image.version != 0 else "" + xmp_filename = base + version + ext + '.xmp' + return os.path.join(self.image_local_folder(image), xmp_filename) + + def image_local_iphone_movie(self, image: Image) -> str: + base, _ = os.path.splitext(image.filename) + return os.path.join(self.image_local_folder(image), "." + base + ".mov") + + def potential_extras(self, image): + yield self.image_local_iphone_movie(image) + +from dataclasses import dataclass +from typing import Dict, List, Optional +from enum import Enum + +class SyncReq(Enum): + EXISTS = 0 + REQUIRED_SOFT = 1 + REQUIRED = 2 + + @staticmethod + def max(a: "SyncReq", b: "SyncReq"): + if a.value < b.value: + return b + return a + @staticmethod + def required(v: 'SyncReq'): + return v == SyncReq.REQUIRED or v == SyncReq.REQUIRED_SOFT + + @staticmethod + def make(required, optional): + if required: + if optional: + return SyncReq.REQUIRED_SOFT + return SyncReq.REQUIRED + return SyncReq.EXISTS + +@dataclass +class FileInfo: + required: bool = False + optional: bool = False + +class SyncManager: + + def __init__(self, config) -> None: + self._config = config + self._files: Dict[str, SyncReq] = {} + + def register_file(self, fn: str, requirement: SyncReq): + if fn in self._files: + requirement = SyncReq.max(self._files[fn], requirement) + self._files[fn] = requirement + + def register_image(self, image: Image): + required = self._config.should_be_local(image) + self.register_file( + self._config.image_local_image(image), + SyncReq.make(required, False) + ) + self.register_file( + self._config.image_local_image_xmp(image), + SyncReq.make(required, False) + ) + + for extra in self._config.potential_extras(image): + self.register_file( + extra, + SyncReq.make(required, True) + ) + + def partition(self): + result = {k: [] for k in SyncReq} + for file, req in self._files.items(): + result[req].append(file) + return result + +config = Config() + +sync_manager = SyncManager(config) + +query = Image.filter() +pw.prefetch(query, FilmRoll) +for image in query: + sync_manager.register_image(image) + + +actions = sync_manager.partition() + +for key in actions: + actions[key] = list(sorted(map( + lambda fn: os.path.relpath(fn, config.d_local) + "\n", + actions[key] + ))) + +with open("required.txt", "w") as f: + f.writelines(actions[SyncReq.REQUIRED]) +import subprocess +user = "oke" +host = "192.168.20.2" + +args = [ + "rsync", + # "--dry-run", + "--info=progress", + "--ignore-existing", + "-av", + "--files-from", + "required.txt", + f"{user}@{host}:lenovo-darktable{config.d_local}", + os.path.abspath(config.d_local) +] +import shlex +print(shlex.join(args)) +subprocess.call(args) + +with open("required_optional.txt", "w") as f: + f.writelines(actions[SyncReq.REQUIRED_SOFT]) + +with open("remove.txt", "w") as f: + f.writelines(actions[SyncReq.EXISTS]) + +exit() +for fn in actions[SyncReq.REQUIRED_SOFT]: + if not os.path.exists(fn): + print("FETCH opt: ", fn) + +for fn in sync_manager.files_to_fetch_required(): + print("FETCH: ", fn) + +for fn in sync_manager.files_to_remove(): + print("RM: ", fn) +n_total = 0 +n_not_needed = 0 +needs_removal = [] +extra_files_to_remove = [] +bytes_to_remove = 0 + +needs_fetching = [] + +for file, usages in file_usages.items(): + if not file.startswith(d_root): + print("SKIPPNG", file) + continue + assert file.startswith(d_root) + remote_file = file.replace(d_root, d_remote) + local_file = file.replace(d_root, d_local) + + if usages > 0: + if not os.path.exists(local_file): + needs_fetching.append((remote_file, local_file)) + else: + n_not_needed += 1 + if os.path.exists(local_file): + needs_removal.append(local_file) + bytes_to_remove += os.stat(local_file).st_size + + for extra_fn in potential_extras(local_file): + if os.path.exists(extra_fn): + extra_files_to_remove.append(extra_fn) + bytes_to_remove += os.stat(extra_fn).st_size + +files_to_remove = list(sorted(needs_removal + extra_files_to_remove)) + +needs_fetching = sorted(needs_fetching) +import pprint +print("Files to fetch: ") +pprint.pprint(needs_fetching) +print("Extra files to remove:") +pprint.pprint(extra_files_to_remove) +print("First files to remove:") +pprint.pprint(needs_removal[:10]) +print("Last files to remove:") +pprint.pprint(needs_removal[-10:]) + +with open("remove.txt", "w") as f: + pprint.pprint(sorted(set(needs_removal).union(extra_files_to_remove)), f) + +print("Not needed: %6d / %6d (%3d%%)" % (n_not_needed, len(file_usages), 100 * n_not_needed / len(file_usages))) +print() +print("Not yet deleted: %6d / %6d (%3d%%)" % (len(needs_removal), n_not_needed, 100 * len(needs_removal) / n_not_needed)) +print("Bytes to free: %5.2fGB" % (bytes_to_remove / 1024. / 1024 / 1024)) +print("Total files to remove: %d" % len(files_to_remove)) +print() +print("Total files to fetch: %3d" % len(needs_fetching)) +print() + +reply = input("Type YES to remove all these files. This is not reversible... : ") + +if reply == 'YES': + + for fn in files_to_remove: + assert fn.startswith(d_local) + os.remove(fn) + + for f, t in needs_fetching: + assert os.path.exists(f), "Backup corruped, file not found: " + f + assert not os.path.exists(t) + print(f, t) + shutil.copy(f, t) + +else: + print("You did not type 'YES'. Not doing anything!") diff --git a/vmgr/actions.py b/vmgr/actions.py new file mode 100644 index 0000000..6228caa --- /dev/null +++ b/vmgr/actions.py @@ -0,0 +1,278 @@ +import subprocess +from dataclasses import dataclass +from typing import List, Callable, Any, Optional +import functools +import inspect +import sys +import os +import datetime +import traceback + +from video import VideoModel, Video, BASE_DIR +from typing import Tuple + + +TRASH_DT_FORMAT = "%Y-%m-%dT%H:%M:%S" +TRASH = "/home/oke/.local/share/Trash" +TRASH_INFO = os.path.join(TRASH, "info") +TRASH_FILES = os.path.join(TRASH, "files") + +import json +from typing import Generic, TypeVar + +T = TypeVar('T') +V = TypeVar('V') + +class ActionException(Exception): + + def __init__(self, message, *args: object) -> None: + self.message = message + super().__init__(*args) + + def __repr__(self) -> str: + return f"" + +class ActionFailed(ActionException): + ... + +class ActionIncomplete(ActionException): + ... + +import uuid +class UndoableAction(Generic[T]): + + ID = "OverrideMe" + + def __init__(self, args: T, group_leader=None, uid=None): + self.args: T = args + self.uuid = uid + if self.uuid is None: + self.uuid = group_leader.uuid if group_leader else uuid.uuid4().hex + + def undo_action(self) -> "UndoableAction[Any]": ... + + def run(self, model: VideoModel) -> Optional[bool]: ... + + +class RestoreFromTrash(UndoableAction[str]): + + def undo_action(self) -> UndoableAction[Any]: + return Trash(self.args) + + def __str__(self) -> str: + return f"Restoring {self.args}" + + @staticmethod + def trashed_files(): + for fn in os.listdir(TRASH_INFO): + if fn.endswith('.trashinfo'): + with open(os.path.join(TRASH_INFO, fn), 'r') as f: + lines_iter = iter(f) + if next(lines_iter) != '[Trash Info]\n': + print("Invalid trashinfo file: ", fn) + continue + path = None + date = None + for line in f: + line = line.strip() + if path is None and line.startswith('Path='): + path = line[5:] + if date is None and line.startswith('DeletionDate='): + date = datetime.datetime.strptime(line[13:], TRASH_DT_FORMAT) + if path is None or date is None: + print("Invalid trashinfo file: ", fn) + continue + trash_file = os.path.join(TRASH_FILES, fn[:-10]) + if not os.path.exists(trash_file): + print("Trashed file does not exist: ", trash_file) + continue + yield (date, trash_file, path) + + + def run(self, model): + file = self.args + matches = sorted(((date, trash_file) for (date, trash_file, path) in self.trashed_files() if path == self.args), reverse=True) + if len(matches) == 0: + raise ActionFailed("Could not find trashed file") + try: + os.rename(matches[0][1], file) + except Exception as e: + traceback.print_exception(e) + raise ActionFailed("Could not move file to original destination...") + try: + os.remove(os.path.join(TRASH_INFO, os.path.basename(file) + ".trashinfo")) + except Exception as e: + traceback.print_exception(e) + + return True + + + +class Trash(UndoableAction[str]): + + def undo_action(self) -> UndoableAction[Any]: + return RestoreFromTrash(self.args) + + def __str__(self) -> str: + return f"Trashing {self.args}" + + def run(self, model: VideoModel): + file = os.path.abspath(os.path.expanduser(self.args)) + i = 0 + suffix = "" + while True: + info_fn = os.path.join(TRASH_INFO, os.path.basename(file) + suffix + ".trashinfo") + try: + with open(info_fn, 'x') as f: + f.write( + "[Trash Info]\n" + f"Path={file}\n" + f"DeletionDate={datetime.datetime.now().strftime(TRASH_DT_FORMAT)}\n" + ) + except FileExistsError: + i += 1 + suffix = f"_{i}" + continue + + trash_fn = os.path.join(TRASH_FILES, os.path.basename(file) + suffix) + if os.path.exists(trash_fn): + os.remove(info_fn) + i += 1 + suffix = f"_{i}" + continue + os.rename(file, trash_fn) + index, video = model.find_video(Video(self.args)) + if index: + model.remove(index) + break + +class SetVideoStars(UndoableAction[Tuple[str, bool, int, bool, int]]): + + def run(self, model: VideoModel): + file, old_reject, old_rating, new_reject, new_rating = self.args + index, video = model.find_video(Video(file)) + video.metadata.rejected = new_reject + video.metadata.rating = new_rating + video.save_metadata() + if index: + model.dataChanged.emit(index, index) + + def undo_action(self) -> UndoableAction[Any]: + file, old_reject, old_rating, new_reject, new_rating = self.args + return SetVideoStars((file, new_reject, new_rating, old_reject, old_rating)) + + def __str__(self): + file, old_reject, old_rating, new_reject, new_rating = self.args + return f"{file}: rejected status ({old_reject} -> {new_reject}) and rating ({old_rating} -> {new_rating})" + +class UndoableActionList: + + def __init__(self, fn: str) -> None: + self._data: List[UndoableAction] = [] + self._fn: str = fn + self._load() + + def _load(self): + with open(self._fn, "a") as f: + pass + + self.f = open(self._fn, 'r+') + + for line in self.f: + ID, uuid, args = json.loads(line.strip()) + + revert_fns = [obj for name, obj in inspect.getmembers(sys.modules[__name__]) + if inspect.isclass(obj) and name == ID] + assert len(revert_fns) == 1 + self._data.append(revert_fns[0](args, uid=uuid)) + self.f.seek(0, os.SEEK_END) + + def append(self, action: UndoableAction): + json.dump((action.__class__.__name__, action.uuid, action.args), self.f) + self.f.write("\n") + self.f.flush() + self._data.append(action) + + def pop(self, uuid=None): + if not self._data: + return None + if uuid and self._data[-1].uuid != uuid: + return None + action = self._data.pop() + self._remove_line() + return action + + def clear(self): + self._data = [] + self.f.seek(0, os.SEEK_SET) + self.f.truncate() + self.f.flush() + + def _remove_line(self): + pos = self.f.tell() - 1 + + if pos == -1: + return + + while pos > 0 and self.f.read(1) != "\n": + pos -= 1 + self.f.seek(pos, os.SEEK_SET) + if pos != 0: + pos += 1 + self.f.seek(pos, os.SEEK_SET) + self.f.truncate() + self.f.flush() + +class HistoryManager: + + def __init__(self) -> None: + self.history = UndoableActionList(os.path.join(BASE_DIR, "vmgr.history")) + self.future = UndoableActionList(os.path.join(BASE_DIR, "vmgr.redo")) + + def apply(self, model, *actions: UndoableAction): + should_reload = False + self.future.clear() + for action in actions: + print(action) + should_reload = action.run(model) or should_reload + self.history.append(action) + if should_reload: + model.reload() + else: + model.update() + + @staticmethod + def move(stack_1: UndoableActionList, stack_2, model, undo): + action: Optional[UndoableAction] = stack_1.pop() + should_reload = False + while action is not None: + try: + if undo: + print(action.undo_action()) + should_reload = action.undo_action().run(model) or should_reload + else: + print(action) + should_reload = action.run(model) or should_reload + except: + stack_1.append(action) + raise + stack_2.append(action) + action = stack_1.pop(action and action.uuid) + return should_reload + + def undo(self, model): + should_reload = self.move(self.history, self.future, model, True) + if should_reload: + model.reload() + else: + model.update() + + def redo(self, model: VideoModel): + should_reload = self.move(self.future, self.history, model, False) + if should_reload: + model.reload() + else: + model.update() + + + diff --git a/vmgr/main.py b/vmgr/main.py new file mode 100644 index 0000000..0a91c45 --- /dev/null +++ b/vmgr/main.py @@ -0,0 +1,445 @@ +import vlc +import sys +import os.path +import vlc +from PyQt5 import QtWidgets, QtCore, QtGui +import subprocess +from video import VideoModel +from superqt import QLabeledRangeSlider +from actions import UndoableAction, HistoryManager +from typing import List +import os +from datetime import datetime + + +history = HistoryManager() + +class Player(QtWidgets.QMainWindow): + """A simple Media Player using VLC and Qt + """ + def __init__(self, master=None): + QtWidgets.QMainWindow.__init__(self, master) + self.setWindowTitle("Media Player") + + # creating a basic vlc instance + self.instance = vlc.Instance() + # creating an empty vlc media player + self.mediaplayer = self.instance.media_player_new() + self.createUI() + self.isPaused = False + + + def createUI(self): + """Set up the user interface, signals & slots + """ + self.widget = QtWidgets.QWidget(self) + self.setCentralWidget(self.widget) + + # In this widget, the video will be drawn + if sys.platform == "darwin": # for MacOS + self.videoframe = QtWidgets.QMacCocoaViewContainer(0) + else: + self.videoframe = QtWidgets.QFrame() + self.palette = self.videoframe.palette() + self.palette.setColor (QtGui.QPalette.Window, + QtGui.QColor(0,0,0)) + self.videoframe.setPalette(self.palette) + self.videoframe.setAutoFillBackground(True) + + self.positionslider = QtWidgets.QSlider(QtCore.Qt.Horizontal, self) + self.positionslider.setToolTip("Position") + self.positionslider.setMaximum(1000) + + def mousePressEvent(event): + if event.button() == QtCore.Qt.LeftButton: + val = self.pixelPosToRangeValue(event.pos()) + self.setValue(val) + + self.positionslider.sliderMoved.connect(self.setPosition) + self.positionslider.sliderPressed.connect(lambda *args: print(args)) + + + self.hbuttonbox = QtWidgets.QHBoxLayout() + self.playbutton = QtWidgets.QPushButton("Play") + self.hbuttonbox.addWidget(self.playbutton) + self.playbutton.clicked.connect(self.PlayPause) + + self.hbuttonbox.addStretch(1) + self.setVolume(100) + + self.videoplayerlayout = QtWidgets.QVBoxLayout() + self.videoplayerlayout.addWidget(self.videoframe) + self.videoplayerlayout.addWidget(self.positionslider) + self.videoplayerlayout.addLayout(self.hbuttonbox) + + + self.videoplayerlayoutWidget = QtWidgets.QWidget(); + self.videoplayerlayoutWidget.setLayout(self.videoplayerlayout); + + + + import dataclasses + def move_current_frame(d_frames): + before = self.mediaplayer.get_time() + + mspf = int(1000 // (self.mediaplayer.get_fps() or 25)) + d_time_ms = d_frames * mspf + target = self.mediaplayer.get_time() + d_time_ms + result = self.mediaplayer.set_time(target) + + print(before, target, result, self.mediaplayer.get_time()) + + + def pp(): + self.PlayPause() + print(self.mediaplayer.get_time()) + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Space"), self) + self.shortcut.activated.connect(pp) + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Right"), self) + self.shortcut.activated.connect(lambda : move_current_frame(1)) + + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Left"), self) + self.shortcut.activated.connect(lambda : move_current_frame(-1)) + + + def export_frame(): + print("XX") + t = self.mediaplayer.get_time() + print(t) + ms = t % 1000 + t = t // 1000 + s = t % 60 + t = t // 60 + m = t % 60 + t = t // 60 + h = t % 60 + t = t // 60 + t_str = f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + print("Exporting: ", self.filename, t_str) + import os + from datetime import timedelta + + def apply(filename, t_str): + base_dir = os.path.expanduser(os.path.join("~", "Pictures", "video_exports")) + + print("W", filename, t_str) + if not os.path.exists(base_dir): + os.makedirs(base_dir) + print("W", filename, t_str) + begin = datetime.now() + base_name = os.path.splitext(os.path.basename(filename))[0] + + print("W", filename, t_str) + print(base_name[:13]) + frame_datetime = datetime.strptime(base_name[:13], "%Y%m%d_%H%M") + timedelta(hours=h, minutes=m, seconds=s, milliseconds=ms) + print(base_name[:13]) + frame_datetime_str = frame_datetime.strftime("%Y:%m:%d_%H:%M:%S") + print(base_name[:13]) + + print("W", filename, t_str) + out = os.path.join(base_dir, + base_name + + f"{h:02d}{m:02d}{s:02d}{ms:03d}.png") + print(out) + subprocess.call([ + "ffmpeg", + "-ss", t_str, + "-i", filename, + "-vframes", "1", + "-c:v", "png", + out + ]) + subprocess.call([ + "exiftool", + f"-CreateDate={frame_datetime_str}", + f"-DateTimeOriginal={frame_datetime_str}", + f"-PNG:CreationTime={frame_datetime_str}", + "-overwrite_original", + out + ]) + print("Duration: ", datetime.now() - begin) + from video import thumbnailer + + thumbnailer.pool.apply_async(apply, args=(self.filename, t_str)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+e"), self) + self.shortcut.activated.connect(lambda : export_frame()) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("f"), self) + self.shortcut.activated.connect(lambda : self.mediaplayer.toggle_fullscreen()) + + def slowmo(): + if self.mediaplayer.get_rate() < 0.7: + print("rate", 1) + self.mediaplayer.set_rate(1) + else: + print("rate", .5) + self.mediaplayer.set_rate(0.5) + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("s"), self) + self.shortcut.activated.connect(slowmo) + + def change_size(diff): + self.icon_size += diff + print(self.icon_size < 64) + if self.icon_size < 64: + self.icon_size = 64 + if self.icon_size > 256: + self.icon_size = 256 + print(self.icon_size) + self.list_view.setIconSize(QtCore.QSize(self.icon_size, self.icon_size)) + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("+"), self) + self.shortcut.activated.connect(lambda : change_size(64)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("-"), self) + self.shortcut.activated.connect(lambda : change_size(-64)) + from typing import Optional + def undo(): + history.undo(self.model) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+z"), self) + self.shortcut.activated.connect(lambda : undo()) + def redo(): + history.redo(self.model) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+Shift+z"), self) + self.shortcut.activated.connect(lambda : redo()) + + + def set_rating(value): + selected = self.list_view.selectedIndexes() + + stars = None + actions = [] + for i, index in enumerate(sorted(selected, key=lambda r: -r.row())): + video = self.model.video_files[index.row()] + from actions import SetVideoStars + stars = SetVideoStars(( + video.fn, + video.metadata.rejected, + video.metadata.rating, + False if value >= 0 else not video.metadata.rejected, + value if value >= 0 else video.metadata.rating + ), stars) + stars.run(self.model) + actions.append(stars) + history.apply(self.model, *actions) + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("0"), self) + self.shortcut.activated.connect(lambda : set_rating(0)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("1"), self) + self.shortcut.activated.connect(lambda : set_rating(1)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("2"), self) + self.shortcut.activated.connect(lambda : set_rating(2)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("3"), self) + self.shortcut.activated.connect(lambda : set_rating(3)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("4"), self) + self.shortcut.activated.connect(lambda : set_rating(4)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("5"), self) + self.shortcut.activated.connect(lambda : set_rating(5)) + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("r"), self) + self.shortcut.activated.connect(lambda : set_rating(-1)) + + def link(): + directory = QtWidgets.QFileDialog.getExistingDirectory( + self, "Open File", os.path.expanduser('~')) + + if not os.path.isdir(directory): + QtWidgets.QMessageBox.warning( + self, + "Not a directory", + "The selected item was not a directory. " + "You need to select an empty folder. ") + return + + if os.listdir(directory): + QtWidgets.QMessageBox.warning( + self, + "Non-empty directory selected", + "The selected directory is not empty. " + "You need to select an empty folder. ") + return + + print(directory) + failure = False + for index in self.list_view.selectedIndexes(): + video = self.model.video_files[index.row()] + failure = failure or ( + 0 != subprocess.call([ + "ln", + "-s", + video.fn, + os.path.join(directory, os.path.basename(video.fn)), + ]) + ) + if failure: + QtWidgets.QMessageBox.warning( + self, + "Linkage failure", + "Some files could not be linked.") + + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+L"), self) + self.shortcut.activated.connect(link) + + def delete(): + from actions import Trash + selected = self.list_view.selectedIndexes() + response = QtWidgets.QMessageBox.question(self, "Delete Items", "Do you really want to delete %d items?" % len(selected), defaultButton=QtWidgets.QMessageBox.No) + if response == QtWidgets.QMessageBox.Yes: + actions = [] + for i, index in enumerate(sorted(selected, key=lambda r: -r.row())): + video = self.model.video_files[index.row()] + action = Trash(video.fn, actions[-1] if len(actions) else None) + actions.append(action) + history.apply(self.model, *actions) + + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Del"), self) + self.shortcut.activated.connect(lambda : delete()) + + self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("F5"), self) + self.shortcut.activated.connect(lambda : self.model.reload()) + + self.model = VideoModel() + + def open_file(index): + self.OpenFile(self.model.video_files[index.row()].fn) + + self.list_view = QtWidgets.QListView() + self.list_view.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.ExtendedSelection) + self.list_view.doubleClicked.connect(open_file) + self.list_view.setViewMode(QtWidgets.QListView.IconMode) + self.list_view.setResizeMode(QtWidgets.QListView.Adjust) + self.list_view.setFlow(QtWidgets.QListView.LeftToRight) + self.list_view.setMovement(QtWidgets.QListView.Snap) + self.list_view.setModel(self.model) + # self.list_view.setGridSize(QtCore.QSize(256 + 10, 256 + 10)) + self.icon_size = 256 + self.list_view.setIconSize(QtCore.QSize(self.icon_size, self.icon_size)) + self.list_view.setWordWrap(True) + + self.hboxlayout = QtWidgets.QSplitter() + self.hboxlayout.addWidget(self.list_view) + self.hboxlayout.addWidget(self.videoplayerlayoutWidget) + + self.filter_slider = QLabeledRangeSlider(QtCore.Qt.Horizontal) + self.filter_slider.setRange(0, 6) + self.filter_slider.setValue((0, 6)) + self.filter_slider.setMaximum(6) + self.filter_slider.setMinimum(-1) + + def set_rating_filter(v): + self.model.filter_rating = v + self.filter_slider.sliderMoved.connect(set_rating_filter) + self.parentLayout = QtWidgets.QVBoxLayout() + self.parentLayout.addWidget(self.filter_slider, 0) + self.parentLayout.addWidget(self.hboxlayout, 2) + self.widget.setLayout(self.parentLayout) + + exit = QtWidgets.QAction("&Exit", self) + exit.triggered.connect(sys.exit) + menubar = self.menuBar() + filemenu = menubar.addMenu("&File") + filemenu.addAction(exit) + + self.timer = QtCore.QTimer(self) + self.timer.setInterval(200) + self.timer.timeout.connect(self.updateUI) + + def PlayPause(self): + """Toggle play/pause status + """ + if self.mediaplayer.is_playing(): + self.mediaplayer.pause() + self.playbutton.setText("Play") + self.isPaused = True + else: + if self.mediaplayer.play() == -1: + self.OpenFile() + return + self.mediaplayer.play() + self.playbutton.setText("Pause") + self.timer.start() + self.isPaused = False + + def Stop(self): + """Stop player + """ + self.mediaplayer.stop() + self.playbutton.setText("Play") + + def OpenFile(self, filename=None): + """Open a media file in a MediaPlayer + """ + if filename is None: + return + filename = QtWidgets.QFileDialog.getOpenFileName(self, "Open File", os.path.expanduser('~')) + if not filename: + return + + # create the media + if sys.version < '3': + filename = unicode(filename) + self.filename = filename + self.media = self.instance.media_new(filename) + # put the media in the media player + self.mediaplayer.set_media(self.media) + + # parse the metadata of the file + self.media.parse() + # set the title of the track as window title + self.setWindowTitle(self.media.get_meta(0)) + + # the media player has to be 'connected' to the QFrame + # (otherwise a video would be displayed in it's own window) + # this is platform specific! + # you have to give the id of the QFrame (or similar object) to + # vlc, different platforms have different functions for this + if sys.platform.startswith('linux'): # for Linux using the X Server + self.mediaplayer.set_xwindow(int(self.videoframe.winId())) + elif sys.platform == "win32": # for Windows + self.mediaplayer.set_hwnd(self.videoframe.winId()) + elif sys.platform == "darwin": # for MacOS + self.mediaplayer.set_nsobject(self.videoframe.winId()) + self.PlayPause() + + def setVolume(self, Volume): + """Set the volume + """ + self.mediaplayer.audio_set_volume(Volume) + + def setPosition(self, position): + """Set the position + """ + # setting the position to where the slider was dragged + self.mediaplayer.set_position(position / 1000.0) + # the vlc MediaPlayer needs a float value between 0 and 1, Qt + # uses integer variables, so you need a factor; the higher the + # factor, the more precise are the results + # (1000 should be enough) + + def updateUI(self): + """updates the user interface""" + # setting the slider to the desired position + self.positionslider.setValue(int(self.mediaplayer.get_position() * 1000)) + + if not self.mediaplayer.is_playing(): + # no need to call this function if nothing is played + self.timer.stop() + if not self.isPaused: + # after the video finished, the play button stills shows + # "Pause", not the desired behavior of a media player + # this will fix it + self.Stop() + +if __name__ == "__main__": + app = QtWidgets.QApplication(sys.argv) + + + + player = Player() + player.show() + player.resize(640, 480) + if sys.argv[1:]: + player.OpenFile(sys.argv[1]) + sys.exit(app.exec_()) diff --git a/vmgr/thumbnails.py b/vmgr/thumbnails.py new file mode 100644 index 0000000..79b809f --- /dev/null +++ b/vmgr/thumbnails.py @@ -0,0 +1,74 @@ +import os +import multiprocessing +import hashlib +import subprocess +from PyQt5 import QtGui +import multiprocessing.pool +from typing import Dict, Optional + +class CachedThumbnailGenerator: + + def __init__(self, cache_folder) -> None: + self.cache_folder = cache_folder + self.memory_cache: Dict[str, Optional[QtGui.QIcon]] = {} + self.pool = multiprocessing.pool.ThreadPool(1) + + def create_thumbnail(self, video_file, callback=None, error_callback=None): + self.pool.apply_async( + self._get_thumbnail_file, (video_file,), + callback=callback, error_callback=error_callback) + + def get_thumbnail(self, video_file, callback=None, error_callback=None): + cache_key = video_file + + if cache_key not in self.memory_cache: + self.memory_cache[cache_key] = None + + def inner_callback(thumbnail_file): + if not thumbnail_file: + return + image = QtGui.QImage(thumbnail_file) + pixmap = QtGui.QPixmap.fromImage(image) + self.memory_cache[cache_key] = QtGui.QIcon(pixmap) + if callback: + callback() + + self.create_thumbnail( + video_file, inner_callback, error_callback) + + if cache_key in self.memory_cache and self.memory_cache[cache_key]: + return self.memory_cache[cache_key] + return None + + def _cache_file(self, video_file): + abs_path = os.path.abspath(os.path.expanduser(video_file)) # + str(sec) + digest = hashlib.md5(abs_path.encode()).hexdigest() + return os.path.join(self.cache_folder, digest[:1], digest[1:2], digest + ".jpg") + + def _load_cache(self, video_file): + pass + + @staticmethod + def _generate_thumbnail(video_fn, thumbnail_fn): + subprocess.check_call([ + 'ffmpeg', '-i', + video_fn, + "-threads", "1", + '-ss', '00:00:00.000', + '-c:v', 'mjpeg', + '-vframes', '1', + '-frames:v', '1', + '-filter:v', 'scale=300:300:force_original_aspect_ratio=increase,crop=300:300', + # '-f', 'image2pipe', + # '-' + thumbnail_fn + ]) + + def _get_thumbnail_file(self, video_file): + cache_file = self._cache_file(video_file) + if not os.path.exists(cache_file): + if not os.path.exists(os.path.dirname(cache_file)): + os.makedirs(os.path.dirname(cache_file)) + self._generate_thumbnail(video_file, cache_file) + assert os.path.exists(cache_file) + return cache_file diff --git a/vmgr/video.py b/vmgr/video.py new file mode 100644 index 0000000..1a35cf9 --- /dev/null +++ b/vmgr/video.py @@ -0,0 +1,252 @@ +import os +from PyQt5 import QtWidgets, QtCore, QtGui +import itertools +from typing import Any, List, Dict +from thumbnails import CachedThumbnailGenerator + +from typing import Tuple, Optional +import yaml + +BASE_DIR = "/home/oke/Pictures/DarktableLocal/" +CACHE_FOLDER = os.path.expanduser("~/.cache/video_manager") + +if not os.path.exists(CACHE_FOLDER): + os.makedirs(CACHE_FOLDER) + +thumbnailer = CachedThumbnailGenerator(CACHE_FOLDER) + + +class Metadata: + + def __init__(self, rating=0, rejected=False): + assert type(rating) == int + self.rating = rating + self.rejected = rejected + + @classmethod + def from_file(cls, fn): + with open(fn) as f: + data = yaml.load(f, yaml.SafeLoader) + return Metadata(**data) + + def save(self, fn): + with open(fn, "w") as f: + yaml.dump(self.__dict__, f) + +class Video: + + def __init__(self, fn: str) -> None: + self.fn = fn + self._metadata = None + + def __repr__(self): + return os.path.basename(self.fn) + + @property + def metadata(self): + if not self._metadata: + if os.path.exists(self.metadata_file()): + self._metadata = Metadata.from_file(self.metadata_file()) + else: + self._metadata = Metadata() + return self._metadata + + def metadata_file(self): + return self.fn + '.metadata' + + def save_metadata(self): + self.metadata.save(self.metadata_file()) + + def load_thumbnail(self, thumbnailer, callback=None, error_callback=None): + thumbnailer.create_thumbnail(self.fn, callback, error_callback) + + def __lt__(self, other): + return self.fn > other.fn + + def __eq__(self, other): + return self.fn == other.fn + +class VideoModel(QtCore.QAbstractListModel): + + + def __init__(self): + super().__init__() + self._filter_rating = (0, 6) + self.show_up_to = 10 + + self.video_obj_cache = {} + self.all_files: List[Video] = self.collect_files() + self.video_files: List[Video] = self.filter_files(self.all_files) + + + pixmap = QtGui.QPixmap(256, 256) + pixmap.fill(QtCore.Qt.transparent) + painter = QtGui.QPainter(pixmap) + painter.setRenderHint(QtGui.QPainter.Antialiasing) + painter.setBrush(QtGui.QColor("red")) + painter.drawEllipse(pixmap.rect()) + painter.end() + self.default_icon = QtGui.QIcon(pixmap) + + def collect_files(self, base_dir=BASE_DIR): + def _(): + for film_roll in os.listdir(base_dir): + roll_dir = os.path.join(base_dir, film_roll) + if os.path.isdir(roll_dir): + for fn in os.listdir(roll_dir): + if fn[0] != '.' and os.path.splitext(fn)[-1].lower() in [".mp4", ".mov", ".avi"]: + path = os.path.join(roll_dir, fn) + if path not in self.video_obj_cache: + self.video_obj_cache[path] = Video(path) + yield self.video_obj_cache[path] + return sorted(_()) + + @property + def filter_rating(self): + return self._filter_rating + + @filter_rating.setter + def filter_rating(self, value): + self._filter_rating = value + self.update() + + def filter_files(self, videos: List[Video]): + def flt(video: Video): + minimum, maximum = self._filter_rating + rating = -1 if video.metadata.rejected else video.metadata.rating + return rating >= minimum and rating < maximum + return list(filter(flt, videos))[:self.show_up_to] + + + def find_video(self, video: Video)-> Tuple[Optional[QtCore.QModelIndex], Video]: + try: + index = self.all_files.index(video) + video = self.all_files[index] + except ValueError: + pass + + try: + index = self.video_files.index(video) + index = self.index(index, 0) + except ValueError: + index = None + return index, video + + + + def update(self): + i = 0 + parent = QtCore.QModelIndex() + before = list(self.video_files) + action_list = [] + after = self.filter_files(self.all_files) + + while i < len(after) and i < len(before): + if before[i] == after[i]: + i += 1 + elif before[i] < after[i]: + action_list.append(('r', i)) + before.pop(i) + else: + action_list.append(('a', i, after[i])) + before.insert(i, after[i]) + + j = i + while j < len(before): + action_list.append(('r', j)) + before.pop(j) + + for j in range(i, len(after)): + action_list.append(('a', j, after[j])) + before.insert(j, after[j]) + # assert before == after + + if len(action_list) == 0: + return + def work_batch(): + batch = action_list[b:e] + if batch[0][0] == 'r': + self.beginRemoveRows(parent, batch[0][1], batch[0][1] + len(batch) - 1) + del self.video_files[batch[0][1]:batch[0][1] + len(batch)] + self.endRemoveRows() + else: + self.beginInsertRows(parent, batch[0][1], batch[0][1] + len(batch) - 1) + self.video_files = ( + self.video_files[0:batch[0][1]] + + list(map(lambda a: a[2], batch)) + + self.video_files[batch[0][1]:] + ) + self.endInsertRows() + + b = 0 + e = 1 + for e in range(1, len(action_list)): + if ( + action_list[e][0] == 'r' and + action_list[e - 1][0] == 'r' and + action_list[e][1] == action_list[e - 1][1] + ): + continue + if ( + action_list[e][0] == 'a' and + action_list[e - 1][0] == 'a' and + action_list[e][1] == action_list[e - 1][1] + 1 + ): + continue + work_batch() + b = e + e = len(action_list) + work_batch() + assert len(self.video_files) == len(after) + # assert self.video_files == after + + def reload(self): + self.all_files = self.collect_files() + self.update() + + def invalidateFilter(self, index=None, index_to=None): + self.update() + + def canFetchMore(self, parent: QtCore.QModelIndex) -> bool: + return self.show_up_to < len(self.all_files) + + def fetchMore(self, parent: QtCore.QModelIndex) -> None: + self.show_up_to = min(self.show_up_to + 5, len(self.all_files)) + self.update() + + def remove(self, index): + self.all_files.remove(self.video_files[index.row()]) + self.update() + + def data(self, index: QtCore.QModelIndex, role: int=0) -> Any: + video = self.video_files[index.row()] + if role == 0: + size = 0 + try: + size = os.stat(video.fn).st_size / 1024. / 1024. / 1024. + except FileNotFoundError: + pass + return '%s %s %.2f' % ( + os.path.basename(video.fn), + ('★' * video.metadata.rating + '☆' * (5 - video.metadata.rating)) + if not video.metadata.rejected else + 'xxxxx' + , + size + ) + if role == 1: + def callback(): + self.dataChanged.emit(index, index) + return ( + thumbnailer.get_thumbnail(video.fn, callback) + or self.default_icon + ) + + + def rowCount(self, index=None): + return len(self.video_files) + + def columnCount(self, index): + return 1 + +