Initial commit

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+**/__pycache__

backup.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/bash
+shopt -s nocaseglob
+shopt -s dotglob
+
+trap "exit" INT
+
+read -r -s -p "Password: " PWD
+echo ""
+
+host="192.168.20.2"
+user="oke"
+extra_args=("--info=progress2" "-h" "${@:1}")
+do_backup() (
+        set -e
+        echo "Waiting for '$host' to be up ..."
+        until ping -c1 "$host" >/dev/null 2>&1; do :; done
+        echo "... done waiting"
+
+        sshpass -p "$PWD" \
+        rsync -av "${extra_args[@]}" --relative "$HOME/Pictures/DarktableLocal" "$user@$host":lenovo-darktable
+
+        sshpass -p "$PWD" \
+        rsync -av "${extra_args[@]}" --relative "$HOME/.config/darktable" "$user@$host":lenovo-darktable
+
+    )
+
+do_backup

check_videos.py

@@ -0,0 +1,187 @@
+import subprocess
+import json
+import os
+import datetime
+import glob
+from PIL import Image
+import shutil
+import re
+
+def get_time(fn):
+        try:
+                output = subprocess.check_output([
+                        "ffprobe", "-v", "quiet", fn, "-print_format", "json",
+                        "-show_entries",
+                        "stream=index,codec_type:stream_tags=creation_time:format_tags=creation_time"
+                ])
+        except subprocess.CalledProcessError as e:
+                print("FFPROBE failed for ", fn)
+                return ValueError()
+        fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
+        data = json.loads(output)
+        if "streams" in data:
+                for d in data["streams"]:
+                        if "tags" in d and "creation_time" in d["tags"]:
+                                return datetime.datetime.strptime(d["tags"]["creation_time"], fmt)
+        if "format" in data and "tags" in data["format"] and "creation_time" in data["format"]["tags"]:
+                        return datetime.datetime.strptime(data["format"]["tags"]["creation_time"], fmt)
+        # import sys
+        # import zoneinfo
+        # dt = datetime.datetime.fromtimestamp(
+        #         os.stat(fn).st_ctime, zoneinfo.ZoneInfo("UTC"))
+        # dt = dt.replace(tzinfo=None)
+        # return dt
+        return ValueError()
+def get_time_image(fn):
+        fmt = "%Y:%m:%d %H:%M:%S"
+        return datetime.datetime.strptime(Image.open(fn)._getexif()[36867], fmt)
+
+def get_new_fn(fn):
+        base, ext = os.path.splitext(fn)
+        directory, base_fn = os.path.split(base)
+        ext = ext.lower()
+
+        image_files = list(
+                filter(lambda fn: os.path.splitext(fn)[1].lower() in [".jpeg", ".jpg", ".png"],
+                       glob.iglob(glob.escape(base) + "*")))
+        assert len(image_files) < 2
+        fmt_folder = "%Y%m%d"
+        fmt_base = "%Y%m%d_%H%M"
+        if image_files:
+                f = image_files[0]
+                image_ext = os.path.splitext(f)[-1]
+                image_dt = get_time_image(f)
+                image_size = os.path.getsize(f)
+                new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), image_dt.strftime(fmt_folder), image_dt.strftime(fmt_base))
+
+                g = glob.escape(new_base) + "_*" + image_ext
+                matches = [g for g in glob.glob(g) if image_size == os.path.getsize(g)]
+                if len(matches) != 1:
+                        print()
+                        print("NO MATCH")
+                        print(image_files)
+                        print(matches)
+                        print(g)
+                        print(image_dt)
+
+                        return None
+                assert len(matches) == 1, fn
+                nfn = os.path.splitext(matches[0])[0] + ext
+                nfn = list(os.path.split(nfn))
+                nfn[-1] = "." + nfn[-1]
+                return os.path.join(*nfn)
+
+        time = get_time(fn)
+        if isinstance(time, ValueError):
+                print("NO TIME")
+                return None
+        new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), time.strftime(fmt_folder), time.strftime(fmt_base))
+        g = glob.glob(glob.escape(new_base) + "_*" + ext)
+        if len(g) == 0:
+                return new_base + "_0000" + ext
+        n = 0
+        s = os.path.getsize(fn)
+        for f in g:
+                if os.path.getsize(f) == s:
+                        return f
+                p = r".*_(\d+)" + ext
+                nn = re.match(p, f)
+                n = max(n, int(nn.group(1)))
+        n = new_base + "_%04d" % (n + 1) + ext
+        return n
+
+def main(dir, dry):
+        from datetime import timedelta, datetime
+        print(dir)
+        offsets = {}
+        for date in sorted(os.listdir(dir)):
+            path = os.path.join(dir, date)
+            if not os.path.isdir(path):
+                continue
+            print(date)
+            date = datetime.strptime(date, '%Y%m%d')
+
+            offset = None
+            for fn in sorted(os.listdir(path)):
+
+                fn = os.path.join(path, fn)
+                _, ext = os.path.splitext(fn)
+                if ext.lower() not in ['.jpeg', '.jpg']:
+                    continue
+
+                import pprint
+                from PIL import ExifTags
+                import exifread
+                import re
+
+                with open(fn, 'rb') as f:
+
+                    tags = exifread.process_file(f)
+
+                    try:
+                        model = str(tags['Image Model'])
+                    except KeyError:
+                        continue
+
+                    if str(model) != 'Pixel 6' or str(model).startswith("iPhone"):
+                        continue
+
+                    offset_tags = ['EXIF OffsetTime', 'EXIF OffsetTimeOriginal', 'EXIF OffsetTimeDigitized', 'EXIF TimeZoneOffset']
+
+                    for tag in offset_tags:
+                        try:
+                            offset_str = str(tags[tag])
+                        except KeyError:
+                            continue
+                        assert re.match(r"(\+|-)\d\d:\d\d", offset_str)
+                        offset = timedelta(hours=int(offset_str[1:3]), minutes=int(offset_str[4:6]))
+                        if offset_str[0] == '-':
+                            offset = -offset
+
+                        break
+
+                    break
+            import pprint
+
+            if offset:
+                offsets[date] = offset
+
+        for p, folders, files in os.walk(os.path.expanduser(dir)):
+                for f in sorted(files):
+                        fn = os.path.join(p, f)
+
+                        base, ext = os.path.splitext(f)
+                        if ext.lower() not in [".mp4", ".mov", ".mts"]:
+                            continue
+
+                        time = get_time(fn)
+                        if isinstance(time, ValueError):
+                            continue
+
+                        diff = sorted(offsets.keys(), key=lambda d: abs((d - time).total_seconds()))
+
+                        # print(diff)
+                        # print(diff[0])
+                        # print(offsets[diff[0]])
+
+                        print(time)
+                        print(time + offsets[diff[0]])
+                        print()
+                        continue
+                        nfn = get_new_fn(fn)
+                        if not nfn:
+                                print("SKIP: ", fn)
+                                continue
+                        assert nfn
+
+                        print(os.path.basename(fn))
+                        print(os.path.basename(nfn))
+                        print()
+
+
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument("--dry", action="store_true")
+options = parser.parse_args()
+main("/home/oke/Pictures/DarktableLocal", options.dry)

copy_videos.py

@@ -0,0 +1,133 @@
+import subprocess
+import json
+import os
+import datetime
+import glob
+from PIL import Image
+import shutil
+import re
+
+def get_time(fn):
+        try:
+                output = subprocess.check_output([
+                        "ffprobe", "-v", "quiet", fn, "-print_format", "json",
+                        "-show_entries",
+                        "stream=index,codec_type:stream_tags=creation_time:format_tags=creation_time"
+                ])
+        except subprocess.CalledProcessError as e:
+                print("FFPROBE failed for ", fn)
+                return ValueError()
+        fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
+        data = json.loads(output)
+        if "streams" in data:
+                for d in data["streams"]:
+                        if "tags" in d and "creation_time" in d["tags"]:
+                                return datetime.datetime.strptime(d["tags"]["creation_time"], fmt)
+        if "format" in data and "tags" in data["format"] and "creation_time" in data["format"]["tags"]:
+                        return datetime.datetime.strptime(data["format"]["tags"]["creation_time"], fmt)
+        #import sys
+        #import zoneinfo
+        #dt = datetime.datetime.fromtimestamp(
+        #        os.stat(fn).st_ctime, zoneinfo.ZoneInfo("UTC"))
+        #dt = dt.replace(tzinfo=None)
+        #return dt
+        return ValueError()
+def get_time_image(fn):
+        fmt = "%Y:%m:%d %H:%M:%S"
+        return datetime.datetime.strptime(Image.open(fn)._getexif()[36867], fmt)
+
+def get_new_fn(fn):
+        base, ext = os.path.splitext(fn)
+        directory, base_fn = os.path.split(base)
+        ext = ext.lower()
+
+        image_files = list(
+                filter(lambda fn: os.path.splitext(fn)[1].lower() in [".jpeg", ".jpg", ".png"],
+                       glob.iglob(glob.escape(base) + "*")))
+        assert len(image_files) < 2
+        fmt_folder = "%Y%m%d"
+        fmt_base = "%Y%m%d_%H%M"
+        if image_files:
+                f = image_files[0]
+                image_ext = os.path.splitext(f)[-1]
+                image_dt = get_time_image(f)
+                image_size = os.path.getsize(f)
+                new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), image_dt.strftime(fmt_folder), image_dt.strftime(fmt_base))
+
+                g = glob.escape(new_base) + "_*" + image_ext
+                matches = [g for g in glob.glob(g) if image_size == os.path.getsize(g)]
+                if len(matches) != 1:
+                        print()
+                        print("NO MATCH")
+                        print(image_files)
+                        print(matches)
+                        print(g)
+                        print(image_dt)
+
+                        return None
+                assert len(matches) == 1, fn
+                nfn = os.path.splitext(matches[0])[0] + ext
+                nfn = list(os.path.split(nfn))
+                nfn[-1] = "." + nfn[-1]
+                return os.path.join(*nfn)
+
+        time = get_time(fn)
+        if isinstance(time, ValueError):
+                print("NO TIME")
+                return None
+        new_base = os.path.join(os.path.expanduser("~/Pictures/Darktable"), time.strftime(fmt_folder), time.strftime(fmt_base))
+        g = glob.glob(glob.escape(new_base) + "_*" + ext)
+        if len(g) == 0:
+                return new_base + "_0000" + ext
+        n = 0
+        s = os.path.getsize(fn)
+        for f in g:
+                if os.path.getsize(f) == s:
+                        return f
+                p = r".*_(\d+)" + ext
+                nn = re.match(p, f)
+                n = max(n, int(nn.group(1)))
+        n = new_base + "_%04d" % (n + 1) + ext
+        return n
+
+def main(dir, dry):
+        print(dir)
+        for p, folders, files in os.walk(os.path.expanduser(dir)):
+                for f in sorted(files):
+                        base, ext = os.path.splitext(f)
+
+                        ext = ext.lower()
+                        if ext not in [".mp4", ".mov", ".mts"]:
+                                if ext not in [".jpg", ".jpeg", ".png"]:
+                                        pass # print("Not handled: ", os.path.join(p, f))
+                                continue
+                        fn = os.path.join(p, f)
+                        nfn = get_new_fn(fn)
+                        if not nfn:
+                                print("SKIP: ", fn)
+                                continue
+                        assert nfn
+
+                        if not os.path.exists(nfn):
+                                if not os.path.exists(os.path.dirname(nfn)):
+                                        os.makedirs(os.path.dirname(nfn))
+                                print(fn, nfn)
+                                if not dry:
+                                        shutil.copy(fn, nfn)
+                        else:
+                                print(fn, nfn, "EXIST")
+
+
+
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument("--dry", action="store_true")
+options = parser.parse_args()
+main("/run/media/oke/3634-3063", options.dry)
+main("/run/media/oke/3138-3162", options.dry)
+main("/run/media/oke/01D5-9878/DCIM/", options.dry)
+main("/run/media/oke/E5B5-DBF0/DCIM", options.dry)
+main("/run/media/oke/disk/", options.dry)
+main("~/Nextcloud/InstantUpload", options.dry)
+main("/home/oke/Desktop/360/", options.dry)

dt_auto_group.py

@@ -0,0 +1,241 @@
+from pytable.models import Image, ImageFlags, FilmRoll
+from datetime import datetime, timedelta
+import os
+
+def is_raw(image):
+    return image.filename.split(".")[-1].lower() not in ["jpeg", "jpg", "png"]
+
+def consume_group_pixel_6(images):
+    # print("NEXT: ", images[0])
+    i = 1
+    for i in range(1, len(images)):
+        # print(images[i], images[i].datetime_taken, images[0].datetime_taken, images[i].datetime_taken - images[0].datetime_taken)
+        if images[i].datetime_taken - images[0].datetime_taken > timedelta(milliseconds=600):
+            break
+    else:
+        i = len(images)
+    # print(i)
+    if i == 1:
+        # print(images[:1])
+        return images[:1], images[1:]
+
+    def priority(image):
+        return - (
+            image.iso - images[0].iso +
+            image.aperture - images[0].aperture +
+            image.exposure - images[0].exposure
+        )
+
+    sorted_images = sorted(filter(lambda image: is_raw(image) != is_raw(images[0]), images[1:i]), key=priority)
+
+    if len(sorted_images) == 0:
+        # print(images[:1])
+        return images[:1], images[1:]
+
+    group = [images[0], sorted_images[0]]
+
+    images = images[1:]
+    images.remove(sorted_images[0])
+    # print(group)
+    return group, images
+
+
+def consume_group_burst(images, delta=None):
+    if delta is None:
+        delta = timedelta(seconds=1)
+    i = -1
+    for i in range(len(images)):
+        # print(i, images[0], images[0].datetime_taken, images[i], images[i].datetime_taken, (images[i].datetime_taken - images[0].datetime_taken))
+        if (images[i].datetime_taken - images[0].datetime_taken) > delta:
+            break
+    else:
+        i += 1
+
+    return images[:i], images[i:]
+
+def consume_group_exposure_bracketing(images):
+    seen_exposures = set([])
+    seen_raw_exposures = set([])
+
+    def trace_exposure(image):
+        bias = int(image.exposure_bias * 100)
+        info((image, image.datetime_taken, bias))
+        if is_raw(image):
+            is_new = bias not in seen_raw_exposures
+            if not is_new:
+                info(seen_raw_exposures, seen_exposures, bias)
+            seen_raw_exposures.add(bias)
+        else:
+            is_new = bias not in seen_exposures
+            if not is_new:
+                info(seen_raw_exposures, seen_exposures, bias)
+            seen_exposures.add(bias)
+        return is_new
+
+    info(images[0])
+    trace_exposure(images[0])
+    for i in range(1, len(images)):
+        if images[i].datetime_taken - images[i - 1].datetime_taken > timedelta(seconds=1):
+            info("::: Out of time")
+            break
+
+        if not trace_exposure(images[i]):
+            info("::: Exposure seen")
+            break
+        info(images[i])
+
+    else:
+        i = len(images)
+
+    return images[:i], images[i:]
+
+
+def select_leader(images, prefer_raw):
+
+    def leader_preference(image):
+        return (
+            image.flag(ImageFlags.REJECTED),
+            -image.stars,
+            -votes[image],
+            abs(image.exposure_bias),
+            (-int(is_raw(image))) if prefer_raw else int(is_raw(image)),
+            image.datetime_taken
+        )
+
+    votes = {}
+    for image in images:
+        if image not in votes:
+            votes[image] = 0
+        if image.group in images:
+            if image.group not in votes:
+                votes[image.group] = 0
+            votes[image.group] += 1
+
+    leader = min(images, key=leader_preference)
+
+    return leader
+
+def info(*args):
+    pass
+    # print(*args)
+
+def validate_group(images, prefer_raw):
+    will_degrade = False
+    has_changes = False
+
+    leader = select_leader(images, prefer_raw)
+
+    for image in images:
+        if image.group != leader:
+            if image.group == image and not image.flag(ImageFlags.REJECTED):
+                # if image was group leader and was not rejected, changing the
+                # group leader would degrate image
+                will_degrade = True
+                print("Below change would degrate dataset: MANUAL ATTENTION NEEDED.")
+            print("Change group leader of {}: {} => {}".format(image, image.group, leader))
+            image.group = leader
+            has_changes = True
+
+    if has_changes:
+        stars = max(map(lambda image: image.stars, images))
+        rejected = min(map(lambda image: image.flag(ImageFlags.REJECTED), images))
+        for image in images:
+            if image.flag(ImageFlags.REJECTED) != rejected:
+                print("Change rejected status of {}: {} => {}".format(image, image.flag(ImageFlags.REJECTED), rejected))
+                image.set_flag(ImageFlags.REJECTED, rejected)
+            if image.stars != stars:
+                print("Change stars of {}: {} => {}".format(image, image.stars, stars))
+                image.stars = stars
+        print()
+    return has_changes, will_degrade
+
+
+def main(date_from, force, dry_run):
+
+    images = set([])
+    for r in FilmRoll.filter():
+        try:
+            if datetime.strptime(os.path.split(r.folder)[-1], "%Y%m%d").date() >= date_from:
+                # print(r)
+                for image in Image.filter(film=r):
+                    images.add(image)
+            else:
+                pass
+                # print("not", r)
+        except ValueError as e:
+            print(e)
+            print("Ignoring", r)
+
+    images = sorted(filter(lambda image: image.datetime_taken, images), key=lambda image: (image.datetime_taken, -is_raw(image), image.filename))
+
+    by_model = {}
+    for image in images:
+        key = (image.maker, image.model)
+        if key not in by_model:
+            by_model[key] = []
+        by_model[key].append(image)
+
+    n = 0
+    for model, images in by_model.items():
+
+        images = images[:]
+        prefer_raw = True
+        while len(images) > 0:
+            if model[1].name == "DSC-RX100":
+                group, images = consume_group_exposure_bracketing(images)
+            elif model[1].name == "ZV-1" or model[1].name == "ILCE-6700":
+                original_images = images
+                group, images = consume_group_exposure_bracketing(original_images)
+                group_burst, images_burst = consume_group_burst(original_images)
+                if len(group_burst) > len(group):
+                    group, images = group_burst, images_burst
+            elif model[1].name == "Pixel 6":
+                group, images = consume_group_pixel_6(images)
+                prefer_raw = False
+
+            elif (
+                model[1].name == "HERO9 Black" or
+                model[1].name == "HERO11 Black"
+            ):
+                group, images = consume_group_burst(images, timedelta(seconds=3))
+                prefer_raw = False
+            elif (
+                model[1].name == "HERO3+ Black Edition" or
+                model[1].name == "iPhone 12 mini" or
+                model[1].name == "iPhone 15 Pro Max"
+            ):
+                group, images = images[:1], images[1:]
+            else:
+                if model[1].name != '':
+                    for image in images:
+                        print(image)
+                    print("Unkown model {} - {}".format(*model))
+                break
+
+            has_changes, will_degrade = validate_group(group, prefer_raw)
+            if has_changes:
+                n += 1
+
+                if not dry_run and (not will_degrade or force):
+                    for image in group:
+                        print("Saving {}".format(image))
+                        image.save()
+
+    print("incorrect", n)
+
+import argparse
+if __name__ == '__main__':
+
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--force", action="store_true")
+    parser.add_argument("--save", action="store_true")
+    parser.add_argument("--from", dest="from_date", default=None)
+
+    options = parser.parse_args()
+
+    if options.from_date is None:
+        from_date = datetime.now().date()
+    else:
+        from_date = datetime.strptime(options.from_date, "%Y%m%d").date()
+    main(from_date, options.force, not options.save)

dt_cleanup.py

@@ -0,0 +1,51 @@
+from pytable.models import Image, ImageFlags, FilmRoll
+
+import peewee as pw
+import os
+import subprocess
+
+base_path = "/home/oke/Pictures/Darktable"
+
+subprocess.check_call(["mountpoint", "/home/oke/Pictures/DarktableRemote"])
+real_folders = [
+    "/home/oke/Pictures/DarktableLocal",
+    "/home/oke/Pictures/DarktableRemote",
+]
+
+query: list[Image] = Image.filter()
+pw.prefetch(query, FilmRoll)
+images = list(query)
+import itertools
+
+
+required_files = list(itertools.chain(*[
+    [os.path.join(image.film.folder, image.filename),
+     os.path.join(image.film.folder,
+                    os.path.splitext(image.filename)[0] +
+                    (("_%02d" % image.version) if image.version != 0 else "") +
+                    os.path.splitext(image.filename)[1]) + ".xmp"
+    ]
+    for image in images]))
+
+allowed_files = list(itertools.chain(*[
+    [os.path.join(image.film.folder, "." + os.path.splitext(image.filename)[0] + ".mov")]
+    for image in images])) + required_files
+
+
+
+for fn in allowed_files:
+    assert fn.startswith(base_path)
+
+for folder in real_folders:
+    real_files = set([os.path.join(path, file) for path, folders, files in os.walk(folder) for file in files if file[-3:] not in ["mov", "mp4", "mts", "ata"]])
+    mapped_allowed = [fn.replace(base_path, folder) for fn in allowed_files]
+
+    print(":::: Files to remove:")
+    for fn in sorted(real_files.difference(set(mapped_allowed))):
+        print(fn)
+
+    if folder == "/home/oke/Pictures/DarktableRemote":
+        print(":::: Missing files:")
+        mapped_required = [fn.replace(base_path, folder) for fn in required_files]
+        for fn in sorted(set(mapped_required).difference(set(real_files))):
+            print(fn)

dt_fix_file_structure.py

@@ -0,0 +1,100 @@
+from pytable.models import Image, ImageFlags, FilmRoll
+
+from datetime import datetime
+import peewee as pw
+import os
+from pprint import pprint
+import subprocess
+
+base_path = "/home/oke/Pictures/Darktable"
+real_folders = [
+    "/home/oke/Pictures/DarktableRemote",
+    "/home/oke/Pictures/DarktableLocal",
+]
+
+subprocess.check_call(["mountpoint", "/home/oke/Pictures/DarktableRemote"])
+
+query = Image.filter()
+pw.prefetch(query, FilmRoll)
+dry = True
+
+move_files = []
+
+
+d_root = "/home/oke/Pictures/Darktable"
+d_local = "/home/oke/Pictures/DarktableLocal"
+d_remote = "/home/oke/Pictures/DarktableRemote"
+
+
+def me_and_related_dt(fn, new_folder, new_basename):
+    directory, filename = os.path.split(fn)
+    basename, ext = os.path.splitext(filename)
+    yield fn, os.path.join(new_folder, new_basename + ext)
+    yield os.path.join(directory, "." + basename + ".mov"), os.path.join(new_folder, "." + new_basename + ".mov"),
+    yield fn + ".xmp", os.path.join(new_folder, new_basename + ext + ".xmp")
+
+
+def me_and_related_potential(fn, new_folder, new_basename):
+    for local_fn, local_fn_related in me_and_related_dt(fn, new_folder, new_basename):
+        assert local_fn.startswith(d_root)
+        yield local_fn.replace(d_root, d_local), local_fn_related.replace(d_root, d_local)
+        yield local_fn.replace(d_root, d_remote), local_fn_related.replace(d_root, d_remote)
+
+def me_and_related(fn, new_folder, new_basename):
+    for fn, fn_related in me_and_related_potential(fn, new_folder, new_basename):
+        if os.path.exists(fn):
+            assert not os.path.exists(fn_related), fn + "=>" + fn_related
+            yield fn, fn_related
+
+changes = []
+
+for image in query:
+    try:
+        n = 0
+        dt_fn = datetime.strptime(image.filename[:13],"%Y%m%d_%H%M")
+        if image.datetime_taken:
+            dt_internal = datetime(image.datetime_taken.year, image.datetime_taken.month, image.datetime_taken.day, image.datetime_taken.hour, image.datetime_taken.minute)
+            expected_folder = os.path.join(base_path, dt_internal.strftime("%Y%m%d"))
+            if dt_fn != dt_internal:
+                d = ((dt_internal - dt_fn).total_seconds() / 3600)
+                dt_formatted = image.datetime_taken.strftime("%Y%m%d_%H%M")
+                new_film_roll = image.film
+                if expected_folder != image.film.folder:
+                    new_film_roll = FilmRoll.get(folder=expected_folder)
+                    assert new_film_roll
+
+                new_fn = dt_formatted + image.filename[13:]
+                new_basename, _ = os.path.splitext(new_fn)
+
+                changes.append((
+                    image, new_film_roll, new_fn,
+                    list(me_and_related(os.path.join(image.film.folder, image.filename), new_film_roll.folder, new_basename))
+                ))
+
+                print("%s: %.2fH" % (image.filename, d))
+                for src, dst in changes[-1][-1]:
+                    print("\t%s => %s" % (src, dst))
+    except Exception as e:
+        print("ERROR:", e)
+
+
+
+reply = input("Type YES to remove all these files. This is not reversible... : ")
+
+if reply != 'YES':
+    print("You did not type 'YES'. Not doing anything!")
+    exit(-1)
+
+for image, film_roll, new_fn, move in changes:
+    print(image.filename, "=>", new_fn)
+
+    for src, dst in move:
+        assert not os.path.exists(dst)
+
+    for src, dst in move:
+        print("Moving %s => %s" % (src, dst))
+        os.rename(src, dst)
+    print("Saving image into sql")
+    image.film = film_roll
+    image.filename = new_fn
+    image.save()

dt_remove_secondary copy.py

@@ -0,0 +1,114 @@
+from pytable.models import Image, ImageFlags, FilmRoll
+
+from datetime import datetime
+import peewee as pw
+import os
+import shutil
+
+remove_three_stars_before = datetime(2024, 4, 1)
+remove_secondary_before = datetime(2024, 6, 15)
+remove_rejected_before = datetime(2024, 6, 15)
+
+query = Image.filter()
+pw.prefetch(query, FilmRoll)
+
+def is_older_than(image, cmp):
+   return (image.datetime_taken or image.datetime_imported) < cmp
+
+def should_remove(image):
+    return (
+        (is_older_than(image, remove_three_stars_before) and image.stars < 3) or
+        (image.flag(ImageFlags.REJECTED) and is_older_than(image, remove_rejected_before)) or
+        (image.group_id != image.id and is_older_than(image, remove_secondary_before))
+    )
+
+file_usages = {}
+for x in query:
+    path = os.path.join(x.film.folder, x.filename)
+    if path not in file_usages:
+        file_usages[path] = 0
+    if not should_remove(x):
+        file_usages[path] += 1
+
+n_total = 0
+n_not_needed = 0
+needs_removal = []
+extra_files_to_remove = []
+bytes_to_remove = 0
+
+d_root = "/home/oke/Pictures/Darktable"
+d_local = "/home/oke/Pictures/DarktableLocal"
+d_remote = "/home/oke/Pictures/DarktableRemote"
+
+def potential_extras(fn):
+    directory, filename = os.path.split(fn)
+    basename, _ = os.path.splitext(filename)
+
+    iphone_movie = os.path.join(directory, "." + basename + ".mov")
+    yield iphone_movie
+
+needs_fetching = []
+
+for file, usages in file_usages.items():
+    if not file.startswith(d_root):
+        print("SKIPPNG", file)
+        continue
+    assert file.startswith(d_root)
+    remote_file = file.replace(d_root, d_remote)
+    local_file = file.replace(d_root, d_local)
+
+    if usages > 0:
+        if not os.path.exists(local_file):
+            needs_fetching.append((remote_file, local_file))
+    else:
+        n_not_needed += 1
+        if os.path.exists(local_file):
+            needs_removal.append(local_file)
+            bytes_to_remove += os.stat(local_file).st_size
+
+        for extra_fn in potential_extras(local_file):
+            if os.path.exists(extra_fn):
+                extra_files_to_remove.append(extra_fn)
+                bytes_to_remove += os.stat(extra_fn).st_size
+
+files_to_remove = list(sorted(needs_removal + extra_files_to_remove))
+
+needs_fetching = sorted(needs_fetching)
+import pprint
+print("Files to fetch: ")
+pprint.pprint(needs_fetching)
+print("Extra files to remove:")
+pprint.pprint(extra_files_to_remove)
+print("First files to remove:")
+pprint.pprint(needs_removal[:10])
+print("Last files to remove:")
+pprint.pprint(needs_removal[-10:])
+
+with open("remove.txt", "w") as f:
+    pprint.pprint(sorted(set(needs_removal).union(extra_files_to_remove)), f)
+
+print("Not needed:            %6d / %6d (%3d%%)" % (n_not_needed, len(file_usages), 100 * n_not_needed / len(file_usages)))
+print()
+print("Not yet deleted:       %6d / %6d (%3d%%)" % (len(needs_removal), n_not_needed, 100 * len(needs_removal) / n_not_needed))
+print("Bytes to free:         %5.2fGB" % (bytes_to_remove / 1024. / 1024 / 1024))
+print("Total files to remove: %d" % len(files_to_remove))
+print()
+print("Total files to fetch: %3d" % len(needs_fetching))
+print()
+
+reply = input("Type YES to remove all these files. This is not reversible... : ")
+
+if reply == 'YES':
+
+    for fn in files_to_remove:
+        assert fn.startswith(d_local)
+        os.remove(fn)
+
+    for f, t in needs_fetching:
+        assert os.path.exists(f), "Backup corruped, file not found: " + f
+        assert not os.path.exists(t)
+        print(f, t)
+        shutil.copy(f, t)
+
+else:
+    print("You did not type 'YES'. Not doing anything!")

dt_render.py

@@ -0,0 +1,34 @@
+from pytable.models import Image, ImageFlags, FilmRoll
+
+from datetime import datetime
+import peewee as pw
+import os
+import shutil
+
+
+query = Image.filter()
+
+output_folder = "output"
+for image in query:
+    assert isinstance(image, Image)
+    if image.stars >= 3 and (not image.group or image.group == image):
+
+        print(image)
+
+        image_path = os.path.join(image.film.folder, image.filename)
+
+        output_path = os.path.join(output_folder, os.path.splitext(image.filename)[0] + ".jpg")
+
+
+        if os.path.exists(output_path):
+            continue
+        command = [
+            'darktable-cli',
+            '--width', '1920',
+            '--hq', 'true',
+            image_path,
+            output_path
+        ]
+
+        import subprocess
+        subprocess.call(command)

export.py

@@ -0,0 +1,52 @@
+from pytable.models import Color, Image, ImageFlags, FilmRoll, ColorLabel, TaggedImages
+
+from datetime import datetime
+import peewee as pw
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from typing import Tuple, Optional, Generic, TypeVar, Iterable
+
+
+export_base = "/home/oke/Nextcloud/Blog/"
+query = (
+    Image.filter()
+    .join(ColorLabel)
+    .where(ColorLabel.color == Color.BLUE)
+    .where(Image.datetime_taken > datetime(2024, 6, 14, 22))
+)
+pw.prefetch(query, FilmRoll)
+
+def is_blog(image: Image):
+    return (
+        (image.group == image or image.group == None) and
+        not image.flag(ImageFlags.REJECTED)
+    )
+
+images = list(filter(is_blog, query))
+for image in sorted(images, key=lambda i: i.datetime_taken or i.datetime_imported):
+    assert isinstance(image, Image)
+    fn = os.path.join(image.film.folder, image.filename)
+    version = "" if image.version == 0 else ("_%02d" % image.version)
+    base, ext = os.path.splitext(image.filename)
+    xmp = os.path.join(image.film.folder, base + version + ext + ".xmp")
+    out_fn = os.path.join(export_base, base + version + ".jpg")
+
+    if os.path.exists(out_fn):
+        if not image.datetime_changed or datetime.fromtimestamp(os.stat(out_fn).st_ctime) > image.datetime_changed:
+            continue
+        os.remove(out_fn)
+    subprocess.call([
+        'darktable-cli',
+        fn,
+        xmp,
+        '--width', '1920',
+        '--out-ext', 'jpg',
+        '--upscale', 'false',
+        '--hq', 'true',
+        out_fn
+    ])
+
+print(len(images))
+

pytable/database.py

@@ -0,0 +1,17 @@
+import peewee as pw
+import sys
+import os
+
+
+if sys.platform == "linux":
+    db_library = pw.SqliteDatabase(
+        os.path.expanduser('~/.config/darktable/library.db'))
+    db_data = pw.SqliteDatabase(
+        os.path.expanduser('~/.config/darktable/data.db'))
+else:
+    db = pw.SqliteDatabase(None)
+    db_data = pw.SqliteDatabase(None)
+
+def open_sqlite_db(sqlite_fn):
+    global db
+    db = pw.init(sqlite_fn)

pytable/fields.py

@@ -0,0 +1,69 @@
+import peewee as pw
+
+import datetime
+import itertools
+
+class DarktableTimestampField(pw.TimestampField):
+    """
+    Darktable not always stores timestamps as UNIX timestamps with origin of
+    1970-01-01, but sometimes 0001-01-01. This timestamp field lets the user
+    specifiy what origin to use for the timestamp.
+    """
+
+    def __init__(self, *args, origin=datetime.datetime(1970, 1, 1), **kwargs) -> None:
+        kwargs['resolution'] = 10**6
+        self.epoch_diff = (datetime.datetime(1970, 1, 1) - origin).total_seconds()
+        super().__init__(*args, **kwargs)
+
+    def python_value(self, value):
+        if value is None or value == -1:
+            return None
+        return super().python_value(value - self.epoch_diff * self.resolution)
+
+    def db_value(self, value):
+        if value is None or value == -1:
+            return -1
+
+        if isinstance(value, datetime.datetime):
+            pass
+        elif isinstance(value, datetime.date):
+            value = datetime.datetime(value.year, value.month, value.day)
+        else:
+            raise ValueError()
+
+        return super().db_value(value) + self.epoch_diff * self.resolution
+
+class ModuleOrderListField(pw.CharField):
+
+    def db_value(self, value):
+        print("db_value")
+        value = super().db_value(value)
+        if hasattr(value, "__item__"):
+            new_value = ",".join([str(v) for v in itertools.chain(*value)])
+            print(value, new_value)
+            return new_value
+        print(value)
+        return value
+
+    def python_value(self, value):
+        value = super().python_value(value)
+        if isinstance(value, str):
+            values = value.split(",")
+            assert len(values) % 2 == 0
+            new_value = [
+                (values[i * 2], int(values[i * 2 + 1]))
+                for i in range(int(len(values) / 2))]
+            return new_value
+        return value
+
+class EnumField(pw.IntegerField):
+
+    def __init__(self, enum_type, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_type = enum_type
+
+    def db_value(self, value):
+        return value.value
+
+    def python_value(self, value):
+        return self.enum_type(super().python_value(value))

pytable/models.py

@@ -0,0 +1,304 @@
+from .database import db_library, db_data
+from .fields import DarktableTimestampField, ModuleOrderListField, EnumField
+from .types import ImageFlags, v30_jpg_order, v30_order, IOPOrderType, legacy_order, Color
+from .modules import DT_MODULES, Module
+
+from typing import List
+import peewee as pw
+
+import datetime
+
+class FilmRoll(pw.Model):
+    """
+    A darktable filmroll.
+    """
+    access_timestamp: datetime.datetime = DarktableTimestampField() # type: ignore
+    folder: str = pw.CharField(1024) # type: ignore
+
+    def __str__(self):
+        return self.folder
+
+    class Meta:
+        db_table = 'film_rolls'
+        database = db_library
+
+class Maker(pw.Model):
+
+    name: str = pw.CharField() # type: ignore
+
+    def __str__(self) -> str:
+        return self.name
+
+    class Meta:
+        db_table = 'makers'
+        database = db_library
+
+class Model(pw.Model):
+
+    name: str = pw.CharField() # type: ignore
+
+    def __str__(self) -> str:
+        return self.name
+
+    class Meta:
+        db_table = 'models'
+        database = db_library
+
+class Lens(pw.Model):
+
+    name: str = pw.CharField() # type: ignore
+
+    def __str__(self) -> str:
+        return self.name
+
+    class Meta:
+        db_table = 'lens'
+        database = db_library
+
+class Camera(pw.Model):
+
+    name: str = pw.CharField() # type: ignore
+
+    class Meta:
+        db_table = 'cameras'
+        database = db_library
+
+class Image(pw.Model):
+    """
+    A darktable image.
+    """
+    id: int = pw.IntegerField(primary_key=True) # type: ignore
+    group = pw.ForeignKeyField('self')
+    film = pw.ForeignKeyField(FilmRoll)
+
+    # Image width in pixels
+    width: int = pw.IntegerField() # type: ignore
+    # Image height in pixels
+    height: int = pw.IntegerField() # type: ignore
+
+    # The filename of the image
+    filename: str = pw.CharField() # type: ignore
+
+    # The maker of the camera used to capture this picture
+    maker = pw.ForeignKeyField(Maker)
+    # The model of the camera used to capture this picture
+    model = pw.ForeignKeyField(Model)
+    # The lens used to capture this picture
+    lens = pw.ForeignKeyField(Lens)
+    # The camera used to capture this picture
+    camera = pw.ForeignKeyField(Camera)
+
+    # The exposure used to capture this picture
+    exposure = pw.FloatField()
+    # The aperture used to capture this picture
+    aperture = pw.FloatField()
+    # The iso used to capture this picture
+    iso = pw.FloatField()
+    # The focal length used to capture this picture
+    focal_length = pw.FloatField()
+    # The focus distance used to capture this picture
+    focus_distance = pw.FloatField()
+
+    # The date this picture was taken
+    datetime_taken: datetime.datetime = DarktableTimestampField(utc=True, origin=datetime.datetime(1, 1, 1)) # type: ignore
+
+    # The flags attached to this picture. To read individual flags, see 'flag()' function
+    flags: int = pw.IntegerField() # type: ignore
+
+    output_width: int = pw.IntegerField() # type: ignore
+    output_height: int = pw.IntegerField() # type: ignore
+    crop = pw.DoubleField()
+
+    raw_parameters: int = pw.IntegerField() # type: ignore
+    # raw_denoise_threshold = pw.DoubleField()
+    # raw_auto_bright_threshold = pw.DoubleField()
+    raw_black: int = pw.IntegerField() # type: ignore
+    raw_maximum: int = pw.IntegerField() # type: ignore
+
+    # license: str = pw.CharField() # type: ignore
+
+    # sha1sum = pw.FixedCharField(40)
+
+    orientation: int = pw.IntegerField() # type: ignore
+
+    # histogram = pw.BlobField()
+    # lightmap = pw.BlobField()
+
+    longitude = pw.DoubleField()
+    latitude = pw.DoubleField()
+    altitude = pw.DoubleField()
+
+    color_matrix = pw.BlobField()
+
+    colorspace: int = pw.IntegerField() # type: ignore
+    version: int = pw.IntegerField() # type: ignore
+    max_version: int = pw.IntegerField() # type: ignore
+    write_timestamp: datetime.datetime = DarktableTimestampField() # type: ignore
+    history_end: int = pw.IntegerField() # type: ignore
+    position: int = pw.IntegerField() # type: ignore
+
+    aspect_ratio = pw.DoubleField()
+    exposure_bias = pw.DoubleField()
+
+    datetime_imported: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="import_timestamp") # type: ignore
+    datetime_changed: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="change_timestamp") # type: ignore
+    datetime_exported: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="export_timestamp") # type: ignore
+    datetime_printed: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="print_timestamp") # type: ignore
+    datetime_thumb: datetime.datetime = DarktableTimestampField(origin=datetime.datetime(1, 1, 1), column_name="thumb_timestamp") # type: ignore
+    thumb_maxmip: int = pw.IntegerField() # type: ignore
+
+    def flag(self, flag):
+        return bool(self.flags & flag.value)
+
+    def set_flag(self, flag, value):
+        b = -1
+        assert flag
+        flag = flag.value
+        while flag:
+            b += 1
+            flag = flag >> 1
+        self.flags = (self.flags & ~flag) | (value & flag)
+
+
+
+    @property
+    def stars(self):
+        return self.flags & 0x7
+
+    @stars.setter
+    def stars(self, value):
+        self.flags = (self.flags & ~0x7) | (value & 0x7)
+
+    def __str__(self):
+        return self.filename
+
+    history: pw.BackrefAccessor
+    module_order: pw.BackrefAccessor
+
+    def get_ordered_active_modules(self):
+
+        # nextline: type: ignore
+        active_history = (self.history
+            .order_by(HistoryEntry.num.asc()) # type: ignore
+            .where(HistoryEntry.num < self.history_end))
+
+        if len(active_history) == 0:
+            return []
+
+        module_orders = self.module_order.select()
+        module_order = module_orders[0]
+        iop_list = None
+        if module_order.version == IOPOrderType.CUSTOM:
+            iop_list = module_order.iop_list
+        elif module_order.version == IOPOrderType.V30:
+            iop_list = [(v, 0) for v in v30_order]
+        elif module_order.version == IOPOrderType.V30_JPG:
+            iop_list = [(v, 0) for v in v30_jpg_order]
+        elif module_order.version == IOPOrderType.LEGACY:
+            iop_list = [(v, 0) for v in legacy_order]
+        else:
+            raise NotImplementedError()
+
+        # Override if iop_list exists for the case that
+        # 'module_order.version != CUSTOM'... Is this a bug in dt?
+        if module_order.iop_list:
+            iop_list = module_order.iop_list
+
+        iop_list = {iop_list[i]: i for i in range(len(iop_list))}
+
+        active_modules = {}
+        for he in active_history:
+            if he.enabled:
+                active_modules[(he.module_name, he.instance)] = he.module
+            elif (he.module_name, he.instance) in active_modules:
+                del active_modules[(he.module_name, he.instance)]
+
+        active_modules = sorted(active_modules.items(), key=lambda kv: iop_list[kv[0]])
+        return list(map(lambda kv: kv[1], active_modules))
+
+    class Meta:
+        db_table = 'images'
+        database = db_library # This model uses the "people.db" database``
+
+class ColorLabel(pw.Model):
+
+    image: Image = pw.ForeignKeyField(Image, column_name='imgid') # type: ignore
+    color: Color = EnumField(Color) # type: ignore
+
+    class Meta:
+        primary_key = False
+        db_table = 'color_labels'
+        database = db_library
+
+class HistoryEntry(pw.Model):
+    """
+    A darktable filmroll.
+    """
+    image: Image = pw.ForeignKeyField(Image, column_name="imgid", backref="history") # type: ignore
+    num: int = pw.IntegerField() # type: ignore
+
+    module_name: str = pw.CharField(256, column_name="operation") # type: ignore
+    version: int = pw.IntegerField(column_name="module") # type: ignore
+    params: bytes = pw.BlobField(column_name="op_params") # type: ignore
+
+    enabled: bool = pw.BooleanField() # type: ignore
+
+    blendop_version: int = pw.IntegerField() # type: ignore
+    blendop_params: bytes = pw.BlobField() # type: ignore
+
+    instance: int = pw.IntegerField(column_name="multi_priority") # type: ignore
+    name: str = pw.CharField(256, column_name="multi_name") # type: ignore
+    name_hand_edited: int = pw.IntegerField(column_name="multi_name_hand_edited") # type: ignore
+
+    @property
+    def module(self):
+        for cls in DT_MODULES:
+            if cls.NAME == self.module_name and cls.VERSION == self.version:
+                return cls(self.instance, self.params)
+        return Module(self.instance, self.params, module_name=self.module_name)
+
+    def __str__(self):
+        return "%3d - %sv%d-%s" % (
+            self.num, self.module_name, self.version, self.instance)
+
+    class Meta:
+        primary_key = False
+        db_table = 'history'
+        database = db_library
+
+
+class ModuleOrderEntry(pw.Model):
+    """
+    A darktable filmroll.
+    """
+    image: Image = pw.ForeignKeyField(Image, column_name="imgid", backref="module_order") # type: ignore
+    version: IOPOrderType = EnumField(IOPOrderType) # type: ignore
+    iop_list = ModuleOrderListField()
+
+    def __str__(self):
+        return str((self.version, self.iop_list))
+
+    class Meta:
+        primary_key = False
+        db_table = 'module_order'
+        database = db_library
+
+class Tag(pw.Model):
+
+    name: str = pw.CharField() # type: ignore
+    synonyms: str = pw.CharField() # type: ignore
+    flags: int = pw.IntegerField() # type: ignore
+
+    class Meta:
+        db_table = 'tags'
+        database = db_data
+
+class TaggedImages(pw.Model):
+
+    image: Image = pw.ForeignKeyField(Image, backref='tags', column_name='imgid') # type: ignore
+    tag: Tag = pw.ForeignKeyField(Tag, backref='images', column_name='tagid') # type: ignore
+
+    class Meta:
+        primary_key = False
+        db_table = 'tagged_images'
+        database = db_library

pytable/modules.py

@@ -0,0 +1,420 @@
+from .types import Adaptation, ColorIntent, ColorSpacesColorProfile, \
+    Illuminant, IlluminantLED, ImageOrientation
+
+import struct
+import enum
+
+DT_MODULES = []
+
+
+def decode_bytes(bytes):
+    return bytes.decode().rstrip("\0")
+
+
+class ModuleMeta(type):
+
+    def __new__(cls, name, bases, dct):
+        new_cls = super().__new__(cls, name, bases, dct)
+        if name != "Module":
+            DT_MODULES.append(new_cls)
+        return new_cls
+
+
+class Module(metaclass=ModuleMeta):
+
+    NAME = ""
+    VERSION = -1
+    PARAMS_NAMES = ()
+    PARAMS_FORMAT = ""
+    PARAMS_TYPES = ()
+
+    def __init__(self, instance, raw_params, module_name=None) -> None:
+        self.module_name = module_name or self.__class__.__name__
+        self.instance = instance
+        self.params = self.parse_params(raw_params)
+
+    def parse_params(self, raw_params):
+        if not self.PARAMS_FORMAT:
+            return {}
+        values = struct.unpack(self.PARAMS_FORMAT, raw_params)
+        params = {}
+        pts = list(self.PARAMS_TYPES) + [None] * (len(self.PARAMS_NAMES) - len(self.PARAMS_TYPES))
+        for name, type in zip(self.PARAMS_NAMES, pts):
+            n = 1
+            if "*" in name:
+                name, n = name.split("*")
+                n = int(n)
+            params[name] = values[:n]
+            assert len(params[name]) == n
+            if type:
+                params[name] = [type(v) for v in params[name]]
+            if n == 1:
+                params[name] = params[name][0]
+            values = values[n:]
+        assert not values, values
+        return params
+
+    def __str__(self):
+        import pprint
+        return "%s-%d: %s" % (self.module_name, self.instance, pprint.pformat(
+            self.params, indent=0, compact=True, width=10000, depth=1
+        ))
+
+    def __repr__(self) -> str:
+        return "%s-%d" % (self.module_name, self.instance)
+
+class ColorBalanceRGBSaturation(enum.Enum):
+    JZAZBZ = 0
+    DTUCS = 1
+
+
+class ColorBalanceRGB(Module):
+    NAME = "colorbalancergb"
+    VERSION = 5
+    PARAMS_NAMES = ("shadows_Y", "shadows_C", "shadows_H", "midtones_Y",
+                    "midtones_C", "midtones_H", "highlights_Y", "highlights_C",
+                    "highlights_H", "global_Y", "global_C", "global_H",
+                    "shadows_weight", "white_fulcrum", "highlights_weight",
+                    "chroma_shadows", "chroma_highlights", "chroma_global",
+                    "chroma_midtones", "saturation_global",
+                    "saturation_highlights", "saturation_midtones",
+                    "saturation_shadows", "hue_angle", "brilliance_global",
+                    "brilliance_highlights", "brilliance_midtones",
+                    "brilliance_shadows", "mask_grey_fulcrum", "vibrance",
+                    "grey_fulcrum", "contrast", "saturation_formula")
+
+    PARAMS_FORMAT = "32fi"
+    PARAMS_TYPES = [None] * 32 + [ColorBalanceRGBSaturation]
+
+
+class ColorCalibrationV2(Module):
+
+    NAME = "channelmixerrgb"
+    VERSION = 3
+    PARAMS_NAMES = ("red*4", "green*4", "blue*4", "saturation*4",
+                    "lightness*4", "grey*4", "normalize_red",
+                    "normalize_green", "normalize_blue",
+                    "normalize_saturation", "normalize_lightness",
+                    "normalize_grey", "illuminant", "illum_fluo",
+                    "illum_led", "adaptation", "x", "y", "temperature",
+                    "gamut", "clip", "version")
+    PARAMS_FORMAT = "24f6i4iffffii"
+    PARAMS_TYPES = ([None] * 6 + [bool] * 6 +
+                    [Illuminant, None, IlluminantLED, Adaptation,
+                     None, None, None, None, bool])
+
+
+class ColorInputProfileNormalize(enum.Enum):
+    OFF = 0
+    SRGB = 1
+    ADOBE_RGB = 2
+    LINEAR_REC709_RGB = 3
+    LINEAR_REC2020_RGB = 4
+
+
+class ColorInputProfileV7(Module):
+
+    NAME = "colorin"
+    VERSION = 7
+    PARAMS_NAMES = (
+        "type", "filename", "intent", "normalize", "blue_mapping", "type_work",
+        "filename_work")
+    PARAMS_FORMAT = "i512siiii512s"
+    PARAMS_TYPES = (ColorSpacesColorProfile, decode_bytes, ColorIntent,
+                    ColorInputProfileNormalize, bool, ColorSpacesColorProfile,
+                    decode_bytes)
+
+
+class ColorOutputProfileV5(Module):
+
+    NAME = "colorout"
+    VERSION = 5
+    PARAMS_NAMES = (
+        "type", "filename", "intent")
+    PARAMS_FORMAT = "i512si"
+    PARAMS_TYPES = (ColorSpacesColorProfile, decode_bytes, ColorIntent)
+
+
+class ColorZonesChannel(enum.Enum):
+    lightness = 0
+    chroma = 1
+    hue = 2
+
+class ColorZonesV5(Module):
+    NAME = "colorzones"
+    VERSION = 5
+    PARAMS_NAMES = ("channel", "curve*120", "curve_num_nodes*3",
+                    "curve_type*3", "strength", "mode", "spline_version")
+
+    PARAMS_FORMAT = "i120f6ifii"
+
+    def parse_params(self, raw_params):
+        params = super().parse_params(raw_params)
+        curve = params["curve"]
+        curve = [
+            [(curve[channel * 20 * 2 + node * 2],
+              curve[channel * 20 * 2 + node * 2 + 1]) for node in range(20)]
+            for channel in range(3)
+        ]
+        curve[0] = curve[0][:params["curve_num_nodes"][0]]
+        curve[1] = curve[1][:params["curve_num_nodes"][1]]
+        curve[2] = curve[2][:params["curve_num_nodes"][2]]
+        params["curve"] = curve
+        return params
+
+
+class DemosaicMethod(enum.Enum):
+    PPG = 0
+    AMAZE = 1
+    VNG4 = 2
+    RCD = 5
+    LMMSE = 6
+    RCD_VNG = 2048 | 5
+    AMAZE_VNG = 2048 | 6
+    PASSTHROUGH_MONOCHROME = 3
+    PASSTHROUGH_COLOR = 4
+    VNG = 1024 | 0
+    MARKESTEIJN = 1024 | 1
+    MARKESTEIJN_3 = 1024 | 2
+    FCD = 1024 | 4
+    MARKEST3_VNG = 2048 | 1024 | 2
+    PASSTHR_MONOX = 1024 | 3
+    PASSTHR_COLORX = 1024 | 5
+
+
+class DemosaicGreenEQ(enum.Enum):
+    NO = 0
+    LOCAL = 1
+    FULL = 2
+    BOTH = 3
+
+
+class DemosaicQualFlags(enum.Enum):
+    DEFAULT = 0
+    FULL_SCALE = 1
+    ONLY_VNG_LINEAR = 2
+
+
+class DemosaicSmooth(enum.Enum):
+    OFF = 0
+    ONCE = 1
+    TWICE = 2
+    THREE_TIMES = 3
+    FOUR_TIMES = 4
+    FIVE_TIMES = 5
+
+
+class DemosaicLMMSE(enum.Enum):
+    BASIC = 0
+    MEDIAN = 1
+    MEDIANX3 = 2
+    REFINE_AND_MEDIANS = 3
+    REFINEx2_AND_MEDIANS = 4
+
+
+class DemosaicV4(Module):
+
+    NAME = "demosaic"
+    VERSION = 4
+    PARAMS_NAMES = (
+        "green_eq", "median_thrs", "color_smoothing", "demosaicing_method",
+        "lmmse_refine", "dual_thrs")
+    PARAMS_FORMAT = "ifiiif"
+    PARAMS_TYPES = (DemosaicGreenEQ, None, DemosaicSmooth, DemosaicMethod, DemosaicLMMSE)
+
+
+class DenoiseProfiledWaveletMode(enum.Enum):
+    RGB = 0
+    Y0U0V0 = 1
+
+class DenoiseProfiled(Module):
+    NAME = "denoiseprofile"
+    VERSION = 11
+    PARAMS_NAMES = ("radius", "nbhood", "strength", "shadows",
+                    "bias", "scattering", "central_pixel_weight",
+                    "overshooting", "a*3", "b*3", "mode", "x*42", "y*42",
+                    "wb_adaptive_anscombe", "fix_anscombe_and_nlmeans_norm",
+                    "use_new_vst", "wavelet_color_mode")
+
+    PARAMS_FORMAT = "8f6fi42f42fiiii"
+    PARAMS_TYPES = [None] * 16 + [DenoiseProfiledWaveletMode]
+
+    def parse_params(self, raw_params):
+        params = super().parse_params(raw_params)
+        params["x"] = [[params["x"][profile * 7 + band] for band in range(7)] for profile in range(6)]
+        params["y"] = [[params["y"][profile * 7 + band] for band in range(7)] for profile in range(6)]
+        return params
+
+
+class DiffuseOrSharpenV2(Module):
+
+    NAME = "diffuse"
+    VERSION = 2
+    PARAMS_NAMES = (
+        "iterations", "sharpness", "radius", "regularization",
+        "variance_threshold", "anisotropy_first", "anisotropy_second",
+        "anisotropy_third", "anisotropy_fourth", "threshold",
+        "first", "second", "third", "fourth", "radius_center")
+    PARAMS_FORMAT = "ififffffffffffi"
+
+
+class DisplayEncodingV1(Module):
+
+    NAME = "gamma"
+    VERSION = 1
+    PARAMS_NAMES = ("gamma", "linear")
+    PARAMS_FORMAT = "ff"
+
+
+class ExposureMode(enum.Enum):
+    MANUAL = 0
+    DEFLICKER = 1
+
+
+class ExposureV6(Module):
+
+    NAME = "exposure"
+    VERSION = 6
+    PARAMS_NAMES = ("mode", "black", "exposure", "deflicker_percentile",
+                    "deflicker_target_level", "compensate_exposure_bias")
+    PARAMS_FORMAT = "iffffi"
+    PARAMS_TYPES = (ExposureMode, None, None, None, None, bool)
+
+
+class FilmicRGBMethod(enum.Enum):
+    NONE = 0
+    MAX_RGB = 1
+    LUMINANCE = 2
+    POWER_NORM = 3
+    EUCLIDEAN_NORM_V1 = 4
+    EUCLIDEAN_NORM_V2 = 5
+
+
+class FilmicRGBCurve(enum.Enum):
+    POLY_4 = 0
+    POLY_3 = 1
+    RATIONAL = 2
+
+
+class FilmicRGBReconstruction(enum.Enum):
+    RECONSTRUCT_RGB = 0
+    RECONSTRUCT_RATIOS = 1
+
+
+class FilmicRGBNoiseDistribution(enum.Enum):
+    UNIFORM = 0
+    GAUSSIAN = 1
+    POISSONIAN = 2
+
+
+class FilmicRGB(Module):
+
+    NAME = "filmicrgb"
+    VERSION = 6
+    PARAMS_NAMES = ("grey_point_source", "black_point_source",
+                    "white_point_source", "reconstruct_threshold",
+                    "reconstruct_feather", "reconstruct_bloom_vs_details",
+                    "reconstruct_grey_vs_color",
+                    "reconstruct_structure_vs_texture", "security_factor",
+                    "grey_point_target", "black_point_target",
+                    "white_point_target", "output_power", "latitude",
+                    "contrast", "saturation", "balance", "noise_level",
+                    "preserve_color", "version", "auto_hardness", "custom_grey",
+                    "high_quality_reconstruction", "noise_distribution",
+                    "shadows", "highlights", "compensate_icc_black",
+                    "spline_version", "enable_highlight_reconstruction")
+    PARAMS_FORMAT = "18f11i"
+    PARAMS_TYPES = [None] * 18 + [
+        FilmicRGBMethod, None, bool, bool, None, FilmicRGBNoiseDistribution,
+        FilmicRGBCurve, FilmicRGBCurve, bool, None, bool]
+
+
+class HighlightsMode(enum.Enum):
+    OPPOSED = 5
+    LCH = 1
+    CLIP = 0
+    SEGMENTS = 4
+    LAPLACIAN = 3
+    INPAINT = 2
+
+class HighlightsAtrousWaveletsScales(enum.Enum):
+    PX2 = 0
+    PX4 = 1
+    PX8 = 2
+    PX16 = 3
+    PX32 = 4
+    PX64 = 5
+    PX128 = 6
+    PX256 = 7
+    PX512 = 8
+    PX1024 = 9
+    PX2048 = 10
+    PX4096 = 11
+
+class HighlightsRecoveryMode(enum.Enum):
+    MODE_OFF = 0
+    MODE_ADAPT = 5
+    MODE_ADAPTF = 6
+    MODE_SMALL = 1
+    MODE_LARGE = 2
+    MODE_SMALLF = 3
+    MODE_LARGEF = 4
+
+
+class HighlightsV4(Module):
+
+    NAME = "highlights"
+    VERSION = 4
+    PARAMS_NAMES = ("mode", "blendL", "blendC", "strength", "clip",
+                    "noise_level", "iterations", "scales", "candidating",
+                    "combine", "recovery", "solid_color")
+    PARAMS_FORMAT = "ifffffiiffif"
+    PARAMS_TYPES = (HighlightsMode, None, None, None, None, None, None,
+                    HighlightsAtrousWaveletsScales, None, None,
+                    HighlightsRecoveryMode)
+
+
+class LocalContrastMode(enum.Enum):
+    BILATERAL = 0
+    LOCAL_LAPLACIAN = 1
+
+class LocalContrastV3(Module):
+    NAME = "bilat"
+    VERSION = 3
+    PARAMS_NAMES = ("mode", "sigma_r", "sigma_s", "detail", "midtone")
+    PARAMS_FORMAT = "i4f"
+    PARAMS_TYPES = (LocalContrastMode,)
+
+
+class OrientationV2(Module):
+    NAME = "flip"
+    VERSION = 2
+    PARAMS_NAMES = ("orientation",)
+    PARAMS_FORMAT = "i"
+    PARAMS_TYPES = (ImageOrientation,)
+
+
+class SharpenV1(Module):
+    NAME = "sharpen"
+    VERSION = 1
+    PARAMS_NAMES = ("radius", "amount", "threshold")
+    PARAMS_FORMAT = "3f"
+
+
+class RawBlackWhitePointV2(Module):
+
+    NAME = "rawprepare"
+    VERSION = 2
+    PARAMS_NAMES = (
+        "left", "top", "right", "bottom", "raw_black_level_separate0",
+        "raw_black_level_separate1", "raw_black_level_separate2",
+        "raw_black_level_separate3", "raw_white_point", "flat_field")
+    PARAMS_FORMAT = "iiiiHHHHHi"
+
+
+class WhiteBalance(Module):
+
+    NAME = "temperature"
+    VERSION = 3
+    PARAMS_NAMES = ("red", "green", "blue", "g2")
+    PARAMS_FORMAT = "ffff"

pytable/types.py

@@ -0,0 +1,202 @@
+import enum
+
+
+class Adaptation(enum.Enum):
+    LINEAR_BRADFORD = 0
+    CAT16           = 1
+    FULL_BRADFORD   = 2
+    XYZ             = 3
+    RGB             = 4
+
+
+class ColorSpacesColorProfile(enum.Enum):
+    NONE = -1
+    FILE = 0
+    SRGB = 1
+    ADOBERGB = 2
+    LIN_REC709 = 3
+    LIN_REC2020 = 4
+    XYZ = 5
+    LAB = 6
+    INFRARED = 7
+    DISPLAY = 8
+    EMBEDDED_ICC = 9
+    EMBEDDED_MATRIX = 10
+    STANDARD_MATRIX = 11
+    ENHANCED_MATRIX = 12
+    VENDOR_MATRIX = 13
+    ALTERNATE_MATRIX = 14
+    BRG = 15
+    EXPORT = 16
+    SOFTPROOF = 17
+    WORK = 18
+    DISPLAY2 = 19
+    REC709 = 20
+    PROPHOTO_RGB = 21
+    PQ_REC2020 = 22
+    HLG_REC2020 = 23
+    PQ_P3 = 24
+    HLG_P3 = 25
+    LAST = 26
+
+
+class ColorIntent(enum.Enum):
+    PERCEPTUAL = 0
+    RELATIVE_COLORIMETRIC = 1
+    SATURATION = 2
+    ABSOLUTE_COLORIMETRIC = 3
+
+
+class Illuminant(enum.Enum):
+    PIPE            = 0
+    A               = 1
+    D               = 2
+    E               = 3
+    F               = 4
+    LED             = 5
+    BB              = 6
+    CUSTOM          = 7
+    DETECT_SURFACES = 8
+    DETECT_EDGES    = 9
+    CAMERA          = 10
+
+
+class IlluminantLED(enum.Enum):
+    LED_B1  = 0
+    LED_B2  = 1
+    LED_B3  = 2
+    LED_B4  = 3
+    LED_B5  = 4
+    LED_BH1 = 5
+    LED_RGB1= 6
+    LED_V1  = 7
+    LED_V2  = 8
+
+
+class IOPOrderType(enum.Enum):
+  CUSTOM  = 0
+  LEGACY  = 1
+  V30     = 2
+  V30_JPG = 3
+
+class Color(enum.Enum):
+    RED = 0
+    YELLOW = 1
+    GREEN = 2
+    BLUE = 3
+    PURPLE = 4
+
+class ImageFlags(enum.Enum):
+    """
+    Flags that are stored in Image::flags. Extract flags with 'image.flag(flag)'
+
+    Extracted from:
+    https:  """
+    # Image is rejected
+    REJECTED = 8
+    THUMBNAIL_DEPRECATED = 16
+    # set during import if the image is low-dynamic range, i.e. doesn't
+    # need demosaic, wb, highlight clipping etc.
+    LDR = 32,
+    # set during import if the image is raw data, i.e. it needs demosaicing.
+    RAW = 64
+    # set during import if images is a high-dynamic range image..
+    HDR = 128
+    # set when marked for deletion
+    REMOVE = 256
+    # set when auto-applying presets have been applied to this image.
+    AUTO_PRESETS_APPLIED = 512
+    # legacy flag. is set for all new images. i hate to waste a bit on this :(
+    NO_LEGACY_PRESETS = 1024
+    # local copy status
+    LOCAL_COPY = 2048
+    # image has an associated .txt file for overlay
+    HAS_TXT = 4096
+    # image has an associated wav file
+    HAS_WAV = 8192
+    # image is a bayer pattern with 4 colors (e.g., CYGM or RGBE)
+    BAYER = 16384
+    # image was detected as monochrome
+    MONOCHROME = 32768
+    # DNG image has exif tags which are not cached in the database but
+    # must be read and stored in t when the image is loaded.
+    HAS_ADDITIONAL_EXIF_TAGS = 65536
+    # image is an sraw
+    S_RAW = 1 << 17
+    # image has a monochrome preview tested
+    MONOCHROME_PREVIEW = 1 << 18
+    # image has been set to monochrome via demosaic module
+    MONOCHROME_BAYER = 1 << 19
+    # image has a flag set to use the monochrome workflow in the modules supporting it
+    MONOCHROME_WORKFLOW = 1 << 20
+
+class ImageOrientation(enum.Enum):
+    NULL    = -1
+    NONE    = 0
+    FLIP_Y  = 1 << 0
+    FLIP_X  = 1 << 1
+    SWAP_XY = 1 << 2
+    FLIP_HORIZONTALLY = FLIP_X
+    FLIP_VERTICALLY   = FLIP_Y
+    ROTATE_180_DEG    = FLIP_Y | FLIP_X
+    TRANSPOSE         = SWAP_XY
+    ROTATE_CCW_90_DEG = FLIP_X | SWAP_XY
+    ROTATE_CW_90_DEG  = FLIP_Y | SWAP_XY
+    TRANSVERSE        = FLIP_Y | FLIP_X | SWAP_XY
+
+
+legacy_order = (
+  "rawprepare", "invert", "temperature", "highlights", "cacorrect",
+  "hotpixels", "rawdenoise", "demosaic", "mask_manager", "denoiseprofile",
+  "tonemap", "exposure", "spots", "retouch", "lens", "cacorrectrgb", "ashift",
+  "liquify", "rotatepixels", "scalepixels", "flip", "clipping", "toneequal",
+  "crop", "graduatednd", "basecurve", "bilateral", "profile_gamma",
+  "hazeremoval", "colorin", "channelmixerrgb", "diffuse", "censorize",
+  "negadoctor", "blurs", "basicadj", "colorreconstruct", "colorchecker",
+  "defringe", "equalizer", "vibrance", "colorbalance", "colorbalancergb",
+  "colorize", "colortransfer", "colormapping", "bloom", "nlmeans",
+  "globaltonemap", "shadhi", "atrous", "bilat", "colorzones", "lowlight",
+  "monochrome", "sigmoid", "filmic", "filmicrgb", "colisa", "zonesystem",
+  "tonecurve", "levels", "rgblevels", "rgbcurve", "relight", "colorcorrection",
+  "sharpen", "lowpass", "highpass", "grain", "lut3d", "colorcontrast",
+  "colorout", "channelmixer", "soften", "vignette", "splittoning", "velvia",
+  "clahe", "finalscale", "overexposed", "rawoverexposed", "dither", "borders",
+  "watermark", "gamma"
+)
+
+v30_order = (
+  "rawprepare", "invert", "temperature", "highlights", "cacorrect", "hotpixels",
+  "rawdenoise", "demosaic", "denoiseprofile", "bilateral", "rotatepixels",
+  "scalepixels", "lens", "cacorrectrgb","hazeremoval", "ashift", "flip",
+  "clipping", "liquify", "spots", "retouch", "exposure", "mask_manager",
+  "tonemap", "toneequal","crop","graduatednd", "profile_gamma", "equalizer",
+  "colorin", "channelmixerrgb", "diffuse", "censorize", "negadoctor","blurs",
+  "nlmeans","colorchecker","defringe","atrous", "lowpass", "highpass", "sharpen",
+  "colortransfer","colormapping", "channelmixer","basicadj", "colorbalance",
+  "colorbalancergb", "rgbcurve", "rgblevels", "basecurve","filmic", "sigmoid",
+   "filmicrgb", "lut3d", "colisa", "tonecurve", "levels", "shadhi",
+   "zonesystem", "globaltonemap", "relight", "bilat","colorcorrection",
+   "colorcontrast", "velvia", "vibrance", "colorzones", "bloom", "colorize",
+   "lowlight", "monochrome", "grain", "soften", "splittoning", "vignette",
+   "colorreconstruct", "colorout", "clahe", "finalscale", "overexposed",
+   "rawoverexposed", "dither", "borders", "watermark", "gamma"
+)
+
+v30_jpg_order = (
+  "rawprepare", "invert", "temperature", "highlights", "cacorrect", "hotpixels",
+  "rawdenoise", "demosaic", "colorin", "denoiseprofile", "bilateral",
+  "rotatepixels", "scalepixels", "lens", "cacorrectrgb", "hazeremoval",
+  "ashift", "flip", "clipping", "liquify", "spots", "retouch", "exposure",
+  "mask_manager", "tonemap", "toneequal", "crop", "graduatednd",
+  "profile_gamma", "equalizer", "channelmixerrgb", "diffuse", "censorize",
+  "negadoctor", "blurs", "nlmeans", "colorchecker","defringe","atrous",
+  "lowpass", "highpass", "sharpen", "colortransfer","colormapping",
+  "channelmixer","basicadj", "colorbalance", "colorbalancergb", "rgbcurve",
+  "rgblevels", "basecurve","filmic", "sigmoid", "filmicrgb", "lut3d", "colisa",
+  "tonecurve", "levels", "shadhi", "zonesystem", "globaltonemap", "relight",
+  "bilat","colorcorrection","colorcontrast", "velvia", "vibrance",
+  "colorzones", "bloom", "colorize", "lowlight", "monochrome", "grain",
+  "soften", "splittoning", "vignette", "colorreconstruct","colorout", "clahe",
+  "finalscale", "overexposed", "rawoverexposed", "dither", "borders",
+  "watermark", "gamma"
+)

synchronise.py

@@ -0,0 +1,297 @@
+from pytable.models import Image, ImageFlags, FilmRoll
+
+from datetime import datetime
+import peewee as pw
+import os
+import shutil
+from dataclasses import dataclass
+from typing import Tuple, Optional, Generic, TypeVar, Iterable
+
+T = TypeVar('T')
+
+Range = Tuple[Optional[T], Optional[T]]
+
+class Filter:
+
+
+    def __call__(self, image: Image):
+        return self.matches(image)
+
+    def matches(self, image: Image) -> bool:
+        ...
+
+@dataclass
+class LocalRegion(Filter):
+
+    date_range: Range[datetime] = (None, None)
+    stars_range: Range[int] = (None, None)
+
+    def matches(self, image: Image) -> bool:
+
+        stars = image.stars if not image.flag(ImageFlags.REJECTED) else -1
+        if self.stars_range[0] is not None and stars < self.stars_range[0]:
+            return False
+        if self.stars_range[1] is not None and stars > self.stars_range[1]:
+            return False
+
+        dt = (image.datetime_taken or image.datetime_imported)
+        if self.date_range[0] is not None and dt < self.date_range[0]:
+            return False
+        if self.date_range[1] is not None and dt > self.date_range[1]:
+            return False
+        return True
+
+@dataclass
+class Or(Filter):
+
+    filters: Iterable[Filter]
+
+    def matches(self, image: Image) -> bool:
+        return any(f(image) for f in self.filters)
+
+@dataclass
+class And(Filter):
+
+    filters: Iterable[Filter]
+
+    def matches(self, image: Image) -> bool:
+        return all(f(image) for f in self.filters)
+
+
+class Config:
+
+    def __init__(self) -> None:
+        self.filter = Or([
+            LocalRegion((datetime(2024, 6, 14), None), (0, None)),
+            LocalRegion((datetime(2024, 6, 22, 14), datetime(2024, 6, 22, 14, 24)))
+        ])
+
+        self.d_root = "/home/oke/Pictures/Darktable"
+        self.d_local = "/home/oke/Pictures/DarktableLocal"
+        self.d_remote = "/home/oke/Pictures/DarktableRemote"
+
+
+    @staticmethod
+    def _is_older_than(image: Image, date: datetime):
+        return (image.datetime_taken or image.datetime_imported) < date
+
+    def should_not_be_local(self, image: Image):
+        return not self.filter(image)
+
+    def should_be_local(self, image: Image):
+        return self.filter(image)
+
+    def image_local_folder(self, image: Image):
+        folder = image.film.folder
+        if not folder.startswith(self.d_root):
+            raise NotImplementedError()
+        assert folder.startswith(self.d_root)
+        return folder.replace(self.d_root, self.d_local)
+
+    def image_local_image(self, image: Image) -> str:
+        return os.path.join(self.image_local_folder(image), image.filename)
+
+    def image_local_image_xmp(self, image: Image) -> str:
+        base, ext = os.path.splitext(image.filename)
+        version = ("_%02d" % image.version) if image.version != 0 else ""
+        xmp_filename = base + version + ext + '.xmp'
+        return os.path.join(self.image_local_folder(image), xmp_filename)
+
+    def image_local_iphone_movie(self, image: Image) -> str:
+        base, _ = os.path.splitext(image.filename)
+        return os.path.join(self.image_local_folder(image), "." + base + ".mov")
+
+    def potential_extras(self, image):
+        yield self.image_local_iphone_movie(image)
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+from enum import Enum
+
+class SyncReq(Enum):
+    EXISTS = 0
+    REQUIRED_SOFT = 1
+    REQUIRED = 2
+
+    @staticmethod
+    def max(a: "SyncReq", b: "SyncReq"):
+        if a.value < b.value:
+            return b
+        return a
+    @staticmethod
+    def required(v: 'SyncReq'):
+        return v == SyncReq.REQUIRED or v == SyncReq.REQUIRED_SOFT
+
+    @staticmethod
+    def make(required, optional):
+        if required:
+            if optional:
+                return SyncReq.REQUIRED_SOFT
+            return SyncReq.REQUIRED
+        return SyncReq.EXISTS
+
+@dataclass
+class FileInfo:
+    required: bool = False
+    optional: bool = False
+
+class SyncManager:
+
+    def __init__(self, config) -> None:
+        self._config = config
+        self._files: Dict[str, SyncReq] = {}
+
+    def register_file(self, fn: str, requirement: SyncReq):
+        if fn in self._files:
+            requirement = SyncReq.max(self._files[fn], requirement)
+        self._files[fn] = requirement
+
+    def register_image(self, image: Image):
+        required = self._config.should_be_local(image)
+        self.register_file(
+            self._config.image_local_image(image),
+            SyncReq.make(required, False)
+        )
+        self.register_file(
+            self._config.image_local_image_xmp(image),
+            SyncReq.make(required, False)
+        )
+
+        for extra in self._config.potential_extras(image):
+            self.register_file(
+                extra,
+                SyncReq.make(required, True)
+            )
+
+    def partition(self):
+        result = {k: [] for k in SyncReq}
+        for file, req in self._files.items():
+            result[req].append(file)
+        return result
+
+config = Config()
+
+sync_manager = SyncManager(config)
+
+query = Image.filter()
+pw.prefetch(query, FilmRoll)
+for image in query:
+    sync_manager.register_image(image)
+
+
+actions = sync_manager.partition()
+
+for key in actions:
+    actions[key] = list(sorted(map(
+        lambda fn: os.path.relpath(fn, config.d_local) + "\n",
+        actions[key]
+    )))
+
+with open("required.txt", "w") as f:
+    f.writelines(actions[SyncReq.REQUIRED])
+import subprocess
+user = "oke"
+host = "192.168.20.2"
+
+args = [
+    "rsync",
+    # "--dry-run",
+    "--info=progress",
+    "--ignore-existing",
+    "-av",
+    "--files-from",
+    "required.txt",
+    f"{user}@{host}:lenovo-darktable{config.d_local}",
+    os.path.abspath(config.d_local)
+]
+import shlex
+print(shlex.join(args))
+subprocess.call(args)
+
+with open("required_optional.txt", "w") as f:
+    f.writelines(actions[SyncReq.REQUIRED_SOFT])
+
+with open("remove.txt", "w") as f:
+    f.writelines(actions[SyncReq.EXISTS])
+
+exit()
+for fn in actions[SyncReq.REQUIRED_SOFT]:
+    if not os.path.exists(fn):
+        print("FETCH opt: ", fn)
+
+for fn in sync_manager.files_to_fetch_required():
+    print("FETCH: ", fn)
+
+for fn in sync_manager.files_to_remove():
+    print("RM: ", fn)
+n_total = 0
+n_not_needed = 0
+needs_removal = []
+extra_files_to_remove = []
+bytes_to_remove = 0
+
+needs_fetching = []
+
+for file, usages in file_usages.items():
+    if not file.startswith(d_root):
+        print("SKIPPNG", file)
+        continue
+    assert file.startswith(d_root)
+    remote_file = file.replace(d_root, d_remote)
+    local_file = file.replace(d_root, d_local)
+
+    if usages > 0:
+        if not os.path.exists(local_file):
+            needs_fetching.append((remote_file, local_file))
+    else:
+        n_not_needed += 1
+        if os.path.exists(local_file):
+            needs_removal.append(local_file)
+            bytes_to_remove += os.stat(local_file).st_size
+
+        for extra_fn in potential_extras(local_file):
+            if os.path.exists(extra_fn):
+                extra_files_to_remove.append(extra_fn)
+                bytes_to_remove += os.stat(extra_fn).st_size
+
+files_to_remove = list(sorted(needs_removal + extra_files_to_remove))
+
+needs_fetching = sorted(needs_fetching)
+import pprint
+print("Files to fetch: ")
+pprint.pprint(needs_fetching)
+print("Extra files to remove:")
+pprint.pprint(extra_files_to_remove)
+print("First files to remove:")
+pprint.pprint(needs_removal[:10])
+print("Last files to remove:")
+pprint.pprint(needs_removal[-10:])
+
+with open("remove.txt", "w") as f:
+    pprint.pprint(sorted(set(needs_removal).union(extra_files_to_remove)), f)
+
+print("Not needed:            %6d / %6d (%3d%%)" % (n_not_needed, len(file_usages), 100 * n_not_needed / len(file_usages)))
+print()
+print("Not yet deleted:       %6d / %6d (%3d%%)" % (len(needs_removal), n_not_needed, 100 * len(needs_removal) / n_not_needed))
+print("Bytes to free:         %5.2fGB" % (bytes_to_remove / 1024. / 1024 / 1024))
+print("Total files to remove: %d" % len(files_to_remove))
+print()
+print("Total files to fetch: %3d" % len(needs_fetching))
+print()
+
+reply = input("Type YES to remove all these files. This is not reversible... : ")
+
+if reply == 'YES':
+
+    for fn in files_to_remove:
+        assert fn.startswith(d_local)
+        os.remove(fn)
+
+    for f, t in needs_fetching:
+        assert os.path.exists(f), "Backup corruped, file not found: " + f
+        assert not os.path.exists(t)
+        print(f, t)
+        shutil.copy(f, t)
+
+else:
+    print("You did not type 'YES'. Not doing anything!")

vmgr/actions.py

@@ -0,0 +1,278 @@
+import subprocess
+from dataclasses import dataclass
+from typing import List, Callable, Any, Optional
+import functools
+import inspect
+import sys
+import os
+import datetime
+import traceback
+
+from video import VideoModel, Video, BASE_DIR
+from typing import Tuple
+
+
+TRASH_DT_FORMAT = "%Y-%m-%dT%H:%M:%S"
+TRASH = "/home/oke/.local/share/Trash"
+TRASH_INFO = os.path.join(TRASH, "info")
+TRASH_FILES = os.path.join(TRASH, "files")
+
+import json
+from typing import Generic, TypeVar
+
+T = TypeVar('T')
+V = TypeVar('V')
+
+class ActionException(Exception):
+
+    def __init__(self, message, *args: object) -> None:
+        self.message = message
+        super().__init__(*args)
+
+    def __repr__(self) -> str:
+        return f"<Exception: '{self.message}'>"
+
+class ActionFailed(ActionException):
+    ...
+
+class ActionIncomplete(ActionException):
+    ...
+
+import uuid
+class UndoableAction(Generic[T]):
+
+    ID = "OverrideMe"
+
+    def __init__(self, args: T, group_leader=None, uid=None):
+        self.args: T = args
+        self.uuid = uid
+        if self.uuid is None:
+            self.uuid = group_leader.uuid if group_leader else uuid.uuid4().hex
+
+    def undo_action(self) -> "UndoableAction[Any]": ...
+
+    def run(self, model: VideoModel) -> Optional[bool]: ...
+
+
+class RestoreFromTrash(UndoableAction[str]):
+
+    def undo_action(self) -> UndoableAction[Any]:
+        return Trash(self.args)
+
+    def __str__(self) -> str:
+        return f"Restoring {self.args}"
+
+    @staticmethod
+    def trashed_files():
+        for fn in os.listdir(TRASH_INFO):
+            if fn.endswith('.trashinfo'):
+                with open(os.path.join(TRASH_INFO, fn), 'r') as f:
+                    lines_iter = iter(f)
+                    if next(lines_iter) != '[Trash Info]\n':
+                        print("Invalid trashinfo file: ", fn)
+                        continue
+                    path = None
+                    date = None
+                    for line in f:
+                        line = line.strip()
+                        if path is None and line.startswith('Path='):
+                            path = line[5:]
+                        if date is None and line.startswith('DeletionDate='):
+                            date = datetime.datetime.strptime(line[13:], TRASH_DT_FORMAT)
+                    if path is None or date is None:
+                        print("Invalid trashinfo file: ", fn)
+                        continue
+                    trash_file = os.path.join(TRASH_FILES, fn[:-10])
+                    if not os.path.exists(trash_file):
+                        print("Trashed file does not exist: ", trash_file)
+                        continue
+                    yield (date, trash_file, path)
+
+
+    def run(self, model):
+        file = self.args
+        matches = sorted(((date, trash_file) for (date, trash_file, path) in self.trashed_files() if path == self.args), reverse=True)
+        if len(matches) == 0:
+            raise ActionFailed("Could not find trashed file")
+        try:
+            os.rename(matches[0][1], file)
+        except Exception as e:
+            traceback.print_exception(e)
+            raise ActionFailed("Could not move file to original destination...")
+        try:
+            os.remove(os.path.join(TRASH_INFO, os.path.basename(file) + ".trashinfo"))
+        except Exception as e:
+            traceback.print_exception(e)
+
+        return True
+
+
+
+class Trash(UndoableAction[str]):
+
+    def undo_action(self) -> UndoableAction[Any]:
+        return RestoreFromTrash(self.args)
+
+    def __str__(self) -> str:
+        return f"Trashing {self.args}"
+
+    def run(self, model: VideoModel):
+        file = os.path.abspath(os.path.expanduser(self.args))
+        i = 0
+        suffix = ""
+        while True:
+            info_fn = os.path.join(TRASH_INFO, os.path.basename(file) + suffix + ".trashinfo")
+            try:
+                with open(info_fn, 'x') as f:
+                    f.write(
+                        "[Trash Info]\n"
+                        f"Path={file}\n"
+                        f"DeletionDate={datetime.datetime.now().strftime(TRASH_DT_FORMAT)}\n"
+                    )
+            except FileExistsError:
+                i += 1
+                suffix = f"_{i}"
+                continue
+
+            trash_fn = os.path.join(TRASH_FILES, os.path.basename(file) + suffix)
+            if os.path.exists(trash_fn):
+                os.remove(info_fn)
+                i += 1
+                suffix = f"_{i}"
+                continue
+            os.rename(file, trash_fn)
+            index, video = model.find_video(Video(self.args))
+            if index:
+                model.remove(index)
+            break
+
+class SetVideoStars(UndoableAction[Tuple[str, bool, int, bool, int]]):
+
+    def run(self, model: VideoModel):
+        file, old_reject, old_rating, new_reject, new_rating = self.args
+        index, video = model.find_video(Video(file))
+        video.metadata.rejected = new_reject
+        video.metadata.rating = new_rating
+        video.save_metadata()
+        if index:
+            model.dataChanged.emit(index, index)
+
+    def undo_action(self) -> UndoableAction[Any]:
+        file, old_reject, old_rating, new_reject, new_rating = self.args
+        return SetVideoStars((file, new_reject, new_rating, old_reject, old_rating))
+
+    def __str__(self):
+        file, old_reject, old_rating, new_reject, new_rating = self.args
+        return f"{file}: rejected status ({old_reject} -> {new_reject}) and rating ({old_rating} -> {new_rating})"
+
+class UndoableActionList:
+
+    def __init__(self, fn: str) -> None:
+        self._data: List[UndoableAction] = []
+        self._fn: str = fn
+        self._load()
+
+    def _load(self):
+        with open(self._fn, "a") as f:
+            pass
+
+        self.f = open(self._fn, 'r+')
+
+        for line in self.f:
+            ID, uuid, args = json.loads(line.strip())
+
+            revert_fns = [obj for name, obj in inspect.getmembers(sys.modules[__name__])
+                          if inspect.isclass(obj) and name == ID]
+            assert len(revert_fns) == 1
+            self._data.append(revert_fns[0](args, uid=uuid))
+        self.f.seek(0, os.SEEK_END)
+
+    def append(self, action: UndoableAction):
+        json.dump((action.__class__.__name__, action.uuid, action.args), self.f)
+        self.f.write("\n")
+        self.f.flush()
+        self._data.append(action)
+
+    def pop(self, uuid=None):
+        if not self._data:
+            return None
+        if uuid and self._data[-1].uuid != uuid:
+            return None
+        action = self._data.pop()
+        self._remove_line()
+        return action
+
+    def clear(self):
+        self._data = []
+        self.f.seek(0, os.SEEK_SET)
+        self.f.truncate()
+        self.f.flush()
+
+    def _remove_line(self):
+        pos = self.f.tell() - 1
+
+        if pos == -1:
+            return
+
+        while pos > 0 and self.f.read(1) != "\n":
+            pos -= 1
+            self.f.seek(pos, os.SEEK_SET)
+        if pos != 0:
+            pos += 1
+        self.f.seek(pos, os.SEEK_SET)
+        self.f.truncate()
+        self.f.flush()
+
+class HistoryManager:
+
+    def __init__(self) -> None:
+        self.history = UndoableActionList(os.path.join(BASE_DIR, "vmgr.history"))
+        self.future = UndoableActionList(os.path.join(BASE_DIR, "vmgr.redo"))
+
+    def apply(self, model, *actions: UndoableAction):
+        should_reload = False
+        self.future.clear()
+        for action in actions:
+            print(action)
+            should_reload = action.run(model) or should_reload
+            self.history.append(action)
+        if should_reload:
+            model.reload()
+        else:
+            model.update()
+
+    @staticmethod
+    def move(stack_1: UndoableActionList, stack_2, model, undo):
+        action: Optional[UndoableAction] = stack_1.pop()
+        should_reload = False
+        while action is not None:
+            try:
+                if undo:
+                    print(action.undo_action())
+                    should_reload = action.undo_action().run(model) or should_reload
+                else:
+                    print(action)
+                    should_reload = action.run(model) or should_reload
+            except:
+                stack_1.append(action)
+                raise
+            stack_2.append(action)
+            action = stack_1.pop(action and action.uuid)
+        return should_reload
+
+    def undo(self, model):
+        should_reload = self.move(self.history, self.future, model, True)
+        if should_reload:
+            model.reload()
+        else:
+            model.update()
+
+    def redo(self, model: VideoModel):
+        should_reload = self.move(self.future, self.history, model, False)
+        if should_reload:
+            model.reload()
+        else:
+            model.update()
+
+
+

vmgr/main.py

@@ -0,0 +1,445 @@
+import vlc
+import sys
+import os.path
+import vlc
+from PyQt5 import QtWidgets, QtCore, QtGui
+import subprocess
+from video import  VideoModel
+from superqt import QLabeledRangeSlider
+from actions import UndoableAction, HistoryManager
+from typing import List
+import os
+from datetime import datetime
+
+
+history = HistoryManager()
+
+class Player(QtWidgets.QMainWindow):
+    """A simple Media Player using VLC and Qt
+    """
+    def __init__(self, master=None):
+        QtWidgets.QMainWindow.__init__(self, master)
+        self.setWindowTitle("Media Player")
+
+        # creating a basic vlc instance
+        self.instance = vlc.Instance()
+        # creating an empty vlc media player
+        self.mediaplayer = self.instance.media_player_new()
+        self.createUI()
+        self.isPaused = False
+
+
+    def createUI(self):
+        """Set up the user interface, signals & slots
+        """
+        self.widget = QtWidgets.QWidget(self)
+        self.setCentralWidget(self.widget)
+
+        # In this widget, the video will be drawn
+        if sys.platform == "darwin": # for MacOS
+            self.videoframe = QtWidgets.QMacCocoaViewContainer(0)
+        else:
+            self.videoframe = QtWidgets.QFrame()
+        self.palette = self.videoframe.palette()
+        self.palette.setColor (QtGui.QPalette.Window,
+                               QtGui.QColor(0,0,0))
+        self.videoframe.setPalette(self.palette)
+        self.videoframe.setAutoFillBackground(True)
+
+        self.positionslider = QtWidgets.QSlider(QtCore.Qt.Horizontal, self)
+        self.positionslider.setToolTip("Position")
+        self.positionslider.setMaximum(1000)
+
+        def mousePressEvent(event):
+            if event.button() == QtCore.Qt.LeftButton:
+                val = self.pixelPosToRangeValue(event.pos())
+                self.setValue(val)
+
+        self.positionslider.sliderMoved.connect(self.setPosition)
+        self.positionslider.sliderPressed.connect(lambda *args: print(args))
+
+
+        self.hbuttonbox = QtWidgets.QHBoxLayout()
+        self.playbutton = QtWidgets.QPushButton("Play")
+        self.hbuttonbox.addWidget(self.playbutton)
+        self.playbutton.clicked.connect(self.PlayPause)
+
+        self.hbuttonbox.addStretch(1)
+        self.setVolume(100)
+
+        self.videoplayerlayout = QtWidgets.QVBoxLayout()
+        self.videoplayerlayout.addWidget(self.videoframe)
+        self.videoplayerlayout.addWidget(self.positionslider)
+        self.videoplayerlayout.addLayout(self.hbuttonbox)
+
+
+        self.videoplayerlayoutWidget = QtWidgets.QWidget();
+        self.videoplayerlayoutWidget.setLayout(self.videoplayerlayout);
+
+
+
+        import dataclasses
+        def move_current_frame(d_frames):
+            before = self.mediaplayer.get_time()
+
+            mspf = int(1000 // (self.mediaplayer.get_fps() or 25))
+            d_time_ms = d_frames * mspf
+            target = self.mediaplayer.get_time() + d_time_ms
+            result = self.mediaplayer.set_time(target)
+
+            print(before, target, result, self.mediaplayer.get_time())
+
+
+        def pp():
+            self.PlayPause()
+            print(self.mediaplayer.get_time())
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Space"), self)
+        self.shortcut.activated.connect(pp)
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Right"), self)
+        self.shortcut.activated.connect(lambda : move_current_frame(1))
+
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Left"), self)
+        self.shortcut.activated.connect(lambda : move_current_frame(-1))
+
+
+        def export_frame():
+            print("XX")
+            t = self.mediaplayer.get_time()
+            print(t)
+            ms = t % 1000
+            t = t // 1000
+            s = t % 60
+            t = t // 60
+            m = t % 60
+            t = t // 60
+            h = t % 60
+            t = t // 60
+            t_str = f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}"
+            print("Exporting: ", self.filename, t_str)
+            import os
+            from datetime import timedelta
+
+            def apply(filename, t_str):
+                base_dir = os.path.expanduser(os.path.join("~", "Pictures", "video_exports"))
+
+                print("W", filename, t_str)
+                if not os.path.exists(base_dir):
+                    os.makedirs(base_dir)
+                print("W", filename, t_str)
+                begin = datetime.now()
+                base_name = os.path.splitext(os.path.basename(filename))[0]
+
+                print("W", filename, t_str)
+                print(base_name[:13])
+                frame_datetime = datetime.strptime(base_name[:13], "%Y%m%d_%H%M") + timedelta(hours=h, minutes=m, seconds=s, milliseconds=ms)
+                print(base_name[:13])
+                frame_datetime_str = frame_datetime.strftime("%Y:%m:%d_%H:%M:%S")
+                print(base_name[:13])
+
+                print("W", filename, t_str)
+                out =  os.path.join(base_dir,
+                                 base_name +
+                                                 f"{h:02d}{m:02d}{s:02d}{ms:03d}.png")
+                print(out)
+                subprocess.call([
+                    "ffmpeg",
+                    "-ss", t_str,
+                    "-i",  filename,
+                    "-vframes", "1",
+                    "-c:v", "png",
+                    out
+                   ])
+                subprocess.call([
+                    "exiftool",
+                    f"-CreateDate={frame_datetime_str}",
+                    f"-DateTimeOriginal={frame_datetime_str}",
+                    f"-PNG:CreationTime={frame_datetime_str}",
+                    "-overwrite_original",
+                    out
+                ])
+                print("Duration: ", datetime.now() - begin)
+            from video import thumbnailer
+
+            thumbnailer.pool.apply_async(apply, args=(self.filename, t_str))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+e"), self)
+        self.shortcut.activated.connect(lambda : export_frame())
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("f"), self)
+        self.shortcut.activated.connect(lambda : self.mediaplayer.toggle_fullscreen())
+
+        def slowmo():
+            if self.mediaplayer.get_rate() < 0.7:
+                print("rate", 1)
+                self.mediaplayer.set_rate(1)
+            else:
+                print("rate", .5)
+                self.mediaplayer.set_rate(0.5)
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("s"), self)
+        self.shortcut.activated.connect(slowmo)
+
+        def change_size(diff):
+            self.icon_size += diff
+            print(self.icon_size < 64)
+            if self.icon_size < 64:
+                self.icon_size = 64
+            if self.icon_size > 256:
+                self.icon_size = 256
+            print(self.icon_size)
+            self.list_view.setIconSize(QtCore.QSize(self.icon_size, self.icon_size))
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("+"), self)
+        self.shortcut.activated.connect(lambda : change_size(64))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("-"), self)
+        self.shortcut.activated.connect(lambda : change_size(-64))
+        from typing import Optional
+        def undo():
+            history.undo(self.model)
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+z"), self)
+        self.shortcut.activated.connect(lambda : undo())
+        def redo():
+            history.redo(self.model)
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+Shift+z"), self)
+        self.shortcut.activated.connect(lambda : redo())
+
+
+        def set_rating(value):
+            selected = self.list_view.selectedIndexes()
+
+            stars = None
+            actions = []
+            for i, index in enumerate(sorted(selected, key=lambda r: -r.row())):
+                video = self.model.video_files[index.row()]
+                from actions import SetVideoStars
+                stars = SetVideoStars((
+                    video.fn,
+                    video.metadata.rejected,
+                    video.metadata.rating,
+                    False if value >= 0 else not video.metadata.rejected,
+                    value if value >= 0 else video.metadata.rating
+                ), stars)
+                stars.run(self.model)
+                actions.append(stars)
+            history.apply(self.model, *actions)
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("0"), self)
+        self.shortcut.activated.connect(lambda : set_rating(0))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("1"), self)
+        self.shortcut.activated.connect(lambda : set_rating(1))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("2"), self)
+        self.shortcut.activated.connect(lambda : set_rating(2))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("3"), self)
+        self.shortcut.activated.connect(lambda : set_rating(3))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("4"), self)
+        self.shortcut.activated.connect(lambda : set_rating(4))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("5"), self)
+        self.shortcut.activated.connect(lambda : set_rating(5))
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("r"), self)
+        self.shortcut.activated.connect(lambda : set_rating(-1))
+
+        def link():
+            directory = QtWidgets.QFileDialog.getExistingDirectory(
+                self, "Open File", os.path.expanduser('~'))
+
+            if not os.path.isdir(directory):
+                QtWidgets.QMessageBox.warning(
+                    self,
+                    "Not a directory",
+                    "The selected item was not a directory. "
+                    "You need to select an empty folder. ")
+                return
+
+            if os.listdir(directory):
+                QtWidgets.QMessageBox.warning(
+                    self,
+                    "Non-empty directory selected",
+                    "The selected directory is not empty. "
+                    "You need to select an empty folder. ")
+                return
+
+            print(directory)
+            failure = False
+            for index in self.list_view.selectedIndexes():
+                video = self.model.video_files[index.row()]
+                failure = failure or (
+                    0 != subprocess.call([
+                        "ln",
+                        "-s",
+                        video.fn,
+                        os.path.join(directory, os.path.basename(video.fn)),
+                    ])
+                )
+            if failure:
+                QtWidgets.QMessageBox.warning(
+                    self,
+                    "Linkage failure",
+                    "Some files could not be linked.")
+
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Ctrl+L"), self)
+        self.shortcut.activated.connect(link)
+
+        def delete():
+            from actions import Trash
+            selected = self.list_view.selectedIndexes()
+            response = QtWidgets.QMessageBox.question(self, "Delete Items", "Do you really want to delete %d items?" % len(selected), defaultButton=QtWidgets.QMessageBox.No)
+            if response == QtWidgets.QMessageBox.Yes:
+                actions = []
+                for i, index in enumerate(sorted(selected, key=lambda r: -r.row())):
+                    video = self.model.video_files[index.row()]
+                    action = Trash(video.fn, actions[-1] if len(actions) else None)
+                    actions.append(action)
+                history.apply(self.model, *actions)
+
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("Del"), self)
+        self.shortcut.activated.connect(lambda : delete())
+
+        self.shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("F5"), self)
+        self.shortcut.activated.connect(lambda : self.model.reload())
+
+        self.model = VideoModel()
+
+        def open_file(index):
+            self.OpenFile(self.model.video_files[index.row()].fn)
+
+        self.list_view = QtWidgets.QListView()
+        self.list_view.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.ExtendedSelection)
+        self.list_view.doubleClicked.connect(open_file)
+        self.list_view.setViewMode(QtWidgets.QListView.IconMode)
+        self.list_view.setResizeMode(QtWidgets.QListView.Adjust)
+        self.list_view.setFlow(QtWidgets.QListView.LeftToRight)
+        self.list_view.setMovement(QtWidgets.QListView.Snap)
+        self.list_view.setModel(self.model)
+        # self.list_view.setGridSize(QtCore.QSize(256 + 10, 256 + 10))
+        self.icon_size = 256
+        self.list_view.setIconSize(QtCore.QSize(self.icon_size, self.icon_size))
+        self.list_view.setWordWrap(True)
+
+        self.hboxlayout = QtWidgets.QSplitter()
+        self.hboxlayout.addWidget(self.list_view)
+        self.hboxlayout.addWidget(self.videoplayerlayoutWidget)
+
+        self.filter_slider = QLabeledRangeSlider(QtCore.Qt.Horizontal)
+        self.filter_slider.setRange(0, 6)
+        self.filter_slider.setValue((0, 6))
+        self.filter_slider.setMaximum(6)
+        self.filter_slider.setMinimum(-1)
+
+        def set_rating_filter(v):
+            self.model.filter_rating = v
+        self.filter_slider.sliderMoved.connect(set_rating_filter)
+        self.parentLayout = QtWidgets.QVBoxLayout()
+        self.parentLayout.addWidget(self.filter_slider, 0)
+        self.parentLayout.addWidget(self.hboxlayout, 2)
+        self.widget.setLayout(self.parentLayout)
+
+        exit = QtWidgets.QAction("&Exit", self)
+        exit.triggered.connect(sys.exit)
+        menubar = self.menuBar()
+        filemenu = menubar.addMenu("&File")
+        filemenu.addAction(exit)
+
+        self.timer = QtCore.QTimer(self)
+        self.timer.setInterval(200)
+        self.timer.timeout.connect(self.updateUI)
+
+    def PlayPause(self):
+        """Toggle play/pause status
+        """
+        if self.mediaplayer.is_playing():
+            self.mediaplayer.pause()
+            self.playbutton.setText("Play")
+            self.isPaused = True
+        else:
+            if self.mediaplayer.play() == -1:
+                self.OpenFile()
+                return
+            self.mediaplayer.play()
+            self.playbutton.setText("Pause")
+            self.timer.start()
+            self.isPaused = False
+
+    def Stop(self):
+        """Stop player
+        """
+        self.mediaplayer.stop()
+        self.playbutton.setText("Play")
+
+    def OpenFile(self, filename=None):
+        """Open a media file in a MediaPlayer
+        """
+        if filename is None:
+            return
+            filename = QtWidgets.QFileDialog.getOpenFileName(self, "Open File", os.path.expanduser('~'))
+        if not filename:
+            return
+
+        # create the media
+        if sys.version < '3':
+            filename = unicode(filename)
+        self.filename = filename
+        self.media = self.instance.media_new(filename)
+        # put the media in the media player
+        self.mediaplayer.set_media(self.media)
+
+        # parse the metadata of the file
+        self.media.parse()
+        # set the title of the track as window title
+        self.setWindowTitle(self.media.get_meta(0))
+
+        # the media player has to be 'connected' to the QFrame
+        # (otherwise a video would be displayed in it's own window)
+        # this is platform specific!
+        # you have to give the id of the QFrame (or similar object) to
+        # vlc, different platforms have different functions for this
+        if sys.platform.startswith('linux'): # for Linux using the X Server
+            self.mediaplayer.set_xwindow(int(self.videoframe.winId()))
+        elif sys.platform == "win32": # for Windows
+            self.mediaplayer.set_hwnd(self.videoframe.winId())
+        elif sys.platform == "darwin": # for MacOS
+            self.mediaplayer.set_nsobject(self.videoframe.winId())
+        self.PlayPause()
+
+    def setVolume(self, Volume):
+        """Set the volume
+        """
+        self.mediaplayer.audio_set_volume(Volume)
+
+    def setPosition(self, position):
+        """Set the position
+        """
+        # setting the position to where the slider was dragged
+        self.mediaplayer.set_position(position / 1000.0)
+        # the vlc MediaPlayer needs a float value between 0 and 1, Qt
+        # uses integer variables, so you need a factor; the higher the
+        # factor, the more precise are the results
+        # (1000 should be enough)
+
+    def updateUI(self):
+        """updates the user interface"""
+        # setting the slider to the desired position
+        self.positionslider.setValue(int(self.mediaplayer.get_position() * 1000))
+
+        if not self.mediaplayer.is_playing():
+            # no need to call this function if nothing is played
+            self.timer.stop()
+            if not self.isPaused:
+                # after the video finished, the play button stills shows
+                # "Pause", not the desired behavior of a media player
+                # this will fix it
+                self.Stop()
+
+if __name__ == "__main__":
+    app = QtWidgets.QApplication(sys.argv)
+
+
+
+    player = Player()
+    player.show()
+    player.resize(640, 480)
+    if sys.argv[1:]:
+        player.OpenFile(sys.argv[1])
+    sys.exit(app.exec_())

vmgr/thumbnails.py

@@ -0,0 +1,74 @@
+import os
+import multiprocessing
+import hashlib
+import subprocess
+from PyQt5 import QtGui
+import multiprocessing.pool
+from typing import Dict, Optional
+
+class CachedThumbnailGenerator:
+
+    def __init__(self, cache_folder) -> None:
+        self.cache_folder = cache_folder
+        self.memory_cache: Dict[str, Optional[QtGui.QIcon]] = {}
+        self.pool = multiprocessing.pool.ThreadPool(1)
+
+    def create_thumbnail(self, video_file, callback=None, error_callback=None):
+        self.pool.apply_async(
+            self._get_thumbnail_file, (video_file,),
+            callback=callback, error_callback=error_callback)
+
+    def get_thumbnail(self, video_file, callback=None, error_callback=None):
+        cache_key = video_file
+
+        if cache_key not in self.memory_cache:
+            self.memory_cache[cache_key] = None
+
+            def inner_callback(thumbnail_file):
+                if not thumbnail_file:
+                    return
+                image = QtGui.QImage(thumbnail_file)
+                pixmap = QtGui.QPixmap.fromImage(image)
+                self.memory_cache[cache_key] = QtGui.QIcon(pixmap)
+                if callback:
+                    callback()
+
+            self.create_thumbnail(
+                video_file, inner_callback, error_callback)
+
+        if cache_key in self.memory_cache and self.memory_cache[cache_key]:
+            return self.memory_cache[cache_key]
+        return None
+
+    def _cache_file(self, video_file):
+        abs_path = os.path.abspath(os.path.expanduser(video_file)) #  + str(sec)
+        digest = hashlib.md5(abs_path.encode()).hexdigest()
+        return os.path.join(self.cache_folder, digest[:1], digest[1:2], digest + ".jpg")
+
+    def _load_cache(self, video_file):
+        pass
+
+    @staticmethod
+    def _generate_thumbnail(video_fn, thumbnail_fn):
+        subprocess.check_call([
+            'ffmpeg', '-i',
+            video_fn,
+            "-threads", "1",
+            '-ss', '00:00:00.000',
+            '-c:v', 'mjpeg',
+            '-vframes', '1',
+            '-frames:v', '1',
+            '-filter:v', 'scale=300:300:force_original_aspect_ratio=increase,crop=300:300',
+            # '-f', 'image2pipe',
+            # '-'
+            thumbnail_fn
+        ])
+
+    def _get_thumbnail_file(self, video_file):
+        cache_file = self._cache_file(video_file)
+        if not os.path.exists(cache_file):
+            if not os.path.exists(os.path.dirname(cache_file)):
+                os.makedirs(os.path.dirname(cache_file))
+            self._generate_thumbnail(video_file, cache_file)
+            assert os.path.exists(cache_file)
+        return cache_file

vmgr/video.py

@@ -0,0 +1,252 @@
+import os
+from PyQt5 import QtWidgets, QtCore, QtGui
+import itertools
+from typing import Any, List, Dict
+from thumbnails import CachedThumbnailGenerator
+
+from typing import Tuple, Optional
+import yaml
+
+BASE_DIR = "/home/oke/Pictures/DarktableLocal/"
+CACHE_FOLDER = os.path.expanduser("~/.cache/video_manager")
+
+if not os.path.exists(CACHE_FOLDER):
+    os.makedirs(CACHE_FOLDER)
+
+thumbnailer = CachedThumbnailGenerator(CACHE_FOLDER)
+
+
+class Metadata:
+
+    def __init__(self, rating=0, rejected=False):
+        assert type(rating) == int
+        self.rating = rating
+        self.rejected = rejected
+
+    @classmethod
+    def from_file(cls, fn):
+        with open(fn) as f:
+            data = yaml.load(f, yaml.SafeLoader)
+        return Metadata(**data)
+
+    def save(self, fn):
+        with open(fn, "w") as f:
+            yaml.dump(self.__dict__, f)
+
+class Video:
+
+    def __init__(self, fn: str) -> None:
+        self.fn = fn
+        self._metadata = None
+
+    def __repr__(self):
+        return os.path.basename(self.fn)
+
+    @property
+    def metadata(self):
+        if not self._metadata:
+            if os.path.exists(self.metadata_file()):
+                self._metadata = Metadata.from_file(self.metadata_file())
+            else:
+                self._metadata = Metadata()
+        return self._metadata
+
+    def metadata_file(self):
+        return self.fn + '.metadata'
+
+    def save_metadata(self):
+        self.metadata.save(self.metadata_file())
+
+    def load_thumbnail(self, thumbnailer, callback=None, error_callback=None):
+        thumbnailer.create_thumbnail(self.fn, callback, error_callback)
+
+    def __lt__(self, other):
+        return self.fn > other.fn
+
+    def __eq__(self, other):
+        return self.fn == other.fn
+
+class VideoModel(QtCore.QAbstractListModel):
+
+
+    def __init__(self):
+        super().__init__()
+        self._filter_rating = (0, 6)
+        self.show_up_to = 10
+
+        self.video_obj_cache = {}
+        self.all_files: List[Video] = self.collect_files()
+        self.video_files: List[Video] = self.filter_files(self.all_files)
+
+
+        pixmap = QtGui.QPixmap(256, 256)
+        pixmap.fill(QtCore.Qt.transparent)
+        painter = QtGui.QPainter(pixmap)
+        painter.setRenderHint(QtGui.QPainter.Antialiasing)
+        painter.setBrush(QtGui.QColor("red"))
+        painter.drawEllipse(pixmap.rect())
+        painter.end()
+        self.default_icon = QtGui.QIcon(pixmap)
+
+    def collect_files(self, base_dir=BASE_DIR):
+        def _():
+            for film_roll in os.listdir(base_dir):
+                roll_dir = os.path.join(base_dir, film_roll)
+                if os.path.isdir(roll_dir):
+                    for fn in os.listdir(roll_dir):
+                        if fn[0] != '.' and os.path.splitext(fn)[-1].lower() in [".mp4", ".mov", ".avi"]:
+                            path = os.path.join(roll_dir, fn)
+                            if path not in self.video_obj_cache:
+                                self.video_obj_cache[path] = Video(path)
+                            yield self.video_obj_cache[path]
+        return sorted(_())
+
+    @property
+    def filter_rating(self):
+        return self._filter_rating
+
+    @filter_rating.setter
+    def filter_rating(self, value):
+        self._filter_rating = value
+        self.update()
+
+    def filter_files(self, videos: List[Video]):
+        def flt(video: Video):
+            minimum, maximum = self._filter_rating
+            rating =  -1 if video.metadata.rejected else video.metadata.rating
+            return rating >= minimum and rating < maximum
+        return list(filter(flt, videos))[:self.show_up_to]
+
+
+    def find_video(self, video: Video)-> Tuple[Optional[QtCore.QModelIndex], Video]:
+        try:
+            index = self.all_files.index(video)
+            video = self.all_files[index]
+        except ValueError:
+            pass
+
+        try:
+            index = self.video_files.index(video)
+            index = self.index(index, 0)
+        except ValueError:
+            index = None
+        return index, video
+
+
+
+    def update(self):
+        i = 0
+        parent = QtCore.QModelIndex()
+        before = list(self.video_files)
+        action_list = []
+        after = self.filter_files(self.all_files)
+
+        while i < len(after) and i < len(before):
+            if before[i] == after[i]:
+                i += 1
+            elif before[i] < after[i]:
+                action_list.append(('r', i))
+                before.pop(i)
+            else:
+                action_list.append(('a', i, after[i]))
+                before.insert(i, after[i])
+
+        j = i
+        while j < len(before):
+            action_list.append(('r', j))
+            before.pop(j)
+
+        for j in range(i, len(after)):
+                action_list.append(('a', j, after[j]))
+                before.insert(j, after[j])
+        # assert before == after
+
+        if len(action_list) == 0:
+            return
+        def work_batch():
+            batch = action_list[b:e]
+            if batch[0][0] == 'r':
+                self.beginRemoveRows(parent, batch[0][1], batch[0][1] + len(batch) - 1)
+                del self.video_files[batch[0][1]:batch[0][1] + len(batch)]
+                self.endRemoveRows()
+            else:
+                self.beginInsertRows(parent, batch[0][1], batch[0][1] + len(batch) - 1)
+                self.video_files = (
+                    self.video_files[0:batch[0][1]] +
+                    list(map(lambda a: a[2], batch)) +
+                    self.video_files[batch[0][1]:]
+                )
+                self.endInsertRows()
+
+        b = 0
+        e = 1
+        for e in range(1, len(action_list)):
+            if (
+                action_list[e][0] == 'r' and
+                action_list[e - 1][0] == 'r' and
+                action_list[e][1] == action_list[e - 1][1]
+            ):
+                continue
+            if (
+                action_list[e][0] == 'a' and
+                action_list[e - 1][0] == 'a' and
+                action_list[e][1] == action_list[e - 1][1] + 1
+            ):
+                continue
+            work_batch()
+            b = e
+        e = len(action_list)
+        work_batch()
+        assert len(self.video_files) == len(after)
+        # assert self.video_files == after
+
+    def reload(self):
+        self.all_files = self.collect_files()
+        self.update()
+
+    def invalidateFilter(self, index=None, index_to=None):
+        self.update()
+
+    def canFetchMore(self, parent: QtCore.QModelIndex) -> bool:
+        return self.show_up_to < len(self.all_files)
+
+    def fetchMore(self, parent: QtCore.QModelIndex) -> None:
+        self.show_up_to = min(self.show_up_to + 5, len(self.all_files))
+        self.update()
+
+    def remove(self, index):
+        self.all_files.remove(self.video_files[index.row()])
+        self.update()
+
+    def data(self, index: QtCore.QModelIndex, role: int=0) -> Any:
+        video = self.video_files[index.row()]
+        if role == 0:
+            size = 0
+            try:
+                size = os.stat(video.fn).st_size / 1024. / 1024. / 1024.
+            except FileNotFoundError:
+                pass
+            return '%s %s %.2f' % (
+                os.path.basename(video.fn),
+                ('★' * video.metadata.rating + '☆' * (5 - video.metadata.rating))
+                if not video.metadata.rejected else
+                'xxxxx'
+                ,
+                size
+            )
+        if role == 1:
+            def callback():
+                self.dataChanged.emit(index, index)
+            return (
+                thumbnailer.get_thumbnail(video.fn, callback)
+                or self.default_icon
+            )
+
+
+    def rowCount(self, index=None):
+        return len(self.video_files)
+
+    def columnCount(self, index):
+        return 1
+
+