arrdem/source
1
0
Fork 0
Code Issues 11 Pull requests Projects Releases Packages Wiki Activity

[NO TESTS] WIP

Browse source
This commit is contained in:
Reid 'arrdem' McKenzie 2021-09-25 00:43:50 -06:00
parent 840e764b91
commit 9aae534971
8 changed files with 402 additions and 377 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.bazelrc
View file

@ -1,5 +1,5 @@
test --test_output=errors test --test_output=errors
# To enable flake8 on all build steps, uncomment this - # To enable flake8 on all build steps, uncomment this -
build --aspects="//tools/flake8:flake8.bzl%flake8_aspect" --output_groups=flake8_checks test --aspects="//tools/flake8:flake8.bzl%flake8_aspect" --output_groups=flake8_checks
build --aspects="//tools/black:black.bzl%black_aspect" --output_groups=black_checks test --aspects="//tools/black:black.bzl%black_aspect" --output_groups=black_checks

6
projects/archiver/BUILD
View file

@ -1,11 +1,17 @@
zapp_binary( zapp_binary(
name = "hash_copy", name = "hash_copy",
main = "hash_copy.py", main = "hash_copy.py",
srcs = [
"util.py"
],
) )
zapp_binary( zapp_binary(
name = "org_photos", name = "org_photos",
main = "org_photos.py", main = "org_photos.py",
srcs = [
"util.py"
],
deps = [ deps = [
py_requirement("ExifRead"), py_requirement("ExifRead"),
] ]

28
projects/archiver/hash_copy.py
View file

@ -3,38 +3,16 @@ A tree deduplicator and archiver tool.
""" """
import argparse import argparse
from hashlib import sha256
from pathlib import Path from pathlib import Path
from shutil import copy2 as copyfile from shutil import copy2 as copyfile
from .util import *
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("from_dir", type=Path) parser.add_argument("from_dir", type=Path)
parser.add_argument("to_dir", type=Path) parser.add_argument("to_dir", type=Path)
def checksum(p: Path, sum=sha256) -> str:
"""Compute block-wise checksums of a file.
Inspired by the Dropbox content-hashing interface -
https://www.dropbox.com/developers/reference/content-hash
"""
def iter_chunks(fp):
yield from iter(lambda: fp.read(4096), b"")
def _helper():
with open(p, "rb") as fp:
for chunk in iter_chunks(fp):
digest = sum()
digest.update(chunk)
yield digest.hexdigest()
return list(_helper())
def main(): def main():
opts, args = parser.parse_known_args() opts, args = parser.parse_known_args()
@ -62,8 +40,8 @@ def main():
): ):
pass pass
elif (src_checksum := checksum(abs_src_path)) != ( elif (src_checksum := checksum_path(abs_src_path)) != (
dest_checksum := checksum(abs_dest_path) dest_checksum := checksum_path(abs_dest_path)
): ):
print( print(
f"file conflict (src {src_checksum}, dest {dest_checksum}), correcting..." f"file conflict (src {src_checksum}, dest {dest_checksum}), correcting..."

288
projects/archiver/notes.md Normal file
View file

@ -0,0 +1,288 @@
# EXIF tags dataset (exifread edition)
- 'EXIF ApertureValue'
- 'EXIF BodySerialNumber'
- 'EXIF BrightnessValue'
- 'EXIF CVAPattern'
- 'EXIF CameraOwnerName'
- 'EXIF ColorSpace'
- 'EXIF ComponentsConfiguration'
- 'EXIF CompressedBitsPerPixel'
- 'EXIF Contrast'
- 'EXIF CustomRendered'
- 'EXIF DateTimeDigitized'
- 'EXIF DateTimeOriginal'
- 'EXIF DeviceSettingDescription'
- 'EXIF DigitalZoomRatio'
- 'EXIF ExifImageLength'
- 'EXIF ExifImageWidth'
- 'EXIF ExifVersion'
- 'EXIF ExposureBiasValue'
- 'EXIF ExposureIndex'
- 'EXIF ExposureMode'
- 'EXIF ExposureProgram'
- 'EXIF ExposureTime'
- 'EXIF FNumber'
- 'EXIF FileSource'
- 'EXIF Flash'
- 'EXIF FlashEnergy'
- 'EXIF FlashPixVersion'
- 'EXIF FocalLength'
- 'EXIF FocalLengthIn35mmFilm'
- 'EXIF FocalPlaneResolutionUnit'
- 'EXIF FocalPlaneXResolution'
- 'EXIF FocalPlaneYResolution'
- 'EXIF GainControl'
- 'EXIF ISOSpeedRatings'
- 'EXIF ImageUniqueID'
- 'EXIF InteroperabilityOffset'
- 'EXIF JPEGInterchangeFormat'
- 'EXIF JPEGInterchangeFormatLength'
- 'EXIF LensMake'
- 'EXIF LensModel'
- 'EXIF LensSerialNumber'
- 'EXIF LensSpecification'
- 'EXIF LightSource'
- 'EXIF MakerNote'
- 'EXIF MaxApertureValue'
- 'EXIF MeteringMode'
- 'EXIF OffsetSchema'
- 'EXIF OffsetTime'
- 'EXIF OffsetTimeDigitized'
- 'EXIF OffsetTimeOriginal'
- 'EXIF Padding'
- 'EXIF RecommendedExposureIndex'
- 'EXIF Saturation'
- 'EXIF SceneCaptureType'
- 'EXIF SceneType'
- 'EXIF SensingMethod'
- 'EXIF SensitivityType'
- 'EXIF Sharpness'
- 'EXIF ShutterSpeedValue'
- 'EXIF SubSecTime'
- 'EXIF SubSecTimeDigitized'
- 'EXIF SubSecTimeOriginal'
- 'EXIF SubjectArea'
- 'EXIF SubjectDistance'
- 'EXIF SubjectDistanceRange'
- 'EXIF UserComment'
- 'EXIF WhiteBalance'
- 'GPS GPSAltitude'
- 'GPS GPSAltitudeRef'
- 'GPS GPSDOP'
- 'GPS GPSDate'
- 'GPS GPSImgDirection'
- 'GPS GPSImgDirectionRef'
- 'GPS GPSLatitude'
- 'GPS GPSLatitudeRef'
- 'GPS GPSLongitude'
- 'GPS GPSLongitudeRef'
- 'GPS GPSMapDatum'
- 'GPS GPSMeasureMode'
- 'GPS GPSProcessingMethod'
- 'GPS GPSTimeStamp'
- 'GPS GPSVersionID'
- 'GPS Tag 0xEA1C'
- 'Image Artist'
- 'Image BitsPerSample'
- 'Image Copyright'
- 'Image DateTime'
- 'Image DateTimeDigitized'
- 'Image ExifOffset'
- 'Image ExposureMode'
- 'Image ExposureProgram'
- 'Image ExposureTime'
- 'Image FNumber'
- 'Image Flash'
- 'Image FocalLength'
- 'Image GPSInfo'
- 'Image ISOSpeedRatings'
- 'Image ImageDescription'
- 'Image ImageLength'
- 'Image ImageWidth'
- 'Image JPEGInterchangeFormat'
- 'Image JPEGInterchangeFormatLength'
- 'Image LightSource'
- 'Image Make'
- 'Image MeteringMode'
- 'Image Model'
- 'Image Orientation'
- 'Image Padding'
- 'Image PhotometricInterpretation'
- 'Image PrintIM'
- 'Image ResolutionUnit'
- 'Image SamplesPerPixel'
- 'Image Software'
- 'Image UserComment'
- 'Image WhiteBalance'
- 'Image XPComment'
- 'Image XPKeywords'
- 'Image XPTitle'
- 'Image XResolution'
- 'Image YCbCrPositioning'
- 'Image YResolution'
- 'Interoperability InteroperabilityIndex'
- 'Interoperability InteroperabilityVersion'
- 'JPEGThumbnail'
- 'MakerNote AEBracketCompensationApplied'
- 'MakerNote AESetting'
- 'MakerNote AFAreaMode'
- 'MakerNote AFInfo2'
- 'MakerNote AFPointSelected'
- 'MakerNote AFPointUsed'
- 'MakerNote ActiveDLighting'
- 'MakerNote AspectInfo'
- 'MakerNote AutoBracketRelease'
- 'MakerNote AutoFlashMode'
- 'MakerNote BracketMode'
- 'MakerNote BracketShotNumber'
- 'MakerNote BracketValue'
- 'MakerNote BracketingMode'
- 'MakerNote CanonImageWidth'
- 'MakerNote ColorBalance'
- 'MakerNote ColorSpace'
- 'MakerNote ContinuousDriveMode'
- 'MakerNote Contrast'
- 'MakerNote CropHiSpeed'
- 'MakerNote CropInfo'
- 'MakerNote DigitalVariProgram'
- 'MakerNote DigitalZoom'
- 'MakerNote DustRemovalData'
- 'MakerNote EasyShootingMode'
- 'MakerNote ExposureDifference'
- 'MakerNote ExposureMode'
- 'MakerNote ExposureTuning'
- 'MakerNote ExternalFlashExposureComp'
- 'MakerNote FileInfo'
- 'MakerNote FileNumber'
- 'MakerNote FilterEffect'
- 'MakerNote FirmwareVersion'
- 'MakerNote FlashActivity'
- 'MakerNote FlashBias'
- 'MakerNote FlashBracketCompensationApplied'
- 'MakerNote FlashCompensation'
- 'MakerNote FlashDetails'
- 'MakerNote FlashExposureLock'
- 'MakerNote FlashInfo'
- 'MakerNote FlashMode'
- 'MakerNote FlashSetting'
- 'MakerNote FocalLength'
- 'MakerNote FocalType'
- 'MakerNote FocalUnitsPerMM'
- 'MakerNote FocusMode'
- 'MakerNote FocusType'
- 'MakerNote HDRImageType'
- 'MakerNote HighISONoiseReduction'
- 'MakerNote ISO'
- 'MakerNote ISOInfo'
- 'MakerNote ISOSetting'
- 'MakerNote ISOSpeedRequested'
- 'MakerNote ImageDataSize'
- 'MakerNote ImageSize'
- 'MakerNote ImageStabilization'
- 'MakerNote ImageType'
- 'MakerNote InternalSerialNumber'
- 'MakerNote LensData'
- 'MakerNote LensFStops'
- 'MakerNote LensMinMaxFocalMaxAperture'
- 'MakerNote LensModel'
- 'MakerNote LensType'
- 'MakerNote LiveViewShooting'
- 'MakerNote LongExposureNoiseReduction2'
- 'MakerNote LongFocalLengthOfLensInFocalUnits'
- 'MakerNote MacroMagnification'
- 'MakerNote Macromode'
- 'MakerNote MakernoteVersion'
- 'MakerNote ManualFlashOutput'
- 'MakerNote MeteringMode'
- 'MakerNote ModelID'
- 'MakerNote MultiExposure'
- 'MakerNote NikonPreview'
- 'MakerNote NoiseReduction'
- 'MakerNote NumAFPoints'
- 'MakerNote OwnerName'
- 'MakerNote PhotoCornerCoordinates'
- 'MakerNote PictureControl'
- 'MakerNote PowerUpTime'
- 'MakerNote ProgramShift'
- 'MakerNote Quality'
- 'MakerNote RawJpgQuality'
- 'MakerNote RawJpgSize'
- 'MakerNote RecordMode'
- 'MakerNote RetouchHistory'
- 'MakerNote Saturation'
- 'MakerNote SelfTimer'
- 'MakerNote SequenceNumber'
- 'MakerNote SerialNumber'
- 'MakerNote Sharpness'
- 'MakerNote ShortFocalLengthOfLensInFocalUnits'
- 'MakerNote ShotInfo'
- 'MakerNote SlowShutter'
- 'MakerNote SpotMeteringMode'
- 'MakerNote SubjectDistance'
- 'MakerNote Tag 0x0001'
- 'MakerNote Tag 0x0002'
- 'MakerNote Tag 0x0003'
- 'MakerNote Tag 0x0004'
- 'MakerNote Tag 0x0005'
- 'MakerNote Tag 0x0006'
- 'MakerNote Tag 0x0007'
- 'MakerNote Tag 0x0008'
- 'MakerNote Tag 0x0009'
- 'MakerNote Tag 0x000E'
- 'MakerNote Tag 0x0014'
- 'MakerNote Tag 0x0015'
- 'MakerNote Tag 0x0019'
- 'MakerNote Tag 0x002B'
- 'MakerNote Tag 0x002C'
- 'MakerNote Tag 0x002D'
- 'MakerNote Tag 0x0083'
- 'MakerNote Tag 0x0099'
- 'MakerNote Tag 0x009D'
- 'MakerNote Tag 0x00A0'
- 'MakerNote Tag 0x00A3'
- 'MakerNote Tag 0x00AA'
- 'MakerNote Tag 0x00BB'
- 'MakerNote Tag 0x00D0'
- 'MakerNote Tag 0x00E0'
- 'MakerNote Tag 0x4001'
- 'MakerNote Tag 0x4008'
- 'MakerNote Tag 0x4009'
- 'MakerNote Tag 0x4010'
- 'MakerNote Tag 0x4011'
- 'MakerNote Tag 0x4012'
- 'MakerNote Tag 0x4015'
- 'MakerNote Tag 0x4016'
- 'MakerNote Tag 0x4017'
- 'MakerNote Tag 0x4018'
- 'MakerNote Tag 0x4019'
- 'MakerNote Tag 0x4020'
- 'MakerNote ThumbnailImageValidArea'
- 'MakerNote ToningEffect'
- 'MakerNote TotalShutterReleases'
- 'MakerNote Unknown'
- 'MakerNote VRInfo'
- 'MakerNote ValidAFPoints'
- 'MakerNote WBBracketMode'
- 'MakerNote WBBracketValueAB'
- 'MakerNote WBBracketValueGM'
- 'MakerNote WhiteBalance'
- 'MakerNote WhiteBalanceBias'
- 'MakerNote WhiteBalanceRBCoeff'
- 'MakerNote Whitebalance'
- 'MakerNote WorldTime'
- 'Thumbnail Compression'
- 'Thumbnail DateTime'
- 'Thumbnail ImageDescription'
- 'Thumbnail ImageLength'
- 'Thumbnail ImageWidth'
- 'Thumbnail JPEGInterchangeFormat'
- 'Thumbnail JPEGInterchangeFormatLength'
- 'Thumbnail Make'
- 'Thumbnail Model'
- 'Thumbnail Orientation'
- 'Thumbnail ResolutionUnit'
- 'Thumbnail Software'
- 'Thumbnail XResolution'
- 'Thumbnail YCbCrPositioning'
- 'Thumbnail YResolution'

379
projects/archiver/org_photos.py
View file

@ -25,6 +25,8 @@ from shutil import copy2 as copyfile
import sys import sys
import typing as t import typing as t
from .util import *
# FIXME: use piexif, which supports writeback not exifread. # FIXME: use piexif, which supports writeback not exifread.
import exifread import exifread
@ -32,351 +34,18 @@ import exifread
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("src_dir", type=Path) parser.add_argument("src_dir", type=Path)
parser.add_argument("dest_dir", type=Path) parser.add_argument("dest_dir", type=Path)
parser.add_option("destructive", action="store_true", default=False)
MODIFIED_ISO_DATE = "%Y:%m:%dT%H:%M:%SF%f" MODIFIED_ISO_DATE = "%Y:%m:%dT%H:%M:%SF%f"
def take(n, iter):
"""Take the first N items lazily off of an iterable."""
for _ in range(0, n):
try:
yield next(iter)
except StopIteration:
break
def exif_tags(p: Path) -> object: def exif_tags(p: Path) -> object:
"""Return the EXIF tags on an image.""" """Return the EXIF tags on an image."""
with open(p, "rb") as fp: with open(p, "rb") as fp:
return exifread.process_file(fp) return exifread.process_file(fp)
# EXIF tags dataset (exifread edition) -
#
# ---
# - 'EXIF ApertureValue'
# - 'EXIF BodySerialNumber'
# - 'EXIF BrightnessValue'
# - 'EXIF CVAPattern'
# - 'EXIF CameraOwnerName'
# - 'EXIF ColorSpace'
# - 'EXIF ComponentsConfiguration'
# - 'EXIF CompressedBitsPerPixel'
# - 'EXIF Contrast'
# - 'EXIF CustomRendered'
# - 'EXIF DateTimeDigitized'
# - 'EXIF DateTimeOriginal'
# - 'EXIF DeviceSettingDescription'
# - 'EXIF DigitalZoomRatio'
# - 'EXIF ExifImageLength'
# - 'EXIF ExifImageWidth'
# - 'EXIF ExifVersion'
# - 'EXIF ExposureBiasValue'
# - 'EXIF ExposureIndex'
# - 'EXIF ExposureMode'
# - 'EXIF ExposureProgram'
# - 'EXIF ExposureTime'
# - 'EXIF FNumber'
# - 'EXIF FileSource'
# - 'EXIF Flash'
# - 'EXIF FlashEnergy'
# - 'EXIF FlashPixVersion'
# - 'EXIF FocalLength'
# - 'EXIF FocalLengthIn35mmFilm'
# - 'EXIF FocalPlaneResolutionUnit'
# - 'EXIF FocalPlaneXResolution'
# - 'EXIF FocalPlaneYResolution'
# - 'EXIF GainControl'
# - 'EXIF ISOSpeedRatings'
# - 'EXIF ImageUniqueID'
# - 'EXIF InteroperabilityOffset'
# - 'EXIF JPEGInterchangeFormat'
# - 'EXIF JPEGInterchangeFormatLength'
# - 'EXIF LensMake'
# - 'EXIF LensModel'
# - 'EXIF LensSerialNumber'
# - 'EXIF LensSpecification'
# - 'EXIF LightSource'
# - 'EXIF MakerNote'
# - 'EXIF MaxApertureValue'
# - 'EXIF MeteringMode'
# - 'EXIF OffsetSchema'
# - 'EXIF OffsetTime'
# - 'EXIF OffsetTimeDigitized'
# - 'EXIF OffsetTimeOriginal'
# - 'EXIF Padding'
# - 'EXIF RecommendedExposureIndex'
# - 'EXIF Saturation'
# - 'EXIF SceneCaptureType'
# - 'EXIF SceneType'
# - 'EXIF SensingMethod'
# - 'EXIF SensitivityType'
# - 'EXIF Sharpness'
# - 'EXIF ShutterSpeedValue'
# - 'EXIF SubSecTime'
# - 'EXIF SubSecTimeDigitized'
# - 'EXIF SubSecTimeOriginal'
# - 'EXIF SubjectArea'
# - 'EXIF SubjectDistance'
# - 'EXIF SubjectDistanceRange'
# - 'EXIF UserComment'
# - 'EXIF WhiteBalance'
# - 'GPS GPSAltitude'
# - 'GPS GPSAltitudeRef'
# - 'GPS GPSDOP'
# - 'GPS GPSDate'
# - 'GPS GPSImgDirection'
# - 'GPS GPSImgDirectionRef'
# - 'GPS GPSLatitude'
# - 'GPS GPSLatitudeRef'
# - 'GPS GPSLongitude'
# - 'GPS GPSLongitudeRef'
# - 'GPS GPSMapDatum'
# - 'GPS GPSMeasureMode'
# - 'GPS GPSProcessingMethod'
# - 'GPS GPSTimeStamp'
# - 'GPS GPSVersionID'
# - 'GPS Tag 0xEA1C'
# - 'Image Artist'
# - 'Image BitsPerSample'
# - 'Image Copyright'
# - 'Image DateTime'
# - 'Image DateTimeDigitized'
# - 'Image ExifOffset'
# - 'Image ExposureMode'
# - 'Image ExposureProgram'
# - 'Image ExposureTime'
# - 'Image FNumber'
# - 'Image Flash'
# - 'Image FocalLength'
# - 'Image GPSInfo'
# - 'Image ISOSpeedRatings'
# - 'Image ImageDescription'
# - 'Image ImageLength'
# - 'Image ImageWidth'
# - 'Image JPEGInterchangeFormat'
# - 'Image JPEGInterchangeFormatLength'
# - 'Image LightSource'
# - 'Image Make'
# - 'Image MeteringMode'
# - 'Image Model'
# - 'Image Orientation'
# - 'Image Padding'
# - 'Image PhotometricInterpretation'
# - 'Image PrintIM'
# - 'Image ResolutionUnit'
# - 'Image SamplesPerPixel'
# - 'Image Software'
# - 'Image UserComment'
# - 'Image WhiteBalance'
# - 'Image XPComment'
# - 'Image XPKeywords'
# - 'Image XPTitle'
# - 'Image XResolution'
# - 'Image YCbCrPositioning'
# - 'Image YResolution'
# - 'Interoperability InteroperabilityIndex'
# - 'Interoperability InteroperabilityVersion'
# - 'JPEGThumbnail'
# - 'MakerNote AEBracketCompensationApplied'
# - 'MakerNote AESetting'
# - 'MakerNote AFAreaMode'
# - 'MakerNote AFInfo2'
# - 'MakerNote AFPointSelected'
# - 'MakerNote AFPointUsed'
# - 'MakerNote ActiveDLighting'
# - 'MakerNote AspectInfo'
# - 'MakerNote AutoBracketRelease'
# - 'MakerNote AutoFlashMode'
# - 'MakerNote BracketMode'
# - 'MakerNote BracketShotNumber'
# - 'MakerNote BracketValue'
# - 'MakerNote BracketingMode'
# - 'MakerNote CanonImageWidth'
# - 'MakerNote ColorBalance'
# - 'MakerNote ColorSpace'
# - 'MakerNote ContinuousDriveMode'
# - 'MakerNote Contrast'
# - 'MakerNote CropHiSpeed'
# - 'MakerNote CropInfo'
# - 'MakerNote DigitalVariProgram'
# - 'MakerNote DigitalZoom'
# - 'MakerNote DustRemovalData'
# - 'MakerNote EasyShootingMode'
# - 'MakerNote ExposureDifference'
# - 'MakerNote ExposureMode'
# - 'MakerNote ExposureTuning'
# - 'MakerNote ExternalFlashExposureComp'
# - 'MakerNote FileInfo'
# - 'MakerNote FileNumber'
# - 'MakerNote FilterEffect'
# - 'MakerNote FirmwareVersion'
# - 'MakerNote FlashActivity'
# - 'MakerNote FlashBias'
# - 'MakerNote FlashBracketCompensationApplied'
# - 'MakerNote FlashCompensation'
# - 'MakerNote FlashDetails'
# - 'MakerNote FlashExposureLock'
# - 'MakerNote FlashInfo'
# - 'MakerNote FlashMode'
# - 'MakerNote FlashSetting'
# - 'MakerNote FocalLength'
# - 'MakerNote FocalType'
# - 'MakerNote FocalUnitsPerMM'
# - 'MakerNote FocusMode'
# - 'MakerNote FocusType'
# - 'MakerNote HDRImageType'
# - 'MakerNote HighISONoiseReduction'
# - 'MakerNote ISO'
# - 'MakerNote ISOInfo'
# - 'MakerNote ISOSetting'
# - 'MakerNote ISOSpeedRequested'
# - 'MakerNote ImageDataSize'
# - 'MakerNote ImageSize'
# - 'MakerNote ImageStabilization'
# - 'MakerNote ImageType'
# - 'MakerNote InternalSerialNumber'
# - 'MakerNote LensData'
# - 'MakerNote LensFStops'
# - 'MakerNote LensMinMaxFocalMaxAperture'
# - 'MakerNote LensModel'
# - 'MakerNote LensType'
# - 'MakerNote LiveViewShooting'
# - 'MakerNote LongExposureNoiseReduction2'
# - 'MakerNote LongFocalLengthOfLensInFocalUnits'
# - 'MakerNote MacroMagnification'
# - 'MakerNote Macromode'
# - 'MakerNote MakernoteVersion'
# - 'MakerNote ManualFlashOutput'
# - 'MakerNote MeteringMode'
# - 'MakerNote ModelID'
# - 'MakerNote MultiExposure'
# - 'MakerNote NikonPreview'
# - 'MakerNote NoiseReduction'
# - 'MakerNote NumAFPoints'
# - 'MakerNote OwnerName'
# - 'MakerNote PhotoCornerCoordinates'
# - 'MakerNote PictureControl'
# - 'MakerNote PowerUpTime'
# - 'MakerNote ProgramShift'
# - 'MakerNote Quality'
# - 'MakerNote RawJpgQuality'
# - 'MakerNote RawJpgSize'
# - 'MakerNote RecordMode'
# - 'MakerNote RetouchHistory'
# - 'MakerNote Saturation'
# - 'MakerNote SelfTimer'
# - 'MakerNote SequenceNumber'
# - 'MakerNote SerialNumber'
# - 'MakerNote Sharpness'
# - 'MakerNote ShortFocalLengthOfLensInFocalUnits'
# - 'MakerNote ShotInfo'
# - 'MakerNote SlowShutter'
# - 'MakerNote SpotMeteringMode'
# - 'MakerNote SubjectDistance'
# - 'MakerNote Tag 0x0001'
# - 'MakerNote Tag 0x0002'
# - 'MakerNote Tag 0x0003'
# - 'MakerNote Tag 0x0004'
# - 'MakerNote Tag 0x0005'
# - 'MakerNote Tag 0x0006'
# - 'MakerNote Tag 0x0007'
# - 'MakerNote Tag 0x0008'
# - 'MakerNote Tag 0x0009'
# - 'MakerNote Tag 0x000E'
# - 'MakerNote Tag 0x0014'
# - 'MakerNote Tag 0x0015'
# - 'MakerNote Tag 0x0019'
# - 'MakerNote Tag 0x002B'
# - 'MakerNote Tag 0x002C'
# - 'MakerNote Tag 0x002D'
# - 'MakerNote Tag 0x0083'
# - 'MakerNote Tag 0x0099'
# - 'MakerNote Tag 0x009D'
# - 'MakerNote Tag 0x00A0'
# - 'MakerNote Tag 0x00A3'
# - 'MakerNote Tag 0x00AA'
# - 'MakerNote Tag 0x00BB'
# - 'MakerNote Tag 0x00D0'
# - 'MakerNote Tag 0x00E0'
# - 'MakerNote Tag 0x4001'
# - 'MakerNote Tag 0x4008'
# - 'MakerNote Tag 0x4009'
# - 'MakerNote Tag 0x4010'
# - 'MakerNote Tag 0x4011'
# - 'MakerNote Tag 0x4012'
# - 'MakerNote Tag 0x4015'
# - 'MakerNote Tag 0x4016'
# - 'MakerNote Tag 0x4017'
# - 'MakerNote Tag 0x4018'
# - 'MakerNote Tag 0x4019'
# - 'MakerNote Tag 0x4020'
# - 'MakerNote ThumbnailImageValidArea'
# - 'MakerNote ToningEffect'
# - 'MakerNote TotalShutterReleases'
# - 'MakerNote Unknown'
# - 'MakerNote VRInfo'
# - 'MakerNote ValidAFPoints'
# - 'MakerNote WBBracketMode'
# - 'MakerNote WBBracketValueAB'
# - 'MakerNote WBBracketValueGM'
# - 'MakerNote WhiteBalance'
# - 'MakerNote WhiteBalanceBias'
# - 'MakerNote WhiteBalanceRBCoeff'
# - 'MakerNote Whitebalance'
# - 'MakerNote WorldTime'
# - 'Thumbnail Compression'
# - 'Thumbnail DateTime'
# - 'Thumbnail ImageDescription'
# - 'Thumbnail ImageLength'
# - 'Thumbnail ImageWidth'
# - 'Thumbnail JPEGInterchangeFormat'
# - 'Thumbnail JPEGInterchangeFormatLength'
# - 'Thumbnail Make'
# - 'Thumbnail Model'
# - 'Thumbnail Orientation'
# - 'Thumbnail ResolutionUnit'
# - 'Thumbnail Software'
# - 'Thumbnail XResolution'
# - 'Thumbnail YCbCrPositioning'
# - 'Thumbnail YResolution'
def checksum(p: Path, sum=sha256) -> str:
"""Compute a chunked checksum of a file.
Does not produce individual block checksums.
"""
def iter_blocks(fp):
yield from iter(lambda: fp.read(4096), b"")
with open(p, "rb") as fp:
digest = sum()
for chunk in iter_blocks(fp):
digest.update(chunk)
return digest.hexdigest()
def checksum_list(iter, sum=sha256):
"""Compute the checksum of a bunch of stuff from an iterable."""
sum = sum()
for i in iter:
sum.update(b";") # Merkle tree salting.
if isinstance(i, str):
i = str.encode(i, "utf-8")
try:
sum.update(i)
except Exception as e:
print(i, type(i))
raise e
return sum.hexdigest()
def sanitize(s: str) -> str: def sanitize(s: str) -> str:
"""Something like b64encode; sanitize a string to a path-friendly version.""" """Something like b64encode; sanitize a string to a path-friendly version."""
@ -524,10 +193,10 @@ class ImgInfo(t.NamedTuple):
return self.file_sha256sum()[: self.shasum_prefix] return self.file_sha256sum()[: self.shasum_prefix]
def file_sha256sum(self): def file_sha256sum(self):
return checksum(self.file_path, sha256) return checksum_path(self.file_path, sha256)
def file_sha512sum(self): def file_sha512sum(self):
return checksum(self.file_path, sha512) return checksum_path(self.file_path, sha512)
def img_info(p: Path) -> ImgInfo: def img_info(p: Path) -> ImgInfo:
@ -620,11 +289,27 @@ def img_info(p: Path) -> ImgInfo:
def main(): def main():
opts, args = parser.parse_known_args() opts, args = parser.parse_known_args()
def _copy():
print(f" rename: {target}")
try:
if not opts.destructive:
raise OSError()
src.rename(target) # Execute the rename
except OSError: # cross-device move
copyfile(src, target)
if opts.destructive:
src.chmod(0o644)
src.unlink()
print("---") print("---")
for src in list(opts.src_dir.glob("**/*")): for src in list(opts.src_dir.glob("**/*")):
if src.is_dir(): if src.is_dir():
continue continue
print(f"{src}:")
info = img_info(src) info = img_info(src)
year_dir = Path(opts.dest_dir / str(info.date.year)) year_dir = Path(opts.dest_dir / str(info.date.year))
year_dir.mkdir(exist_ok=True) # Ignore existing and continue year_dir.mkdir(exist_ok=True) # Ignore existing and continue
@ -636,20 +321,20 @@ def main():
continue # Just skip fucked up files continue # Just skip fucked up files
target = Path(year_dir / f"{stable_name}{ext}") target = Path(year_dir / f"{stable_name}{ext}")
print(f"{src}:")
print(f" rename: {target}")
if not target.exists(): if not target.exists():
try: # src & !target => copy
src.rename(target) # Execute the rename _copy()
except OSError:
copyfile(src, target)
target.chmod(0o644)
src.unlink()
elif src == target: elif src == target:
pass # Nothing to do # src == target; skip DO NOT DELETE SRC
pass
elif checksum_path_blocks(src) == checksum_path_blocks(target):
# src != target && id(src) == id(target); delete src
if opts.destructive:
src.chmod(0o644)
src.unlink()
else: else:
target.chmod(0o644) # src != target && id(src) != id(target); replace target with src?
src.unlink() # Delete the source print(f" warning: {target} is a content-id collision with a different checksum")
if __name__ == "__main__": if __name__ == "__main__":

69
projects/archiver/util.py Normal file
View file

@ -0,0 +1,69 @@
import typing as t
from hashlib import sha256
from pathlib import Path
def iter_chunks(fp):
yield from iter(lambda: fp.read(4096), b"")
def take(n, iter):
"""Take the first N items lazily off of an iterable."""
for _ in range(0, n):
try:
yield next(iter)
except StopIteration:
break
def checksum_list(iter, sum=sha256, salt=b";"):
"""Compute the checksum of a bunch of stuff from an iterable."""
sum = sum()
for i in iter:
if salt:
sum.update(salt) # Merkle tree salting.
if isinstance(i, str):
i = str.encode(i, "utf-8")
try:
sum.update(i)
except Exception as e:
print(i, type(i))
raise e
return sum.hexdigest()
def checksum_path_blocks(p: Path, sum=sha256) -> t.Iterable[str]:
"""Compute block-wise checksums of a file.
Inspired by the Dropbox content-hashing interface -
https://www.dropbox.com/developers/reference/content-hash
"""
def _helper():
with open(p, "rb") as fp:
for chunk in iter_chunks(fp):
digest = sum()
digest.update(chunk)
yield digest.hexdigest()
return list(_helper())
def checksum_path(p: Path, sum=sha256) -> str:
"""Compute 'the' checksum of an entire file.
Note that this does semi-streaming I/O.
"""
sum = sum()
with open(p, "rb") as fp:
for chunk in iter_chunks(fp):
sum.update(chunk)
return sum.hexdigest()

2
tools/autoflake/BUILD
View file

@ -1,4 +1,4 @@
py_binary( zapp_binary(
name = "autoflake", name = "autoflake",
main = "__main__.py", main = "__main__.py",
deps = [ deps = [

3
tools/fmt.sh
View file

@ -9,8 +9,7 @@ DIRS=(projects tools)
function brl() { function brl() {
bin="$1" bin="$1"
shift shift
bazel build "//${bin}" bazel run "//${bin}" -- "$@"
"bazel-bin/${bin}/$(basename ${bin})" "$@"
return "$?" return "$?"
} }