arrdem/source
1
0
Fork 0
Code Issues 11 Pull requests Projects Releases Packages Wiki Activity

Fmt.

Browse source
This commit is contained in:
Reid 'arrdem' McKenzie 2021-09-24 22:37:38 -06:00
parent 3bedbf7f05
commit caf3acae12
5 changed files with 90 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.bazelrc
View file

@ -1,5 +1,5 @@
test --test_output=errors test --test_output=errors
# To enable flake8 on all build steps, uncomment this - # To enable flake8 on all build steps, uncomment this -
# build --aspects="//tools/flake8:flake8.bzl%flake8_aspect" --output_groups=flake8_checks build --aspects="//tools/flake8:flake8.bzl%flake8_aspect" --output_groups=flake8_checks
# build --aspects="//tools/black:black.bzl%black_aspect" --output_groups=black_checks build --aspects="//tools/black:black.bzl%black_aspect" --output_groups=black_checks

15
projects/archiver/hash_copy.py
View file

@ -56,11 +56,18 @@ def main():
# If you trust mtime, this can go a lot faster # If you trust mtime, this can go a lot faster
trust_mtime = False trust_mtime = False
if trust_mtime and abs_dest_path.stat().st_mtime < abs_src_path.stat().st_mtime: if (
pass trust_mtime
and abs_dest_path.stat().st_mtime < abs_src_path.stat().st_mtime
):
pass
elif (src_checksum := checksum(abs_src_path)) != (dest_checksum := checksum(abs_dest_path)): elif (src_checksum := checksum(abs_src_path)) != (
print(f"file conflict (src {src_checksum}, dest {dest_checksum}), correcting...") dest_checksum := checksum(abs_dest_path)
):
print(
f"file conflict (src {src_checksum}, dest {dest_checksum}), correcting..."
)
copyfile(abs_src_path, abs_dest_path) copyfile(abs_src_path, abs_dest_path)
abs_src_path.unlink() abs_src_path.unlink()

119
projects/archiver/org_photos.py
View file

@ -52,6 +52,7 @@ def exif_tags(p: Path) -> object:
with open(p, "rb") as fp: with open(p, "rb") as fp:
return exifread.process_file(fp) return exifread.process_file(fp)
# EXIF tags dataset (exifread edition) - # EXIF tags dataset (exifread edition) -
# #
# --- # ---
@ -389,7 +390,7 @@ def sanitize(s: str) -> str:
def safe_strptime(date, format): def safe_strptime(date, format):
try: try:
return datetime.strptime(date, format) return datetime.strptime(date, format)
except: except ValueError:
return None return None
@ -417,56 +418,49 @@ def date_from_name(p: Path):
# A couple of date formats use _ as field separators, consistently choice " " instead so that we can write fewer # A couple of date formats use _ as field separators, consistently choice " " instead so that we can write fewer
# date patterns and be more correct. # date patterns and be more correct.
fname = fname.replace("_", " ") fname = fname.replace("_", " ")
fname = re.sub(r"(-\d+)(-\d+)*$", r"\1", fname) # deal with -1-2 etc. crap from Dropbox fname = re.sub(
r"(-\d+)(-\d+)*$", r"\1", fname
) # deal with -1-2 etc. crap from Dropbox
# Try to guess the date # Try to guess the date
# File date formats: # File date formats:
for fmt in [ for fmt in [
# Our date format # Our date format
MODIFIED_ISO_DATE, MODIFIED_ISO_DATE,
# A bug
# A bug # 2014:08:21T19:4640F1408672000
# 2014:08:21T19:4640F1408672000 # 2015:12:14T23:0933F1450159773
# 2015:12:14T23:0933F1450159773 "%Y:%m:%dT%H:%M%SF%f",
"%Y:%m:%dT%H:%M%SF%f", # 2020-12-21 17.15.09.0
"%Y-%m-%d %H.%M.%S.%f",
# 2020-12-21 17.15.09.0 # 2020-12-21 17.15.09
"%Y-%m-%d %H.%M.%S.%f", "%Y-%m-%d %H.%M.%S",
# 2019-02-09 12.45.32-6
# 2020-12-21 17.15.09 # 2019-01-13 13.43.45-16
"%Y-%m-%d %H.%M.%S", "%Y-%m-%d %H.%M.%S-%f",
# Note the _1 or such may not be millis, but we assume it is.
# 2019-02-09 12.45.32-6 # 20171113_130826_1
# 2019-01-13 13.43.45-16 # 20171113 130826 1
"%Y-%m-%d %H.%M.%S-%f", "%Y%m%d %H%M%S %f",
# 20180404_114639
# Note the _1 or such may not be millis, but we assume it is. # 20180404 114639
# 20171113_130826_1 "%Y%m%d %H%M%S",
# 20171113 130826 1 # 2017-11-05_15:15:55
"%Y%m%d %H%M%S %f", # 2017-11-05 15:15:55
"%Y-%m-%d %H:%M:%S",
# 20180404_114639 # 20210417_220753284
# 20180404 114639 # 20210417 220753284
"%Y%m%d %H%M%S", # 20210304 204755545
"%Y%m%d %h%m%s%f",
# 2017-11-05_15:15:55
# 2017-11-05 15:15:55
"%Y-%m-%d %H:%M:%S",
# 20210417_220753284
# 20210417 220753284
# 20210304 204755545
"%Y%m%d %h%m%s%f",
]: ]:
try: try:
return datetime.strptime(fname, fmt) return datetime.strptime(fname, fmt)
except: except ValueError:
continue continue
else: else:
print(f"Warning: Unable to infer datetime from {fname!r}", file=sys.stderr) print(f"Warning: Unable to infer datetime from {fname!r}", file=sys.stderr)
def normalize_ext(p: Path): def normalize_ext(p: Path):
renaming = { renaming = {
".jpg": ".jpeg", ".jpg": ".jpeg",
@ -509,15 +503,17 @@ class ImgInfo(t.NamedTuple):
def device_fingerprint(self): def device_fingerprint(self):
"""Compute a stable 'fingerprint' for the device that took the shot.""" """Compute a stable 'fingerprint' for the device that took the shot."""
return checksum_list([ return checksum_list(
self.camera_make, [
self.camera_model, self.camera_make,
self.camera_sn, self.camera_model,
self.lens_make, self.camera_sn,
self.lens_model, self.lens_make,
self.lens_sn, self.lens_model,
self.software, self.lens_sn,
])[:self.shasum_prefix] self.software,
]
)[: self.shasum_prefix]
def file_fingerprint(self): def file_fingerprint(self):
"""Compute a 'fingerprint' for the file itself. """Compute a 'fingerprint' for the file itself.
@ -525,7 +521,7 @@ class ImgInfo(t.NamedTuple):
Note that this hash DOES include EXIF data, and is not stable. Note that this hash DOES include EXIF data, and is not stable.
""" """
return self.file_sha256sum()[:self.shasum_prefix] return self.file_sha256sum()[: self.shasum_prefix]
def file_sha256sum(self): def file_sha256sum(self):
return checksum(self.file_path, sha256) return checksum(self.file_path, sha256)
@ -540,7 +536,7 @@ def img_info(p: Path) -> ImgInfo:
tags = exif_tags(p) tags = exif_tags(p)
def get_tag(tag, default=None): def get_tag(tag, default=None):
if (v := tags.get(tag)): if v := tags.get(tag):
if isinstance(v.values, list): if isinstance(v.values, list):
return v.values[0] return v.values[0]
elif isinstance(v.values, str): elif isinstance(v.values, str):
@ -578,10 +574,14 @@ def img_info(p: Path) -> ImgInfo:
stat = p.stat() stat = p.stat()
# 2019:03:31 15:59:26 # 2019:03:31 15:59:26
date = get_tag("Image DateTime") or get_tag("EXIF DateTimeOriginal") or get_tag("EXIF DateTimeDigitized") date = (
get_tag("Image DateTime")
or get_tag("EXIF DateTimeOriginal")
or get_tag("EXIF DateTimeDigitized")
)
if date and (date := safe_strptime(date, "%Y:%m:%d %H:%M:%S")): if date and (date := safe_strptime(date, "%Y:%m:%d %H:%M:%S")):
pass pass
elif (date := date_from_name(p)): elif date := date_from_name(p):
dirty |= True dirty |= True
else: else:
# The oldest of the mtime and the ctime # The oldest of the mtime and the ctime
@ -589,10 +589,15 @@ def img_info(p: Path) -> ImgInfo:
dirty |= True dirty |= True
# 944404 # 944404
subsec = int(get_tag("EXIF SubSecTime") or get_tag("EXIF SubSecTimeOriginal") or get_tag("EXIF SubSecTimeDigitized") or "0") subsec = int(
get_tag("EXIF SubSecTime")
or get_tag("EXIF SubSecTimeOriginal")
or get_tag("EXIF SubSecTimeDigitized")
or "0"
)
# GoPro burst format is G%f.JPG or something close to it # GoPro burst format is G%f.JPG or something close to it
if subsec == 0 and (m := re.match("g.*(\d{6}).jpe?g", p.name.lower())): if subsec == 0 and (m := re.match(r"g.*(\d{6}).jpe?g", p.name.lower())):
subsec = int(m.group(1)) subsec = int(m.group(1))
date = date.replace(microsecond=subsec) date = date.replace(microsecond=subsec)
@ -614,11 +619,6 @@ def img_info(p: Path) -> ImgInfo:
def main(): def main():
opts, args = parser.parse_known_args() opts, args = parser.parse_known_args()
skip_tags = [
0x927C, # MakerNote
0x9286, # UserComment
0xC4A5, # PrintIM
]
print("---") print("---")
for src in list(opts.src_dir.glob("**/*")): for src in list(opts.src_dir.glob("**/*")):
@ -651,5 +651,6 @@ def main():
target.chmod(0o644) target.chmod(0o644)
src.unlink() # Delete the source src.unlink() # Delete the source
if __name__ == "__main__": if __name__ == "__main__":
main() main()

1
setup.cfg
View file

@ -27,6 +27,7 @@ shitlist = [
extend-ignore = [ extend-ignore = [
E203, E203,
E227, E227,
E266,
E306, E306,
E501, E501,
E731, E731,

20
tools/python/test_licenses.py
View file

@ -67,7 +67,9 @@ APPROVED_PACKAGES = [
def bash_license(ln): def bash_license(ln):
while True: while True:
lnn = re.sub( lnn = re.sub(
r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)|(or later)", "", ln.lower() r"[(),]|( version)|( license)|( ?v(?=\d))|([ -]clause)|(or later)",
"",
ln.lower(),
) )
if ln != lnn: if ln != lnn:
ln = lnn ln = lnn
@ -111,8 +113,14 @@ def licenses(dist: DistInfoDistribution):
print(name, version, type(dist)) print(name, version, type(dist))
meta = dist.get_metadata(dist.PKG_INFO).split("\n") meta = dist.get_metadata(dist.PKG_INFO).split("\n")
classifiers = [l.replace("Classifier: ", "", 1) for l in meta if l.startswith("Classifier: ")] classifiers = [
license = bash_license(next((l for l in meta if l.startswith("License:")), "License: UNKNOWN").replace("License: ", "", 1)) l.replace("Classifier: ", "", 1) for l in meta if l.startswith("Classifier: ")
]
license = bash_license(
next((l for l in meta if l.startswith("License:")), "License: UNKNOWN").replace(
"License: ", "", 1
)
)
lics.extend(l for l in classifiers if l.startswith("License ::")) lics.extend(l for l in classifiers if l.startswith("License ::"))
if not lics: if not lics:
@ -121,7 +129,11 @@ def licenses(dist: DistInfoDistribution):
return lics return lics
@pytest.mark.parametrize("dist", (w for w in working_set if w.location.find("arrdem_source_pypi") != -1), ids=lambda dist: dist.project_name) @pytest.mark.parametrize(
"dist",
(w for w in working_set if w.location.find("arrdem_source_pypi") != -1),
ids=lambda dist: dist.project_name,
)
def test_approved_license(dist: DistInfoDistribution): def test_approved_license(dist: DistInfoDistribution):
"""Ensure that a given package is either allowed by name or uses an approved license.""" """Ensure that a given package is either allowed by name or uses an approved license."""