Skip to content

Commit

Permalink
Restore download tracker (#68)
Browse files Browse the repository at this point in the history
  • Loading branch information
mdellabitta authored Dec 18, 2024
1 parent 065117e commit d988092
Showing 1 changed file with 8 additions and 7 deletions.
15 changes: 8 additions & 7 deletions tools/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,12 @@ def process_media(

sha1 = get_file_hash(temp_file_name)
upload_file_to_s3(temp_file_name, destination_path, content_type, sha1)
tracker.increment(Result.DOWNLOADED)
tracker.increment(Result.BYTES, os.stat(temp_file_name).st_size)

except Exception as e:
tracker.increment(Result.FAILED)
logging.warning(f"Failed: {dpla_id} {ordinal}", exc_info=e)

finally:
if temp_file:
Expand Down Expand Up @@ -244,13 +250,8 @@ def process_item(
if media_url.startswith("https/"):
media_url = media_url.replace("https/", "https:/")
logging.info(f"Downloading {partner} {dpla_id} {count} from {media_url}")
try:
if not dry_run:
process_media(partner, dpla_id, count, media_url, overwrite, sleep_secs)

except Exception as e:
tracker.increment(Result.FAILED)
logging.warning(f"Failed: {dpla_id} {count}", exc_info=e)
if not dry_run:
process_media(partner, dpla_id, count, media_url, overwrite, sleep_secs)


@click.command()
Expand Down

0 comments on commit d988092

Please sign in to comment.