From ed819c6f3dde945841863d7e14f968c0a4aac0bf Mon Sep 17 00:00:00 2001 From: Neil Massey Date: Tue, 26 Sep 2023 10:27:37 +0100 Subject: [PATCH 1/7] Added functions to get other users files if they are in the same workspace --- nlds_client/clientlib/config.py | 3 +- nlds_client/clientlib/nlds_client_setup.py | 8 +- nlds_client/clientlib/transactions.py | 23 +++++- nlds_client/nlds_client.py | 94 ++++++++++++++-------- 4 files changed, 88 insertions(+), 40 deletions(-) diff --git a/nlds_client/clientlib/config.py b/nlds_client/clientlib/config.py index c683753..dbe448b 100644 --- a/nlds_client/clientlib/config.py +++ b/nlds_client/clientlib/config.py @@ -2,8 +2,9 @@ import os.path from click import option -from nlds_client.clientlib.nlds_client_setup import CONFIG_FILE_LOCATION +from nlds_client.clientlib.nlds_client_setup import get_config_file_location from nlds_client.clientlib.exceptions import ConfigError +CONFIG_FILE_LOCATION = get_config_file_location() def validate_config_file(json_config): """Validate the JSON config file to match the schema in load_config_file.""" diff --git a/nlds_client/clientlib/nlds_client_setup.py b/nlds_client/clientlib/nlds_client_setup.py index 25508a1..bcbfba4 100644 --- a/nlds_client/clientlib/nlds_client_setup.py +++ b/nlds_client/clientlib/nlds_client_setup.py @@ -1 +1,7 @@ -CONFIG_FILE_LOCATION="~/.nlds-config" +import os + +def get_config_file_location(): + CONFIG_DIR = os.environ["HOME"] + return CONFIG_DIR + "/.nlds-config" + +CONFIG_FILE_LOCATION = get_config_file_location() \ No newline at end of file diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index 5417c9d..bd25787 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -326,6 +326,7 @@ def put_filelist(filelist: List[str]=[], def get_filelist(filelist: List[str]=[], user: str=None, group: str=None, + groupall: bool=False, target: str = None, job_label: str = None, label: str=None, @@ -409,6 +410,7 @@ def get_filelist(filelist: List[str]=[], input_params = {"transaction_id" : transaction_id, "user" : user, "group" : group, + "groupall" : groupall, "access_key" : access_key, "secret_key" : secret_key, "tenancy" : tenancy, @@ -453,6 +455,7 @@ def get_filelist(filelist: List[str]=[], def list_holding(user: str, group: str, + groupall: bool=False, label: str=None, holding_id: int=None, transaction_id: str=None, @@ -464,6 +467,9 @@ def list_holding(user: str, :param group: the group to get the holding(s) for :type group: string + :param groupall: list holdings for the entire group, not just the user + :type groupall: bool + :param label: the label of an existing holding to get the details for :type label: str, optional @@ -492,7 +498,8 @@ def list_holding(user: str, # user: str # group: str input_params = {"user" : user, - "group" : group} + "group" : group, + "groupall" : groupall} # add additional / optional components to input params if label is not None: @@ -524,6 +531,7 @@ def list_holding(user: str, def find_file(user: str, group: str, + groupall: bool=False, label: str=None, holding_id: int=None, transaction_id: str=None, @@ -536,6 +544,9 @@ def find_file(user: str, :param group: the group to get the holding(s) for :type group: string + :param groupall: list files for the entire group, not just the user + :type groupall: bool + :param label: the label of an existing holding to get the details for :type label: str, optional @@ -566,7 +577,8 @@ def find_file(user: str, # user: str # group: str input_params = {"user" : user, - "group" : group} + "group" : group, + "groupall" : groupall} # add additional / optional components to input params if label is not None: @@ -600,6 +612,7 @@ def find_file(user: str, def monitor_transactions(user: str, group: str, + groupall: bool=False, idd: int=None, transaction_id: str=None, job_label: str=None, @@ -616,6 +629,9 @@ def monitor_transactions(user: str, :param group: the group to get the transaction state(s) for :type group: string + :param groupall: list transactions for the entire group, not just the user + :type groupall: bool + :param idd: the numeric id (primary key) of the transaction :type idd: int @@ -658,7 +674,8 @@ def monitor_transactions(user: str, # user: str # group: str input_params = {"user" : user, - "group" : group} + "group" : group, + "groupall" : groupall} # add additional / optional components to input params if idd is not None: diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index 9965c15..75b5d55 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -69,10 +69,10 @@ def pretty_size(size): return round(size / float(multipler), 2).__str__() + suf -def format_request_details(user, group, label=None, holding_id=None, - tag=None, id=None, transaction_id=None, sub_id=None, - state=None, retry_count=None, api_action=None, - job_label=None): +def format_request_details(user, group, groupall=False, label=None, + holding_id=None, tag=None, id=None, + transaction_id=None, sub_id=None, state=None, + retry_count=None, api_action=None, job_label=None): config = load_config() out = "" user = get_user(config, user) @@ -81,6 +81,8 @@ def format_request_details(user, group, label=None, holding_id=None, group = get_group(config, group) out += f"group:{group}, " + if groupall: + out += f"groupall:{groupall}, " if label: out += f"label:{label}, " if id: @@ -122,10 +124,10 @@ def print_list(response: dict, req_details): click.echo(list_string) if n_holdings == 1: h = response['data']['holdings'][0] + click.echo(f"{'':<4}{'user':<16}: {h['user']}") click.echo(f"{'':<4}{'id':<16}: {h['id']}") click.echo(f"{'':<4}{'label':<16}: {h['label']}") click.echo(f"{'':<4}{'ingest time':<16}: {h['date'].replace('T',' ')}") - # click.echo(f"{'':<4}{'user':<16}: {h['user']}") # click.echo(f"{'':<4}{'group':<16}: {h['group']}") if 'transactions' in h: trans_str = "" @@ -136,12 +138,10 @@ def print_list(response: dict, req_details): tags_str = _tags_to_str(h['tags']) click.echo(f"{'':<4}{'tags':<16}: {tags_str[:-23]}") else: - # click.echo(f"{'':<4}{'id':<6}{'label':<16}{'user':<16}{'group':<16}") - click.echo(f"{'':<4}{'id':<6}{'label':<16}{'ingest time':<32}") + click.echo(f"{'':<4}{'user':<16}{'id':<6}{'label':<16}{'ingest time':<32}") for h in response['data']['holdings']: click.echo( - # f"{'':<4}{h['id']:<6}{h['label']:<16}{h['user']:<16}{h['group']:<16}" - f"{'':<4}{h['id']:<6}{h['label']:<16}{h['date'].replace('T',' '):<32}" + f"{'':<4}{h['user']:<16}{h['id']:<6}{h['label']:<16}{h['date'].replace('T',' '):<32}" ) @@ -191,7 +191,7 @@ def print_multi_stat(response: dict, req_details): stat_string = "State of transactions for " stat_string += req_details click.echo(stat_string) - click.echo(f"{'':<4}{'id':<6}{'action':<16}{'job label':<16}" + click.echo(f"{'':<4}{'user':<16}{'id':<6}{'action':<16}{'job label':<16}" f"{'label':<16}{'state':<23}{'last update':<20}") for tr in response['data']['records']: state, time = get_transaction_state(tr) @@ -206,7 +206,7 @@ def print_multi_stat(response: dict, req_details): job_label = tr['job_label'] else: job_label = "" #tr['transaction_id'][0:8] - click.echo(f"{'':<4}{tr['id']:<6}{tr['api_action']:<16}" + click.echo(f"{'':<4}{tr['user']:<16}{tr['id']:<6}{tr['api_action']:<16}" f"{job_label:16}{label:16}" f"{state:<23}{time:<20}") @@ -267,7 +267,7 @@ def print_single_file(response): def print_multi_file(response): - click.echo(f"{'':<4}{'h-id':<6}{'h-label':<16}{'path':<64}{'size':<8}{'time':<12}") + click.echo(f"{'':<4}{'user':<16}{'h-id':<6}{'h-label':<16}{'size':<8}{'date':<12}{'path'}") for hkey in response['data']['holdings']: h = response['data']['holdings'][hkey] for tkey in h['transactions']: @@ -275,9 +275,9 @@ def print_multi_file(response): time = t['ingest_time'].replace("T", " ") for f in t['filelist']: size = pretty_size(f['size']) - click.echo(f"{'':4}{h['holding_id']:<6}{h['label']:<16}" - f"{f['original_path']:<64}{size:<8}" - f"{time:<12}") + click.echo(f"{'':4}{h['user']:<16}" + f"{h['holding_id']:<6}{h['label']:<16}" + f"{size:<8}{time[:11]:<12}{f['original_path']}") def print_find(response:dict, req_details): @@ -390,6 +390,9 @@ def put(filepath, user, group, job_label, help="The username to get a file for.") @click.option("-g", "--group", default=None, type=str, help="The group to get a file for.") +@click.option("-G", "--groupall", default=False, is_flag=True, + help="Get a file that belongs to a group, rather than a single " + "user") @click.option("-r", "--target", default=None, type=click.Path(exists=True), help="The target path for the retrieved file. Default is to " "retrieve the file to its original path.") @@ -406,11 +409,11 @@ def put(filepath, user, group, job_label, @click.option("-j", "--json", default=False, type=bool, help="Output the result as JSON.") @click.argument("filepath", type=str) -def get(filepath, user, group, target, job_label, +def get(filepath, user, group, groupall, target, job_label, label, holding_id, tag, json): try: - response = get_filelist([filepath], user, group, target, job_label, - label, holding_id, tag) + response = get_filelist([filepath], user, group, groupall, target, + job_label, label, holding_id, tag) if json: click.echo(response) else: @@ -484,6 +487,9 @@ def putlist(filelist, user, group, label, job_label, help="The username to get files for.") @click.option("-g", "--group", default=None, type=str, help="The group to get files for.") +@click.option("-G", "--groupall", default=False, is_flag=True, + help="Get files that belong to a group, rather than a single " + "user") @click.option("-t", "--target", default=None, type=click.Path(exists=True), help="The target path for the retrieved files. Default is to " "retrieve files to their original path.") @@ -500,7 +506,7 @@ def putlist(filelist, user, group, label, job_label, @click.option("-j", "--json", default=False, type=bool, help="Output the result as JSON.") @click.argument("filelist", type=str) -def getlist(filelist, user, group, target, job_label, +def getlist(filelist, user, group, groupall, target, job_label, label, holding_id, tag, json): # read the filelist from the file try: @@ -511,7 +517,7 @@ def getlist(filelist, user, group, target, job_label, raise click.UsageError(fe) try: - response = get_filelist(files, user, group, target, job_label, + response = get_filelist(files, user, group, groupall, target, job_label, label, holding_id, tag) if json: click.echo(response) @@ -532,6 +538,9 @@ def getlist(filelist, user, group, target, job_label, help="The username to list holdings for.") @click.option("-g", "--group", default=None, type=str, help="The group to list holdings for.") +@click.option("-G", "--groupall", default=False, is_flag=True, + help="List holdings that belong to a group, rather than a single " + "user") @click.option("-l", "--label", default=None, type=str, help="The label of the holding(s) to list. This can be a regular" "expression (regex).") @@ -543,14 +552,16 @@ def getlist(filelist, user, group, target, job_label, help="The tag(s) of the holding(s) to list.") @click.option("-j", "--json", default=False, is_flag=True, help="Output the result as JSON.") -def list(user, group, label, holding_id, transaction_id, tag, json): +def list(user, group, groupall, label, holding_id, transaction_id, tag, json): # try: response = list_holding( - user, group, label, holding_id, transaction_id, tag + user, group, groupall=groupall, label=label, + holding_id=holding_id, transaction_id=transaction_id, tag=tag ) req_details = format_request_details( - user, group, label=label, holding_id=holding_id, tag=tag + user, group, groupall=groupall, label=label, + holding_id=holding_id, tag=tag ) if response['success']: if json: @@ -579,6 +590,9 @@ def list(user, group, label, holding_id, transaction_id, tag, json): help="The username to list transactions for.") @click.option("-g", "--group", default=None, type=str, help="The group to list transactions for.") +@click.option("-G", "--groupall", default=False, is_flag=True, + help="List transactions that belong to a group, rather than a " + "single user") @click.option("-i", "--id", default=None, type=int, help="The numeric id of the transaction to list.") @click.option("-n", "--transaction_id", default=None, type=str, @@ -595,15 +609,19 @@ def list(user, group, label, holding_id, transaction_id, tag, json): "TRANSFER_GETTING | COMPLETE | FAILED") @click.option("-j", "--json", default=False, type=bool, is_flag=True, help="Output the result as JSON.") -def stat(user, group, id, transaction_id, job_label, api_action, state, json): +def stat(user, group, groupall, id, transaction_id, job_label, api_action, + state, json): try: response = monitor_transactions( - user, group, id, transaction_id, job_label, api_action, state, + user, group, groupall=groupall, idd=id, + transaction_id=transaction_id, job_label=job_label, + api_action=api_action, state=state, ) req_details = format_request_details( - user, group, id=id, transaction_id=transaction_id, state=state, - api_action=api_action, - ) + user, group, groupall=groupall, id=id, + transaction_id=transaction_id, state=state, + api_action=api_action, + ) if response['success']: if json: click.echo(response) @@ -630,6 +648,9 @@ def stat(user, group, id, transaction_id, job_label, api_action, state, json): help="The username to find files for.") @click.option("-g", "--group", default=None, type=str, help="The group to find files for.") +@click.option("-G", "--groupall", default=False, is_flag=True, + help="Find files that belong to a group, rather than a single " + "user") @click.option("-l", "--label", default=None, type=str, help="The label of the holding which the files belong to. This " "can be a regular expression (regex).") @@ -644,15 +665,18 @@ def stat(user, group, id, transaction_id, job_label, api_action, state, json): help="The tag(s) of the holding(s) to find files within.") @click.option("-j", "--json", default=False, type=bool, is_flag=True, help="Output the result as JSON.") -def find(user, group, label, holding_id, transaction_id, path, tag, json): +def find(user, group, groupall, label, holding_id, transaction_id, path, tag, + json): # try: - response = find_file(user, group, label, holding_id, - transaction_id, path, tag) + response = find_file( + user, group, groupall=groupall, label=label, holding_id=holding_id, + transaction_id=transaction_id, path=path, tag=tag + ) req_details = format_request_details( - user, group, label=label, holding_id=holding_id, tag=tag, - transaction_id=transaction_id - ) + user, group, groupall=groupall, label=label, + holding_id=holding_id, tag=tag, transaction_id=transaction_id + ) if response['success']: if json: click.echo(response) @@ -698,7 +722,7 @@ def meta(user, group, label, holding_id, tag, new_label, new_tag, del_tag, json) # try: req_details = format_request_details( - user, group, label, holding_id, tag + user, group, label=label, holding_id=holding_id, tag=tag ) response = change_metadata(user, group, label, holding_id, tag, new_label, new_tag, del_tag) From 09a9eef7451c7a59113971a9f7472727dde30fba Mon Sep 17 00:00:00 2001 From: Neil Massey Date: Tue, 26 Sep 2023 11:56:30 +0100 Subject: [PATCH 2/7] Added simple, single line per file output --- nlds_client/nlds_client.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index 75b5d55..201836d 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -266,6 +266,15 @@ def print_single_file(response): click.echo(f"{'':<4}{'storage location':<16}:{stls[0:-2]}") +def print_simple_file(response): + for hkey in response['data']['holdings']: + h = response['data']['holdings'][hkey] + for tkey in h['transactions']: + t = h['transactions'][tkey] + for f in t['filelist']: + click.echo(f"{f['original_path']}") + + def print_multi_file(response): click.echo(f"{'':<4}{'user':<16}{'h-id':<6}{'h-label':<16}{'size':<8}{'date':<12}{'path'}") for hkey in response['data']['holdings']: @@ -280,21 +289,24 @@ def print_multi_file(response): f"{size:<8}{time[:11]:<12}{f['original_path']}") -def print_find(response:dict, req_details): +def print_find(response:dict, req_details, simple): """Print out the response from the find command""" n_holdings = len(response['data']['holdings']) - list_string = "Listing files for holding" - if n_holdings > 1: - list_string += "s" - list_string += " for " - list_string += req_details - click.echo(list_string) + if not simple: + list_string = "Listing files for holding" + if n_holdings > 1: + list_string += "s" + list_string += " for " + list_string += req_details + click.echo(list_string) # get total number of files n_files = __count_files(response) if (n_files == 1): print_single_file(response) + elif simple: + print_simple_file(response) else: - print_multi_file(response) + print_multi_file(response, simple) def print_meta(response:dict, req_details:str): @@ -665,8 +677,10 @@ def stat(user, group, groupall, id, transaction_id, job_label, api_action, help="The tag(s) of the holding(s) to find files within.") @click.option("-j", "--json", default=False, type=bool, is_flag=True, help="Output the result as JSON.") +@click.option("-1", "--simple", default=False, type=bool, is_flag=True, + help="Output the list of files, one per line, filepath only.") def find(user, group, groupall, label, holding_id, transaction_id, path, tag, - json): + json, simple): # try: response = find_file( @@ -681,7 +695,7 @@ def find(user, group, groupall, label, holding_id, transaction_id, path, tag, if json: click.echo(response) else: - print_find(response, req_details) + print_find(response, req_details, simple) else: fail_string = "Failed to list files with " fail_string += req_details From f32905b11b5ddbf7b7e437fe6e05b2041782e062 Mon Sep 17 00:00:00 2001 From: Neil Massey Date: Mon, 2 Oct 2023 15:10:36 +0100 Subject: [PATCH 3/7] Retrieve url --- nlds_client/clientlib/transactions.py | 1 + nlds_client/nlds_client.py | 47 +++++++++++++++++++-------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index bd25787..48485cf 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -669,6 +669,7 @@ def monitor_transactions(user: str, group = get_group(config, group) url = construct_server_url(config, "status") MAX_LOOPS = 2 + print(group, groupall) # build the parameters. monitoring->get requires # user: str diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index 201836d..84ee6b5 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -233,7 +233,7 @@ def __count_files(response:dict): return n_files -def print_single_file(response): +def print_single_file(response, print_url=False): """Print (full) details of one file""" # NRM - note: still using loops over dictionary keys as its # 1. easier than trying to just use the first key @@ -260,22 +260,36 @@ def print_single_file(response): click.echo(f"{'':<4}{'ingest time':<16}: {time}") # locations stls = " " + url = _get_url_from_file(f) for s in f['locations']: stls += s['storage_type']+", " click.echo(f"{'':<4}{'storage location':<16}:{stls[0:-2]}") + if url is not None and print_url: + click.echo(f"{'':<4}{'url':<16}: {url}") +def _get_url_from_file(f): + url = None + for s in f['locations']: + if s['storage_type'] == "OBJECT_STORAGE": + url = s['url'] + return url -def print_simple_file(response): + +def print_simple_file(response, print_url=False): for hkey in response['data']['holdings']: h = response['data']['holdings'][hkey] for tkey in h['transactions']: t = h['transactions'][tkey] for f in t['filelist']: - click.echo(f"{f['original_path']}") + url = _get_url_from_file(f) + if print_url and url: + click.echo(url) + else: + click.echo(f"{f['original_path']}") -def print_multi_file(response): +def print_multi_file(response, print_url): click.echo(f"{'':<4}{'user':<16}{'h-id':<6}{'h-label':<16}{'size':<8}{'date':<12}{'path'}") for hkey in response['data']['holdings']: h = response['data']['holdings'][hkey] @@ -284,12 +298,17 @@ def print_multi_file(response): time = t['ingest_time'].replace("T", " ") for f in t['filelist']: size = pretty_size(f['size']) + url = _get_url_from_file(f) + if url and print_url: + path_print = _get_url_from_file(f) + else: + path_print = f['original_path'] click.echo(f"{'':4}{h['user']:<16}" f"{h['holding_id']:<6}{h['label']:<16}" - f"{size:<8}{time[:11]:<12}{f['original_path']}") + f"{size:<8}{time[:11]:<12}{path_print}") -def print_find(response:dict, req_details, simple): +def print_find(response:dict, req_details, simple, url): """Print out the response from the find command""" n_holdings = len(response['data']['holdings']) if not simple: @@ -301,12 +320,12 @@ def print_find(response:dict, req_details, simple): click.echo(list_string) # get total number of files n_files = __count_files(response) - if (n_files == 1): - print_single_file(response) - elif simple: - print_simple_file(response) + if (simple): + print_simple_file(response, url) + elif (n_files == 1): + print_single_file(response, url) else: - print_multi_file(response, simple) + print_multi_file(response, url) def print_meta(response:dict, req_details:str): @@ -679,8 +698,10 @@ def stat(user, group, groupall, id, transaction_id, job_label, api_action, help="Output the result as JSON.") @click.option("-1", "--simple", default=False, type=bool, is_flag=True, help="Output the list of files, one per line, filepath only.") +@click.option("-U", "--url", default=False, type=bool, is_flag=True, + help="Output the URL for the file on the object storage.") def find(user, group, groupall, label, holding_id, transaction_id, path, tag, - json, simple): + json, simple, url): # try: response = find_file( @@ -695,7 +716,7 @@ def find(user, group, groupall, label, holding_id, transaction_id, path, tag, if json: click.echo(response) else: - print_find(response, req_details, simple) + print_find(response, req_details, simple, url) else: fail_string = "Failed to list files with " fail_string += req_details From 33aa77b6a3b570b635c9d2bd12ce9862a74826d4 Mon Sep 17 00:00:00 2001 From: Neil Massey Date: Thu, 5 Oct 2023 14:54:31 +0100 Subject: [PATCH 4/7] Changed -G option to -A --- nlds_client/nlds_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index 84ee6b5..0471f08 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -421,7 +421,7 @@ def put(filepath, user, group, job_label, help="The username to get a file for.") @click.option("-g", "--group", default=None, type=str, help="The group to get a file for.") -@click.option("-G", "--groupall", default=False, is_flag=True, +@click.option("-A", "--groupall", default=False, is_flag=True, help="Get a file that belongs to a group, rather than a single " "user") @click.option("-r", "--target", default=None, type=click.Path(exists=True), @@ -518,7 +518,7 @@ def putlist(filelist, user, group, label, job_label, help="The username to get files for.") @click.option("-g", "--group", default=None, type=str, help="The group to get files for.") -@click.option("-G", "--groupall", default=False, is_flag=True, +@click.option("-A", "--groupall", default=False, is_flag=True, help="Get files that belong to a group, rather than a single " "user") @click.option("-t", "--target", default=None, type=click.Path(exists=True), @@ -569,7 +569,7 @@ def getlist(filelist, user, group, groupall, target, job_label, help="The username to list holdings for.") @click.option("-g", "--group", default=None, type=str, help="The group to list holdings for.") -@click.option("-G", "--groupall", default=False, is_flag=True, +@click.option("-A", "--groupall", default=False, is_flag=True, help="List holdings that belong to a group, rather than a single " "user") @click.option("-l", "--label", default=None, type=str, @@ -621,7 +621,7 @@ def list(user, group, groupall, label, holding_id, transaction_id, tag, json): help="The username to list transactions for.") @click.option("-g", "--group", default=None, type=str, help="The group to list transactions for.") -@click.option("-G", "--groupall", default=False, is_flag=True, +@click.option("-A", "--groupall", default=False, is_flag=True, help="List transactions that belong to a group, rather than a " "single user") @click.option("-i", "--id", default=None, type=int, @@ -679,7 +679,7 @@ def stat(user, group, groupall, id, transaction_id, job_label, api_action, help="The username to find files for.") @click.option("-g", "--group", default=None, type=str, help="The group to find files for.") -@click.option("-G", "--groupall", default=False, is_flag=True, +@click.option("-A", "--groupall", default=False, is_flag=True, help="Find files that belong to a group, rather than a single " "user") @click.option("-l", "--label", default=None, type=str, From 979f9ca747916fe0518ee0e796f7d5db40fa183c Mon Sep 17 00:00:00 2001 From: Jack Leland Date: Wed, 11 Oct 2023 11:59:54 +0100 Subject: [PATCH 5/7] Add a tests-specific requirements file --- .github/workflows/testing.yml | 3 ++- tests/requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/requirements.txt diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5834634..2fd4f93 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -29,6 +29,7 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi python -m pip install -e . - name: Lint with flake8 run: | diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..d00689e --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1 @@ +pytest==7.1.2 From 9f0a30c7882e616876781ac6dbbf721fa57a56e4 Mon Sep 17 00:00:00 2001 From: Jack Leland Date: Mon, 16 Oct 2023 16:52:34 +0100 Subject: [PATCH 6/7] Update get_transaction_state to use new values --- nlds_client/clientlib/transactions.py | 81 +++++++++++++-------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index 48485cf..9cf44bb 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -740,52 +740,51 @@ def get_transaction_state(transaction: dict): } possible values of state are: - INITIALISING = -1 - ROUTING = 0 - SPLITTING = 1 - INDEXING = 2 - TRANSFER_PUTTING = 3 - CATALOG_PUTTING = 4 - CATALOG_GETTING = 5 - TRANSFER_GETTING = 6 - COMPLETE = 8 - FAILED = 9 - COMPLETE_WITH_ERRORS = 10 - COMPLETE_WITH_WARNINGS = 11 + INITIALISING = -1 + ROUTING = 0 + SPLITTING = 1 + INDEXING = 2 + CATALOG_PUTTING = 3 + TRANSFER_PUTTING = 4 + CATALOG_ROLLBACK = 5 + CATALOG_GETTING = 10 + ARCHIVE_GETTING = 11 + TRANSFER_GETTING = 12 + ARCHIVE_INIT = 20 + CATALOG_ARCHIVE_AGGREGATING = 21 + ARCHIVE_PUTTING = 22 + CATALOG_ARCHIVE_UPDATING = 23 + CATALOG_ARCHIVE_ROLLBACK = 40 + COMPLETE = 100 + FAILED = 101 + COMPLETE_WITH_ERRORS = 102 + COMPLETE_WITH_WARNINGS = 103 The overall state is the minimum of these """ state_mapping = { - "INITIALISING" : -1, - "ROUTING" : 0, - "SPLITTING" : 1, - "INDEXING" : 2, - "CATALOG_PUTTING" : 3, - "TRANSFER_PUTTING" : 4, + "INITIALISING": -1, + "ROUTING": 0, + "SPLITTING": 1, + "INDEXING": 2, + "CATALOG_PUTTING": 3, + "TRANSFER_PUTTING": 4, "CATALOG_ROLLBACK": 5, - "CATALOG_GETTING" : 6, - "TRANSFER_GETTING" : 7, - "COMPLETE" : 8, - "FAILED" : 9, - "COMPLETE_WITH_ERRORS" : 10, - "COMPLETE_WITH_WARNINGS": 11, - } - state_mapping_reverse = { - -1 : "INITIALISING", - 0 : "ROUTING", - 1 : "SPLITTING", - 2 : "INDEXING", - 3 : "CATALOG_PUTTING", - 4 : "TRANSFER_PUTTING" , - 5 : "CATALOG_ROLLBACK", - 6 : "CATALOG_GETTING", - 7 : "TRANSFER_GETTING", - 8 : "COMPLETE", - 9 : "FAILED", - 10: "COMPLETE_WITH_ERRORS", - 11: "COMPLETE_WITH_WARNINGS" + "CATALOG_GETTING": 10, + "ARCHIVE_GETTING": 11, + "TRANSFER_GETTING": 12, + "ARCHIVE_INIT": 20, + "CATALOG_ARCHIVE_AGGREGATING": 21, + "ARCHIVE_PUTTING": 22, + "CATALOG_ARCHIVE_UPDATING": 23, + "CATALOG_ARCHIVE_ROLLBACK": 40, + "COMPLETE": 100, + "FAILED": 101, + "COMPLETE_WITH_ERRORS" : 102, + "COMPLETE_WITH_WARNINGS": 103, } + state_mapping_reverse = {v: k for k, v in state_mapping.items()} - min_state = 100 + min_state = 200 min_time = datetime(1970,1,1) error_count = 0 for sr in transaction["sub_records"]: @@ -798,7 +797,7 @@ def get_transaction_state(transaction: dict): if sr_state == "FAILED": error_count += 1 - if min_state == 100: + if min_state == 200: return None, None if min_state == state_mapping["COMPLETE"] and error_count > 0: From 4e0dffa3e774d79d49bbd3a4c0166f5f792639cf Mon Sep 17 00:00:00 2001 From: Jack Leland Date: Mon, 18 Dec 2023 16:02:50 +0000 Subject: [PATCH 7/7] Remove extraneous print statement --- nlds_client/clientlib/transactions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index 9cf44bb..dc858cd 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -669,7 +669,6 @@ def monitor_transactions(user: str, group = get_group(config, group) url = construct_server_url(config, "status") MAX_LOOPS = 2 - print(group, groupall) # build the parameters. monitoring->get requires # user: str