From 3879f0e64143b2e80a2df7263a0e3abb863f003b Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Tue, 23 Oct 2018 20:16:02 +1100 Subject: [PATCH 1/8] Add exception handling for issue #49 --- CHANGELOG.md | 1 + clean_content_views.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be3a3d..56e2057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - clean_content_views raised an exception if a CV version was included in a composite view. - Default org view was assumed to be version 1.0. Correct version is now extracted (Issue #43) - Org name and label do not always match. Issue with mixed case and spaces in org name (Issue #42) +- clean_content_views did not handle exception if API returns null value (Issue #49) ### Added - Option to define the tar split size (Issue #44) diff --git a/clean_content_views.py b/clean_content_views.py index 8a595fe..04adfa9 100755 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -98,7 +98,7 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir # Check if there is a publish/promote already running on this content view locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid]) - msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'" + msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'INFO') print helpers.HEADER + msg + helpers.ENDC @@ -217,6 +217,10 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir msg = "Failed to initiate removal" helpers.log_msg(msg, 'WARNING') + except KeyError: + msg = "Failed to initiate removal" + helpers.log_msg(msg, 'WARNING') + # Exit in the case of a dry-run if dry_run: msg = "Dry run - not actually performing removal" @@ -299,5 +303,3 @@ def main(args): except KeyboardInterrupt, e: print >> sys.stderr, ("\n\nExiting on user cancel.") sys.exit(1) - - From 65a8c289333f8d3ce975943c8bc55aef3a7326f1 Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Sun, 4 Nov 2018 10:31:48 +1100 Subject: [PATCH 2/8] Add missing float type for CV --- clean_content_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clean_content_views.py b/clean_content_views.py index 04adfa9..7875553 100755 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -178,7 +178,7 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir print msg continue else: - if float(version['version']) < (lastver - float(ver_keep[cvid])): + if float(version['version']) < (float(lastver) - float(ver_keep[cvid])): msg = "Removing version " + str(version['version']) helpers.log_msg(msg, 'INFO') print helpers.HEADER + msg + helpers.ENDC @@ -220,7 +220,7 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir except KeyError: msg = "Failed to initiate removal" helpers.log_msg(msg, 'WARNING') - + # Exit in the case of a dry-run if dry_run: msg = "Dry run - not actually performing removal" From da855dd1d766f764ad6d39383cc654265f5add4f Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Mon, 12 Nov 2018 07:10:29 +1100 Subject: [PATCH 3/8] Remove duplicate calculation of CV to delete. Change method from an integer count. --- clean_content_views.py | 63 +++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/clean_content_views.py b/clean_content_views.py index 7875553..c1e3eba 100755 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -110,6 +110,8 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir version_list_all = [] for version in cvinfo['versions']: if not version['environment_ids']: + # These are the versions that don't belong to an environment + # i.e. orphans. version_list_all.append(float(version['version'])) continue else: @@ -133,10 +135,20 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir msg = "Oldest NOT-in-use version is " + str(min(version_list_all)) helpers.log_msg(msg, 'DEBUG') - # Find version to delete (based on keep parameter) if --ignorefirstpromoted + # Find versions to delete (based on keep parameter) + # Make sure the version list is in order version_list_all.sort() - todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))] - msg = "Versions to remove if --ignorefirstpromoted: " + str(todelete) + + if cleanall: + # Remove all orphaned versions + todelete = version_list_all + elif ignorefirstpromoted: + # Remove the last 'keep' elements from the orphans list (from PR #26) + todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))] + else: + # Need to find 'lastver' in the list and then calculate keep from that + todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))] + msg = "Versions to remove: " + str(todelete) helpers.log_msg(msg, 'DEBUG') for version in cvinfo['versions']: @@ -149,44 +161,21 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'DEBUG') - if ignorefirstpromoted: - if cleanall: + if cleanall: + msg = "Removing version " + str(version['version']) + helpers.log_msg(msg, 'INFO') + print helpers.HEADER + msg + helpers.ENDC + else: + if float(version['version']) in todelete: + # If ignorefirstpromoted delete CV msg = "Removing version " + str(version['version']) helpers.log_msg(msg, 'INFO') print helpers.HEADER + msg + helpers.ENDC else: - if float(version['version']) in todelete: - # If ignorefirstpromoted delete CV - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) + " due to --keep value" - helpers.log_msg(msg, 'INFO') - print msg - continue - else: - if float(version['version']) > float(lastver): - # If we have chosen to remove all orphans - if cleanall: - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print msg - continue - else: - if float(version['version']) < (float(lastver) - float(ver_keep[cvid])): - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) + " due to --keep value" - helpers.log_msg(msg, 'INFO') - print msg - continue + msg = "Skipping delete of version " + str(version['version']) + " due to --keep value" + helpers.log_msg(msg, 'INFO') + print msg + continue # Delete the view version from the content view if not dry_run and not locked: From f5e1f13e79941f732f1def20985cbf237830c7b9 Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Mon, 12 Nov 2018 21:13:11 +1100 Subject: [PATCH 4/8] Fixed standard behaviour. Updated documentation. --- CHANGELOG.md | 1 + README.md | 15 +++++++++++++++ clean_content_views.py | 33 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56e2057..4f761ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Default org view was assumed to be version 1.0. Correct version is now extracted (Issue #43) - Org name and label do not always match. Issue with mixed case and spaces in org name (Issue #42) - clean_content_views did not handle exception if API returns null value (Issue #49) +- clean_content_views now correctly handles incremental content view deletion (Issue #49) ### Added - Option to define the tar split size (Issue #44) diff --git a/README.md b/README.md index 4736f94..9eb1871 100644 --- a/README.md +++ b/README.md @@ -366,6 +366,21 @@ Content views to clean can be defined by either: - All content views (-a) - All content views, ignoring the first promoted one (-i) +The option to use will depend on the historic (old) content views you wish to keep. +An example of the different options with a keep value of '1' is shown below: + ++-----------------+------+------+------+ +| version | -i | -c | none | ++-----------------+------+------+------+ +| 110.0 (Library) | | | | +| 109.0 | KEEP | DEL | KEEP | +| 108.3 | DEL | DEL | KEEP | +| 108.2 (Quality) | | | | +| 108.1 | DEL | DEL | KEEP | +| 108.0 | DEL | DEL | DEL | +| 107.0 | DEL | DEL | DEL | ++-----------------+------+------+------+ + The dry run (-d) option can be used to see what would be published for a given command input. Use this option to see the difference in behaviour between (-a) and (-i) options, with and without (-c) diff --git a/clean_content_views.py b/clean_content_views.py index c1e3eba..613aa1c 100755 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -107,12 +107,14 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir # Find the oldest published version version_list = [] - version_list_all = [] + orphan_versions = [] + all_versions = [] for version in cvinfo['versions']: + all_versions.append(float(version['version'])) if not version['environment_ids']: # These are the versions that don't belong to an environment # i.e. orphans. - version_list_all.append(float(version['version'])) + orphan_versions.append(float(version['version'])) continue else: msg = "Found version " + str(version['version']) @@ -129,25 +131,38 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir helpers.log_msg(msg, 'DEBUG') # Find the oldest 'NOT in use' version id - if not version_list_all: + if not orphan_versions: msg = "No oldest NOT-in-use version found" else: - msg = "Oldest NOT-in-use version is " + str(min(version_list_all)) + msg = "Oldest NOT-in-use version is " + str(min(orphan_versions)) helpers.log_msg(msg, 'DEBUG') + # Find the element position in the all_versions list of the oldest in-use version + # e.g. vers 102.0 is oldest in-use and is element [5] in the all_versions list + list_position = [i for i,x in enumerate(all_versions) if x == lastver] + # Remove the number of views to keep from the element position of the oldest in-use + # e.g. keep=2 results in an adjusted list element position [3] + num_to_delete = list_position[0] - int(ver_keep[cvid]) + # Delete from position [0] to the first 'keep' position + # e.g. first keep element is [3] so list of elements [0, 1, 2] is created + list_pos_to_delete = [i for i in range(num_to_delete)] + # Find versions to delete (based on keep parameter) # Make sure the version list is in order - version_list_all.sort() + orphan_versions.sort() if cleanall: # Remove all orphaned versions - todelete = version_list_all + todelete = orphan_versions elif ignorefirstpromoted: # Remove the last 'keep' elements from the orphans list (from PR #26) - todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))] + todelete = orphan_versions[:(len(orphan_versions) - int(ver_keep[cvid]))] else: - # Need to find 'lastver' in the list and then calculate keep from that - todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))] + todelete = [] + # Remove the element numbers for deletion from the list all versions + for i in sorted(list_pos_to_delete, reverse=True): + todelete.append(orphan_versions[i]) + msg = "Versions to remove: " + str(todelete) helpers.log_msg(msg, 'DEBUG') From 2a8b05d2d47481149c6e2e3076ceab25bff23196 Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Mon, 12 Nov 2018 21:20:42 +1100 Subject: [PATCH 5/8] Formatting change --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9eb1871..8b5b3a2 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,7 @@ Content views to clean can be defined by either: The option to use will depend on the historic (old) content views you wish to keep. An example of the different options with a keep value of '1' is shown below: +``` +-----------------+------+------+------+ | version | -i | -c | none | +-----------------+------+------+------+ @@ -380,6 +381,7 @@ An example of the different options with a keep value of '1' is shown below: | 108.0 | DEL | DEL | DEL | | 107.0 | DEL | DEL | DEL | +-----------------+------+------+------+ +``` The dry run (-d) option can be used to see what would be published for a given command input. Use this option to see the difference in behaviour between From e12bc55768ef410cfa17104a9a718b807c26dd83 Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Mon, 12 Nov 2018 21:29:32 +1100 Subject: [PATCH 6/8] Update table and definition of -i flag --- README.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8b5b3a2..1266c89 100644 --- a/README.md +++ b/README.md @@ -360,27 +360,31 @@ Any orphaned versions older than the last in-use version are purged (orphans between in-use versions will NOT be purged, unless the cleanall (-c) option is used). There is a keep (-k) option that allows a specific number of versions older than the last in-use to be kept as well, allowing for possible rollback of versions. +Note that the (-c) option will delete ALL orphaned versions, regardless of the (-k) +value. An alternative option (-i) slightly alters the calculation of the versions +that will be deleted, in that the (-k) option defines how many versions after the +newest (last promoted) version will be kept, rather than use the oldest (first +promoted) version. Content views to clean can be defined by either: - Specific content views defined in the main config file - All content views (-a) - - All content views, ignoring the first promoted one (-i) The option to use will depend on the historic (old) content views you wish to keep. An example of the different options with a keep value of '1' is shown below: ``` -+-----------------+------+------+------+ -| version | -i | -c | none | -+-----------------+------+------+------+ -| 110.0 (Library) | | | | -| 109.0 | KEEP | DEL | KEEP | -| 108.3 | DEL | DEL | KEEP | -| 108.2 (Quality) | | | | -| 108.1 | DEL | DEL | KEEP | -| 108.0 | DEL | DEL | DEL | -| 107.0 | DEL | DEL | DEL | -+-----------------+------+------+------+ ++-----------------+----------+-----------------------+------------+ +| version | no flags | --ignorefirstpromoted | --cleanall | ++-----------------+----------+-----------------------+------------+ +| 110.0 (Library) | | | | +| 109.0 | KEEP | KEEP | DEL | +| 108.3 | KEEP | DEL | DEL | +| 108.2 (Quality) | | | | +| 108.1 | KEEP | DEL | DEL | +| 108.0 | DEL | DEL | DEL | +| 107.0 | DEL | DEL | DEL | ++-----------------+----------+-----------------------+------------+ ``` The dry run (-d) option can be used to see what would be published for a From dededcb5df7f4c11c0d2c5dfe13d4a5f75dcfa31 Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Tue, 20 Nov 2018 21:11:00 +1100 Subject: [PATCH 7/8] Rewrite of version deletion calculations --- CHANGELOG.md | 5 +- clean_content_views.py | 160 +++++++++++++++++++++++++---------------- helpers.py | 18 ++++- 3 files changed, 118 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f761ae..9502685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Fixed -- clean_content_views raised an exception if a CV version was included in a composite view. +- clean_content_views raised an exception if a CV version was included in a composite view - Default org view was assumed to be version 1.0. Correct version is now extracted (Issue #43) - Org name and label do not always match. Issue with mixed case and spaces in org name (Issue #42) - clean_content_views did not handle exception if API returns null value (Issue #49) - clean_content_views now correctly handles incremental content view deletion (Issue #49) +- Tasks in 'planning' state were not being considered when checking for locks +- foreman_tasks API returns action as 'Promote' instead of 'Promotion' in Sat 6.3 + ### Added - Option to define the tar split size (Issue #44) diff --git a/clean_content_views.py b/clean_content_views.py index 613aa1c..abdf454 100755 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -62,11 +62,11 @@ def get_cv(org_id, cleanup_list, keep): return ver_list, ver_descr, ver_keep -def get_content_view_version(cvid): - cvv = helpers.get_json( - helpers.KATELLO_API + "content_view_versions/" + str(cvid)) - - return cvv +# def get_content_view_version(cvid): +# cvv = helpers.get_json( +# helpers.KATELLO_API + "content_view_versions/" + str(cvid)) +# +# return cvv def get_content_view_info(cvid): """ @@ -78,6 +78,32 @@ def get_content_view_info(cvid): return cvinfo +def check_version_views(version_id): + """ + Check if our version ID belongs to any views, including CCV + """ + version_in_use = False + version_in_ccv = False + + # Extract a list of content views that the CV version belongs to + viewlist = helpers.get_json( + helpers.KATELLO_API + "content_view_versions/" + str(version_id)) + + # If the list is not empty we need to return this fact. A CV that belongs + # to NO versions will be a candidate for cleanup. + viewlist['composite_content_view_ids'] + if viewlist['katello_content_views']: + version_in_use = True + msg = "Version " + str(viewlist['version']) + " is associated with published CV" + helpers.log_msg(msg, 'DEBUG') + + # We can go further and see if this is associated with a CCV + if viewlist['composite_content_view_ids']: + version_in_ccv = True + + return version_in_use, version_in_ccv + + def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefirstpromoted): """Clean Content Views""" @@ -109,13 +135,24 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir version_list = [] orphan_versions = [] all_versions = [] + ccv_versions = [] for version in cvinfo['versions']: + + # Check if the version is part of a published view. + # This is not returned in cvinfo, and we need to see if we are part of a CCV + version_in_use, version_in_ccv = check_version_views(version['id']) + + # Build a list of ALL version numbers all_versions.append(float(version['version'])) + # Add any version numbers that are part of a CCV to a list + if version_in_ccv: + ccv_versions.append(float(version['version'])) if not version['environment_ids']: - # These are the versions that don't belong to an environment - # i.e. orphans. - orphan_versions.append(float(version['version'])) - continue + # These are the versions that don't belong to an environment (i.e. orphans) + # We also cross-check for versions that may be in a CCV here. + if not version_in_use: + orphan_versions.append(float(version['version'])) + continue else: msg = "Found version " + str(version['version']) helpers.log_msg(msg, 'DEBUG') @@ -166,64 +203,63 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir msg = "Versions to remove: " + str(todelete) helpers.log_msg(msg, 'DEBUG') - for version in cvinfo['versions']: - # Get composite content views for version - cvv = get_content_view_version(version['id']) - # Find versions that are not in any environment and not in any composite content view - if not version['environment_ids'] and not cvv['composite_content_view_ids']: - if not locked: - msg = "Orphan view version " + str(version['version']) + " found in '" +\ + for version in all_versions: + if not locked: + if version in todelete: + msg = "Orphan view version " + str(version) + " found in '" +\ str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'DEBUG') - if cleanall: - msg = "Removing version " + str(version['version']) + msg = "Removing version " + str(version) + helpers.log_msg(msg, 'INFO') + print helpers.HEADER + msg + helpers.ENDC + else: + if version in ccv_versions: + msg = "Skipping delete of version " + str(version) + " (member of a CCV)" + elif version in orphan_versions: + msg = "Skipping delete of version " + str(version) + " (due to keep value)" + else: + msg = "Skipping delete of version " + str(version) + helpers.log_msg(msg, 'INFO') + print msg + continue + else: + msg = "Version " + str(version) + " is locked" + helpers.log_msg(msg, 'WARNING') + continue + + # Delete the view version from the content view + if not dry_run and not locked: + try: + task_id = helpers.put_json( + helpers.KATELLO_API + "content_views/" + str(cvid) + "/remove/", + json.dumps( + { + "id": cvid, + "content_view_version_ids": version['id'] + } + ))['id'] + + # Wait for the task to complete + helpers.wait_for_task(task_id,'clean') + + # Check if the deletion completed successfully + tinfo = helpers.get_task_status(task_id) + if tinfo['state'] != 'running' and tinfo['result'] == 'success': + msg = "Removal of content view version OK" helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC + print helpers.GREEN + "OK" + helpers.ENDC else: - if float(version['version']) in todelete: - # If ignorefirstpromoted delete CV - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) + " due to --keep value" - helpers.log_msg(msg, 'INFO') - print msg - continue - - # Delete the view version from the content view - if not dry_run and not locked: - try: - task_id = helpers.put_json( - helpers.KATELLO_API + "content_views/" + str(cvid) + "/remove/", - json.dumps( - { - "id": cvid, - "content_view_version_ids": version['id'] - } - ))['id'] - - # Wait for the task to complete - helpers.wait_for_task(task_id,'clean') - - # Check if the deletion completed successfully - tinfo = helpers.get_task_status(task_id) - if tinfo['state'] != 'running' and tinfo['result'] == 'success': - msg = "Removal of content view version OK" - helpers.log_msg(msg, 'INFO') - print helpers.GREEN + "OK" + helpers.ENDC - else: - msg = "Failed" - helpers.log_msg(msg, 'ERROR') - - except Warning: - msg = "Failed to initiate removal" - helpers.log_msg(msg, 'WARNING') - - except KeyError: - msg = "Failed to initiate removal" - helpers.log_msg(msg, 'WARNING') + msg = "Failed" + helpers.log_msg(msg, 'ERROR') + + except Warning: + msg = "Failed to initiate removal" + helpers.log_msg(msg, 'WARNING') + + except KeyError: + msg = "Failed to initiate removal (KeyError)" + helpers.log_msg(msg, 'WARNING') # Exit in the case of a dry-run if dry_run: diff --git a/helpers.py b/helpers.py index 48365f7..454ee3c 100644 --- a/helpers.py +++ b/helpers.py @@ -414,6 +414,13 @@ def check_running_publish(cvid, desc): # From the list of tasks, look for any running sync jobs. # If e have any we exit, as we can't trigger a new sync in this state. for task_result in tasks['results']: + if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Publish': + if task_result['input']['content_view']['id'] == cvid: + msg = "Unable to start '" + desc + "': content view is locked by another task" + log_msg(msg, 'WARNING') + locked = True + return locked if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Publish': if task_result['input']['content_view']['id'] == cvid: @@ -428,15 +435,22 @@ def check_running_publish(cvid, desc): log_msg(msg, 'WARNING') locked = True return locked + if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': + if task_result['input']['content_view']['id'] == cvid: + msg = "Unable to start '" + desc + "': content view is locked by another task" + log_msg(msg, 'WARNING') + locked = True + return locked if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': - if task_result['humanized']['action'] == 'Promotion': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': if task_result['input']['content_view']['id'] == cvid: msg = "Unable to start '" + desc + "': content view is locked by another task" log_msg(msg, 'WARNING') locked = True return locked if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': - if task_result['humanized']['action'] == 'Promotion': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': if task_result['input']['content_view']['id'] == cvid: msg = "Unable to start '" + desc + "': content view is locked by a paused task" log_msg(msg, 'WARNING') From 28143963a198a954df19cf3a3b6285ad7e5bd2d8 Mon Sep 17 00:00:00 2001 From: Geoff Gatward Date: Sun, 25 Nov 2018 13:17:55 +1100 Subject: [PATCH 8/8] Fix clean functionality, pre-release updates --- CHANGELOG.md | 7 +++++-- README.md | 8 ++++++-- bin/auto_content | 0 clean_content_views.py | 23 ++++++++++++--------- helpers.py | 43 ++++++++++++++++++++++++++++----------- rel-eng/sat6_scripts.spec | 17 +++++++++------- 6 files changed, 65 insertions(+), 33 deletions(-) mode change 100644 => 100755 bin/auto_content mode change 100755 => 100644 clean_content_views.py mode change 100755 => 100644 rel-eng/sat6_scripts.spec diff --git a/CHANGELOG.md b/CHANGELOG.md index 9502685..8eb52cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Fixed + +## [1.2.4] - 2018-11-25 +### Fixed - clean_content_views raised an exception if a CV version was included in a composite view - Default org view was assumed to be version 1.0. Correct version is now extracted (Issue #43) - Org name and label do not always match. Issue with mixed case and spaces in org name (Issue #42) @@ -13,7 +16,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Tasks in 'planning' state were not being considered when checking for locks - foreman_tasks API returns action as 'Promote' instead of 'Promotion' in Sat 6.3 - ### Added - Option to define the tar split size (Issue #44) @@ -97,7 +99,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## 0.6 - 2017-02-27 - Last of a series of pre-release betas -[Unreleased]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.3...HEAD +[Unreleased]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.4...HEAD +[1.2.4]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.3...1.2.4 [1.2.3]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.2...1.2.3 [1.2.2]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.1...1.2.2 [1.2.1]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.0...1.2.1 diff --git a/README.md b/README.md index 1266c89..5cca346 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,19 @@ # Overview Importing content in a disconnected environment can be a challenge. -These scripts make use of the Inter-Satellite Sync capability in Satellite 6.2 to +These scripts make use of the Inter-Satellite Sync capability in Satellite 6 to allow for full and incremental export/import of content between environments. These scripts have been written and tested using Satellite 6.x on RHEL7. (RHEL6 not supported) Export/Import testing has been performed on the following version combinations: * 6.2 -> 6.2 * 6.2 -> 6.3 -* 6.3 -> 6.2 * 6.3 -> 6.3 +* 6.3 -> 6.2 +* 6.3 -> 6.4 +* 6.4 -> 6.4 +* 6.4 -> 6.3 + ## Definitions Throughout these scripts the following references are used: diff --git a/bin/auto_content b/bin/auto_content old mode 100644 new mode 100755 diff --git a/clean_content_views.py b/clean_content_views.py old mode 100755 new mode 100644 index abdf454..0f38015 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -62,11 +62,6 @@ def get_cv(org_id, cleanup_list, keep): return ver_list, ver_descr, ver_keep -# def get_content_view_version(cvid): -# cvv = helpers.get_json( -# helpers.KATELLO_API + "content_view_versions/" + str(cvid)) -# -# return cvv def get_content_view_info(cvid): """ @@ -121,19 +116,22 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir sys.exit(1) for cvid in ver_list.keys(): - # Check if there is a publish/promote already running on this content view - locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid]) - msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'INFO') print helpers.HEADER + msg + helpers.ENDC + # Check if there is a publish/promote already running on this content view + locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid]) + if locked: + continue + # For the given content view we need to find the orphaned versions cvinfo = get_content_view_info(cvid) # Find the oldest published version version_list = [] orphan_versions = [] + orphan_dict = {} all_versions = [] ccv_versions = [] for version in cvinfo['versions']: @@ -150,8 +148,10 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir if not version['environment_ids']: # These are the versions that don't belong to an environment (i.e. orphans) # We also cross-check for versions that may be in a CCV here. + # We add the version name and id into a dictionary so we can delete by id. if not version_in_use: orphan_versions.append(float(version['version'])) + orphan_dict[version['version']] = version['id'] continue else: msg = "Found version " + str(version['version']) @@ -210,6 +210,9 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'DEBUG') + # Lookup the version_id from our orphan_dict + delete_id = orphan_dict.get(str(version)) + msg = "Removing version " + str(version) helpers.log_msg(msg, 'INFO') print helpers.HEADER + msg + helpers.ENDC @@ -219,7 +222,7 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir elif version in orphan_versions: msg = "Skipping delete of version " + str(version) + " (due to keep value)" else: - msg = "Skipping delete of version " + str(version) + msg = "Skipping delete of version " + str(version) + " (in use)" helpers.log_msg(msg, 'INFO') print msg continue @@ -236,7 +239,7 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir json.dumps( { "id": cvid, - "content_view_version_ids": version['id'] + "content_view_version_ids": delete_id } ))['id'] diff --git a/helpers.py b/helpers.py index 454ee3c..17c0d87 100644 --- a/helpers.py +++ b/helpers.py @@ -416,48 +416,67 @@ def check_running_publish(cvid, desc): for task_result in tasks['results']: if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Publish': - if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by another task" - log_msg(msg, 'WARNING') - locked = True - return locked + msg = "Unable to start '" + desc + "': A publish task is in planning state, cannot determine if it is for this CV" + log_msg(msg, 'WARNING') + locked = True + return locked if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Publish': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by another task" + msg = "Unable to start '" + desc + "': content view is locked by a running Publish task" log_msg(msg, 'WARNING') locked = True return locked if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Publish': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by a paused task" + msg = "Unable to start '" + desc + "': content view is locked by a paused Publish task" log_msg(msg, 'WARNING') locked = True return locked if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': + msg = "Unable to start '" + desc + "': A promotion task is in planning state, cannot determine if it is for this CV" + log_msg(msg, 'WARNING') + locked = True + return locked + if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by another task" + msg = "Unable to start '" + desc + "': content view is locked by a running Promotion task" log_msg(msg, 'WARNING') locked = True return locked - if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': + if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by another task" + msg = "Unable to start '" + desc + "': content view is locked by a paused Promotion task" + log_msg(msg, 'WARNING') + locked = True + return locked + if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Remove Versions and Associations': + msg = "Unable to start '" + desc + "': A remove task is in planning state, cannot determine if it is for this CV" + log_msg(msg, 'WARNING') + locked = True + return locked + if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Remove Versions and Associations': + if task_result['input']['content_view']['id'] == cvid: + msg = "Unable to start '" + desc + "': content view is locked by a running CV deletion task" log_msg(msg, 'WARNING') locked = True return locked if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': - if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': + if task_result['humanized']['action'] == 'Remove Versions and Associations': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by a paused task" + msg = "Unable to start '" + desc + "': content view is locked by a paused CV deletion task" log_msg(msg, 'WARNING') locked = True return locked + def query_yes_no(question, default="yes"): """ Ask a yes/no question via raw_input() and return their answer. diff --git a/rel-eng/sat6_scripts.spec b/rel-eng/sat6_scripts.spec old mode 100755 new mode 100644 index b6317f9..e8b27a0 --- a/rel-eng/sat6_scripts.spec +++ b/rel-eng/sat6_scripts.spec @@ -1,11 +1,11 @@ Name: sat6_scripts -Version: 1.2.3 +Version: 1.2.4 Release: 1%{?dist} Summary: Scripts to automate Satellite 6 tasks License: GPL -URL: https://github.com/ggatward/sat6_scripts -Source0: sat6_scripts-1.2.3.tar.gz +URL: https://github.com/RedHatSatellite/sat6_scripts +Source0: sat6_scripts-1.2.4.tar.gz Requires: python >= 2.7, PyYAML @@ -130,17 +130,20 @@ mandb -q %changelog +* Sun Nov 25 2018 Geoff Gatward 1.2.4 +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.4/CHANGELOG.md + * Mon Mar 12 2018 Geoff Gatward 1.2.3 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.3/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.3/CHANGELOG.md * Sun Feb 25 2018 Geoff Gatward 1.2.2 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.2/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.2/CHANGELOG.md * Mon Dec 11 2017 Geoff Gatward 1.2.1 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.1/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.1/CHANGELOG.md * Sun Dec 10 2017 Geoff Gatward 1.2.0 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.0/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.0/CHANGELOG.md * Wed Oct 25 2017 Geoff Gatward 1.1.1 - Refer https://github.com/ggatward/sat6_scripts/blob/1.1.1/CHANGELOG.md