diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be3a3d..8eb52cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Fixed -- clean_content_views raised an exception if a CV version was included in a composite view. + +## [1.2.4] - 2018-11-25 +### Fixed +- clean_content_views raised an exception if a CV version was included in a composite view - Default org view was assumed to be version 1.0. Correct version is now extracted (Issue #43) - Org name and label do not always match. Issue with mixed case and spaces in org name (Issue #42) +- clean_content_views did not handle exception if API returns null value (Issue #49) +- clean_content_views now correctly handles incremental content view deletion (Issue #49) +- Tasks in 'planning' state were not being considered when checking for locks +- foreman_tasks API returns action as 'Promote' instead of 'Promotion' in Sat 6.3 ### Added - Option to define the tar split size (Issue #44) @@ -92,7 +99,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## 0.6 - 2017-02-27 - Last of a series of pre-release betas -[Unreleased]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.3...HEAD +[Unreleased]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.4...HEAD +[1.2.4]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.3...1.2.4 [1.2.3]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.2...1.2.3 [1.2.2]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.1...1.2.2 [1.2.1]: https://github.com/RedHatSatellite/sat6_scripts/compare/1.2.0...1.2.1 diff --git a/README.md b/README.md index 4736f94..5cca346 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,19 @@ # Overview Importing content in a disconnected environment can be a challenge. -These scripts make use of the Inter-Satellite Sync capability in Satellite 6.2 to +These scripts make use of the Inter-Satellite Sync capability in Satellite 6 to allow for full and incremental export/import of content between environments. These scripts have been written and tested using Satellite 6.x on RHEL7. (RHEL6 not supported) Export/Import testing has been performed on the following version combinations: * 6.2 -> 6.2 * 6.2 -> 6.3 -* 6.3 -> 6.2 * 6.3 -> 6.3 +* 6.3 -> 6.2 +* 6.3 -> 6.4 +* 6.4 -> 6.4 +* 6.4 -> 6.3 + ## Definitions Throughout these scripts the following references are used: @@ -360,11 +364,32 @@ Any orphaned versions older than the last in-use version are purged (orphans between in-use versions will NOT be purged, unless the cleanall (-c) option is used). There is a keep (-k) option that allows a specific number of versions older than the last in-use to be kept as well, allowing for possible rollback of versions. +Note that the (-c) option will delete ALL orphaned versions, regardless of the (-k) +value. An alternative option (-i) slightly alters the calculation of the versions +that will be deleted, in that the (-k) option defines how many versions after the +newest (last promoted) version will be kept, rather than use the oldest (first +promoted) version. Content views to clean can be defined by either: - Specific content views defined in the main config file - All content views (-a) - - All content views, ignoring the first promoted one (-i) + +The option to use will depend on the historic (old) content views you wish to keep. +An example of the different options with a keep value of '1' is shown below: + +``` ++-----------------+----------+-----------------------+------------+ +| version | no flags | --ignorefirstpromoted | --cleanall | ++-----------------+----------+-----------------------+------------+ +| 110.0 (Library) | | | | +| 109.0 | KEEP | KEEP | DEL | +| 108.3 | KEEP | DEL | DEL | +| 108.2 (Quality) | | | | +| 108.1 | KEEP | DEL | DEL | +| 108.0 | DEL | DEL | DEL | +| 107.0 | DEL | DEL | DEL | ++-----------------+----------+-----------------------+------------+ +``` The dry run (-d) option can be used to see what would be published for a given command input. Use this option to see the difference in behaviour between diff --git a/bin/auto_content b/bin/auto_content old mode 100644 new mode 100755 diff --git a/clean_content_views.py b/clean_content_views.py old mode 100755 new mode 100644 index 8a595fe..0f38015 --- a/clean_content_views.py +++ b/clean_content_views.py @@ -62,11 +62,6 @@ def get_cv(org_id, cleanup_list, keep): return ver_list, ver_descr, ver_keep -def get_content_view_version(cvid): - cvv = helpers.get_json( - helpers.KATELLO_API + "content_view_versions/" + str(cvid)) - - return cvv def get_content_view_info(cvid): """ @@ -78,6 +73,32 @@ def get_content_view_info(cvid): return cvinfo +def check_version_views(version_id): + """ + Check if our version ID belongs to any views, including CCV + """ + version_in_use = False + version_in_ccv = False + + # Extract a list of content views that the CV version belongs to + viewlist = helpers.get_json( + helpers.KATELLO_API + "content_view_versions/" + str(version_id)) + + # If the list is not empty we need to return this fact. A CV that belongs + # to NO versions will be a candidate for cleanup. + viewlist['composite_content_view_ids'] + if viewlist['katello_content_views']: + version_in_use = True + msg = "Version " + str(viewlist['version']) + " is associated with published CV" + helpers.log_msg(msg, 'DEBUG') + + # We can go further and see if this is associated with a CCV + if viewlist['composite_content_view_ids']: + version_in_ccv = True + + return version_in_use, version_in_ccv + + def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefirstpromoted): """Clean Content Views""" @@ -95,23 +116,43 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir sys.exit(1) for cvid in ver_list.keys(): - # Check if there is a publish/promote already running on this content view - locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid]) - - msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'" + msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'INFO') print helpers.HEADER + msg + helpers.ENDC + # Check if there is a publish/promote already running on this content view + locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid]) + if locked: + continue + # For the given content view we need to find the orphaned versions cvinfo = get_content_view_info(cvid) # Find the oldest published version version_list = [] - version_list_all = [] + orphan_versions = [] + orphan_dict = {} + all_versions = [] + ccv_versions = [] for version in cvinfo['versions']: + + # Check if the version is part of a published view. + # This is not returned in cvinfo, and we need to see if we are part of a CCV + version_in_use, version_in_ccv = check_version_views(version['id']) + + # Build a list of ALL version numbers + all_versions.append(float(version['version'])) + # Add any version numbers that are part of a CCV to a list + if version_in_ccv: + ccv_versions.append(float(version['version'])) if not version['environment_ids']: - version_list_all.append(float(version['version'])) - continue + # These are the versions that don't belong to an environment (i.e. orphans) + # We also cross-check for versions that may be in a CCV here. + # We add the version name and id into a dictionary so we can delete by id. + if not version_in_use: + orphan_versions.append(float(version['version'])) + orphan_dict[version['version']] = version['id'] + continue else: msg = "Found version " + str(version['version']) helpers.log_msg(msg, 'DEBUG') @@ -127,95 +168,101 @@ def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefir helpers.log_msg(msg, 'DEBUG') # Find the oldest 'NOT in use' version id - if not version_list_all: + if not orphan_versions: msg = "No oldest NOT-in-use version found" else: - msg = "Oldest NOT-in-use version is " + str(min(version_list_all)) + msg = "Oldest NOT-in-use version is " + str(min(orphan_versions)) helpers.log_msg(msg, 'DEBUG') - # Find version to delete (based on keep parameter) if --ignorefirstpromoted - version_list_all.sort() - todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))] - msg = "Versions to remove if --ignorefirstpromoted: " + str(todelete) + # Find the element position in the all_versions list of the oldest in-use version + # e.g. vers 102.0 is oldest in-use and is element [5] in the all_versions list + list_position = [i for i,x in enumerate(all_versions) if x == lastver] + # Remove the number of views to keep from the element position of the oldest in-use + # e.g. keep=2 results in an adjusted list element position [3] + num_to_delete = list_position[0] - int(ver_keep[cvid]) + # Delete from position [0] to the first 'keep' position + # e.g. first keep element is [3] so list of elements [0, 1, 2] is created + list_pos_to_delete = [i for i in range(num_to_delete)] + + # Find versions to delete (based on keep parameter) + # Make sure the version list is in order + orphan_versions.sort() + + if cleanall: + # Remove all orphaned versions + todelete = orphan_versions + elif ignorefirstpromoted: + # Remove the last 'keep' elements from the orphans list (from PR #26) + todelete = orphan_versions[:(len(orphan_versions) - int(ver_keep[cvid]))] + else: + todelete = [] + # Remove the element numbers for deletion from the list all versions + for i in sorted(list_pos_to_delete, reverse=True): + todelete.append(orphan_versions[i]) + + msg = "Versions to remove: " + str(todelete) helpers.log_msg(msg, 'DEBUG') - for version in cvinfo['versions']: - # Get composite content views for version - cvv = get_content_view_version(version['id']) - # Find versions that are not in any environment and not in any composite content view - if not version['environment_ids'] and not cvv['composite_content_view_ids']: - if not locked: - msg = "Orphan view version " + str(version['version']) + " found in '" +\ + for version in all_versions: + if not locked: + if version in todelete: + msg = "Orphan view version " + str(version) + " found in '" +\ str(ver_descr[cvid]) + "'" helpers.log_msg(msg, 'DEBUG') - if ignorefirstpromoted: - if cleanall: - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - if float(version['version']) in todelete: - # If ignorefirstpromoted delete CV - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) + " due to --keep value" - helpers.log_msg(msg, 'INFO') - print msg - continue + # Lookup the version_id from our orphan_dict + delete_id = orphan_dict.get(str(version)) + + msg = "Removing version " + str(version) + helpers.log_msg(msg, 'INFO') + print helpers.HEADER + msg + helpers.ENDC + else: + if version in ccv_versions: + msg = "Skipping delete of version " + str(version) + " (member of a CCV)" + elif version in orphan_versions: + msg = "Skipping delete of version " + str(version) + " (due to keep value)" + else: + msg = "Skipping delete of version " + str(version) + " (in use)" + helpers.log_msg(msg, 'INFO') + print msg + continue + else: + msg = "Version " + str(version) + " is locked" + helpers.log_msg(msg, 'WARNING') + continue + + # Delete the view version from the content view + if not dry_run and not locked: + try: + task_id = helpers.put_json( + helpers.KATELLO_API + "content_views/" + str(cvid) + "/remove/", + json.dumps( + { + "id": cvid, + "content_view_version_ids": delete_id + } + ))['id'] + + # Wait for the task to complete + helpers.wait_for_task(task_id,'clean') + + # Check if the deletion completed successfully + tinfo = helpers.get_task_status(task_id) + if tinfo['state'] != 'running' and tinfo['result'] == 'success': + msg = "Removal of content view version OK" + helpers.log_msg(msg, 'INFO') + print helpers.GREEN + "OK" + helpers.ENDC else: - if float(version['version']) > float(lastver): - # If we have chosen to remove all orphans - if cleanall: - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print msg - continue - else: - if float(version['version']) < (lastver - float(ver_keep[cvid])): - msg = "Removing version " + str(version['version']) - helpers.log_msg(msg, 'INFO') - print helpers.HEADER + msg + helpers.ENDC - else: - msg = "Skipping delete of version " + str(version['version']) + " due to --keep value" - helpers.log_msg(msg, 'INFO') - print msg - continue - - # Delete the view version from the content view - if not dry_run and not locked: - try: - task_id = helpers.put_json( - helpers.KATELLO_API + "content_views/" + str(cvid) + "/remove/", - json.dumps( - { - "id": cvid, - "content_view_version_ids": version['id'] - } - ))['id'] - - # Wait for the task to complete - helpers.wait_for_task(task_id,'clean') - - # Check if the deletion completed successfully - tinfo = helpers.get_task_status(task_id) - if tinfo['state'] != 'running' and tinfo['result'] == 'success': - msg = "Removal of content view version OK" - helpers.log_msg(msg, 'INFO') - print helpers.GREEN + "OK" + helpers.ENDC - else: - msg = "Failed" - helpers.log_msg(msg, 'ERROR') - - except Warning: - msg = "Failed to initiate removal" - helpers.log_msg(msg, 'WARNING') + msg = "Failed" + helpers.log_msg(msg, 'ERROR') + + except Warning: + msg = "Failed to initiate removal" + helpers.log_msg(msg, 'WARNING') + + except KeyError: + msg = "Failed to initiate removal (KeyError)" + helpers.log_msg(msg, 'WARNING') # Exit in the case of a dry-run if dry_run: @@ -299,5 +346,3 @@ def main(args): except KeyboardInterrupt, e: print >> sys.stderr, ("\n\nExiting on user cancel.") sys.exit(1) - - diff --git a/helpers.py b/helpers.py index 48365f7..17c0d87 100644 --- a/helpers.py +++ b/helpers.py @@ -414,36 +414,69 @@ def check_running_publish(cvid, desc): # From the list of tasks, look for any running sync jobs. # If e have any we exit, as we can't trigger a new sync in this state. for task_result in tasks['results']: + if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Publish': + msg = "Unable to start '" + desc + "': A publish task is in planning state, cannot determine if it is for this CV" + log_msg(msg, 'WARNING') + locked = True + return locked if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Publish': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by another task" + msg = "Unable to start '" + desc + "': content view is locked by a running Publish task" log_msg(msg, 'WARNING') locked = True return locked if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': if task_result['humanized']['action'] == 'Publish': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by a paused task" + msg = "Unable to start '" + desc + "': content view is locked by a paused Publish task" + log_msg(msg, 'WARNING') + locked = True + return locked + if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': + msg = "Unable to start '" + desc + "': A promotion task is in planning state, cannot determine if it is for this CV" + log_msg(msg, 'WARNING') + locked = True + return locked + if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': + if task_result['input']['content_view']['id'] == cvid: + msg = "Unable to start '" + desc + "': content view is locked by a running Promotion task" log_msg(msg, 'WARNING') locked = True return locked + if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Promotion' or task_result['humanized']['action'] == 'Promote': + if task_result['input']['content_view']['id'] == cvid: + msg = "Unable to start '" + desc + "': content view is locked by a paused Promotion task" + log_msg(msg, 'WARNING') + locked = True + return locked + if task_result['state'] == 'planning' and task_result['label'] != 'Actions::BulkAction': + if task_result['humanized']['action'] == 'Remove Versions and Associations': + msg = "Unable to start '" + desc + "': A remove task is in planning state, cannot determine if it is for this CV" + log_msg(msg, 'WARNING') + locked = True + return locked if task_result['state'] == 'running' and task_result['label'] != 'Actions::BulkAction': - if task_result['humanized']['action'] == 'Promotion': + if task_result['humanized']['action'] == 'Remove Versions and Associations': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by another task" + msg = "Unable to start '" + desc + "': content view is locked by a running CV deletion task" log_msg(msg, 'WARNING') locked = True return locked if task_result['state'] == 'paused' and task_result['label'] != 'Actions::BulkAction': - if task_result['humanized']['action'] == 'Promotion': + if task_result['humanized']['action'] == 'Remove Versions and Associations': if task_result['input']['content_view']['id'] == cvid: - msg = "Unable to start '" + desc + "': content view is locked by a paused task" + msg = "Unable to start '" + desc + "': content view is locked by a paused CV deletion task" log_msg(msg, 'WARNING') locked = True return locked + def query_yes_no(question, default="yes"): """ Ask a yes/no question via raw_input() and return their answer. diff --git a/rel-eng/sat6_scripts.spec b/rel-eng/sat6_scripts.spec old mode 100755 new mode 100644 index b6317f9..e8b27a0 --- a/rel-eng/sat6_scripts.spec +++ b/rel-eng/sat6_scripts.spec @@ -1,11 +1,11 @@ Name: sat6_scripts -Version: 1.2.3 +Version: 1.2.4 Release: 1%{?dist} Summary: Scripts to automate Satellite 6 tasks License: GPL -URL: https://github.com/ggatward/sat6_scripts -Source0: sat6_scripts-1.2.3.tar.gz +URL: https://github.com/RedHatSatellite/sat6_scripts +Source0: sat6_scripts-1.2.4.tar.gz Requires: python >= 2.7, PyYAML @@ -130,17 +130,20 @@ mandb -q %changelog +* Sun Nov 25 2018 Geoff Gatward 1.2.4 +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.4/CHANGELOG.md + * Mon Mar 12 2018 Geoff Gatward 1.2.3 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.3/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.3/CHANGELOG.md * Sun Feb 25 2018 Geoff Gatward 1.2.2 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.2/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.2/CHANGELOG.md * Mon Dec 11 2017 Geoff Gatward 1.2.1 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.1/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.1/CHANGELOG.md * Sun Dec 10 2017 Geoff Gatward 1.2.0 -- Refer https://github.com/ggatward/sat6_scripts/blob/1.2.0/CHANGELOG.md +- Refer https://github.com/RedHatSatellite/sat6_scripts/blob/1.2.0/CHANGELOG.md * Wed Oct 25 2017 Geoff Gatward 1.1.1 - Refer https://github.com/ggatward/sat6_scripts/blob/1.1.1/CHANGELOG.md