Allow refresh of all system pillar data via API and spacecmd

containers/proxy-squid-image/proxy-squid-image.changes.mcalmer.api-for-pillar-refresh-all-systems

@@ -0,0 +1 @@
+- Set maximal cache time for metadata to 5 minutes

containers/proxy-squid-image/squid.conf

@@ -25,13 +25,13 @@ cache_replacement_policy heap LFUDA
 memory_replacement_policy heap GDSF
 
 # cache repodata only few minutes and then query parent whether it is fresh
-refresh_pattern /XMLRPC/GET-REQ/.*/repodata/.*$ 0 1% 1440 reload-into-ims refresh-ims
-refresh_pattern /ks/.*/repodata/.*$ 0 1% 1440 reload-into-ims refresh-ims
+refresh_pattern /XMLRPC/GET-REQ/.*/repodata/.*$ 0 1% 5 reload-into-ims refresh-ims
+refresh_pattern /ks/.*/repodata/.*$ 0 1% 5 reload-into-ims refresh-ims
 # salt minions get the repodata via a different URL
-refresh_pattern /rhn/manager/download/.*/repodata/.*$ 0 1% 1440 reload-into-ims refresh-ims
+refresh_pattern /rhn/manager/download/.*/repodata/.*$ 0 1% 5 reload-into-ims refresh-ims
 # bootstrap repos needs to be handled as well
-refresh_pattern /pub/repositories/.*/repodata/.*$ 0 1% 1440 reload-into-ims refresh-ims
-refresh_pattern /pub/repositories/.*/venv-enabled-.*.txt$ 0 1% 1440 reload-into-ims refresh-ims
+refresh_pattern /pub/repositories/.*/repodata/.*$ 0 1% 5 reload-into-ims refresh-ims
+refresh_pattern /pub/repositories/.*/venv-enabled-.*.txt$ 0 1% 5 reload-into-ims refresh-ims
 # rpm will hardly ever change, force to cache it for very long time
 refresh_pattern  \.rpm$  10080 100% 525600 override-expire override-lastmod ignore-reload reload-into-ims
 refresh_pattern  \.deb$  10080 100% 525600 override-expire override-lastmod ignore-reload reload-into-ims

java/code/src/com/redhat/rhn/frontend/xmlrpc/system/SystemHandler.java

@@ -9072,7 +9072,7 @@ public int setPillar(User loggedInUser, String minionId, String category, Map<St
      *
      * @apidoc.doc refresh all the pillar data of a list of systems.
      * @apidoc.param #session_key()
-     * @apidoc.param #array_single("int", "sids")
+     * @apidoc.param #array_single("int", "sids", "System IDs to be refreshed. If empty, all systems will be refreshed")
      * @apidoc.returntype #array_single("int", "skippedIds", "System IDs which couldn't be refreshed")
      */
     public List<Integer> refreshPillar(User loggedInUser, List<Integer> sids) {
@@ -9091,17 +9091,24 @@ public List<Integer> refreshPillar(User loggedInUser, List<Integer> sids) {
      * and can be one of 'general', 'group_membership', 'virtualization' or 'custom_info'.
      * @apidoc.param #session_key()
      * @apidoc.param #param("string", "subset", "subset of the pillar to refresh.")
-     * @apidoc.param #array_single("int", "sids")
+     * @apidoc.param #array_single("int", "sids", "System IDs to be refreshed. If empty, all systems will be refreshed")
      * @apidoc.returntype #array_single("int", "skippedIds", "System IDs which couldn't be refreshed")
      */
     public List<Integer> refreshPillar(User loggedInUser, String subset, List<Integer> sids) {
         List<Integer> skipped = new ArrayList<>();
         MinionPillarManager.PillarSubset subsetValue = subset != null ?
                 MinionPillarManager.PillarSubset.valueOf(subset.toUpperCase()) :
                 null;
-        for (Integer sysId : sids) {
-            if (SystemManager.isAvailableToUser(loggedInUser, sysId.longValue())) {
-                Server system = SystemManager.lookupByIdAndUser(Long.valueOf(sysId), loggedInUser);
+        List<Long> sysids;
+        if (sids == null || sids.isEmpty()) {
+            sysids = MinionServerFactory.lookupVisibleToUser(loggedInUser).map(MinionServer::getId).toList();
+        }
+        else {
+            sysids = sids.stream().map(Integer::longValue).toList();
+        }
+        for (Long sysId : sysids) {
+            if (SystemManager.isAvailableToUser(loggedInUser, sysId)) {
+                Server system = SystemManager.lookupByIdAndUser(sysId, loggedInUser);
                 system.asMinionServer().ifPresentOrElse(
                     minionServer -> {
                         if (subsetValue != null) {
@@ -9113,13 +9120,13 @@ public List<Integer> refreshPillar(User loggedInUser, String subset, List<Intege
                     },
                     () -> {
                         log.warn("system {} is not a salt minion, hence pillar will not be updated", sysId);
-                        skipped.add(sysId);
+                        skipped.add(sysId.intValue());
                     }
                 );
             }
             else {
                 log.warn("system {} is not available to user, hence pillar will not be refreshed", sysId);
-                skipped.add(sysId);
+                skipped.add(sysId.intValue());
             }
         }
         return skipped;

java/spacewalk-java.changes.mcalmer.api-for-pillar-refresh-all-systems

@@ -0,0 +1,2 @@
+- Modify systems refresh Pillar API to allow refresh pillar data
+  of all systems

python/spacewalk/satellite_tools/mgr-sign-metadata-ctl

@@ -29,8 +29,10 @@ function regenerate_metadata {
     systemctl restart taskomatic
     echo "done."
     echo
-    echo "Scheduling repo metadata regeneration..."
     read_spacecmd_user_pass
+    echo "Refreshing pillar data..."
+    spacecmd -q -u $SPACECMD_USER -p $SPACECMD_PASS "system_refreshpillar"
+    echo "Scheduling repo metadata regeneration..."
     spacecmd -q -u $SPACECMD_USER -p $SPACECMD_PASS "softwarechannel_regenerateyumcache -f *"
     RETVAL=$?
     if [ $RETVAL -ne 0 ]; then
@@ -317,5 +319,7 @@ if [[ $ACTION = "enable" || $ACTION = "disable" ]]; then
     echo
     echo "NOTE. For the changes to become effective run:"
     echo "   mgr-sign-metadata-ctl regen-metadata"
+    echo "and schedule a highstate for all your systems to"
+    echo "apply the new repository configuration."
     echo
 fi

python/spacewalk/spacewalk-backend.changes.mcalmer.api-for-pillar-refresh-all-systems

@@ -0,0 +1 @@
+- Refresh pillar data together with regenration of channel metadata

spacecmd/spacecmd.changes.mcalmer.api-for-pillar-refresh-all-systems

@@ -0,0 +1 @@
+- Add command to refresh systems pillar data

spacecmd/src/spacecmd/system.py

@@ -2057,6 +2057,45 @@ def do_system_rename(self, args):
 ####################
 
 
+def help_system_refreshpillar(self):
+    print(_('system_refreshpillar: Refresh pillar data of system(s) '))
+    print(_('usage: system_refreshpillar [<SYSTEMS>]'))
+    print('')
+    print(self.HELP_SYSTEM_OPTS)
+
+def complete_system_refreshpillar(self, text, line, beg, end):
+    return self.tab_complete_systems(text)
+
+def do_system_refreshpillar(self, args):
+    arg_parser = get_argument_parser()
+
+    (args, _options) = parse_command_arguments(args, arg_parser)
+
+    sids = []
+    # use the systems listed in the SSM
+    if args:
+        if re.match('ssm', args[0], re.I):
+            systems = self.ssm.keys()
+        else:
+            systems = self.expand_systems(args)
+
+        for system in sorted(systems):
+            system_id = self.get_system_id(system)
+            if not system_id:
+                continue
+            sids.append(system_id)
+
+    self.client.system.refreshPillar(self.session, sids)
+
+    num = "%s" % len(sids)
+    if (num == "0"):
+        num = "all"
+    print('Refreshed Pillar data for "%s" systems' % (num))
+    return 0
+
+####################
+
+
 def help_system_listcustomvalues(self):
     print(_('system_listcustomvalues: List the custom values for a system'))
     print(_('usage: system_listcustomvalues <SYSTEMS>'))