feat: support for managing RDS Postgres instances

.gitignore

@@ -99,3 +99,7 @@ ENV/
 # End of https://www.gitignore.io/api/python
 *.sw[mnop]
 tmp/
+
+# intellij and friends
+*.iml
+.idea/

pgbedrock/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.4.2'
+__version__ = '0.4.2+5.indigo'
 LOG_FORMAT = '%(levelname)s:%(filename)s:%(funcName)s:%(lineno)s - %(message)s'

pgbedrock/attributes.py

@@ -54,9 +54,9 @@
 COLUMN_NAME_TO_KEYWORD = {v: k for k, v in PG_COLUMN_NAME.items()}
 
 
-def analyze_attributes(spec, cursor, verbose):
+def analyze_attributes(spec, cursor, verbose, attributes_source_table):
     logger.debug('Starting analyze_attributes()')
-    dbcontext = DatabaseContext(cursor, verbose)
+    dbcontext = DatabaseContext(cursor, verbose, attributes_source_table)
 
     # We disable the progress bar when showing verbose output (using '' as our bar_template)
     # or # the bar will get lost in the # output
@@ -109,6 +109,7 @@ def __init__(self, rolename, spec_attributes, dbcontext):
         self.spec_attributes = spec_attributes
 
         self.current_attributes = dbcontext.get_role_attributes(rolename)
+        self.manage_passwords = dbcontext.manage_passwords
 
         # We keep track of password-related SQL separately as we don't want running this to
         # go into the main SQL stream since that could leak password
@@ -211,8 +212,13 @@ def set_all_attributes(self, attributes):
         for attribute, desired_value in attributes.items():
             current_value = self.get_attribute_value(attribute)
             if attribute == 'rolpassword' and not self.is_same_password(desired_value):
-                logger.debug('Altering password for role "{}"'.format(self.rolename))
-                self.set_password(desired_value)
+                # we don't manage password values if we can't reliably read them from the attribute table
+                if self.manage_passwords:
+                    logger.debug('Altering password for role "{}"'.format(self.rolename))
+                    self.set_password(desired_value)
+                else:
+                    logger.debug('Skipping password management for role "{}"'.format(self.rolename))
+                    continue
 
             if attribute == 'rolvaliduntil' \
                and is_valid_forever(desired_value) \

pgbedrock/cli.py

@@ -27,8 +27,9 @@ def entrypoint():
 @click.option('--privileges/--no-privileges', default=True, help='whether to configure privileges (default: --privileges)')
 @click.option('--live/--check', default=False, help='whether to actually make changes ("live") or only show what would be changed ("check") (default: --check)')
 @click.option('--verbose/--no-verbose', default=False, help='whether to show debug-level logging messages while running (default: --no-verbose)')
+@click.option('--alternate-attributes-table/--no-alternate-attributes-table', default=False, help='whether to use pg_roles instead of pg_authid (default: --no-alternate-attributes-table)')
 def configure(spec, host, port, user, password, dbname, prompt, attributes, memberships, ownerships,
-              privileges, live, verbose):
+              privileges, live, verbose, alternate_attributes_table):
     """
     Configure the role attributes, memberships, object ownerships, and/or privileges of a
     database cluster to match a desired spec.
@@ -41,8 +42,11 @@ def configure(spec, host, port, user, password, dbname, prompt, attributes, memb
     In addition, using --verbose will print to STDOUT all debug statements and all SQL queries
     issued by pgbedrock.
     """
+
+    attributes_source_table = 'pg_roles' if alternate_attributes_table else 'pg_authid'
+
     core_configure.configure(spec, host, port, user, password, dbname, prompt, attributes,
-                             memberships, ownerships, privileges, live, verbose)
+                             memberships, ownerships, privileges, live, verbose, attributes_source_table)
 
 
 @entrypoint.command(short_help='Generate a YAML spec for a database')
@@ -53,12 +57,15 @@ def configure(spec, host, port, user, password, dbname, prompt, attributes, memb
 @click.option('-d', '--dbname', default=USER, help='database to connect to (default: "{}")'.format(USER))
 @click.option('--prompt/--no-prompt', default=False, help='prompt the user to input a password (default: --no-prompt)')
 @click.option('--verbose/--no-verbose', default=False, help='whether to show debug-level logging messages while running (default: --no-verbose)')
-def generate(host, port, user, password, dbname, prompt, verbose):
+@click.option('--alternate-attributes-table/--no-alternate-attributes-table', default=False, help='whether to use pg_roles instead of pg_authid (default: --no-alternate-attributes-table)')
+def generate(host, port, user, password, dbname, prompt, verbose, alternate_attributes_table):
     """
     Generate a YAML spec that represents the roles, memberships, ownerships, and/or privileges of a
     database.
     """
-    core_generate.generate(host, port, user, password, dbname, prompt, verbose)
+
+    attributes_source_table = 'pg_roles' if alternate_attributes_table else 'pg_authid'
+    core_generate.generate(host, port, user, password, dbname, prompt, verbose, attributes_source_table)
 
 
 if __name__ == '__main__':

pgbedrock/context.py

@@ -24,7 +24,7 @@
             (aclexplode(def.defaclacl)).privilege_type
         FROM
             pg_default_acl def
-            JOIN pg_authid auth
+            JOIN pg_roles auth
                     ON def.defaclrole = auth.oid
             JOIN pg_namespace nsp
                     ON def.defaclnamespace = nsp.oid
@@ -41,7 +41,7 @@
         subq.privilege_type
     FROM
         subq
-        JOIN pg_authid t_grantee
+        JOIN pg_roles t_grantee
             ON subq.grantee_oid = t_grantee.oid
     WHERE
         subq.grantor_oid != subq.grantee_oid
@@ -65,7 +65,7 @@
             (aclexplode(c.relacl)).privilege_type
         FROM
             pg_class c
-            JOIN pg_authid t_owner
+            JOIN pg_roles t_owner
                 ON c.relowner = t_owner.OID
             JOIN pg_namespace nsp
                 ON c.relnamespace = nsp.oid
@@ -83,7 +83,7 @@
              t_owner.rolname AS owner,
              (aclexplode(nsp.nspacl)).privilege_type
         FROM pg_namespace nsp
-        JOIN pg_authid t_owner
+        JOIN pg_roles t_owner
             ON nsp.nspowner = t_owner.OID
     ), combined AS (
         SELECT *
@@ -100,7 +100,7 @@
         combined.privilege_type
     FROM
         combined
-        JOIN pg_authid t_grantee
+        JOIN pg_roles t_grantee
             ON combined.grantee_oid = t_grantee.oid
         WHERE combined.owner != t_grantee.rolname
     ;
@@ -119,7 +119,7 @@
         rolreplication,
         rolsuper,
         rolvaliduntil
-    FROM pg_authid
+    FROM {source_table}
     WHERE rolname != 'pg_signal_backend'
     ;
     """
@@ -130,9 +130,9 @@
         auth_group.rolname AS group
     FROM
         pg_auth_members link_table
-        JOIN pg_authid auth_member
+        JOIN pg_roles auth_member
             ON link_table.member = auth_member.oid
-        JOIN pg_authid auth_group
+        JOIN pg_roles auth_group
             ON link_table.roleid = auth_group.oid
     ;
     """
@@ -191,7 +191,7 @@
         t_owner.rolname AS owner,
         co.is_dependent
     FROM combined AS co
-    JOIN pg_authid t_owner
+    JOIN pg_roles t_owner
         ON co.owner_id = t_owner.OID
     WHERE
         co.schema NOT LIKE 'pg\_t%'
@@ -201,7 +201,7 @@
 Q_GET_ALL_PERSONAL_SCHEMAS = """
     SELECT nsp.nspname
     FROM pg_namespace nsp
-        JOIN pg_authid auth
+        JOIN pg_roles auth
             ON  nsp.nspname = auth.rolname
     WHERE auth.rolcanlogin IS TRUE
     ;
@@ -233,6 +233,11 @@
          },
 }
 
+# the pg_authid catalog requires superuser privileges to access, the pg_roles view does not
+# https://www.postgresql.org/docs/current/catalog-pg-authid.html
+ATTRIBUTES_TABLE_SUPERUSER = 'pg_authid'
+ATTRIBUTES_TABLE_NONSUPERUSER = 'pg_roles'
+
 ObjectInfo = namedtuple('ObjectInfo', ['kind', 'objname', 'owner', 'is_dependent'])
 ObjectAttributes = namedtuple('ObjectAttributes',
                               ['kind', 'schema', 'objname', 'owner', 'is_dependent'])
@@ -256,10 +261,17 @@ class DatabaseContext(object):
         'get_version_info',
     }
 
-    def __init__(self, cursor, verbose):
+    def __init__(self, cursor, verbose, attributes_source_table):
         self.cursor = cursor
         self.verbose = verbose
         self._cache = dict()
+        self._attributes_source_table = attributes_source_table
+
+        if attributes_source_table == ATTRIBUTES_TABLE_SUPERUSER:
+            # we should be able to read password values and manage them
+            self.manage_passwords = True
+        else:
+            self.manage_passwords = False
 
     def __getattribute__(self, attr):
         """ If the requested attribute should be cached and hasn't, fetch it and cache it. """
@@ -413,9 +425,13 @@ def get_role_current_nondefaults(self, rolename, object_kind, access):
         except KeyError:
             return set()
 
-    def get_all_role_attributes(self):
-        """ Return a dict with key = rolname and values = all fields in pg_authid """
-        common.run_query(self.cursor, self.verbose, Q_GET_ALL_ROLE_ATTRIBUTES)
+    def get_all_role_attributes(self, source_table=None):
+        """ Return a dict with key = rolname and values = all fields in pg_authid/pg_roles """
+
+        if source_table is None:
+            source_table = self._attributes_source_table
+
+        common.run_query(self.cursor, self.verbose, Q_GET_ALL_ROLE_ATTRIBUTES.format(source_table=source_table))
         role_attributes = {row['rolname']: dict(row) for row in self.cursor.fetchall()}
         return role_attributes
 

pgbedrock/core_configure.py

@@ -66,7 +66,7 @@ def run_password_sql(cursor, all_password_sql_to_run):
 
 
 def configure(spec_path, host, port, user, password, dbname, prompt, attributes, memberships,
-              ownerships, privileges, live, verbose):
+              ownerships, privileges, live, verbose, attributes_source_table):
     """
     Configure the role attributes, memberships, object ownerships, and/or privileges of a
     database cluster to match a desired spec.
@@ -106,6 +106,8 @@ def configure(spec_path, host, port, user, password, dbname, prompt, attributes,
 
         verbose - bool; whether to show all queries that are executed and all debug log
             messages during execution
+
+        attributes_source_table - str; the table to read use attributes from (pg_authid or pg_roles)
     """
     if verbose:
         root_logger = logging.getLogger('')
@@ -117,7 +119,7 @@ def configure(spec_path, host, port, user, password, dbname, prompt, attributes,
     db_connection = common.get_db_connection(host, port, dbname, user, password)
     cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor)
 
-    spec = load_spec(spec_path, cursor, verbose, attributes, memberships, ownerships, privileges)
+    spec = load_spec(spec_path, cursor, verbose, attributes, memberships, ownerships, privileges, attributes_source_table)
 
     sql_to_run = []
     password_changed = False  # Initialize this in case the attributes module isn't run
@@ -126,7 +128,7 @@ def configure(spec_path, host, port, user, password, dbname, prompt, attributes,
         sql_to_run.append(create_divider('attributes'))
         # Password changes happen within the attributes.py module itself so we don't leak
         # passwords; as a result we need to see if password changes occurred
-        module_sql, all_password_sql_to_run = analyze_attributes(spec, cursor, verbose)
+        module_sql, all_password_sql_to_run = analyze_attributes(spec, cursor, verbose, attributes_source_table)
         run_module_sql(module_sql, cursor, verbose)
         if all_password_sql_to_run:
             password_changed = True
@@ -136,19 +138,19 @@ def configure(spec_path, host, port, user, password, dbname, prompt, attributes,
 
     if memberships:
         sql_to_run.append(create_divider('memberships'))
-        module_sql = analyze_memberships(spec, cursor, verbose)
+        module_sql = analyze_memberships(spec, cursor, verbose, attributes_source_table)
         run_module_sql(module_sql, cursor, verbose)
         sql_to_run.extend(module_sql)
 
     if ownerships:
         sql_to_run.append(create_divider('ownerships'))
-        module_sql = analyze_ownerships(spec, cursor, verbose)
+        module_sql = analyze_ownerships(spec, cursor, verbose, attributes_source_table)
         run_module_sql(module_sql, cursor, verbose)
         sql_to_run.extend(module_sql)
 
     if privileges:
         sql_to_run.append(create_divider('privileges'))
-        module_sql = analyze_privileges(spec, cursor, verbose)
+        module_sql = analyze_privileges(spec, cursor, verbose, attributes_source_table)
         run_module_sql(module_sql, cursor, verbose)
         sql_to_run.extend(module_sql)
 

pgbedrock/core_generate.py

@@ -382,13 +382,13 @@ def determine_nonschema_privileges_for_schema(role, objkind, objname, dbcontext)
         return all_writes, only_reads
 
 
-def create_spec(host, port, user, password, dbname, verbose):
+def create_spec(host, port, user, password, dbname, verbose, attributes_source_table):
     db_connection = common.get_db_connection(host, port, dbname, user, password)
     # We will only be reading, so it is worth being safe here and ensuring that we can't write
     db_connection.set_session(readonly=True)
     cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor)
 
-    dbcontext = DatabaseContext(cursor, verbose)
+    dbcontext = DatabaseContext(cursor, verbose, attributes_source_table)
     spec = initialize_spec(dbcontext)
     spec = add_attributes(spec, dbcontext)
     spec = add_memberships(spec, dbcontext)
@@ -483,7 +483,7 @@ def sort_sublists(data):
     return data
 
 
-def generate(host, port, user, password, dbname, prompt, verbose):
+def generate(host, port, user, password, dbname, prompt, verbose, attributes_source_table):
     """
     Generate a YAML spec that represents the role attributes, memberships, object ownerships,
     and privileges for all roles in a database.
@@ -508,6 +508,8 @@ def generate(host, port, user, password, dbname, prompt, verbose):
 
         verbose - bool; whether to show all queries that are executed and all debug log
             messages during execution
+
+        attributes_source_table - str; the table to read use attributes from (pg_authid or pg_roles)
     """
     if verbose:
         root_logger = logging.getLogger('')
@@ -516,6 +518,6 @@ def generate(host, port, user, password, dbname, prompt, verbose):
     if prompt:
         password = getpass.getpass()
 
-    spec = create_spec(host, port, user, password, dbname, verbose)
+    spec = create_spec(host, port, user, password, dbname, verbose, attributes_source_table)
     sorted_spec = sort_sublists(spec)
     output_spec(sorted_spec)

pgbedrock/memberships.py

@@ -14,9 +14,9 @@
 Q_REVOKE_MEMBERSHIP = 'REVOKE "{}" FROM "{}";'
 
 
-def analyze_memberships(spec, cursor, verbose):
+def analyze_memberships(spec, cursor, verbose, attributes_source_table):
     logger.debug('Starting analyze_memberships()')
-    dbcontext = DatabaseContext(cursor, verbose)
+    dbcontext = DatabaseContext(cursor, verbose, attributes_source_table)
 
     # We disable the progress bar when showing verbose output (using '' as our bar_template)
     # or # the bar will get lost in the # output

pgbedrock/ownerships.py

@@ -13,9 +13,9 @@
 Q_SET_OBJECT_OWNER = 'ALTER {} {} OWNER TO "{}"; -- Previous owner: "{}"'
 
 
-def analyze_ownerships(spec, cursor, verbose):
+def analyze_ownerships(spec, cursor, verbose, attributes_source_table):
     logger.debug('Starting analyze_ownerships()')
-    dbcontext = DatabaseContext(cursor, verbose)
+    dbcontext = DatabaseContext(cursor, verbose, attributes_source_table)
 
     # We disable the progress bar when showing verbose output (using '' as our bar_template)
     # or # the bar will get lost in the # output

pgbedrock/privileges.py

@@ -30,9 +30,9 @@
     """
 
 
-def analyze_privileges(spec, cursor, verbose):
+def analyze_privileges(spec, cursor, verbose, attributes_source_table):
     logger.debug('Starting analyze_privileges()')
-    dbcontext = DatabaseContext(cursor, verbose)
+    dbcontext = DatabaseContext(cursor, verbose, attributes_source_table)
 
     # We disable the progress bar when showing verbose output (using '' as our bar_template)
     # or # the bar will get lost in the # output
@@ -196,6 +196,7 @@ def __init__(self, rolename, access, object_kind, desired_items, schema_writers,
         self.schema_writers = schema_writers
         self.personal_schemas = personal_schemas
         self.default_acl_possible = self.object_kind in OBJECTS_WITH_DEFAULTS
+        self.dbcontext = dbcontext
 
         self.current_defaults = dbcontext.get_role_current_defaults(rolename, object_kind, access)
         self.current_nondefaults = dbcontext.get_role_current_nondefaults(rolename, object_kind, access)
@@ -276,9 +277,11 @@ def get_schema_objects(self, schema):
         return {objname for objname, attr in object_owners.items() if attr['owner'] != self.rolename}
 
     def grant_default(self, grantor, schema, privilege):
-        query = Q_GRANT_DEFAULT.format(grantor, schema.qualified_name, privilege,
-                                       self.object_kind.upper(), self.rolename)
-        self.sql_to_run.append(query)
+        if self.dbcontext.is_superuser(grantor):
+            return self.sql_to_run.append(SKIP_SUPERUSER_PRIVILEGE_CONFIGURATION_MSG.format(grantor))
+        else:
+            query = Q_GRANT_DEFAULT.format(grantor, schema.qualified_name, privilege, self.object_kind.upper(), self.rolename)
+            return self.sql_to_run.append(query)
 
     def grant_nondefault(self, objname, privilege):
         obj_kind_singular = self.object_kind.upper()[:-1]
@@ -334,9 +337,11 @@ def identify_desired_objects(self):
             self.determine_desired_defaults(schemas)
 
     def revoke_default(self, grantor, schema, privilege):
-        query = Q_REVOKE_DEFAULT.format(grantor, schema.qualified_name, privilege,
-                                        self.object_kind.upper(), self.rolename)
-        self.sql_to_run.append(query)
+        if self.dbcontext.is_superuser(grantor):
+            return self.sql_to_run.append(SKIP_SUPERUSER_PRIVILEGE_CONFIGURATION_MSG.format(grantor))
+        else:
+            query = Q_REVOKE_DEFAULT.format(grantor, schema.qualified_name, privilege, self.object_kind.upper(), self.rolename)
+            return self.sql_to_run.append(query)
 
     def revoke_nondefault(self, objname, privilege):
         obj_kind_singular = self.object_kind.upper()[:-1]