Merge pull request #73 from moj-analytical-services/implement-CaDeT-p…

…olicy

Implement a CADET-specific SuperUser Policy - This breaks the generic nature of the package somewhat, but is an optional flag to provide in configs, and will not affect any existing policies implemented.

README.md

@@ -55,7 +55,7 @@ glue_job: true
 
 secrets: true
 
-secretsmanager: 
+secretsmanager:
   read_only:
     - test_secret
 
@@ -115,6 +115,8 @@ Whilst the example json (`iam_config.json`) looks like this:
   - **write**: Either `true` or `false`. If `false` then only read access to Athena (cannot create, delete or alter tables, databases and partitions). If `true` then the role will also have the ability to do stuff like CTAS queries, `DROP TABLE`, `CREATE DATABASE`, etc.
   - **dump_bucket**: The location in S3 (either an S3 path or a list of S3 paths) for temporarily storing the results of queries. This defaults to `mojap-athena-query-dump` and should not normally need changing.
 
+- **is_cadet_deployer:** Boolean; Gives access to a highly empowered Glue role for Create-A-Derived-Table deployments. Will fail to apply if the `iam_role_name` doesn't include `cadet` in the string. Gives the user full control over all glue and athena structures in the named account.
+
 - **glue_job:** Boolean; must be set to `true` to allow role to run glue jobs. If `false` or absent role will not be able to run glue jobs.
 
 - **secrets:** Boolean or string; must be set to `true` or `"read"` to allow role to access secrets from AWS Parameter Store, and `readwrite` to provide read/write access. If `false` or absent role will not be able to access secrets.

iam_builder/exceptions.py

@@ -3,3 +3,7 @@
 
 class IAMValidationError(ValidationError):
     pass
+
+
+class PrivilegedRoleValidationError(ValueError):
+    pass

iam_builder/iam_builder.py

@@ -15,6 +15,7 @@
     get_secretsmanager_read_only_policy,
 )
 from iam_builder.iam_schema import validate_iam
+from iam_builder.exceptions import PrivilegedRoleValidationError
 
 
 def build_iam_policy(config: dict) -> dict:  # noqa: C901
@@ -119,4 +120,11 @@ def build_iam_policy(config: dict) -> dict:  # noqa: C901
             iam_lookup["cloudwatch_athena_query_executions"]
         )
 
+    if "is_cadet_deployer" in config:
+        if "cadet" not in config["iam_role_name"].lower():
+            raise PrivilegedRoleValidationError(
+                "\'is_cadet_deployer\' is only valid for CaDeT deployment roles"
+            )
+        iam["Statement"].extend(iam_lookup["cadet_deployer"])
+
     return iam

iam_builder/schemas/iam_schema.json

@@ -36,6 +36,10 @@
                 }
             }
         },
+        "is_cadet_deployer": {
+            "description": "is_cadet_deployer should be reserved only for the highly empowered cadet deployment task. Inappropriate for other roles.",
+            "type": "boolean"
+        },
         "glue_job": {
             "description": "glue_job must be set to true to allow role to run glue jobs",
             "type": "boolean"
@@ -106,8 +110,8 @@
             "description": "cloudwatch_athena_query_executions must be set to true to allow",
             "type": "boolean"
         },
-        
-        
+
+
         "role_duration_seconds":{
             "description": "Max duration role can be assumed for in seconds",
             "type": "integer"

iam_builder/templates.py

@@ -98,6 +98,100 @@
             ]
         }
     ],
+    "cadet_deployer": [
+        {
+            "Sid": "GlueCatalogActions",
+            "Effect": "Allow",
+            "Action": [
+                "glue:Get*",
+                "glue:DeleteTable",
+                "glue:DeleteTableVersion",
+                "glue:DeleteSchema",
+                "glue:DeletePartition",
+                "glue:DeleteDatabase",
+                "glue:UpdateTable",
+                "glue:UpdateSchema",
+                "glue:UpdatePartition",
+                "glue:UpdateDatabase",
+                "glue:CreateTable",
+                "glue:CreateSchema",
+                "glue:CreatePartition",
+                "glue:CreatePartitionIndex",
+                "glue:BatchCreatePartition",
+                "glue:CreateDatabase"
+            ],
+            "Resource": [
+                "arn:aws:glue:*:*:schema/*",
+                "arn:aws:glue:*:*:database/*",
+                "arn:aws:glue:*:*:table/*/*",
+                "arn:aws:glue:*:*:catalog"
+            ]
+        },
+        {
+            "Sid": "AthenaActions",
+            "Effect": "Allow",
+            "Action": [
+                "athena:List*",
+                "athena:Get*",
+                "athena:StartQueryExecution",
+                "athena:StopQueryExecution"
+            ],
+            "Resource": [
+                "arn:aws:athena:*:*:datacatalog/*",
+                "arn:aws:athena:*:*:workgroup/*"
+            ]
+        },
+        {
+            "Sid": "AirflowCLIPolicy",
+            "Effect": "Allow",
+            "Action": [
+                "airflow:CreateCliToken"
+            ],
+            "Resource": [
+                "arn:aws:airflow:*:*:environment/dev",
+                "arn:aws:airflow:*:*:environment/prod"
+            ]
+        },
+        {
+            "Sid": "CadetWriteAccess",
+            "Effect": "Allow",
+            "Action": [
+                "s3:List*",
+                "s3:GetObject*",
+                "s3:GetBucket*",
+                "s3:DeleteObject*",
+                "s3:PutObject*"
+            ],
+            "Resource": [
+                "arn:aws:s3:::mojap-derived-tables/*",
+                "arn:aws:s3:::mojap-derived-tables",
+                "arn:aws:s3:::dbt-query-dump/*",
+                "arn:aws:s3:::dbt-query-dump",
+                "arn:aws:s3:::mojap-manage-offences/ho-offence-codes/*",
+                "arn:aws:s3:::mojap-manage-offences",
+                "arn:aws:s3:::mojap-hub-exports/probation_referrals_dump/*",
+                "arn:aws:s3:::mojap-hub-exports",
+                "arn:aws:s3:::alpha-app-opg-lpa-dashboard",
+                "arn:aws:s3:::alpha-app-opg-lpa-dashboard/dev/models/domain_name=opg/*",
+                "arn:aws:s3:::alpha-app-opg-lpa-dashboard/prod/models/domain_name=opg/*",
+                "arn:aws:s3:::alpha-bold-data-shares",
+                "arn:aws:s3:::alpha-bold-data-shares/reducing-reoffending/*"
+            ]
+        },
+        {
+            "Sid": "CadetReadAccess",
+            "Effect": "Allow",
+            "Action": [
+                "s3:List*",
+                "s3:GetObject*",
+                "s3:GetBucket*"
+            ],
+            "Resource": [
+                "arn:aws:s3:::*",
+                "arn:aws:s3:::*/*"
+            ]
+        }
+    ],
     "decrypt_statement": [
         {
             "Sid": "allowDecrypt",

tests/expected_policy/cadet_deployer.json

@@ -0,0 +1,97 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+      {
+          "Sid": "GlueCatalogActions",
+          "Effect": "Allow",
+          "Action": [
+              "glue:Get*",
+              "glue:DeleteTable",
+              "glue:DeleteTableVersion",
+              "glue:DeleteSchema",
+              "glue:DeletePartition",
+              "glue:DeleteDatabase",
+              "glue:UpdateTable",
+              "glue:UpdateSchema",
+              "glue:UpdatePartition",
+              "glue:UpdateDatabase",
+              "glue:CreateTable",
+              "glue:CreateSchema",
+              "glue:CreatePartition",
+              "glue:CreatePartitionIndex",
+              "glue:BatchCreatePartition",
+              "glue:CreateDatabase"
+          ],
+          "Resource": [
+              "arn:aws:glue:*:*:schema/*",
+              "arn:aws:glue:*:*:database/*",
+              "arn:aws:glue:*:*:table/*/*",
+              "arn:aws:glue:*:*:catalog"
+          ]
+      },
+      {
+          "Sid": "AthenaActions",
+          "Effect": "Allow",
+          "Action": [
+              "athena:List*",
+              "athena:Get*",
+              "athena:StartQueryExecution",
+              "athena:StopQueryExecution"
+          ],
+          "Resource": [
+              "arn:aws:athena:*:*:datacatalog/*",
+              "arn:aws:athena:*:*:workgroup/*"
+          ]
+      },
+      {
+          "Sid": "AirflowCLIPolicy",
+          "Effect": "Allow",
+          "Action": [
+              "airflow:CreateCliToken"
+          ],
+          "Resource": [
+              "arn:aws:airflow:*:*:environment/dev",
+              "arn:aws:airflow:*:*:environment/prod"
+          ]
+      },
+      {
+          "Sid": "CadetWriteAccess",
+          "Effect": "Allow",
+          "Action": [
+              "s3:List*",
+              "s3:GetObject*",
+              "s3:GetBucket*",
+              "s3:DeleteObject*",
+              "s3:PutObject*"
+          ],
+          "Resource": [
+              "arn:aws:s3:::mojap-derived-tables/*",
+              "arn:aws:s3:::mojap-derived-tables",
+              "arn:aws:s3:::dbt-query-dump/*",
+              "arn:aws:s3:::dbt-query-dump",
+              "arn:aws:s3:::mojap-manage-offences/ho-offence-codes/*",
+              "arn:aws:s3:::mojap-manage-offences",
+              "arn:aws:s3:::mojap-hub-exports/probation_referrals_dump/*",
+              "arn:aws:s3:::mojap-hub-exports",
+              "arn:aws:s3:::alpha-app-opg-lpa-dashboard",
+              "arn:aws:s3:::alpha-app-opg-lpa-dashboard/dev/models/domain_name=opg/*",
+              "arn:aws:s3:::alpha-app-opg-lpa-dashboard/prod/models/domain_name=opg/*",
+              "arn:aws:s3:::alpha-bold-data-shares",
+              "arn:aws:s3:::alpha-bold-data-shares/reducing-reoffending/*"
+          ]
+      },
+      {
+          "Sid": "CadetReadAccess",
+          "Effect": "Allow",
+          "Action": [
+              "s3:List*",
+              "s3:GetObject*",
+              "s3:GetBucket*"
+          ],
+          "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+          ]
+      }
+  ]
+}

tests/test_config/bad_cadet_deployer.yaml

@@ -0,0 +1,3 @@
+iam_role_name: an_iam_role_name
+
+is_cadet_deployer: true

tests/test_config/cadet_deployer.yaml

@@ -0,0 +1,3 @@
+iam_role_name: cadet_airflow_deployment
+
+is_cadet_deployer: true

tests/test_iam_builder.py

@@ -8,7 +8,7 @@
 import json
 
 from parameterized import parameterized
-from iam_builder.exceptions import IAMValidationError
+from iam_builder.exceptions import IAMValidationError, PrivilegedRoleValidationError
 from yaml.parser import ParserError
 
 
@@ -72,6 +72,7 @@ class TestConfigOutputs(unittest.TestCase):
             "athena_full_access",
             "athena_two_dumps",
             "glue_job",
+            "cadet_deployer",
             "all_config",
             "secrets",
             "secrets_readwrite",
@@ -91,6 +92,7 @@ class TestBadConfigs(unittest.TestCase):
     @parameterized.expand(
         [
             ("bad_athena_config", IAMValidationError),
+            ("bad_cadet_deployer", PrivilegedRoleValidationError),
             ("bad_glue_config", IAMValidationError),
             ("bad_read_only_not_list", IAMValidationError),
             ("bad_s3_config", IAMValidationError),