Merge pull request #704 from tmobile/develop

Release - v1.11.1

.gitignore

@@ -16,3 +16,4 @@ npm-debug.log
 /.idea
 package-lock.json
 
+core/jazz_ui/target/

.travis.yml

@@ -2,16 +2,19 @@ language: node_js
 node_js:
  - "node"
 env:
+ - PLATFORM_SERVICE=core/jazz_admin
  - PLATFORM_SERVICE=core/jazz_asset-event-handler
  - PLATFORM_SERVICE=core/jazz_assets
  - PLATFORM_SERVICE=core/jazz_codeq
  - PLATFORM_SERVICE=core/jazz_create-serverless-service
  - PLATFORM_SERVICE=core/jazz_delete-serverless-service
  - PLATFORM_SERVICE=core/jazz_deployments
  - PLATFORM_SERVICE=core/jazz_deployments-event-handler
- - PLATFORM_SERVICE=core/jazz_environments
+ - PLATFORM_SERVICE=core/jazz_email
  - PLATFORM_SERVICE=core/jazz_environment-event-handler
+ - PLATFORM_SERVICE=core/jazz_environments
  - PLATFORM_SERVICE=core/jazz_events
+ - PLATFORM_SERVICE=core/jazz_events-handler
  - PLATFORM_SERVICE=core/jazz_is-service-available
  - PLATFORM_SERVICE=core/jazz_is-slack-channel-available
  - PLATFORM_SERVICE=core/jazz_login
@@ -24,6 +27,7 @@ env:
  - PLATFORM_SERVICE=core/jazz_slack-channel
  - PLATFORM_SERVICE=core/jazz_slack-event-handler
  - PLATFORM_SERVICE=core/jazz_test-lambda
+ - PLATFORM_SERVICE=core/jazz_usermanagement
 
 before_script:
   - npm prune

README.md

@@ -1,46 +1,58 @@
-#  ![Jazz Logo](misc/logo.png) Jazz Serverless Platform 
+# ![Jazz Logo](misc/logo.png) Jazz Serverless Platform
 
 [![Build Status](https://travis-ci.org/tmobile/jazz.svg?branch=master)](https://travis-ci.org/tmobile/jazz)
 [![License](https://img.shields.io/badge/License-Apache%202.0-yellowgreen.svg)](https://github.com/tmobile/jazz/blob/master/LICENSE)
 [![Gitter](https://img.shields.io/gitter/room/badges/shields.svg)](https://gitter.im/TMO-OSS/Jazz)
 [![Slack Chat](https://img.shields.io/badge/Chat-Slack-ff69b4.svg)](https://tmo-oss-getinvite.herokuapp.com/)
 
-**Seamlessly build, deploy & manage cloud-native applications.**
+**Seamlessly build, deploy & manage cloud-native serverless applications!**
 
-Jazz accelerates adoption of serverless services. Jazz comes with a UI that lets developers quickly create serverless services with the click of a button. Its modular design makes it easy to add new integrations.
+## Introduction
 
-* **Services** - Jazz can help build functions, APIs and static websites. The template-based system makes it easy to define new ones.
-* **Deployment Targets** - Currently we deploy to AWS (Lambda, API gateway and S3). We plan to support Azure Functions and Docker containers in the near future.
-* **Features** - Services seamlessly integrate features like monitoring (CloudWatch), logging (ElasticSearch), authentication (Cognito).
-* **Deployment & CI/CD** - CI/CD platform is driven using Jenkins where it leverages [Serverless Framework](http://www.serverless.com) for deployment of serverless services.
-* **SCM** - Jazz works with Gitlab as well as Bitbucket.
+Jazz, a serverless platform, accelerates adoption of serverless technology within your enterprise. Jazz comes with a beautiful UI that lets developers quickly create serverless applications with a click of a button. Its modular design makes it easy to add new integrations.
 
+## Overview
+
+* **Services** - As of today, Jazz can help build functions, APIs and static websites. Fully customizable template-based design makes it easy to define new ones and expose them as services to the developers.
+* **Deployment Targets** - Currently Jazz can deploy to AWS managed services like (Lambda, API Gateway, S3, CloudFront etc.). We plan to support related services in Azure and GCP in the near future.
+* **Deployment & CI/CD** - Jazz comes with CI/CD by default. It creates a code repository per service and adds a web hook to trigger build/deployment workflows whenever it sees a commit. We leverage [Jenkins](https://github.com/jenkinsci/jenkins) open source for build process and [Serverless Framework](http://www.serverless.com) for deploying these services.
+* **Other Features** - Other useful features/integrations like SCM (Gitlab/Bitbucket), monitoring (CloudWatch), logging (ElasticSearch), authentication (Cognito), code quality metrics (SonarQube) comes with Jazz by default.
+* **Extensions** - Jazz is designed to integrate and work well with other systems that your enterprise needs. You can check out our optional extensions like Slack, Splunk etc. Centralized configuration helps Jazz admins to easily enable/disable these features as per their needs.
 
 Jazz is [open-sourced](http://opensource.t-mobile.com) and under active development by T-Mobile's Cloud Center of Excellence.
 
 [Watch the video preview here.](https://www.youtube.com/watch?v=6Kp1yxMjn1k)
 
-## Install
+## User Guide
+
+For complete user guide, see our [wiki](https://github.com/tmobile/jazz/wiki).
+
+## Architecture
+
+Following is the high level logical architecture of Jazz.
 
-You can [install Jazz](https://github.com/tmobile/jazz-installer) in your account using the automated installer.
+![Jazz Architecture](misc/jazz_logical_architecture.png)
+
+## Installation
+
+You can [install Jazz](https://github.com/tmobile/jazz-installer) in your AWS account using the automated installer.
 
 ## Try Jazz!
-You can try out public preview version of Jazz by registering with your email address [here](http://try.tmo-jazz.net). You will need a registration code which can be requested by joining [slack](https://tmo-oss-getinvite.herokuapp.com/). Once in slack, you can request a registration code.
 
-## User Guide
+You can try out public preview version of Jazz by registering with your email address [here](http://try.tmo-jazz.net). You will need a registration code which can be requested by joining [slack](https://tmo-oss-getinvite.herokuapp.com/).
+
 
-For more details, see the [Wiki](https://github.com/tmobile/jazz/wiki).
+## Platform Development
 
-## Development
-### Branching/release flow
+### Branching/Release Strategy
 
 1. Breaking/nontrivial features first go into named feature branches cut from `develop`
 2. When/if a feature branch is chosen to be included in the next release, it is merged into `develop`
 3. Release testing happens in `develop`
 4. When confirmed/vetted, `develop` is merged into `master`, and `master` becomes the current release.
 5. Small fixes explicitly intended for the next release can be PRed directly into `develop` without first needing a feature branch.
 
-tl;dr `master` is always the current release, `develop` is always the current state of the next release. If you want to contribute a PR, recommend you fork and work in a branch off of `develop`, then PR against `develop`. Project owners will move you into a feature branch if they deem it necessary.
+tl;dr `master` is always the current release, `develop` is always the current state of the next release. If you want to contribute a PR, we recommend you fork and work in a branch off of `develop`, then PR against `develop`. Project owners will move you into a feature branch if they deem it necessary.
 
 ## License
 

builds/delete-serverless-service-build-pack/Jenkinsfile

@@ -132,6 +132,7 @@ node {
 				}
 			} else {
 				error "Invalid project configuration"
+
 			}
 		}
 	}
@@ -335,9 +336,11 @@ def unDeployService(stage) {
 		[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: configLoader.AWS_CREDENTIAL_ID, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
 	]) {
 		try {
-			sh "aws configure set profile.cloud-api.region ${configLoader.AWS.REGION}"
-			sh "aws configure set profile.cloud-api.aws_access_key_id $AWS_ACCESS_KEY_ID"
-			sh "aws configure set profile.cloud-api.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
+			def randomString = utilModule.generateRequestId();
+			def credsId = "jazz-${randomString}";
+			sh "aws configure set profile.${credsId}.region ${configLoader.AWS.REGION}"
+			sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
+			sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 
 			def env_key
 			if (stage.endsWith("-dev")) {
@@ -349,6 +352,8 @@ def unDeployService(stage) {
 			sh "serverless remove --stage ${stage} --verbose  --bucket ${configLoader.AWS.S3[envBucketKey]}"
 
 			echo "Service undeployed"
+
+			resetCredentials(credsId)
 		} catch (ex) {
 			handleFailureEvent(ex.getMessage())
 		}
@@ -449,8 +454,8 @@ def updateSwaggerConfig() {
 			sh "sed -i -- 's/{domain}/${service_config['domain']}/g' swagger/swagger.json"
 
 			def region = "${configLoader.AWS.REGION}"
-			def role = "${configLoader.AWS.ROLEID}"
-			def roleARN = role.replaceAll("/", "\\\\/")
+
+			def roleARN = configLoader.AWS.PLATFORMSERVICES_ROLEID.replaceAll("/", "\\\\/")
 
 			// TODO: the below couple of statements will be replaced with regular expression in very near future;
 			def roleId = roleARN.substring(roleARN.indexOf("::") + 2, roleARN.lastIndexOf(":"))
@@ -493,9 +498,11 @@ def cleanUpApiGatewayResources(stage, path) {
 		[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: configLoader.AWS_CREDENTIAL_ID, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
 	]) {
 		try {
-			sh "aws configure set profile.cloud-api.region ${configLoader.AWS.REGION}"
-			sh "aws configure set profile.cloud-api.aws_access_key_id $AWS_ACCESS_KEY_ID"
-			sh "aws configure set profile.cloud-api.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
+			def randomString = utilModule.generateRequestId();
+			def credsId = "jazz-${randomString}";
+			sh "aws configure set profile.${credsId}.region ${configLoader.AWS.REGION}"
+			sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
+			sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 
 			def resource_id = null
 			def resource_search_key
@@ -541,6 +548,7 @@ def cleanUpApiGatewayResources(stage, path) {
 			} else {
 				echo "Resource Id does not exist in API gateway."
 			}
+			resetCredentials(credsId)
 		} catch (ex) {
 			handleFailureEvent(ex.getMessage())
 		}
@@ -601,11 +609,14 @@ def cleanUpApiDocs(stage) {
 		[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: configLoader.AWS_CREDENTIAL_ID, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
 	]) {
 		try {
-			sh "aws configure set profile.cloud-api.region ${configLoader.AWS.REGION}"
-			sh "aws configure set profile.cloud-api.aws_access_key_id $AWS_ACCESS_KEY_ID"
-			sh "aws configure set profile.cloud-api.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
+			def randomString = utilModule.generateRequestId();
+			def credsId = "jazz-${randomString}";
+			sh "aws configure set profile.${credsId}.region ${configLoader.AWS.REGION}"
+			sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
+			sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 			def apiRootFolder = getApiDocsFolder(stage)
 			sh "aws s3 rm s3://${apiRootFolder}/${service_config['domain']}/${service_config['service']}/${stage} --recursive"
+			resetCredentials(credsId)
 		} catch (ex) {
 			handleFailureEvent(ex.getMessage())
 		}
@@ -705,9 +716,11 @@ def unDeployWebsite(stage) {
 		[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: configLoader.AWS_CREDENTIAL_ID, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
 	]) {
 		try {
-			sh "aws configure set profile.cloud-api.region ${configLoader.AWS.REGION}"
-			sh "aws configure set profile.cloud-api.aws_access_key_id $AWS_ACCESS_KEY_ID"
-			sh "aws configure set profile.cloud-api.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
+			def randomString = utilModule.generateRequestId();
+			def credsId = "jazz-${randomString}";
+			sh "aws configure set profile.${credsId}.region ${configLoader.AWS.REGION}"
+			sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
+			sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 			sh "aws configure set preview.cloudfront true"
 
 			def assetInfo = generateAssetInfo(stage)
@@ -733,6 +746,7 @@ def unDeployWebsite(stage) {
 					sh "aws s3 rm s3://${assetInfo['folder_name']} --recursive"
 				}
 			}
+			resetCredentials(credsId)
 		} catch (ex) {
 			handleFailureEvent(ex.getMessage())
 		}
@@ -791,9 +805,11 @@ def cleanupS3BucketPolicy(stage, assetInfo) {
 		[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: configLoader.AWS_CREDENTIAL_ID, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
 	]) {
 		try {
-			sh "aws configure set profile.cloud-api.region ${configLoader.AWS.REGION}"
-			sh "aws configure set profile.cloud-api.aws_access_key_id $AWS_ACCESS_KEY_ID"
-			sh "aws configure set profile.cloud-api.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
+			def randomString = utilModule.generateRequestId();
+			def credsId = "jazz-${randomString}";
+			sh "aws configure set profile.${credsId}.region ${configLoader.AWS.REGION}"
+			sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
+			sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 			sh "aws configure set preview.cloudfront true"
 			def bucketPolicy = sh(
 				script: "aws s3api get-bucket-policy --bucket ${assetInfo['s3Bucket']} --output json",
@@ -818,7 +834,7 @@ def cleanupS3BucketPolicy(stage, assetInfo) {
 				def policy_json = JsonOutput.toJson(policyObjectUpdated)
 				updateBucketPolicy(policy_json, assetInfo['s3Bucket'])
 			}
-			resetCredentials()
+			resetCredentials(credsId)
 		} catch (ex) {
 			resetCredentials()
 			if (ex.getMessage().indexOf("groovy.json.internal.LazyMap") < 0) {
@@ -828,15 +844,13 @@ def cleanupS3BucketPolicy(stage, assetInfo) {
 	}
 }
 
-/**
-	Reset credentials
-*/
-def resetCredentials() {
-	echo "resetting AWS credentials"
-	sh "aws configure set profile.cloud-api.aws_access_key_id XXXXXXXXXXXXXXXXXXXXXXXXXX"
-	sh "aws configure set profile.cloud-api.aws_access_key_id XXXXXXXXXXXXXXXXXXXXXX"
+/** Reset credentials
+ */
+def resetCredentials(credsId) {
+  echo "resetting AWS credentials"
+	sh "aws configure set profile.${credsId}.aws_access_key_id XXXXXXXXXXXXXXXXXXXXXXXXXX"
+  sh "aws configure set profile.${credsId}.aws_secret_access_key XXXXXXXXXXXXXXXXXXXXXX"
 }
-
 @NonCPS
 def updateBucketPolicy(policy_json, bucketName){
 	try {
@@ -860,9 +874,11 @@ def cleanupCloudFrontDistribution(stage) {
 		[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: configLoader.AWS_CREDENTIAL_ID, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
 	]) {
 		try {
-			sh "aws configure set profile.cloud-api.region ${configLoader.AWS.REGION}"
-			sh "aws configure set profile.cloud-api.aws_access_key_id $AWS_ACCESS_KEY_ID"
-			sh "aws configure set profile.cloud-api.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
+			def randomString = utilModule.generateRequestId();
+			def credsId = "jazz-${randomString}";
+			sh "aws configure set profile.${credsId}.region ${configLoader.AWS.REGION}"
+			sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
+			sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 			sh "aws configure set preview.cloudfront true"
 
 			def distributionID
@@ -875,6 +891,7 @@ def cleanupCloudFrontDistribution(stage) {
 				_Etag = generateDistributionConfigForDisable(distributionConfig)
 				_Etag = disableCloudFrontDistribution(distributionID, _Etag, "disable-cf-distribution-config.json", stage)
 			}
+			resetCredentials(credsId)
 		} catch (ex) {
 			if ((ex.getMessage()).indexOf("getDistributionId Failed") > -1) {
 				echo "Could not find a CloudFront distribution Id for service: ${service_config['service']} and environment $stage"
@@ -1050,6 +1067,8 @@ def loadServerlessConfig() {
 		sh "cp _config/serverless-java.yml ./serverless.yml"
 	} else if (service_config['providerRuntime'].indexOf("python") > -1) {
 		sh "cp _config/serverless-python.yml ./serverless.yml"
+	} else if (service_config['providerRuntime'].indexOf("go") > -1) {
+		sh "cp _config/serverless-go.yml ./serverless.yml"
 	}
 
   if (service_config['event_source_dynamodb'] || service_config['event_source_sqs']) {
@@ -1062,8 +1081,9 @@ def loadServerlessConfig() {
 
 }
 
-def removeEventResources(){
+def removeEventResources() {
    sh "sed -i -- '/#Start:resources/,/#End:resources/d' ./serverless.yml"
+   sh "sed -i -- '/#Start:events/,/#End:events/d' ./serverless.yml"
 }
 
 

builds/jazz-build-module/aws-lambda-events-module.groovy

@@ -40,10 +40,14 @@ def checkKinesisStreamExists(stream_name) {
 }
 
 def updateKinesisResourceServerless(event_stream_arn){
-  sh "sed -i -- 's/resources/resourcesDisabled/g' ./serverless.yml"
+  sh "sed -i -- '/#Start:isKinesisStreamNotExist/,/#End:isKinesisStreamNotExist/d' ./serverless.yml"
   sh "sed -i -- '/#Start:streamGetArn/,/#End:streamGetArn/d' ./serverless.yml"
   sh "sed -i -- 's/arnDisabled/arn/g' ./serverless.yml"
   sh "sed -i -- 's|{event_stream_arn}|${event_stream_arn}|g' ./serverless.yml"
+
+  sh "sed -i -- '/#Start:kinesisStreamGetArn/,/#End:kinesisStreamGetArn/d' ./policyFile.yml"
+  sh "sed -i -- 's|{event_kinesis_stream_arn}|${event_stream_arn}|g' ./policyFile.yml"
+  sh  "sed -i -- 's/#ResourceKinesisDisabled/Resource/g' ./policyFile.yml"
 }
 
 def getRoleArn(role_name) {
@@ -412,7 +416,7 @@ def updateDynamoDbResourceServerless(event_stream_arn){
 
   sh "sed -i -- '/#Start:dynamoDbstreamGetArn/,/#End:dynamoDbstreamGetArn/d' ./policyFile.yml"
   sh "sed -i -- 's|{event_dynamodb_stream_arn}|${event_stream_arn}|g' ./policyFile.yml"
-  sh  "sed -i -- 's/#ResourceDisabled/Resource/g' ./policyFile.yml"
+  sh  "sed -i -- 's/#ResourceDynamoDbDisabled/Resource/g' ./policyFile.yml"
 }
 
 def getDynamoDbStreamDetails(event_source_dynamodb) {

builds/jazz-build-module/events-module.groovy

@@ -209,7 +209,7 @@ def sendEvent(event_name, event_status, message, moreCxtMap){
 		'branch': g_branch,
 		'runtime': service_metadata['runtime'],
 		'domain': service_metadata['domain'],
-		'iam_role': config_loader.AWS.ROLEID,
+		'iam_role': service_metadata['iamRoleARN'],
 		'environment': g_environment,
 		'region': config_loader.AWS.REGION,
 		'message': message