azure-pipelines.yml

# Kubeflow Container Build Pipeline

trigger:
- master

pr: none

stages:
- stage: ContainerConfig
  displayName: 'Configure and Register Containers'
  jobs:
  - job: Containers
    pool:
      name: default 
    steps:
    - task: AzureCLI@1
      inputs:
        azureSubscription: 'Shared Data Platform - R+D (1308a130-d549-44e1-ba66-ce8c487d76e3)'
        scriptLocation: 'inlineScript'
        inlineScript: '
          sudo az acr login -n kubeflowregistry && 
          cd code &&
          cd preprocess &&
          sudo docker build -t kubeflowregistry.azurecr.io/kubeflow/preprocess:$BUILD_SOURCEVERSION . &&
          sudo docker push kubeflowregistry.azurecr.io/kubeflow/preprocess:$BUILD_SOURCEVERSION '
        displayName: 'Build & Push Preprocess Image'

    - task: AzureCLI@1
      inputs:
        azureSubscription: 'Shared Data Platform - R+D (1308a130-d549-44e1-ba66-ce8c487d76e3)'
        scriptLocation: 'inlineScript'
        inlineScript: '
          cd code &&
          cd training &&
          sudo docker build -t kubeflowregistry.azurecr.io/kubeflow/training:$BUILD_SOURCEVERSION . &&
          sudo docker push kubeflowregistry.azurecr.io/kubeflow/training:$BUILD_SOURCEVERSION '
        displayName: 'Build & Push Training Image'

    - task: AzureCLI@1
      inputs:
        azureSubscription: 'Shared Data Platform - R+D (1308a130-d549-44e1-ba66-ce8c487d76e3)'
        scriptLocation: 'inlineScript'
        inlineScript: '
          cd code &&
          cd register &&
          sudo docker build -t kubeflowregistry.azurecr.io/kubeflow/register:$BUILD_SOURCEVERSION . &&
          sudo docker push kubeflowregistry.azurecr.io/kubeflow/register:$BUILD_SOURCEVERSION '
        displayName: 'Build & Push Register Image'


# Moved KF step to build
- stage: KubeflowTrigger
  dependsOn: ContainerConfig
  displayName: 'Trigger Kubeflow Pipeline'
  variables:
  - group: kf-variables
  jobs:
  - job: Kubeflow
    pool:
      name: default 
    steps:
    - task: AzureCLI@1
      env:
        KF_MAPPED_SERVICE_PRINCIPAL_PASSWORD: $(KF_SERVICE_PRINCIPAL_PASSWORD)
      inputs:
        azureSubscription: 'Shared Data Platform - R+D (1308a130-d549-44e1-ba66-ce8c487d76e3)'
        scriptLocation: 'inlineScript'
        inlineScript: |
          az aks get-credentials -g kubeflow-mlops-rg -n kubeflow-mlops-cluster
          kubectl port-forward --namespace kubeflow svc/ml-pipeline 8888:8888 &
          kubepid=$!
          
          sudo apt-get install python3-setuptools
          pip3 install wheel
          pip3 install kfp
          
          touch script.py
          echo "import kfp" >> script.py
          echo "client = kfp.Client(host='localhost:8888')" >> script.py
          echo "client.run_pipeline('$KF_EXPERIMENT_ID', 'Run ${BUILD_BUILDID}', params={'imagetag': '${BUILD_SOURCEVERSION}', 'tenant-id': '$KF_TENANT_ID', 'service-principal-id': '$KF_SERVICE_PRINCIPAL_ID', 'service-principal-password': '$KF_MAPPED_SERVICE_PRINCIPAL_PASSWORD', 'subscription-id': '$KF_SUBSCRIPTION_ID', 'resource-group': '$KF_RESOURCE_GROUP', 'workspace': '$KF_WORKSPACE', 'persistent-volume-name': '$KF_PERSISTENT_VOLUME_NAME', 'persistent-volume-path': '$KF_PERSISTENT_VOLUME_PATH', 'data-download': '$KF_DATA_DOWNLOAD', 'epochs': '$KF_EPOCHS', 'batch': '$KF_BATCH', 'learning-rate': '$KF_LEARNING_RATE', 'model-name': '$KF_MODEL_NAME'}, pipeline_id='$KF_PIPELINE_ID')" >> script.py 
          
          cat script.py
          
          python3 script.py
          
          kill $kubepid
        displayName: 'Trigger Kubeflow Pipeline'