Update examples

in_cluster/artifacts/main.py

@@ -1,3 +1,5 @@
+import logging
+
 import argparse
 import random
 import time
@@ -16,6 +18,9 @@
 from polyaxon import tracking
 from polyaxon.tracking.contrib.keras import PolyaxonKerasCallback
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 
 def get_loss(step):
     result = 10 / (step + 1)
@@ -348,7 +353,7 @@ def main():
     tracking.init()
 
     for i in range(args.steps):
-        print('Step', i)
+        logger.info('Step %s', i)
         # Scalars
         loss = get_loss(i)
         accuracy = get_accuracy(loss)

in_cluster/artifacts/polyaxonfile.yml

@@ -12,10 +12,9 @@ run:
   init:
   - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
   container:
-    image: polyaxon/polyaxon-examples
+    image: polyaxon/polyaxon-examples:artifacts
     workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/artifacts"
     command: ["python", "-u", "main.py"]
-    imagePullPolicy: "Always"
     args: [
       "--steps={{ steps }}",
       "--validate_every={{ validate_every }}",

in_cluster/build_image/build-artifacts.yaml

@@ -0,0 +1,16 @@
+version: 1.1
+kind: operation
+name: build
+params:
+  destination:
+    connection: docker-connection
+    value: polyaxon-examples:artifacts
+runPatch:
+  init:
+  - dockerfile:
+      image: tensorflow/tensorflow:2.2.2
+      run:
+      - 'pip3 install --no-cache-dir -U polyaxon["s3","gcs","azure","polyboard","polytune"]==1.7.0'
+      - pip3 install scikit-learn matplotlib vega-datasets
+      langEnv: 'en_US.UTF-8'
+hubRef: kaniko

in_cluster/build_image/build-ml.yaml

@@ -0,0 +1,16 @@
+version: 1.1
+kind: operation
+name: build
+params:
+  destination:
+    connection: docker-connection
+    value: polyaxon-examples:ml
+runPatch:
+  init:
+  - dockerfile:
+      image: python:3.8.8-buster
+      run:
+      - 'pip3 install --no-cache-dir -U polyaxon["polyboard"]==1.7.0'
+      - pip3 install scikit-learn xgboost matplotlib vega-datasets joblib lightgbm xgboost
+      langEnv: 'en_US.UTF-8'
+hubRef: kaniko

in_cluster/build_image/build-pytorch.yaml

@@ -0,0 +1,20 @@
+version: 1.1
+kind: operation
+name: build
+params:
+  destination:
+    connection: docker-connection
+    value: polyaxon-examples:pytorch
+  nocache:
+    value: true
+runPatch:
+  init:
+  - dockerfile:
+      image: "pytorch/pytorch"
+      run:
+      - 'pip3 install --no-cache-dir -U polyaxon==1.7.0'
+      - pip3 install -U pytorch-lightning
+      - pip3 install -U pytorch-ignite
+      - pip3 install -U "torchvision>=0.9"
+      langEnv: 'en_US.UTF-8'
+hubRef: kaniko

in_cluster/build_image/build-tf.yaml

@@ -0,0 +1,17 @@
+version: 1.1
+kind: operation
+name: build
+params:
+  destination:
+    connection: docker-connection
+    value: polyaxon-examples:tf
+runPatch:
+  init:
+  - dockerfile:
+      image: "tensorflow/tensorflow:2.2.2"
+      run:
+      - 'pip3 install --no-cache-dir -U polyaxon==1.7.0'
+      - pip3 install keras
+      - pip3 install tensorflow_datasets
+      langEnv: 'en_US.UTF-8'
+hubRef: kaniko

in_cluster/fastai/mnist.py

@@ -3,7 +3,7 @@
 from fastai.vision.all import *
 from fastai.basics import *
 
-from polyaxon.tracking.contrib.fastai import PolyaxonFastaiCallback
+from polyaxon.tracking.contrib.fastai import PolyaxonCallback
 
 path = untar_data(URLs.MNIST_SAMPLE)
 items = get_image_files(path)
@@ -15,7 +15,7 @@
     dls,
     resnet18,
     loss_func=CrossEntropyLossFlat(),
-    cbs=[PolyaxonFastaiCallback()]
+    cbs=[PolyaxonCallback()]
 )
 
 if __name__ == '__main__':

in_cluster/fastai/segmentation.py

@@ -2,8 +2,7 @@
 
 from fastai.vision.all import *
 
-from polyaxon.tracking.contrib.fastai import PolyaxonFastaiCallback
-
+from polyaxon.tracking.contrib.fastai import PolyaxonCallback
 
 path = untar_data(URLs.CAMVID_TINY)
 codes = np.loadtxt(path / 'codes.txt', dtype=str)
@@ -18,7 +17,7 @@ def label_func(fn):
     path, bs=8, fnames=fnames, label_func=label_func, codes=codes
 )
 
-learn = unet_learner(dls, resnet18, cbs=[PolyaxonFastaiCallback(log_model=True), SaveModelCallback()])
+learn = unet_learner(dls, resnet18, cbs=[PolyaxonCallback(log_model=True), SaveModelCallback()])
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

in_cluster/fastai/tabular.py

@@ -2,7 +2,7 @@
 
 from fastai.tabular.all import *
 
-from polyaxon.tracking.contrib.fastai import PolyaxonFastaiCallback
+from polyaxon.tracking.contrib.fastai import PolyaxonCallback
 
 path = untar_data(URLs.ADULT_SAMPLE)
 df = pd.read_csv(path / 'adult.csv')
@@ -16,7 +16,7 @@
 )
 
 # create a learner and train
-learn = tabular_learner(dls, metrics=accuracy, cbs=[PolyaxonFastaiCallback()])
+learn = tabular_learner(dls, metrics=accuracy, cbs=[PolyaxonCallback()])
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

in_cluster/ignite/mnist.py

@@ -125,11 +125,11 @@ def log_validation_results(engine):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--epochs', type=int, default=2)
-    parser.add_argument('--batch_size', type=int, default=256)
-    parser.add_argument('--val_batch_size', type=int, default=100)
+    parser.add_argument('--batch-size', type=int, default=256)
+    parser.add_argument('--val-batch-size', type=int, default=100)
     parser.add_argument('--lr', type=float, default=0.001)
     parser.add_argument('--momentum', type=float, default=0.1)
-    parser.add_argument('--log_interval', type=int, default=10)
+    parser.add_argument('--log-interval', type=int, default=10)
     args = parser.parse_args()
     main(
         args.batch_size, args.val_batch_size, args.epochs, args.lr, args.momentum, args.log_interval

in_cluster/ignite/polyaxonfile.yml

@@ -16,10 +16,9 @@ run:
   init:
   - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
   container:
-    image: polyaxon/polyaxon-examples
+    image: polyaxon/polyaxon-examples:pytorch
     workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/ignite"
     command: ["python", "-u", "mnist.py"]
-    imagePullPolicy: "Always"
     args: [
       "{{ params.epochs.as_arg }}",
       "{{ params.batch_size.as_arg }}",

in_cluster/keras/bidirectional_lstm/model.py

@@ -10,7 +10,7 @@
 
 # Polyaxon
 from polyaxon import tracking
-from polyaxon.tracking.contrib.keras import PolyaxonKerasCallback, PolyaxonKerasModelCheckpoint
+from polyaxon.tracking.contrib.keras import PolyaxonCallback
 
 logger = logging.getLogger('bidir-lstm')
 
@@ -55,10 +55,9 @@ def train(max_features,
               callbacks=[
                   EarlyStopping(
                       monitor='val_loss', min_delta=1e-4, patience=3, verbose=1, mode='auto'),
-                  PolyaxonKerasCallback(),
-                  PolyaxonKerasModelCheckpoint(),
+                  PolyaxonCallback(),
                   TensorBoard(log_dir=tracking.get_tensorboard_path(), histogram_freq=1),
-                  ModelCheckpoint(tracking.get_model_path())
+                  ModelCheckpoint(tracking.get_outputs_path("model"))
               ])
 
     return model.evaluate(x_test, y_test)[1]

in_cluster/keras/bidirectional_lstm/polyaxonfile.yml

@@ -19,9 +19,9 @@ run:
   init:
   - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
   container:
-    image: polyaxon/polyaxon-examples
-    command: ["python", "-u", "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/keras/bidirectional_lstm/model.py"]
-    imagePullPolicy: "Always"
+    image: polyaxon/polyaxon-examples:tf
+    workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/keras/bidirectional_lstm"
+    command: ["python", "-u", "model.py"]
     args: [
       "--max_features={{ max_features }}",
       "--skip_top={{ skip_top }}",

in_cluster/keras/cnn_lstm/model.py

@@ -13,7 +13,7 @@
 
 # Polyaxon
 from polyaxon import tracking
-from polyaxon.tracking.contrib.keras import PolyaxonKerasCallback, PolyaxonKerasModelCheckpoint
+from polyaxon.tracking.contrib.keras import PolyaxonCallback
 
 logger = logging.getLogger('cnn-lstm')
 
@@ -61,10 +61,9 @@ def train(max_features,
               epochs=epochs,
               validation_data=(x_test, y_test),
               callbacks=[
-                  PolyaxonKerasCallback(),
-                  PolyaxonKerasModelCheckpoint(),
+                  PolyaxonCallback(),
                   TensorBoard(log_dir=tracking.get_tensorboard_path(), histogram_freq=1),
-                  ModelCheckpoint(tracking.get_model_path())
+                  ModelCheckpoint(tracking.get_outputs_path("model"))
               ])
 
     score, accuracy = model.evaluate(x_test, y_test, batch_size=batch_size)

in_cluster/keras/mnist/model.py

@@ -1,19 +1,15 @@
 import argparse
 
-from keras import utils
-from keras import optimizers
-from keras.datasets import mnist
-from keras.layers import Dense, Conv2D, Dropout, Flatten, MaxPooling2D
-from keras.models import Sequential
+import tensorflow as tf
 
 # Polyaxon
 from polyaxon import tracking
-
+from polyaxon.tracking.contrib.keras import PolyaxonCallback
 
 OPTIMIZERS = {
-    'adam': optimizers.Adam,
-    'rmsprop': optimizers.RMSprop,
-    'sgd': optimizers.SGD,
+    'adam': tf.keras.optimizers.Adam,
+    'rmsprop': tf.keras.optimizers.RMSprop,
+    'sgd': tf.keras.optimizers.SGD,
 }
 
 
@@ -24,34 +20,41 @@ def transform_data(x_train, y_train, x_test, y_test):
     x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
     x_test = x_test.astype('float32') / 255
 
-    y_train = utils.to_categorical(y_train, num_classes=10)
-    y_test = utils.to_categorical(y_test, num_classes=10)
+    y_train = tf.keras.utils.to_categorical(y_train, num_classes=10)
+    y_test = tf.keras.utils.to_categorical(y_test, num_classes=10)
 
     return x_train, y_train, x_test, y_test
 
 
 def train(conv1_size, conv2_size, dropout, hidden1_size, optimizer, log_learning_rate, epochs):
-    model = Sequential()
-    model.add(Conv2D(filters=conv1_size,
-                     kernel_size=(3, 3),
-                     activation='relu',
-                     input_shape=x_train.shape[1:]))
-    model.add(Conv2D(filters=conv2_size,
-                     kernel_size=(3, 3),
-                     activation='relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-    model.add(Dropout(dropout))
-    model.add(Flatten())
-    model.add(Dense(hidden1_size, activation='relu'))
-    model.add(Dense(10, activation='softmax'))
+    model = tf.keras.models.Sequential()
+    model.add(tf.keras.layers.Conv2D(filters=conv1_size,
+                                     kernel_size=(3, 3),
+                                     activation='relu',
+                                     input_shape=x_train.shape[1:]))
+    model.add(tf.keras.layers.Conv2D(filters=conv2_size,
+                                     kernel_size=(3, 3),
+                                     activation='relu'))
+    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
+    model.add(tf.keras.layers.Dropout(dropout))
+    model.add(tf.keras.layers.Flatten())
+    model.add(tf.keras.layers.Dense(hidden1_size, activation='relu'))
+    model.add(tf.keras.layers.Dense(10, activation='softmax'))
+
+    optimizer = OPTIMIZERS[optimizer](lr=10 ** log_learning_rate)
     model.compile(
-        optimizer=OPTIMIZERS[optimizer](lr=10 ** log_learning_rate),
+        optimizer=optimizer,
         loss='categorical_crossentropy',
         metrics=['accuracy'],
     )
 
-    model.fit(x_train, y_train, epochs=epochs, batch_size=100)
-
+    model.fit(
+        x_train,
+        y_train,
+        epochs=epochs,
+        batch_size=100,
+        callbacks=[PolyaxonCallback()],  # Polyaxon
+    )
     return model.evaluate(x_test, y_test)[1]
 
 
@@ -96,7 +99,7 @@ def train(conv1_size, conv2_size, dropout, hidden1_size, optimizer, log_learning
     # Polyaxon
     tracking.init()
 
-    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
 
     # Polyaxon
     tracking.log_data_ref(content=x_train, name='x_train')
@@ -105,6 +108,13 @@ def train(conv1_size, conv2_size, dropout, hidden1_size, optimizer, log_learning
     tracking.log_data_ref(content=y_test, name='y_test')
 
     x_train, y_train, x_test, y_test = transform_data(x_train, y_train, x_test, y_test)
+    model = tf.keras.models.Sequential([
+        tf.keras.layers.Flatten(),
+        tf.keras.layers.Dense(256, activation=tf.keras.activations.relu),
+        tf.keras.layers.Dropout(0.5),
+        tf.keras.layers.Dense(10, activation=tf.keras.activations.softmax)
+    ])
+
     accuracy = train(conv1_size=args.conv1_size,
                      conv2_size=args.conv2_size,
                      dropout=args.dropout,
@@ -114,4 +124,4 @@ def train(conv1_size, conv2_size, dropout, hidden1_size, optimizer, log_learning
                      epochs=args.epochs)
 
     # Polyaxon
-    tracking.log_metrics(accuracy=accuracy)
+    tracking.log_metrics(eval_accuracy=accuracy)