Update

in_cluster/build_image/build.yaml

@@ -3,9 +3,9 @@ kind: operation
 name: build
 params:
   destination:
+    connection: docker-connection
     value:
       name: polyaxon/polyaxon-examples
-      connection: docker-connection
 runPatch:
   init:
   - dockerfile:
@@ -17,5 +17,10 @@ runPatch:
       - pip3 install xgboost
       - pip3 install keras
       - pip3 install tensorflow_datasets
+      - pip3 install torch
+      - pip3 install torchvision
+      - pip3 install pytorch-ignite
+      - pip3 install tensorboard
+      - pip3 install matplotlib
       langEnv: 'en_US.UTF-8'
 hubRef: kaniko

in_cluster/fastai/mnist.py

@@ -0,0 +1,25 @@
+import argparse
+
+from fastai.vision.all import *
+from fastai.basics import *
+
+from polyaxon.tracking.contrib.fastai import PolyaxonFastaiCallback
+
+path = untar_data(URLs.MNIST_SAMPLE)
+items = get_image_files(path)
+tds = Datasets(items, [PILImageBW.create, [parent_label, Categorize()]], splits=GrandparentSplitter()(items))
+dls = tds.dataloaders(bs=32, after_item=[ToTensor(), IntToFloatTensor()])
+
+# create a learner with gradient accumulation
+learn = cnn_learner(
+    dls,
+    resnet18,
+    loss_func=CrossEntropyLossFlat(),
+    cbs=[PolyaxonFastaiCallback()]
+)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--fit', type=int, default=2)
+    args = parser.parse_args()
+    learn.fit(args.fit)

in_cluster/fastai/polyaxonfile-mnist.yml

@@ -0,0 +1,18 @@
+version: 1.1
+kind: component
+name: mnist
+tags: [examples]
+
+inputs:
+- {name: fit, type: int, value: 3, isOptional: true}
+
+run:
+  kind: job
+  init:
+  - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
+  container:
+    image: polyaxon/polyaxon-examples
+    workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/fastai"
+    command: ["python", "-u", "mnist.py"]
+    imagePullPolicy: "Always"
+    args: ["--fit={{ fit }}"]

in_cluster/fastai/polyaxonfile-segmentation.yml

@@ -0,0 +1,18 @@
+version: 1.1
+kind: component
+name: segmentation
+tags: [examples]
+
+inputs:
+- {name: fit, type: int, value: 3, isOptional: true}
+
+run:
+  kind: job
+  init:
+  - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
+  container:
+    image: polyaxon/polyaxon-examples
+    workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/fastai"
+    command: ["python", "-u", "segmentation.py"]
+    imagePullPolicy: "Always"
+    args: ["--fit={{ fit }}"]

in_cluster/fastai/polyaxonfile-tabular.yml

@@ -0,0 +1,18 @@
+version: 1.1
+kind: component
+name: segmentation
+tags: [examples]
+
+inputs:
+- {name: fit, type: int, value: 3, isOptional: true}
+
+run:
+  kind: job
+  init:
+  - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
+  container:
+    image: polyaxon/polyaxon-examples
+    workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/fastai"
+    command: ["python", "-u", "tabular.py"]
+    imagePullPolicy: "Always"
+    args: ["--fit={{ fit }}"]

in_cluster/fastai/segmentation.py

@@ -0,0 +1,27 @@
+import argparse
+
+from fastai.vision.all import *
+
+from polyaxon.tracking.contrib.fastai import PolyaxonFastaiCallback
+
+
+path = untar_data(URLs.CAMVID_TINY)
+codes = np.loadtxt(path / 'codes.txt', dtype=str)
+fnames = get_image_files(path / "images")
+
+
+def label_func(fn):
+    return path / "labels" / f"{fn.stem}_P{fn.suffix}"
+
+
+dls = SegmentationDataLoaders.from_label_func(
+    path, bs=8, fnames=fnames, label_func=label_func, codes=codes
+)
+
+learn = unet_learner(dls, resnet18, cbs=[PolyaxonFastaiCallback(log_model=True), SaveModelCallback()])
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--fit', type=int, default=2)
+    args = parser.parse_args()
+    learn.fit(args.fit)

in_cluster/fastai/tabular.py

@@ -0,0 +1,25 @@
+import argparse
+
+from fastai.tabular.all import *
+
+from polyaxon.tracking.contrib.fastai import PolyaxonFastaiCallback
+
+path = untar_data(URLs.ADULT_SAMPLE)
+df = pd.read_csv(path / 'adult.csv')
+dls = TabularDataLoaders.from_csv(
+    path / 'adult.csv',
+    path=path,
+    y_names="salary",
+    cat_names=['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race'],
+    cont_names=['age', 'fnlwgt', 'education-num'],
+    procs=[Categorify, FillMissing, Normalize]
+)
+
+# create a learner and train
+learn = tabular_learner(dls, metrics=accuracy, cbs=[PolyaxonFastaiCallback()])
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--fit', type=int, default=2)
+    args = parser.parse_args()
+    learn.fit(args.fit)

in_cluster/ignite/mnist.py

@@ -0,0 +1,136 @@
+import argparse
+
+import torch
+from torch import nn
+from torch.optim import SGD
+from torch.utils.data import DataLoader
+import torch.nn.functional as F
+from torchvision.transforms import Compose, ToTensor, Normalize
+from torchvision.datasets import MNIST
+
+from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator
+from ignite.metrics import Accuracy, Loss
+
+from tqdm import tqdm
+
+from polyaxon.tracking.contrib.ignite import PolyaxonIgniteLogger
+
+
+class Net(nn.Module):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
+        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
+        self.conv2_drop = nn.Dropout2d()
+        self.fc1 = nn.Linear(320, 50)
+        self.fc2 = nn.Linear(50, 10)
+
+    def forward(self, x):
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
+        x = x.view(-1, 320)
+        x = F.relu(self.fc1(x))
+        x = F.dropout(x, training=self.training)
+        x = self.fc2(x)
+        return F.log_softmax(x, dim=-1)
+
+
+def get_data_loaders(train_batch_size, val_batch_size):
+    data_transform = Compose([ToTensor(), Normalize((0.1307,), (0.3081,))])
+
+    train_loader = DataLoader(MNIST(download=True, root=".", transform=data_transform, train=True),
+                              batch_size=train_batch_size, shuffle=True)
+
+    val_loader = DataLoader(MNIST(download=False, root=".", transform=data_transform, train=False),
+                            batch_size=val_batch_size, shuffle=False)
+    return train_loader, val_loader
+
+
+def main(train_batch_size, val_batch_size, epochs, lr, momentum, log_interval):
+    train_loader, val_loader = get_data_loaders(train_batch_size, val_batch_size)
+    model = Net()
+    device = 'cpu'
+
+    if torch.cuda.is_available():
+        device = 'cuda'
+    model = model.to(device)
+    optimizer = SGD(model.parameters(), lr=lr, momentum=momentum)
+    trainer = create_supervised_trainer(model, optimizer, F.nll_loss, device=device)
+    evaluator = create_supervised_evaluator(model,
+                                            metrics={'accuracy': Accuracy(),
+                                                     'nll': Loss(F.nll_loss)},
+                                            device=device)
+
+    desc = "ITERATION - loss: {:.2f}"
+    pbar = tqdm(
+        initial=0, leave=False, total=len(train_loader),
+        desc=desc.format(0)
+    )
+
+    @trainer.on(Events.ITERATION_COMPLETED(every=log_interval))
+    def log_training_loss(engine):
+        pbar.desc = desc.format(engine.state.output)
+        pbar.update(log_interval)
+
+    @trainer.on(Events.EPOCH_COMPLETED)
+    def log_training_results(engine):
+        pbar.refresh()
+        evaluator.run(train_loader)
+        metrics = evaluator.state.metrics
+        avg_accuracy = metrics['accuracy']
+        avg_nll = metrics['nll']
+        tqdm.write(
+            "Training Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}"
+                .format(engine.state.epoch, avg_accuracy, avg_nll)
+        )
+
+    @trainer.on(Events.EPOCH_COMPLETED)
+    def log_validation_results(engine):
+        evaluator.run(val_loader)
+        metrics = evaluator.state.metrics
+        avg_accuracy = metrics['accuracy']
+        avg_nll = metrics['nll']
+        tqdm.write(
+            "Validation Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}"
+                .format(engine.state.epoch, avg_accuracy, avg_nll))
+
+        pbar.n = pbar.last_print_n = 0
+
+    # Polyaxon
+    polyaxon_logger = PolyaxonIgniteLogger()
+    polyaxon_logger.attach_output_handler(
+        trainer,
+        event_name=Events.ITERATION_COMPLETED,
+        tag="training",
+        output_transform=lambda loss: {"loss": loss}
+    )
+    polyaxon_logger.attach_output_handler(
+        evaluator,
+        event_name=Events.EPOCH_COMPLETED,
+        tag="training",
+        metric_names=["nll", "accuracy"],
+        global_step_transform=lambda *_: trainer.state.iteration,
+    )
+    polyaxon_logger.attach_opt_params_handler(
+        trainer,
+        event_name=Events.ITERATION_STARTED,
+        optimizer=optimizer,
+        param_name='lr'  # optional
+    )
+
+    trainer.run(train_loader, max_epochs=epochs)
+    pbar.close()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--epochs', type=int, default=2)
+    parser.add_argument('--batch_size', type=int, default=256)
+    parser.add_argument('--val_batch_size', type=int, default=100)
+    parser.add_argument('--lr', type=float, default=0.001)
+    parser.add_argument('--momentum', type=float, default=0.1)
+    parser.add_argument('--log_interval', type=int, default=10)
+    args = parser.parse_args()
+    main(
+        args.batch_size, args.val_batch_size, args.epochs, args.lr, args.momentum, args.log_interval
+    )

in_cluster/ignite/polyaxonfile.yml

@@ -0,0 +1,30 @@
+version: 1.1
+kind: component
+name: mnist
+tags: [examples]
+
+inputs:
+- {name: epochs, type: int, value: 2, isOptional: true}
+- {name: batch_size, type: int, value: 256, isOptional: true}
+- {name: val_batch_size, type: int, value: 100, isOptional: true}
+- {name: lr, type: float, value: 0.001, isOptional: true}
+- {name: momentum, type: float, value: 0.1, isOptional: true}
+- {name: log_interval, type: int, value: 10, isOptional: true}
+
+run:
+  kind: job
+  init:
+  - git: {"url": "https://github.com/polyaxon/polyaxon-examples"}
+  container:
+    image: polyaxon/polyaxon-examples
+    workingDir: "{{ globals.artifacts_path }}/polyaxon-examples/in_cluster/ignite"
+    command: ["python", "-u", "mnist.py"]
+    imagePullPolicy: "Always"
+    args: [
+      "{{ params.steps.as_arg }}",
+      "{{ params.batch_size.as_arg }}",
+      "{{ params.val_batch_size.as_arg }}",
+      "{{ params.lr.as_arg }}",
+      "{{ params.momentum.as_arg }}",
+      "{{ params.log_interval.as_arg }}",
+    ]

in_cluster/sklearn/iris/app.py

@@ -0,0 +1,57 @@
+import streamlit as st
+import pandas as pd
+import joblib
+import argparse
+
+from PIL import Image
+
+
+def load_model(model_path: str):
+    model = open(model_path, "rb")
+    return joblib.load(model)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--model-path',
+        type=str,
+    )
+    args = parser.parse_args()
+
+    setosa = Image.open("images/iris-setosa.png")
+    versicolor = Image.open("images/iris-versicolor.png")
+    virginica = Image.open("images/iris-virginica.png")
+    classifier = load_model(args.model_path)
+    print(classifier)
+
+    st.title("Iris flower species Classification")
+    st.sidebar.title("Features")
+    parameter_list = [
+        "Sepal length (cm)",
+        "Sepal Width (cm)",
+        "Petal length (cm)",
+        "Petal Width (cm)"
+    ]
+    sliders = []
+    for parameter, parameter_df in zip(parameter_list, ['5.2', '3.2', '4.2', '1.2']):
+        values = st.sidebar.slider(
+            label=parameter,
+            key=parameter,
+            value=float(parameter_df),
+            min_value=0.0,
+            max_value=8.0,
+            step=0.1
+        )
+        sliders.append(values)
+
+    input_variables = pd.DataFrame([sliders], columns=parameter_list)
+
+    prediction = classifier.predict(input_variables)
+    if prediction == 0:
+        st.image(setosa)
+    elif prediction == 1:
+        st.image(versicolor)
+    else:
+        st.image(virginica)
+