Example Helm Deployments

Setup Seldon Core

Use the setup notebook to Setup Cluster with Ambassador Ingress and Install Seldon Core. Instructions also online.

!kubectl create namespace seldon

Error from server (AlreadyExists): namespaces "seldon" already exists

!kubectl config set-context $(kubectl config current-context) --namespace=seldon

Context "kind-kind" modified.

Serve Single Model

!helm install mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier:1.5.0-dev'

NAME: mymodel
LAST DEPLOYED: Mon Nov  2 11:18:38 2020
NAMESPACE: seldon
STATUS: deployed
REVISION: 1
TEST SUITE: None

!helm template mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier:1.5.0-dev' | pygmentize -l json

---
# Source: seldon-single-model/templates/seldondeployment.json
{
  "kind": "SeldonDeployment",
  "apiVersion": "machinelearning.seldon.io/v1",
  "metadata": {
    "name": "mymodel",
    "namespace": "seldon",
    "labels": {}
  },
  "spec": {
      "name": "mymodel",
      "protocol": "seldon",
    "annotations": {},
    "predictors": [
      {
        "name": "default",
        "graph": {
          "name": "model",
          "type": "MODEL",
        },
        "componentSpecs": [
          {
            "spec": {
              "containers": [
                {
                  "name": "model",
                  "image": "seldonio/mock_classifier:1.5.0-dev",
                  "env": [
                      {
                        "name": "LOG_LEVEL",
                        "value": "INFO"
                      },
                    ],
                  "resources": {"requests":{"memory":"1Mi"}},
                }
              ]
            },
          }
        ],
        "replicas": 1
      }
    ]
  }
}

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymodel -o jsonpath='{.items[0].metadata.name}')

deployment "mymodel-default-0-model" successfully rolled out

Get predictions

from seldon_core.seldon_client import SeldonClient

sc = SeldonClient(
    deployment_name="mymodel",
    namespace="seldon",
    gateway_endpoint="localhost:8003",
    gateway="ambassador",
)

REST Request

r = sc.predict(transport="rest")
assert r.success == True
print(r)

Success:True message:
Request:
meta {
}
data {
  tensor {
    shape: 1
    shape: 1
    values: 0.0406846384836026
  }
}

Response:
{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.05335370865277927]}}, 'meta': {}}

GRPC Request

r = sc.predict(transport="grpc")
print(r)

Success:True message:
Request:
{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.3321428950191112]}}}
Response:
{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.07014111011256721]}}}

!helm delete mymodel

release "mymodel" uninstalled

Serve REST AB Test

!helm install myabtest ../helm-charts/seldon-abtest

NAME: myabtest
LAST DEPLOYED: Mon Nov  2 11:19:50 2020
NAMESPACE: seldon
STATUS: deployed
REVISION: 1
TEST SUITE: None

!helm template ../helm-charts/seldon-abtest | pygmentize -l json

---
# Source: seldon-abtest/templates/ab_test_2pods.json
{
    "apiVersion": "machinelearning.seldon.io/v1alpha2",
    "kind": "SeldonDeployment",
    "metadata": {
	"labels": {
	    "app": "seldon"
	},
	"name": "RELEASE-NAME"
    },
    "spec": {
	"name": "RELEASE-NAME",
	"predictors": [
	    {
		"name": "default",
		"replicas": 1,
		"componentSpecs": [{
		    "spec": {
			"containers": [
			    {
                                "image": "seldonio/mock_classifier:1.5.0-dev",
				"imagePullPolicy": "IfNotPresent",
				"name": "classifier-1",
				"resources": {
				    "requests": {
					"memory": "1Mi"
				    }
				}
			    }],
			"terminationGracePeriodSeconds": 20
		    }},
	        {
		    "metadata":{
			"labels":{
			    "version":"v2"
			}
		    },    
			"spec":{
			    "containers":[
				{
                                "image": "seldonio/mock_classifier:1.5.0-dev",
				"imagePullPolicy": "IfNotPresent",
				"name": "classifier-2",
				"resources": {
				    "requests": {
					"memory": "1Mi"
				    }
				}
			    }
			],
			"terminationGracePeriodSeconds": 20
				   }
				   }],
		"graph": {
		    "name": "RELEASE-NAME",
		    "implementation":"RANDOM_ABTEST",
		    "parameters": [
			{
			    "name":"ratioA",
			    "value":"0.5",
			    "type":"FLOAT"
			}
		    ],
		    "children": [
			{
			    "name": "classifier-1",
			    "type":"MODEL",
			    "children":[]
			},
			{
			    "name": "classifier-2",
			    "type":"MODEL",
			    "children":[]
			}   
		    ]
		}
	    }
	]
    }
}

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=myabtest -o jsonpath='{.items[0].metadata.name}')
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=myabtest -o jsonpath='{.items[1].metadata.name}')

Waiting for deployment "myabtest-default-0-classifier-1" rollout to finish: 0 of 1 updated replicas are available...
deployment "myabtest-default-0-classifier-1" successfully rolled out
deployment "myabtest-default-1-classifier-2" successfully rolled out

Get predictions

from seldon_core.seldon_client import SeldonClient

sc = SeldonClient(
    deployment_name="myabtest",
    namespace="seldon",
    gateway_endpoint="localhost:8003",
    gateway="ambassador",
)

REST Request

r = sc.predict(transport="rest")
assert r.success == True
print(r)

Success:True message:
Request:
meta {
}
data {
  tensor {
    shape: 1
    shape: 1
    values: 0.8562060281778412
  }
}

Response:
{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.11299965170860979]}}, 'meta': {}}

gRPC Request

r = sc.predict(transport="grpc")
print(r)

Success:True message:
Request:
{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.45187622094165814]}}}
Response:
{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.07836365822139986]}}}

!helm delete myabtest

release "myabtest" uninstalled

Serve REST Multi-Armed Bandit

!helm install mymab ../helm-charts/seldon-mab

NAME: mymab
LAST DEPLOYED: Mon Nov  2 11:22:19 2020
NAMESPACE: seldon
STATUS: deployed
REVISION: 1
TEST SUITE: None

!helm template ../helm-charts/seldon-mab | pygmentize -l json

---
# Source: seldon-mab/templates/mab.json
{
    "apiVersion": "machinelearning.seldon.io/v1alpha2",
    "kind": "SeldonDeployment",
    "metadata": {
		"labels": {"app":"seldon"},
		"name": "RELEASE-NAME"
    },
    "spec": {
	"name": "RELEASE-NAME",
	"predictors": [
	    {
		"name": "default",
		"replicas": 1,
		"componentSpecs": [{
		    "spec": {
			"containers": [
			    {
                                "image": "seldonio/mock_classifier:1.5.0-dev",				
				"imagePullPolicy": "IfNotPresent",
				"name": "classifier-1",
				"resources": {
				    "requests": {
					"memory": "1Mi"
				    }
				}
			    }],
			"terminationGracePeriodSeconds": 20
		    }},
	        {
			"spec":{
			    "containers":[
				{
                                "image": "seldonio/mock_classifier:1.5.0-dev",								    
				"imagePullPolicy": "IfNotPresent",
				"name": "classifier-2",
				"resources": {
				    "requests": {
					"memory": "1Mi"
				    }
				}
			    }
			],
			"terminationGracePeriodSeconds": 20
			}
		},
	        {
		    "spec":{
			"containers": [{
                            "image": "seldonio/mab_epsilon_greedy:1.5.0-dev",								    			    
			    "name": "eg-router"
			}],
			"terminationGracePeriodSeconds": 20
		    }}
	        ],
		"graph": {
		    "name": "eg-router",
		    "type":"ROUTER",
		    "parameters": [
			{
			    "name": "n_branches",
			    "value": "2",
			    "type": "INT"
			},
			{
			    "name": "epsilon",
			    "value": "0.2",
			    "type": "FLOAT"
			},
			{
			    "name": "verbose",
			    "value": "1",
			    "type": "BOOL"
			}
		    ],
		    "children": [
			{
			    "name": "classifier-1",
			    "type":"MODEL",
			    "children":[]
			},
			{
			    "name": "classifier-2",
			    "type":"MODEL",
			    "children":[]
			}   
		    ]
		},
		"svcOrchSpec": {
		"resources": {"requests":{"cpu":"0.1"}},
"env": [
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
},
{
"name": "SELDON_LOG_MESSAGE_TYPE",
"value": "seldon.message.pair"
},
{
"name": "SELDON_LOG_REQUESTS",
"value": "false"
},
{
"name": "SELDON_LOG_RESPONSES",
"value": "false"
},
]
},
		"labels": {"fluentd":"true","version":"1.5.0-dev"}
	    }
	]
    }
}

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[0].metadata.name}')
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[1].metadata.name}')
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[2].metadata.name}')

Waiting for deployment "mymab-default-0-classifier-1" rollout to finish: 0 of 1 updated replicas are available...
deployment "mymab-default-0-classifier-1" successfully rolled out
deployment "mymab-default-1-classifier-2" successfully rolled out
deployment "mymab-default-2-eg-router" successfully rolled out

Get predictions

from seldon_core.seldon_client import SeldonClient

sc = SeldonClient(
    deployment_name="mymab",
    namespace="seldon",
    gateway_endpoint="localhost:8003",
    gateway="ambassador",
)

REST Request

r = sc.predict(transport="rest")
assert r.success == True
print(r)

Success:True message:
Request:
meta {
}
data {
  tensor {
    shape: 1
    shape: 1
    values: 0.1000299187972008
  }
}

Response:
{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.05643175042558145]}}, 'meta': {}}

gRPC Request

r = sc.predict(transport="grpc")
print(r)

Success:True message:
Request:
{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.23579893772394123]}}}
Response:
{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.0641117916962909]}}}

!helm delete mymab

release "mymab" uninstalled