Distributed Tracing with Jaeger

Illustrate the configuration for allowing distributed tracing using Jaeger.

Setup Seldon Core

Install Seldon Core as described in docs

Then port-forward to that ingress on localhost:8003 in a separate terminal either with:

• Ambassador:

kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080

• Istio:

kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8003:80

!kubectl create namespace seldon

Error from server (AlreadyExists): namespaces "seldon" already exists

!kubectl config set-context $(kubectl config current-context) --namespace=seldon

Context "kind-ansible" modified.

Install Jaeger

Follow the Jaeger docs to install on Kubernetes.

!kubectl create namespace observability
!kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.38.0/jaeger-operator.yaml -n observability

namespace/observability created
customresourcedefinition.apiextensions.k8s.io/jaegers.jaegertracing.io created
serviceaccount/jaeger-operator created
role.rbac.authorization.k8s.io/jaeger-operator created
rolebinding.rbac.authorization.k8s.io/jaeger-operator created
deployment.apps/jaeger-operator created

!pygmentize simplest.yaml

apiVersion: jaegertracing.io/v1
kind: Jaeger
metadata:
  name: simplest
spec:
  agent:
    strategy: DaemonSet

!kubectl apply -f simplest.yaml

jaeger.jaegertracing.io/simplest created

Port forward to Jaeger UI

kubectl port-forward $(kubectl get pods -l app.kubernetes.io/name=simplest -n seldon -o jsonpath='{.items[0].metadata.name}') 16686:16686 -n seldon

Run Example REST Deployment

!pygmentize deployment_rest.yaml

apiVersion: machinelearning.seldon.io/v1
kind: SeldonDeployment
metadata:
  name: tracing-example
  namespace: seldon
spec:
  name: tracing-example
  predictors:
  - componentSpecs:
    - spec:
        containers:
        - env:
          - name: TRACING
            value: '0'
          - name: JAEGER_AGENT_HOST
            valueFrom:
              fieldRef:
                fieldPath: status.hostIP
          - name: JAEGER_AGENT_PORT
            value: '5775'
          - name: JAEGER_SAMPLER_TYPE
            value: const
          - name: JAEGER_SAMPLER_PARAM
            value: '1'
          image: seldonio/mock_classifier:1.9.0-dev
          name: model1
        terminationGracePeriodSeconds: 1
    graph:
      children: []
      endpoint:
        type: REST
      name: model1
      type: MODEL
    name: tracing
    replicas: 1
    svcOrchSpec:
      env:
      - name: TRACING
        value: '1'
      - name: JAEGER_AGENT_HOST
        valueFrom:
          fieldRef:
            fieldPath: status.hostIP
      - name: JAEGER_AGENT_PORT
        value: '5775'
      - name: JAEGER_SAMPLER_TYPE
        value: const
      - name: JAEGER_SAMPLER_PARAM
        value: '1'

!kubectl create -f deployment_rest.yaml

seldondeployment.machinelearning.seldon.io/tracing-example created

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tracing-example -o jsonpath='{.items[0].metadata.name}')

deployment "tracing-example-tracing-0-model1" successfully rolled out

!curl -d '{"data": {"ndarray":[[1.0, 2.0, 5.0]]}}' \
   -X POST http://localhost:8003/seldon/seldon/tracing-example/api/v1.0/predictions \
   -H "Content-Type: application/json"

{"data":{"names":["proba"],"ndarray":[[0.43782349911420193]]},"meta":{"requestPath":{"model1":"seldonio/mock_classifier:1.9.0-dev"}}}

Check the Jaeger UI. You should be able to find traces like below:

rest

!kubectl delete -f deployment_rest.yaml

seldondeployment.machinelearning.seldon.io "tracing-example" deleted

Run Example GRPC Deployment

!pygmentize deployment_grpc.yaml

!kubectl create -f deployment_grpc.yaml

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tracing-example -o jsonpath='{.items[0].metadata.name}')

!cd ../../../executor/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' \
         -rpc-header seldon:tracing-example -rpc-header namespace:seldon \
         -plaintext \
         -proto ./prediction.proto  0.0.0.0:8003 seldon.protos.Seldon/Predict

Check the Jaeger UI. You should be able to find traces like below:

grpc

!kubectl delete -f deployment_grpc.yaml

!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/crds/jaegertracing.io_jaegers_crd.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/service_account.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role_binding.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/operator.yaml
!kubectl delete namespace observability