fullnameOverride: seldon-monitoring
kube-state-metrics:
  extraArgs:
    metric-labels-allowlist: pods=[*]

helm upgrade --install prometheus kube-prometheus \
 --version 9.5.12 \
 --namespace seldon-monitoring \
 --values prometheus-values.yaml \
 --repo https://charts.bitnami.com/bitnami

WARNING: There are "resources" sections in the chart not set. Using "resourcesPreset" is not recommended for production. For production installations, please set the following values according to your workload needs:
  - alertmanager.resources
  - blackboxExporter.resources
  - operator.resources
  - prometheus.resources
  - prometheus.thanos.resources
+info https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/

kubectl rollout status -n seldon-monitoring deployment/seldon-monitoring-operator

Waiting for deployment "seldon-monitoring-operator" rollout to finish: 0 of 1 updated replicas are available...
deployment "seldon-monitoring-operator" successfully rolled out

echo "Prometheus URL: http://127.0.0.1:9090/"
kubectl port-forward --namespace seldon-monitoring svc/seldon-monitoring-prometheus 9090:9090

echo "Alertmanager URL: http://127.0.0.1:9093/"
kubectl port-forward --namespace seldon-monitoring svc/seldon-monitoring-alertmanager 9093:9093

prometheus:
 seldon:
   namespaceMetricName: namespace
   activeModelsNamespaceMetricName: exported_namespace
   serviceMetricName: service
   url: http://seldon-monitoring-prometheus.seldon-monitoring:9090/api/v1/
env:
  ALERTMANAGER_URL: http://seldon-monitoring-alertmanager.seldon-monitoring:9093/api/v1/alerts   

PODMONITOR_RESOURCE_LOCATION=https://raw.githubusercontent.com/SeldonIO/seldon-core/v2.8.5/prometheus/monitors

kubectl apply -f ${PODMONITOR_RESOURCE_LOCATION}/agent-podmonitor.yaml
kubectl apply -f ${PODMONITOR_RESOURCE_LOCATION}/envoy-servicemonitor.yaml
kubectl apply -f ${PODMONITOR_RESOURCE_LOCATION}/pipelinegateway-podmonitor.yaml
kubectl apply -f ${PODMONITOR_RESOURCE_LOCATION}/server-podmonitor.yaml

podmonitor.monitoring.coreos.com/agent created
servicemonitor.monitoring.coreos.com/envoy created
podmonitor.monitoring.coreos.com/pipelinegateway created
podmonitor.monitoring.coreos.com/server created

helm upgrade seldon-enterprise seldon-charts/seldon-deploy --namespace seldon-system  -f install-values.yaml --version 2.4.0 --install

kubectl rollout status deployment/seldon-enterprise-seldon-deploy -n seldon-system

deployment "seldon-enterprise-seldon-deploy" successfully rolled out

ISTIO_INGRESS=$(kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
ISTIO_INGRESS+=$(kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')

echo "Seldon Enterprise Platform: http://$ISTIO_INGRESS/seldon-deploy/"

TAG=2.4.0 && \
 docker create --name=tmp-sd-container seldonio/seldon-deploy-server:2.4.0 && \
 docker cp tmp-sd-container:/seldon-deploy-dist/seldon-deploy-install.tar.gz . && \
 docker rm -v tmp-sd-container

tar -xzf seldon-deploy-install.tar.gz

cp seldon-deploy-install/reference-configuration/metrics/seldon-monitor.yaml seldon-monitor.yaml
cp seldon-deploy-install/reference-configuration/metrics/drift-monitor.yaml drift-monitor.yaml
cp seldon-deploy-install/reference-configuration/metrics/deploy-monitor.yaml deploy-monitor.yaml
cp seldon-deploy-install/reference-configuration/metrics/metrics-server-monitor.yaml metrics-server-monitor.yaml
cp seldon-deploy-install/reference-configuration/metrics/deployment-usage-rules.yaml deployment-usage-rules.yaml

kubectl apply -n seldon-monitoring -f seldon-monitor.yaml
kubectl apply -n seldon-monitoring -f drift-monitor.yaml
kubectl apply -n seldon-monitoring -f deploy-monitor.yaml
kubectl apply -n seldon-monitoring -f metrics-server-monitor.yaml
kubectl apply -f deployment-usage-rules.yaml -n seldon-monitoring

podmonitor.monitoring.coreos.com/seldon-core created
podmonitor.monitoring.coreos.com/seldon-drift-detector created
podmonitor.monitoring.coreos.com/seldon-deploy created
podmonitor.monitoring.coreos.com/metrics-server created
prometheusrule.monitoring.coreos.com/seldon-deployment-usage-rules created

export POD_NAME=$(kubectl get pods --namespace seldon-system -l "app.kubernetes.io/name=seldon-deploy,app.kubernetes.io/instance=seldon-enterprise" -o jsonpath="{.items[0].metadata.name}")

kubectl port-forward $POD_NAME 8000:8000 --namespace seldon-system

kubectl create ns seldon-monitoring || echo "Namespace seldon-monitoring already exists"

kubectl delete secret -n seldon-monitoring alertmanager-seldon-monitoring-alertmanager || echo "Does not yet exist"
kubectl apply -f alertmanager.yaml -n seldon-monitoring

secret "alertmanager-seldon-monitoring-alertmanager" deleted
secret/alertmanager-seldon-monitoring-alertmanager created

echo "Alertmanager URL: http://127.0.0.1:9093/"
kubectl port-forward --namespace seldon-monitoring svc/seldon-monitoring-alertmanager 9093:9093

ISTIO_INGRESS=$(kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
ISTIO_INGRESS+=$(kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')

echo "Seldon Enterprise Platform: http://$ISTIO_INGRESS/seldon-deploy/"

 cp seldon-deploy-install/reference-configuration/metrics/user-alerts.yaml user-alerts.yaml
 cp seldon-deploy-install/reference-configuration/metrics/infra-alerts.yaml infra-alerts.yaml
 cp seldon-deploy-install/reference-configuration/metrics/drift-alerts.yaml drift-alerts.yaml

 kubectl apply -n seldon-monitoring -f infra-alerts.yaml
 kubectl apply -n seldon-monitoring -f user-alerts.yaml
 kubectl apply -n seldon-monitoring -f drift-alerts.yaml

 prometheusrule.monitoring.coreos.com/deploy-infra-alerts created
 prometheusrule.monitoring.coreos.com/deploy-user-alerts created
 prometheusrule.monitoring.coreos.com/seldon-drift-alerts created

export POD_NAME=$(kubectl get pods --namespace seldon-system -l "app.kubernetes.io/name=seldon-deploy,app.kubernetes.io/instance=seldon-enterprise" -o jsonpath="{.items[0].metadata.name}")

kubectl port-forward $POD_NAME 8000:8000 --namespace seldon-system

kubectl create -f custom-alert.yaml

webhook_configs:
  - url: "http://seldon-deploy.seldon-system:80/seldon-deploy/api/v1alpha1/webhooks/firing-alert"
    http_config:
      oauth2:
        client_id: "${OIDC_CLIENT_ID}"
        client_secret: "${OIDC_CLIENT_SECRET}"
        scopes: [openid]
        token_url: "${OIDC_HOST}/auth/realms/${OIDC_REALM}/protocol/openid-connect/token"
        # Note: only needed if using a self-signed certificate on your OIDC provider
        tls_config:
          insecure_skip_verify: true

kind: Secret
apiVersion: v1
metadata:
  name: alertmanager-seldon-monitoring-alertmanager
stringData:
  alertmanager.yaml: |
    receivers:
      - name: default-receiver
      - name: deploy-webhook
        webhook_configs:
          - url: "http://seldon-deploy.seldon-system:80/seldon-deploy/api/v1alpha1/webhooks/firing-alert"
    route:
      group_wait: 10s
      group_by: ['alertname']
      group_interval: 5m
      receiver: default-receiver
      repeat_interval: 3h
      routes:
        - receiver: deploy-webhook
          matchers:
            - severity =~ "warning|critical"
            - type =~ "user|infra"