busybox_pod.yml

apiVersion: v1 
kind: Pod
metadata:
  name: busybox
  namespace: kubeflow
spec:
  restartPolicy: Never
  containers:
    - name: volume-check
      image: busybox
      command:
        - sh
        - "-c"
        - "sleep 360"
      volumeMounts:
        - mountPath: "/scripts"
          name: "training"
    - name: install
      image: alpine/git
      command:
        - "git"
        - "clone"
        - "--depth"
        - "1"
        - "https://github.com/vpereira/ml-demo.git"
        - "/scripts/demo"
      volumeMounts:
        - mountPath: "/scripts"
          name: "training"
  volumes:
  - name: "training"
    persistentVolumeClaim:
      claimName: "tfevent-volume"  

tf_job_demo.yml

apiVersion: "kubeflow.org/v1"
kind: "TFJob"
metadata:
  name: "demo"
  namespace: kubeflow
spec:
  cleanPodPolicy: None 
  tfReplicaSpecs:
    Worker:
      replicas: 1 
      restartPolicy: Never
      template:
        spec:
         containers:
            - name: tensorflow
              image: tensorflow/tensorflow
              command:
                - "bash"
                - "/scripts/demo/run_demo.sh"
              volumeMounts:
                - mountPath: "/scripts"
                  name: "training"
         volumes:
           - name: "training"
             persistentVolumeClaim:
               claimName: "tfevent-volume"  

tfevent-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: tfevent-volume
  labels:
    type: local
    app: tfjob
spec:
  capacity:
    storage: 10Gi
  storageClassName: standard  
  accessModes:
    - ReadWriteMany
  hostPath:
    path: /tmp/data

tfevent-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: tfevent-volume
  namespace: kubeflow 
  labels:
    type: local
    app: tfjob
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi