Transport endpoint is not connected

[dyrnq]

kubectl exec -it pod/csi-s3-test-nginx -- bash -c "ls -l /usr/share/nginx/html/s3"
ls: cannot access '/usr/share/nginx/html/s3': Transport endpoint is not connected
command terminated with exit code 2

[dyrnq]

https://zhuanlan.zhihu.com/p/684136809

[dyrnq]

systemctl list-units --no-pager |grep gee

  geesefs-pvc_2d4ba8128b_2d493a_2d4976_2daf85_2dfb24ed18bbfe.service                                                                                                     loaded active running   GeeseFS mount for Kubernetes volume pvc-4ba8128b-493a-4976-af85-fb24ed18bbfe

systemctl cat geesefs-pvc_2d4ba8128b_2d493a_2d4976_2daf85_2dfb24ed18bbfe.service --no-pager
# /run/systemd/transient/geesefs-pvc_2d4ba8128b_2d493a_2d4976_2daf85_2dfb24ed18bbfe.service
# This is a transient unit file, created programmatically via the systemd API. Do not edit.
[Unit]
Description=GeeseFS mount for Kubernetes volume pvc-4ba8128b-493a-4976-af85-fb24ed18bbfe

[Service]
ExecStart=
ExecStart="/var/lib/kubelet/plugins/ru.yandex.s3.csi/geesefs" "-f" "-o" "allow_other" "--endpoint" "http://192.168.1.201:19000" "--setuid" "65534" "--setgid" "65534" "--memory-limit" "1000" "--dir-mode" "0777" "--file-mode" "0666" "pvc-4ba8128b-493a-4976-af85-fb24ed18bbfe:" "/var/lib/kubelet/plugins/kubernetes.io/csi/ru.yandex.s3.csi/50f2cc2e51b1741268ac8799992194dff79c7790166e693d6d8a5eb975efd3e7/globalmount"
Environment="AWS_ACCESS_KEY_ID=xxxxxx" "AWS_SECRET_ACCESS_KEY=xxxxxxxx"

[Unit]
CollectMode=inactive-or-failed

# /run/systemd/system/geesefs-pvc_2d4ba8128b_2d493a_2d4976_2daf85_2dfb24ed18bbfe.service.d/50-ExecStopPost.conf
[Service]
ExecStopPost=/bin/umount -f -l /var/lib/kubelet/plugins/kubernetes.io/csi/ru.yandex.s3.csi/50f2cc2e51b1741268ac8799992194dff79c7790166e693d6d8a5eb975efd3e7/globalmount

[dyrnq]

/var/lib/kubelet/plugins/ru.yandex.s3.csi/geesefs --version
geesefs version 0.40.6

[dyrnq]

    Host Port:      <none>
    State:          Running
      Started:      Fri, 05 Apr 2024 10:41:03 +0800
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Fri, 05 Apr 2024 10:40:28 +0800
      Finished:     Fri, 05 Apr 2024 10:40:47 +0800
    Ready:          True
    Restart Count:  1
    Liveness:       exec [ls /usr/share/nginx/html/s3] delay=5s timeout=1s period=5s #success=1 #failure=3
    Environment:    <none>
    Mounts:
      /usr/share/nginx/html/s3 from webroot (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-gxsrl (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  webroot:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  csi-s3-pvc
    ReadOnly:   false
  kube-api-access-gxsrl:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                From     Message
  ----     ------     ----               ----     -------
  Normal   Pulled     62s                kubelet  Successfully pulled image "nginx" in 15.350299966s
  Warning  Unhealthy  43s (x3 over 53s)  kubelet  Liveness probe failed: ls: cannot open directory '/usr/share/nginx/html/s3': Transport endpoint is not connected
  Normal   Created    28s (x2 over 62s)  kubelet  Created container csi-s3-test-nginx
  Normal   Pulled     28s                kubelet  Successfully pulled image "nginx" in 15.358141747s
  Normal   Started    27s (x2 over 62s)  kubelet  Started container csi-s3-test-nginx
  Normal   Pulling    8s (x3 over 78s)   kubelet  Pulling image "nginx"
  Normal   Killing    8s (x2 over 43s)   kubelet  Container csi-s3-test-nginx failed liveness probe, will be restarted
  Warning  Unhealthy  8s (x3 over 18s)   kubelet  Liveness probe failed: ls: cannot access '/usr/share/nginx/html/s3': Transport endpoint is not connected

[dyrnq]

  Normal   Pulled     2m23s                 kubelet  Successfully pulled image "nginx" in 15.350299966s
  Warning  Unhealthy  2m4s (x3 over 2m14s)  kubelet  Liveness probe failed: ls: cannot open directory '/usr/share/nginx/html/s3': Transport endpoint is not connected
  Normal   Created    109s (x2 over 2m23s)  kubelet  Created container csi-s3-test-nginx
  Normal   Pulled     109s                  kubelet  Successfully pulled image "nginx" in 15.358141747s
  Normal   Started    108s (x2 over 2m23s)  kubelet  Started container csi-s3-test-nginx
  Normal   Killing    89s (x2 over 2m4s)    kubelet  Container csi-s3-test-nginx failed liveness probe, will be restarted
  Warning  Unhealthy  89s (x3 over 99s)     kubelet  Liveness probe failed: ls: cannot access '/usr/share/nginx/html/s3': Transport endpoint is not connected
  Normal   Pulled     74s                   kubelet  Successfully pulled image "nginx" in 15.40625235s
  Warning  Failed     74s                   kubelet  Error: failed to generate container "3aaa09a5e11838139070b67ee1cdcf6d62a0f97725c9ac36dc5002951ef77658" spec: failed to generate spec: failed to stat "/var/lib/kubelet/pods/6c668da5-efee-4b8f-a66e-42eba4a019ab/volumes/kubernetes.io~csi/pvc-7451f1a3-dcd1-4c08-b57b-c146a8b66105/mount": stat /var/lib/kubelet/pods/6c668da5-efee-4b8f-a66e-42eba4a019ab/volumes/kubernetes.io~csi/pvc-7451f1a3-dcd1-4c08-b57b-c146a8b66105/mount: transport endpoint is not connected
  Normal   Pulled     73s                   kubelet  Successfully pulled image "nginx" in 338.14188ms
  Warning  Failed     73s                   kubelet  Error: failed to generate container "b890aca39eeb70089e9edd79e92882d6455b9d7494337ad2a801d54bf9587d44" spec: failed to generate spec: failed to stat "/var/lib/kubelet/pods/6c668da5-efee-4b8f-a66e-42eba4a019ab/volumes/kubernetes.io~csi/pvc-7451f1a3-dcd1-4c08-b57b-c146a8b66105/mount": stat /var/lib/kubelet/pods/6c668da5-efee-4b8f-a66e-42eba4a019ab/volumes/kubernetes.io~csi/pvc-7451f1a3-dcd1-4c08-b57b-c146a8b66105/mount: transport endpoint is not connected
  Normal   Pulling    62s (x5 over 2m39s)   kubelet  Pulling image "nginx"
  Normal   Pulled     46s                   kubelet  Successfully pulled image "nginx" in 15.428806082s
  Warning  Failed     46s                   kubelet  Error: failed to generate container "ee0db2f106a7387053763f306e18314ac88d84e4fb0e0345e6ee8a3d9925442a" spec: failed to generate spec: failed to stat "/var/lib/kubelet/pods/6c668da5-efee-4b8f-a66e-42eba4a019ab/volumes/kubernetes.io~csi/pvc-7451f1a3-dcd1-4c08-b57b-c146a8b66105/mount": stat /var/lib/kubelet/pods/6c668da5-efee-4b8f-a66e-42eba4a019ab/volumes/kubernetes.io~csi/pvc-7451f1a3-dcd1-4c08-b57b-c146a8b66105/mount: transport endpoint is not connected