I0728 15:09:10.718564 1 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0728 15:09:10.718713 1 cephcsi.go:223] Starting driver type: controller with name: rbd.csi.ceph.com I0728 15:09:10.719366 1 leaderelection.go:250] attempting to acquire leader lease kube-system/rbd.csi.ceph.com-kube-system... I0728 15:09:11.496701 1 leaderelection.go:354] lock is held by ceph-csi-rbd-provisioner-69cb97fb6-jmmjn_fa242fc4-1294-4656-b8b2-520efdb49731 and has not yet expired I0728 15:09:11.496752 1 leaderelection.go:255] failed to acquire lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:16.059869 1 leaderelection.go:354] lock is held by ceph-csi-rbd-provisioner-69cb97fb6-jmmjn_fa242fc4-1294-4656-b8b2-520efdb49731 and has not yet expired I0728 15:09:16.059918 1 leaderelection.go:255] failed to acquire lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:20.293935 1 leaderelection.go:354] lock is held by ceph-csi-rbd-provisioner-69cb97fb6-jmmjn_fa242fc4-1294-4656-b8b2-520efdb49731 and has not yet expired I0728 15:09:20.293970 1 leaderelection.go:255] failed to acquire lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:24.288672 1 leaderelection.go:354] lock is held by ceph-csi-rbd-provisioner-69cb97fb6-jmmjn_fa242fc4-1294-4656-b8b2-520efdb49731 and has not yet expired I0728 15:09:24.288790 1 leaderelection.go:255] failed to acquire lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:29.316804 1 leaderelection.go:260] successfully acquired lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:29.320152 1 reflector.go:289] Starting reflector *v1.PersistentVolume (9h1m45.163688555s) from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0728 15:09:29.320173 1 reflector.go:325] Listing and watching *v1.PersistentVolume from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0728 15:09:29.322195 1 reflector.go:351] Caches populated for *v1.PersistentVolume from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0728 15:09:29.420825 1 reflector.go:289] Starting reflector *v1.Secret (9h19m46.289560032s) from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0728 15:09:29.420853 1 reflector.go:325] Listing and watching *v1.Secret from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0728 15:09:29.543654 1 reflector.go:351] Caches populated for *v1.Secret from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0728 15:09:29.643301 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-50c63c5a-fe21-46d5-9406-1f5bf339274b:d8a778c7-bb5f-43c2-b35a-b021dace8fe6] I0728 15:09:29.645680 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.d8a778c7-bb5f-43c2-b35a-b021dace8fe6"): map[csi.imageid:37c6a283bc4c csi.imagename:csi-vol-d8a778c7-bb5f-43c2-b35a-b021dace8fe6 csi.volname:pvc-50c63c5a-fe21-46d5-9406-1f5bf339274b csi.volume.owner:openstack] I0728 15:09:29.906788 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:30.744183 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-b69ca01a-5abe-465a-ade4-0f17735097b6:08c84c32-a100-49ad-918c-6d0874768121] I0728 15:09:30.750697 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.08c84c32-a100-49ad-918c-6d0874768121"): map[csi.imageid:37c6b2a90aac csi.imagename:csi-vol-08c84c32-a100-49ad-918c-6d0874768121 csi.volname:pvc-b69ca01a-5abe-465a-ade4-0f17735097b6 csi.volume.owner:monitoring] I0728 15:09:30.973044 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-92eac0a6-f0b6-4119-814b-278481d229e2:90cb3c02-bf76-4934-8dbf-acd89432ac1d] I0728 15:09:30.974336 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.90cb3c02-bf76-4934-8dbf-acd89432ac1d"): map[csi.imageid:37c658fd8d1 csi.imagename:csi-vol-90cb3c02-bf76-4934-8dbf-acd89432ac1d csi.volname:pvc-92eac0a6-f0b6-4119-814b-278481d229e2 csi.volume.owner:openstack] I0728 15:09:31.817995 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-98cdd5b1-c7eb-4544-8325-9a7e92823c9e:1219cbae-4da0-4aea-9382-a8d5f64b99a9] I0728 15:09:31.820250 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.1219cbae-4da0-4aea-9382-a8d5f64b99a9"): map[csi.imageid:37c681e68c71 csi.imagename:csi-vol-1219cbae-4da0-4aea-9382-a8d5f64b99a9 csi.volname:pvc-98cdd5b1-c7eb-4544-8325-9a7e92823c9e csi.volume.owner:openstack] I0728 15:09:31.929483 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:32.269530 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-de933e38-b02e-4cec-9758-c19dc96d1795:a121c16c-687b-426f-9a14-22e610a0b652] I0728 15:09:32.271037 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.a121c16c-687b-426f-9a14-22e610a0b652"): map[csi.imageid:37c6a20433d1 csi.imagename:csi-vol-a121c16c-687b-426f-9a14-22e610a0b652 csi.volname:pvc-de933e38-b02e-4cec-9758-c19dc96d1795 csi.volume.owner:openstack] I0728 15:09:32.514232 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-6f106af9-a044-4580-8fb8-044b9a87c7ec:cdad4af3-bc8d-4ca4-b535-ebed172bcbc5] I0728 15:09:32.516590 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.cdad4af3-bc8d-4ca4-b535-ebed172bcbc5"): map[csi.imageid:37c695a19006 csi.imagename:csi-vol-cdad4af3-bc8d-4ca4-b535-ebed172bcbc5 csi.volname:pvc-6f106af9-a044-4580-8fb8-044b9a87c7ec csi.volume.owner:openstack] I0728 15:09:32.706167 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-70f0cd11-3999-4e88-9830-c9e6f4df1191:1ca641c9-9ea6-44b1-82ae-b36ca243ebf4] I0728 15:09:32.707575 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.1ca641c9-9ea6-44b1-82ae-b36ca243ebf4"): map[csi.imageid:37c638070366 csi.imagename:csi-vol-1ca641c9-9ea6-44b1-82ae-b36ca243ebf4 csi.volname:pvc-70f0cd11-3999-4e88-9830-c9e6f4df1191 csi.volume.owner:openstack] I0728 15:09:32.834187 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-76fcaeac-28ef-4b05-a2a7-bd7952446b30:deb5257b-f4d9-465e-a272-bfcaad869a87] I0728 15:09:32.835476 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.deb5257b-f4d9-465e-a272-bfcaad869a87"): map[csi.imageid:37c61de17076 csi.imagename:csi-vol-deb5257b-f4d9-465e-a272-bfcaad869a87 csi.volname:pvc-76fcaeac-28ef-4b05-a2a7-bd7952446b30 csi.volume.owner:openstack] I0728 15:09:33.009445 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-8c678a10-138d-4a2b-8403-5cdffae5507c:e8939796-94c4-4c8b-89f8-e48cfa31f84b] I0728 15:09:33.011382 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.e8939796-94c4-4c8b-89f8-e48cfa31f84b"): map[csi.imageid:37c6ac790b15 csi.imagename:csi-vol-e8939796-94c4-4c8b-89f8-e48cfa31f84b csi.volname:pvc-8c678a10-138d-4a2b-8403-5cdffae5507c csi.volume.owner:openstack] I0728 15:09:33.434572 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-59406e5f-c347-4613-b9ee-f54e34b495ee:c9b09818-3992-4ee7-9376-0ccc17a93259] I0728 15:09:33.435861 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.c9b09818-3992-4ee7-9376-0ccc17a93259"): map[csi.imageid:37c6670283f7 csi.imagename:csi-vol-c9b09818-3992-4ee7-9376-0ccc17a93259 csi.volname:pvc-59406e5f-c347-4613-b9ee-f54e34b495ee csi.volume.owner:openstack] I0728 15:09:33.774710 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-6620e010-86f3-4616-b518-1ae104b265b4:9dae4c6a-a380-4e37-98bb-8600eeb6529f] I0728 15:09:33.776013 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.9dae4c6a-a380-4e37-98bb-8600eeb6529f"): map[csi.imageid:37c6cb498fa7 csi.imagename:csi-vol-9dae4c6a-a380-4e37-98bb-8600eeb6529f csi.volname:pvc-6620e010-86f3-4616-b518-1ae104b265b4 csi.volume.owner:openstack] I0728 15:09:33.947428 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0728 15:09:33.949967 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-d6af4bbc-a7db-49db-ae69-eb80198d289e:b6ac04d9-7318-45f9-a9ea-47c6fb3c321b] I0728 15:09:33.950916 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.b6ac04d9-7318-45f9-a9ea-47c6fb3c321b"): map[csi.imageid:37c671d67dd4 csi.imagename:csi-vol-b6ac04d9-7318-45f9-a9ea-47c6fb3c321b csi.volname:pvc-d6af4bbc-a7db-49db-ae69-eb80198d289e csi.volume.owner:monitoring] I0728 15:09:34.181803 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-f84334f5-38be-430d-818b-b3166f18684e:8aa4466a-ef9f-492b-a871-faed1ae87e16] I0728 15:09:34.182949 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.8aa4466a-ef9f-492b-a871-faed1ae87e16"): map[csi.imageid:37c66dac8735 csi.imagename:csi-vol-8aa4466a-ef9f-492b-a871-faed1ae87e16 csi.volname:pvc-f84334f5-38be-430d-818b-b3166f18684e csi.volume.owner:monitoring] I0728 15:09:34.308979 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-a8fc85e1-1364-445c-ad70-f374dc72a730:e406d730-62f1-41f5-b9a8-742b50a78e3f] I0728 15:09:34.309985 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.e406d730-62f1-41f5-b9a8-742b50a78e3f"): map[csi.imageid:37c67c55a853 csi.imagename:csi-vol-e406d730-62f1-41f5-b9a8-742b50a78e3f csi.volname:pvc-a8fc85e1-1364-445c-ad70-f374dc72a730 csi.volume.owner:openstack] E0728 15:09:43.459932 1 leaderelection.go:332] error retrieving resource lock kube-system/rbd.csi.ceph.com-kube-system: etcdserver: request timed out E0728 15:09:45.948397 1 leaderelection.go:332] error retrieving resource lock kube-system/rbd.csi.ceph.com-kube-system: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/rbd.csi.ceph.com-kube-system": context deadline exceeded I0728 15:09:45.948464 1 leaderelection.go:285] failed to renew lease kube-system/rbd.csi.ceph.com-kube-system: timed out waiting for the condition E0728 15:09:45.948544 1 controller.go:85] failed to start manager leader election lost E0728 15:09:45.948561 1 cephcsi.go:314] leader election lost [controller-runtime] log.SetLogger(...) was never called; logs will not be displayed. Detected at: > goroutine 115 [running]: > runtime/debug.Stack() > /usr/local/go/src/runtime/debug/stack.go:24 +0x5e > sigs.k8s.io/controller-runtime/pkg/log.eventuallyFulfillRoot() > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/log/log.go:60 +0xcd > sigs.k8s.io/controller-runtime/pkg/log.(*delegatingLogSink).Enabled(0xc0006395c0, 0x10?) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/log/deleg.go:111 +0x32 > github.com/go-logr/logr.Logger.Info({{0x27c4810?, 0xc0006395c0?}, 0xc0001737a8?}, {0xc00065d4a0, 0x5d}, {0xc000d01020, 0x6, 0x6}) > /go/src/github.com/ceph/ceph-csi/vendor/github.com/go-logr/logr/logr.go:276 +0x72 > sigs.k8s.io/controller-runtime/pkg/internal/recorder.(*Provider).getBroadcaster.func1.1(0xc00081b180) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/internal/recorder/recorder.go:104 +0x1e7 > k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1() > /go/src/github.com/ceph/ceph-csi/vendor/k8s.io/client-go/tools/record/event.go:401 +0x11d > created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher in goroutine 85 > /go/src/github.com/ceph/ceph-csi/vendor/k8s.io/client-go/tools/record/event.go:387 +0x109