Skip to content

Commit 1e53729

Browse files
DmitryRomanovDmitryRomanov
authored
clean meta about deleted k8s pods (#778)
* try to upgrade to 20241010143419 * Revert "try to upgrade to 20241010143419" This reverts commit c0d8118. * k8s.io/client-go v0.29.14 * meta about deleted pods expires too * metric for metric on len k8s meta * Revert "metric for metric on len k8s meta" This reverts commit 2cd40c2. * deletedPodsCounter
1 parent be67ac7 commit 1e53729

File tree

4 files changed

+24
-23
lines changed

4 files changed

+24
-23
lines changed

‎go.mod‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ require (
5151
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
5252
gopkg.in/yaml.v2 v2.4.0
5353
gopkg.in/yaml.v3 v3.0.1
54-
k8s.io/api v0.29.0
55-
k8s.io/apimachinery v0.29.0
56-
k8s.io/client-go v0.29.0
54+
k8s.io/api v0.29.14
55+
k8s.io/apimachinery v0.29.14
56+
k8s.io/client-go v0.29.14
5757
sigs.k8s.io/yaml v1.3.0
5858
)
5959

@@ -81,7 +81,7 @@ require (
8181
github.com/go-openapi/swag v0.22.3 // indirect
8282
github.com/godbus/dbus/v5 v5.0.4 // indirect
8383
github.com/gogo/protobuf v1.3.2 // indirect
84-
github.com/golang/protobuf v1.5.3 // indirect
84+
github.com/golang/protobuf v1.5.4 // indirect
8585
github.com/google/gnostic-models v0.6.8 // indirect
8686
github.com/google/go-cmp v0.6.0 // indirect
8787
github.com/google/gofuzz v1.2.0 // indirect

‎go.sum‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,13 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
117117
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
118118
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
119119
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
120-
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
121-
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
122-
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
120+
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
121+
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
123122
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
124123
github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
125124
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
126125
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
127126
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
128-
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
129127
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
130128
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
131129
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -565,8 +563,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
565563
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
566564
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
567565
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
568-
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
569-
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
570566
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 h1:V7Da7qt0MkY3noVANIMVBk28nOnijADeOR3i5Hcvpj4=
571567
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
572568
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -592,12 +588,12 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
592588
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
593589
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
594590
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
595-
k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
596-
k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
597-
k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
598-
k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
599-
k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
600-
k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
591+
k8s.io/api v0.29.14 h1:JWFh5ufowH3Y6tCgEzY3URVJHb27f0tEDEej0nCjWDw=
592+
k8s.io/api v0.29.14/go.mod h1:IV8YqKxMm8JGLBLlHM13Npn5lCITH10XYipWEW+YEOQ=
593+
k8s.io/apimachinery v0.29.14 h1:IDhwnGNCp836SLOwW1SoEfFNV77wxIklhxeAHX9vmSo=
594+
k8s.io/apimachinery v0.29.14/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y=
595+
k8s.io/client-go v0.29.14 h1:OSnzZ9DClaFRgl3zMAY2kGZhNjdGJkEb+RDz+MW2h6k=
596+
k8s.io/client-go v0.29.14/go.mod h1:XtZt5n5UxKfPJ+sCoTPcEavWgZbLFFxMnAFFRQGK1RY=
601597
k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
602598
k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
603599
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=

‎plugin/input/k8s/meta/gatherer.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ var (
7070
DisableMetaUpdates = false
7171
metaAddedCounter atomic.Int64
7272
expiredItemsCounter atomic.Int64
73+
deletedPodsCounter atomic.Int64
7374

7475
CriType = "docker"
7576
NodeLabels = make(map[string]string)
@@ -163,6 +164,7 @@ func initInformer() {
163164
pod := obj.(*corev1.Pod)
164165
PutMeta(pod)
165166
deletedPodsCache.Add(PodName(pod.Name), true)
167+
deletedPodsCounter.Inc()
166168
},
167169
}, cache.Indexers{})
168170
controller = c
@@ -189,11 +191,19 @@ func removeExpired() {
189191
cleanUpItems(expiredItems)
190192

191193
if MaintenanceInterval > time.Second {
192-
localLogger.Infof("k8s meta stat for last %d seconds: total=%d, updated=%d, expired=%d", MaintenanceInterval/time.Second, getTotalItems(), metaAddedCounter.Load(), expiredItemsCounter.Load())
194+
localLogger.Infof(
195+
"k8s meta stat for last %d seconds: total=%d, updated=%d, expired=%d, deleted=%d",
196+
MaintenanceInterval/time.Second,
197+
getTotalItems(),
198+
metaAddedCounter.Load(),
199+
expiredItemsCounter.Load(),
200+
deletedPodsCounter.Load(),
201+
)
193202
}
194203

195204
metaAddedCounter.Swap(0)
196205
expiredItemsCounter.Swap(0)
206+
deletedPodsCounter.Swap(0)
197207
}
198208

199209
func maintenance() {
@@ -234,11 +244,6 @@ func getExpiredItems(out []*MetaItem) []*MetaItem {
234244
// find pods which aren't in k8s pod list for some time and add them to the expiration list
235245
for ns, podNames := range MetaData {
236246
for pod, containerIDs := range podNames {
237-
isDeleted := deletedPodsCache.Contains(pod)
238-
if isDeleted {
239-
// information about deleted pods will never change again
240-
continue
241-
}
242247
for cid, podData := range containerIDs {
243248
if now.Sub(podData.updateTime) > MetaExpireDuration {
244249
out = append(out, &MetaItem{

‎plugin/input/k8s/pipeline_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ func TestPipeline(t *testing.T) {
4141
k8sContainerID = strings.Clone(e.Root.Dig("k8s_container_id").AsString())
4242
wg.Done()
4343
})
44+
p.Start()
4445

4546
item := &meta.MetaItem{
4647
Namespace: "sre",
@@ -51,7 +52,6 @@ func TestPipeline(t *testing.T) {
5152
meta.PutMeta(getPodInfo(item, true))
5253
filename := getLogFilename(dir, item)
5354

54-
p.Start()
5555
file, err := os.Create(filename)
5656
if err != nil {
5757
logger.Fatalf("Error creating file: %s", err.Error())

0 commit comments

Comments
 (0)