From cf2023c4cd8e7fb81557782f6e0a721c00adf653 Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Tue, 24 Sep 2024 15:21:23 +0300 Subject: [PATCH 1/7] add related.entity field to activitylogs default ingest pipeline --- .../test-activitylogs-edgecases.log-expected.json | 13 +++++++++++++ .../test-activitylogs-identity.log-expected.json | 3 +++ .../test-activitylogs-raw.log-expected.json | 4 ++++ .../elasticsearch/ingest_pipeline/default.yml | 14 ++++++++++++++ .../data_stream/activitylogs/fields/fields.yml | 7 +++++++ packages/azure/docs/activitylogs.md | 1 + 6 files changed, 42 insertions(+) diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json index 278caf0e314..3ce52e2932f 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json @@ -35,6 +35,11 @@ "log": { "level": "Information" }, + "related": { + "entity": [ + "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration" + ] + }, "tags": [ "preserve_original_event" ] @@ -281,6 +286,9 @@ "related": { "ip": [ "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + ], + "entity":[ + "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" ] }, "source": { @@ -529,6 +537,11 @@ "geo": { "name": "GB" }, + "related":{ + "entity":[ + "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" + ] + }, "source": { "address": "127.0.0.0/8" }, diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json index 12b1eae2aff..c3bdd0ecdf4 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json @@ -245,6 +245,9 @@ "related": { "ip": [ "81.2.69.143" + ], + "entity":[ + "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" ] }, "source": { diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index f36213fa546..116e6a411db 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -86,6 +86,10 @@ "related": { "ip": [ "81.2.69.144" + ], + "entity": [ + "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY" ] }, "source": { diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index 41870bb47d5..51953144ea0 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -313,6 +313,20 @@ processors: - set: field: event.kind value: event + - append: + description: Append principal id to `related.entity` + field: related.entity + value: "{{{ azure.activitylogs.identity.authorization.evidence.principal_id }}}" + if: "ctx?.azure?.activitylogs?.identity?.authorization?.evidence?.principal_id != null" + allow_duplicates: false + ignore_failure: true + - append: + description: Append resource id to `related.entity` + field: related.entity + value: "{{{ azure.resource_id }}}" + if: "ctx.azure.resource_id != null" + allow_duplicates: false + ignore_failure: true - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' on_failure: diff --git a/packages/azure/data_stream/activitylogs/fields/fields.yml b/packages/azure/data_stream/activitylogs/fields/fields.yml index 916ebcffc55..c102b324690 100644 --- a/packages/azure/data_stream/activitylogs/fields/fields.yml +++ b/packages/azure/data_stream/activitylogs/fields/fields.yml @@ -147,3 +147,10 @@ Not typically used in automated geolocation.' level: extended +- name: related.entity + description: | + All the entity identifiers related to the document. If the document + contains multiple entities, identifiers belonging to different entities + will be present. Example identifiers include cloud resource IDs, ARNs, + email addresses, or hostnames. + type: keyword \ No newline at end of file diff --git a/packages/azure/docs/activitylogs.md b/packages/azure/docs/activitylogs.md index 384be09ae70..6966fd1e0f8 100644 --- a/packages/azure/docs/activitylogs.md +++ b/packages/azure/docs/activitylogs.md @@ -213,4 +213,5 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.containerized | If the host is a container. | boolean | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | +| related.entity | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. | keyword | From a34bb80505168ee5e1746fef10e3d3526cd4b9f8 Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Thu, 26 Sep 2024 18:11:54 +0300 Subject: [PATCH 2/7] use painless --- ...t-activitylogs-edgecases.log-expected.json | 10 +++---- ...st-activitylogs-identity.log-expected.json | 6 ++-- .../test-activitylogs-raw.log-expected.json | 8 ++--- .../elasticsearch/ingest_pipeline/default.yml | 29 ++++++++++--------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json index 3ce52e2932f..dad023096a9 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json @@ -284,11 +284,11 @@ } }, "related": { + "entity": [ + "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" + ], "ip": [ "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" - ], - "entity":[ - "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" ] }, "source": { @@ -537,8 +537,8 @@ "geo": { "name": "GB" }, - "related":{ - "entity":[ + "related": { + "entity": [ "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" ] }, diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json index c3bdd0ecdf4..45503dbea49 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json @@ -243,11 +243,11 @@ "region_name": "England" }, "related": { + "entity": [ + "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" + ], "ip": [ "81.2.69.143" - ], - "entity":[ - "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam" ] }, "source": { diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index 116e6a411db..685772b74f4 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -84,12 +84,12 @@ "level": "Information" }, "related": { + "entity": [ + "/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "8a4de8b5-095c-47d0-a96f-a75130c61d53" + ], "ip": [ "81.2.69.144" - ], - "entity": [ - "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY" ] }, "source": { diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index 51953144ea0..273d520ff02 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -313,20 +313,23 @@ processors: - set: field: event.kind value: event - - append: - description: Append principal id to `related.entity` - field: related.entity - value: "{{{ azure.activitylogs.identity.authorization.evidence.principal_id }}}" - if: "ctx?.azure?.activitylogs?.identity?.authorization?.evidence?.principal_id != null" - allow_duplicates: false - ignore_failure: true - - append: - description: Append resource id to `related.entity` - field: related.entity - value: "{{{ azure.resource_id }}}" - if: "ctx.azure.resource_id != null" - allow_duplicates: false + - script: + description: Appends principal_id and resource_id to `related.entity` + lang: painless ignore_failure: true + on_failure: + - set: + description: Add error reason + field: error.message + value: "{{{ _ingest.on_failure_message }}}" + source: | + ctx.related = ctx.related ?: [:]; + ctx.related.entity = new HashSet(); + ctx.related.entity.add(field("azure.activitylogs.identity.authorization.evidence.principal_id").get(null)); + ctx.related.entity.add(field("azure.resource_id").get(null)); + ctx.related.entity.remove(""); + ctx.related.entity.remove(null); + - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' on_failure: From 604015a570e71aab3a20d65e9019769ae30b360d Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Tue, 8 Oct 2024 13:07:07 +0300 Subject: [PATCH 3/7] add new line --- packages/azure/data_stream/activitylogs/fields/fields.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/data_stream/activitylogs/fields/fields.yml b/packages/azure/data_stream/activitylogs/fields/fields.yml index c102b324690..136f955dd3c 100644 --- a/packages/azure/data_stream/activitylogs/fields/fields.yml +++ b/packages/azure/data_stream/activitylogs/fields/fields.yml @@ -153,4 +153,4 @@ contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. - type: keyword \ No newline at end of file + type: keyword From ce127a60528e72610663bb4f3a4ff4ebd409b7c3 Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Tue, 8 Oct 2024 15:05:17 +0300 Subject: [PATCH 4/7] use append --- .../elasticsearch/ingest_pipeline/default.yml | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index 273d520ff02..85b1c26dc8b 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -313,23 +313,18 @@ processors: - set: field: event.kind value: event - - script: - description: Appends principal_id and resource_id to `related.entity` - lang: painless - ignore_failure: true - on_failure: - - set: - description: Add error reason - field: error.message - value: "{{{ _ingest.on_failure_message }}}" - source: | - ctx.related = ctx.related ?: [:]; - ctx.related.entity = new HashSet(); - ctx.related.entity.add(field("azure.activitylogs.identity.authorization.evidence.principal_id").get(null)); - ctx.related.entity.add(field("azure.resource_id").get(null)); - ctx.related.entity.remove(""); - ctx.related.entity.remove(null); - + - append: + field: related.entity + value: '{{{ azure.resource_id }}}' + allow_duplicates: false + if: ctx.azure?.resource_id != null && ctx.azure.resource_id != '' + - append: + field: related.entity + value: '{{{ azure.activitylogs.identity.authorization.evidence.principal_id }}}' + allow_duplicates: false + if: > + ctx.azure?.activitylogs?.identity?.authorization?.evidence?.principal_id != null && + ctx.azure.activitylogs.identity.authorization.evidence.principal_id != '' - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' on_failure: @@ -337,3 +332,4 @@ on_failure: field: error.message value: |- Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + From 236c7fc95c4fe21724ac4a97f75ac72676a7d892 Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Wed, 30 Oct 2024 12:03:50 +0200 Subject: [PATCH 5/7] fix formatting in docs --- packages/azure/data_stream/activitylogs/fields/fields.yml | 4 ++-- packages/azure/docs/activitylogs.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/azure/data_stream/activitylogs/fields/fields.yml b/packages/azure/data_stream/activitylogs/fields/fields.yml index 136f955dd3c..ce8a3b760d0 100644 --- a/packages/azure/data_stream/activitylogs/fields/fields.yml +++ b/packages/azure/data_stream/activitylogs/fields/fields.yml @@ -149,8 +149,8 @@ level: extended - name: related.entity description: | - All the entity identifiers related to the document. If the document - contains multiple entities, identifiers belonging to different entities + All the entity identifiers related to the document. If the document + contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. type: keyword diff --git a/packages/azure/docs/activitylogs.md b/packages/azure/docs/activitylogs.md index 6966fd1e0f8..697bd41e9c3 100644 --- a/packages/azure/docs/activitylogs.md +++ b/packages/azure/docs/activitylogs.md @@ -213,5 +213,5 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.containerized | If the host is a container. | boolean | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| related.entity | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. | keyword | +| related.entity | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. | keyword | From 1f35fa8d808dccdfcbd7054f82e48f4c09a6ac87 Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Wed, 6 Nov 2024 12:41:52 +0200 Subject: [PATCH 6/7] rebase and bump version --- packages/azure/changelog.yml | 5 +++++ packages/azure/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 9e70031fa53..83635b007bf 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.19.0" + changes: + - description: Add entity identifiers to `related.entity` in activitylogs. + type: enhancement + link: https://github.com/elastic/integrations/pull/11233 - version: "1.18.0" changes: - description: Add entity identifiers to `related.entity`. diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 26a3bc35108..8180585c881 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: 1.18.0 +version: 1.19.0 description: This Elastic integration collects logs from Azure type: integration icons: From 827bd1354cffa1b1fd83a4c1561cda16910095d8 Mon Sep 17 00:00:00 2001 From: Or Ouziel Date: Wed, 6 Nov 2024 18:09:14 +0200 Subject: [PATCH 7/7] fix empty space --- packages/azure/data_stream/activitylogs/fields/fields.yml | 2 +- packages/azure/docs/activitylogs.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/azure/data_stream/activitylogs/fields/fields.yml b/packages/azure/data_stream/activitylogs/fields/fields.yml index ce8a3b760d0..ae1c9b1b521 100644 --- a/packages/azure/data_stream/activitylogs/fields/fields.yml +++ b/packages/azure/data_stream/activitylogs/fields/fields.yml @@ -151,6 +151,6 @@ description: | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities - will be present. Example identifiers include cloud resource IDs, ARNs, + will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. type: keyword diff --git a/packages/azure/docs/activitylogs.md b/packages/azure/docs/activitylogs.md index 697bd41e9c3..f2035ec5541 100644 --- a/packages/azure/docs/activitylogs.md +++ b/packages/azure/docs/activitylogs.md @@ -213,5 +213,5 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.containerized | If the host is a container. | boolean | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| related.entity | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. | keyword | +| related.entity | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. | keyword |