Merge pull request #19 from superstreamlabs/master

Push to latest

Author: nikola-nedic-sa

.gitignore

@@ -203,6 +203,7 @@ config.production.json
 config.apache-kafka-no-auth.json
 config.aws-msk-iam.json
 test-config-confluent.json
+certs/*
 *.key
 *.pem
 *.crt

Jenkinsfile

@@ -23,6 +23,7 @@ pipeline {
 
         stage('Install dependencies') {
             steps {
+                sh 'sudo dnf install -y nodejs || true'
                 sh 'npm install'
                 sh 'npm pack'
             }

README.md

@@ -218,14 +218,11 @@ The full list is under the `./config-examples/` folder:
     "brokers": ["kafka-xxxxx-aiven-kafka.aivencloud.com:12345"],
     "clientId": "superstream-analyzer",
     "vendor": "aiven",
-    "useSasl": true,
-    "sasl": {
-      "mechanism": "SCRAM-SHA-256",
-      "username": "avnadmin",
-      "password": "YOUR_AVNADMIN_PASSWORD"
-    },
+    "useSasl": false,
     "ssl": {
-      "ca": "./path/to/ca.pem"
+      "ca": "path/to/ca.pem",
+      "cert": "path/to/service.cert",
+      "key": "path/to/service.key"
     }
   },
   "file": {
@@ -549,6 +546,27 @@ The tool includes comprehensive validation that will:
 
 ### Common Issues
 
+**Missing Vendor Field Error**
+- **Error**: "Missing 'vendor' field in kafka configuration"
+- **Solution**: Add the appropriate vendor field to your configuration:
+  - AWS MSK IAM: `"vendor": "aws-msk"`
+  - Confluent Cloud: `"vendor": "confluent-cloud"`
+  - Aiven: `"vendor": "aiven"`
+  - Apache Kafka: `"vendor": "apache"`
+  - Redpanda: `"vendor": "redpanda"`
+- **Why**: The vendor field tells the tool how to handle vendor-specific authentication mechanisms
+
+**AWS MSK IAM Authentication Failed**
+- **Error**: "Failed to generate auth token" or "authenticationProvider is not a function"
+- **Solution**: 
+  1. Ensure AWS credentials are properly configured:
+     - Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables, OR
+     - Include credentials in config file: `"accessKeyId"` and `"secretAccessKey"`
+  2. Verify the IAM user has proper MSK permissions
+  3. Check that the broker URLs are correct (should use port 9198 for IAM)
+  4. Ensure the region matches your MSK cluster
+- **Why**: AWS MSK IAM requires valid AWS credentials and proper IAM permissions
+
 **Connection Timeout**
 - Verify broker URLs are correct
 - Check network connectivity

bin/index.js

@@ -13,6 +13,7 @@ program
   .description('Interactive utility to analyze Kafka clusters health and configuration')
   .version('1.0.0')
   .option('-c, --config <path>', 'Path to configuration file')
+  .option('--nats-config <path>', 'Path to NATS configuration file (uploads to NATS instead of generating files)')
   .option('-b, --bootstrap-servers <servers>', 'Comma-separated list of Kafka bootstrap servers')
   .option('-v, --verbose', 'Enable verbose logging')
   .option('-t, --timeout <seconds>', 'Connection timeout in seconds', '30')

config-examples/README.md

@@ -2,6 +2,16 @@
 
 This directory contains configuration examples for different Kafka vendors. Each vendor has specific authentication requirements and SSL settings.
 
+## ⚠️ Important: Vendor Field Requirement
+
+**The `vendor` field is required for authentication mechanisms that are vendor-specific:**
+
+- **AWS MSK IAM**: Must use `"vendor": "aws-msk"`
+- **OAuth/OIDC**: Must specify the vendor (e.g., `"aws-msk"`, `"confluent-cloud"`, `"oidc"`)
+- **Other mechanisms**: Vendor field helps optimize connection settings
+
+If you get an error about missing vendor field, add the appropriate vendor value to your configuration.
+
 ## 🔐 Authentication Methods Supported
 
 - **PLAINTEXT** - No authentication (development only)
@@ -36,6 +46,34 @@ This directory contains configuration examples for different Kafka vendors. Each
 }
 ```
 
+**AWS MSK IAM Authentication (Port 9198):**
+```json
+{
+  "kafka": {
+    "brokers": ["your-msk-cluster.amazonaws.com:9198"],
+    "clientId": "superstream-analyzer",
+    "vendor": "aws-msk",
+    "useSasl": true,
+    "sasl": {
+      "mechanism": "AWS_MSK_IAM",
+      "accessKeyId": "AKIA...",
+      "secretAccessKey": "...",
+      "authorizationIdentity": "arn:aws:iam::123456789012:user/your-iam-user"
+    }
+  },
+  "file": {
+    "outputDir": "./kafka-analysis",
+    "formats": ["json", "csv", "html"],
+    "includeMetadata": true
+  }
+}
+```
+
+**Note**: For AWS MSK IAM authentication, you can either:
+1. Provide AWS credentials in the config file (as shown above)
+2. Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables
+3. Use AWS IAM roles (if running on EC2 or ECS)
+
 **SCRAM Authentication (Port 9096):**
 ```json
 {

config-examples/config.example.aiven-kafka.json

@@ -3,14 +3,11 @@
     "brokers": ["kafka-xxxxx-aiven-kafka.aivencloud.com:12345"],
     "clientId": "superstream-analyzer",
     "vendor": "aiven",
-    "useSasl": true,
-    "sasl": {
-      "mechanism": "SCRAM-SHA-256",
-      "username": "avnadmin",
-      "password": "YOUR_AVNADMIN_PASSWORD"
-    },
+    "useSasl": false,
     "ssl": {
-      "ca": "/PATH/TO/YOUR/ca.pem"
+      "ca": "path/to/ca.pem",
+      "cert": "path/to/service.cert",
+      "key": "path/to/service.key"
     }
   },
   "file": {
@@ -19,4 +16,4 @@
     "includeMetadata": true
   },
   "email": "[email protected]"
-} 
+}

config-examples/config.example.aws-msk-iam.json

@@ -2,12 +2,13 @@
   "kafka": {
     "brokers": ["b-1.your-cluster.abc123.c2.kafka.us-east-1.amazonaws.com:9098", "b-2.your-cluster.abc123.c2.kafka.us-east-1.amazonaws.com:9098"],
     "clientId": "superstream-analyzer",
+    "vendor": "aws-msk",
     "useSasl": true,
     "sasl": {
       "mechanism": "AWS_MSK_IAM",
-      "authorizationIdentity": "arn:aws:iam::123456789012:user/your-iam-user",
-      "accessKeyId": "AKIAIOSFODNN7EXAMPLE",
-      "secretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+      "authorizationIdentity": "arn:aws:iam::123123123:user/your-iam-user",
+      "accessKeyId": "aaaaaa",
+      "secretAccessKey": "aaaa/K7MDENG/aaaaaaaa"
     }
   },
   "file": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "superstream-kafka-analyzer",
-  "version": "1.0.14",
+  "version": "1.0.15",
   "description": "Interactive utility to analyze Kafka clusters health and configuration",
   "main": "src/cli.js",
   "bin": {
@@ -38,6 +38,7 @@
     "jsonwebtoken": "^9.0.2",
     "jwks-rsa": "^3.1.0",
     "kafkajs": "^2.2.4",
+    "nats": "^2.19.0",
     "ora": "^5.4.1",
     "simple-oauth2": "^5.1.0",
     "superstream-kafka-analyzer": "^1.0.13"