Merge pull request #29 from superstreamlabs/master

release

Author: idanasulin2706

README.md

@@ -85,7 +85,8 @@ The full list is under the `./config-examples/` folder:
 - [AWS MSK (SCRAM)](config-examples/config.example.aws-msk.json) - AWS MSK with SCRAM authentication
 - [Confluent Cloud](config-examples/config.example.confluent-cloud.json) - Confluent Cloud setup
 - [Confluent Platform](config-examples/config.example.confluent-platform.json) - Confluent Platform setup
-- [Aiven Kafka](config-examples/config.example.aiven-kafka.json) - Aiven Kafka setup
+- [Aiven Kafka](config-examples/config.example.aiven-kafka-ssl.json) - Aiven Kafka with SSL authentication
+- [Aiven Kafka](config-examples/config.example.aiven-kafka-sasl.json) - Aiven Kafka with SASL SSL authentication
 - [Redpanda](config-examples/config.example.redpanda.json) - Redpanda setup
 - [OIDC Authentication](config-examples/config.example.oidc.json) - OpenID Connect authentication
 - [Azure AD OAuth](config-examples/config.example.azure-ad-oauth.json) - Azure Active Directory
@@ -243,11 +244,12 @@ The full list is under the `./config-examples/` folder:
     "vendor": "aiven",
     "useSasl": true,
     "sasl": {
-      "mechanism": "oauthbearer",
-      "clientId": "your-client-id",
-      "clientSecret": "your-client-secret",
-      "host": "https://my-oauth-server.com",
-      "path": "/oauth/token",
+      "mechanism": "scram-sha-256",
+      "username": "avnadmin",
+      "password": "*********"
+    },
+    "ssl": {
+      "ca": "./certs/ca.pem"
     }
   },
   "file": {
@@ -376,21 +378,30 @@ The tool performs comprehensive health checks on your Kafka cluster to identify
 - **Quotas Configuration**: Checks if Kafka quotas are configured and being used
 - **Payload Compression**: Checks if payload compression is enabled on user topics
 - **Infinite Retention Policy**: Checks if any topics have infinite retention policy enabled
+- **Unclean Leader Election**: Detects if unclean.leader.election.enable is true
+- **ACL Enforcement**: Verifies authorizer.class.name and allow.everyone.if.no.acl.found settings
+- **Auto Topic Creation**: Detects if auto.create.topics.enable is true
+- **Message Size Consistency**: Validates message.max.bytes < replica.fetch.max.bytes < fetch.max.bytes
+- **Default Topic Replication**: Verifies default.replication.factor >= 3
+- **Controlled Shutdown**: Validates controlled.shutdown.* settings
+- **Consumer Lag Threshold**: Flags groups exceeding lag threshold
+- **Dead Consumer Groups**: Detects groups in DEAD state
+- **Single-Partition High Throughput**: Flags 1-partition topics > 1MB/s
 
 ### Confluent Cloud Health Checks
-- **Replication Factor vs Broker Count**: Ensures topics don't have replication factor > broker count
 - **Topic Partition Distribution**: Checks for balanced partition distribution across topics
 - **Consumer Group Health**: Identifies consumer groups with no active members
 - **Internal Topics Health**: Verifies system topics are healthy
 - **Under-Replicated Partitions**: Checks if topics have fewer in-sync replicas than configured
-- **Rack Awareness**: Checks rack awareness configuration for better availability
-- **Replica Distribution**: Ensures replicas are evenly distributed across brokers
-- **Metrics Configuration**: Verifies metrics accessibility
 - **Logging Configuration**: Confirms built-in logging availability
 - **Authentication Configuration**: Detects if unauthenticated access is enabled (security risk)
 - **Quotas Configuration**: Checks if Kafka quotas are configured and being used
 - **Payload Compression**: Checks if payload compression is enabled on user topics
 - **Infinite Retention Policy**: Checks if any topics have infinite retention policy enabled
+- **ACL Enforcement**: Uses Confluent Cloud API to analyze ACLs for overly permissive rules (requires API credentials)
+- **Consumer Lag Threshold**: Flags groups exceeding lag threshold
+- **Dead Consumer Groups**: Detects groups in DEAD state
+- **Single-Partition High Throughput**: Flags 1-partition topics > 1MB/s
 
 ### Aiven Kafka Health Checks
 - **Replication Factor vs Broker Count**: Ensures topics don't have replication factor > broker count
@@ -401,12 +412,18 @@ The tool performs comprehensive health checks on your Kafka cluster to identify
 - **Min In-Sync Replicas Configuration**: Checks if topics have min.insync.replicas > replication factor
 - **Rack Awareness**: Checks rack awareness configuration for better availability
 - **Replica Distribution**: Ensures replicas are evenly distributed across brokers
-- **Metrics Configuration**: Verifies metrics accessibility
 - **Logging Configuration**: Confirms built-in logging availability
 - **Authentication Configuration**: Detects if unauthenticated access is enabled (security risk)
 - **Quotas Configuration**: Checks if Kafka quotas are configured and being used
 - **Payload Compression**: Checks if payload compression is enabled on user topics
 - **Infinite Retention Policy**: Checks if any topics have infinite retention policy enabled
+- **Auto Topic Creation**: Detects if auto.create.topics.enable is true
+- **Message Size Consistency**: Validates message.max.bytes < replica.fetch.max.bytes < fetch.max.bytes
+- **Controlled Shutdown**: Validates controlled.shutdown.* settings
+- **Consumer Lag Threshold**: Flags groups exceeding lag threshold
+- **Dead Consumer Groups**: Detects groups in DEAD state
+- **Single-Partition High Throughput**: Flags 1-partition topics > 1MB/s
+- **ACL Enforcement**: Verifies authorizer and allow_everyone_if_no_acl_found equivalents
 
 ### Generic Kafka Health Checks
 - **Replication Factor vs Broker Count**: Ensures topics don't have replication factor > broker count
@@ -417,12 +434,21 @@ The tool performs comprehensive health checks on your Kafka cluster to identify
 - **Min In-Sync Replicas Configuration**: Checks if topics have min.insync.replicas > replication factor
 - **Rack Awareness**: Checks rack awareness configuration for better availability
 - **Replica Distribution**: Ensures replicas are evenly distributed across brokers
-- **Metrics Configuration**: Verifies JMX metrics configuration
-- **Logging Configuration**: Checks log4j configuration
+- **Metrics Configuration**: Verifies JMX/metrics exposure in broker metadata
+- **Logging Configuration**: Checks logging configuration
 - **Authentication Configuration**: Detects if unauthenticated access is enabled (security risk)
 - **Quotas Configuration**: Checks if Kafka quotas are configured and being used
 - **Payload Compression**: Checks if payload compression is enabled on user topics
 - **Infinite Retention Policy**: Checks if any topics have infinite retention policy enabled
+- **Unclean Leader Election**: Detects if unclean.leader.election.enable is true
+- **ACL Enforcement**: Verifies authorizer.class.name and allow.everyone.if.no.acl.found settings
+- **Auto Topic Creation**: Detects if auto.create.topics.enable is true
+- **Message Size Consistency**: Validates message.max.bytes < replica.fetch.max.bytes < fetch.max.bytes
+- **Default Topic Replication**: Verifies default.replication.factor >= 3 (when broker count ≥ 3)
+- **Controlled Shutdown**: Validates controlled.shutdown.* settings
+- **Consumer Lag Threshold**: Flags groups exceeding lag threshold
+- **Dead Consumer Groups**: Detects groups in DEAD state
+- **Single-Partition High Throughput**: Flags 1-partition topics > 1MB/s
 
 ### Health Check Status
 - ✅ **Pass**: Configuration is healthy and optimal
@@ -542,6 +568,20 @@ The tool includes comprehensive validation that will:
 |-------|------|----------|-------------|
 | `email` | string | No | Email address for generating report files. If not provided, no file output will be generated |
 
+### Confluent Cloud ACL Analysis (Optional)
+
+For enhanced ACL analysis on Confluent Cloud, provide resource API credentials and your Cluster ID. The REST endpoint is automatically derived from your broker host.
+
+| Field | Type | Required | Where |
+|-------|------|----------|-------|
+| `confluent.resourceApiKey` | string | No | config file or CLI prompt |
+| `confluent.resourceApiSecret` | string | No | config file or CLI prompt |
+| `confluent.clusterId` | string | No | config file or CLI prompt |
+
+Notes:
+- The REST endpoint is inferred from your first broker in `kafka.brokers` (host portion, without port).
+- If credentials are not provided, the Confluent ACL analysis is skipped.
+
 ## 🚨 Troubleshooting
 
 ### Common Issues
@@ -633,6 +673,15 @@ SuperStream Kafka Analyzer performs a comprehensive set of health checks on your
 - **Quotas Configuration:** Checks if Kafka quotas are configured and being used.
 - **Payload Compression:** Checks if payload compression is enabled on user topics.
 - **Infinite Retention Policy:** Checks if any topics have infinite retention policy enabled.
+- **Unclean Leader Election:** Detects if unclean.leader.election.enable is true.
+- **ACL Enforcement:** Verifies authorizer.class.name and allow.everyone.if.no.acl.found settings.
+- **Auto Topic Creation:** Detects if auto.create.topics.enable is true.
+- **Message Size Consistency:** Validates message.max.bytes < replica.fetch.max.bytes < fetch.max.bytes.
+- **Default Topic Replication:** Verifies default.replication.factor >= 3 (when broker count ≥ 3).
+- **Controlled Shutdown:** Validates controlled.shutdown.* settings.
+- **Consumer Lag Threshold:** Flags groups exceeding lag threshold.
+- **Dead Consumer Groups:** Detects groups in DEAD state.
+- **Single-Partition High Throughput:** Flags 1-partition topics > 1MB/s.
 
 Each check provides a clear status (✅ Pass, ⚠️ Warning, ❌ Failed, ℹ️ Info) and actionable recommendations.
 

config-examples/README.md

@@ -22,6 +22,20 @@ If you get an error about missing vendor field, add the appropriate vendor value
 
 📚 **For detailed OIDC setup, see [OIDC-AUTH-GUIDE.md](OIDC-AUTH-GUIDE.md)**
 
+## 🔒 ACL Enforcement Analysis
+
+The analyzer now includes comprehensive ACL (Access Control List) enforcement checks for all major Kafka vendors:
+
+- **Apache/MSK**: Verifies `authorizer.class.name` and `allow.everyone.if.no.acl.found` settings
+- **Aiven**: Checks `kafka_enable_authorizer` and `allow_everyone_if_no_acl_found` configuration
+- **Confluent Cloud**: Uses API to analyze ACLs for overly permissive rules (requires additional API credentials)
+
+**Confluent Cloud ACL Analysis:**
+- Optional feature that requires additional API credentials
+- Analyzes ACLs for wildcard permissions and overly broad operations
+- Provides least-privilege access pattern validation
+- Set `confluentApiKey`, `confluentApiSecret`, and `confluentEnvironmentId` in your config
+
 ## Vendor-Specific Authentication Requirements
 
 ### AWS MSK (Amazon Managed Streaming for Apache Kafka)

config-examples/config.example.confluent-cloud.json

@@ -15,6 +15,11 @@
     "formats": ["json", "csv", "html", "txt"],
     "includeMetadata": true
   },
+  "confluent": {
+    "resourceApiKey": "YOUR_RESOURCE_API_KEY",
+    "resourceApiSecret": "YOUR_RESOURCE_API_SECRET",
+    "clusterId": "lkc-xxxxx"
+  },
   "email": "[email protected]",
-  "_comment": "Confluent Cloud uses official connection methodology with SASL_SSL and PLAIN mechanism. Replace pkc-xxxxx.us-central1.gcp.confluent.cloud:9092 with your actual Confluent Cloud broker URL from your cluster overview. Use your API Key as username and API Secret as password."
+  "_comment": "Confluent Cloud uses official connection methodology with SASL_SSL and PLAIN mechanism. Replace pkc-xxxxx...:9092 with your actual broker URL. Use your API Key as username and API Secret as password. For ACL analysis, provide resource API credentials and Cluster ID (optional). The REST endpoint is derived from your broker host."
 } 

dashboard/README.md

@@ -24,7 +24,7 @@ Interactive web dashboard for monitoring and analyzing Kafka cluster health and
 ### Health Checks
 
 - Comprehensive table of all health check results
-- Color-coded status indicators (✅ Passed, ❌ Failed, ⚠️ Warning)
+- Color-coded status indicators (✅ Passed, ❌ Failed)
 - Recommendations for each check
 
 ## 🛠️ Installation

dashboard/app.py

@@ -311,10 +311,8 @@ def create_health_details_table(self, kafka_data):
     def get_status_emoji(self, status):
         """Get emoji for status"""
         status_map = {
-            'PASSED': '✅ PASSED',
             'FAILED': '❌ FAILED',
-            'WARNING': '⚠️ WARNING',
-            'INFO': 'ℹ️ INFO'
+            'PASSED': '✅ PASSED'
         }
         return status_map.get(status, '? UNKNOWN')
 

dashboard/components/charts.py

@@ -64,11 +64,10 @@ def create_health_score_gauge(health_score: float, title: str = "Cluster Health
     def create_health_checks_summary(health_data: Dict[str, int]) -> go.Figure:
         """Create health checks summary bar chart"""
 
-        categories = ['Passed', 'Failed', 'Warnings']
+        categories = ['Passed', 'Failed']
         values = [
             health_data.get('passedChecks', 0),
             health_data.get('failedChecks', 0),
-            health_data.get('warnings', 0)
         ]
         colors = ['#28a745', '#dc3545', '#ffc107']
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "superstream-kafka-analyzer",
-  "version": "1.0.18",
+  "version": "1.1.0",
   "description": "Interactive utility to analyze Kafka clusters health and configuration",
   "main": "src/cli.js",
   "bin": {

report-examples/kafka-report.html

@@ -217,8 +217,7 @@ <h3>🔍 Health Check Results</h3>
           <strong>Summary:</strong> 
           Total: 16 | 
           ✅ Passed: 10 | 
-          ❌ Failed: 0 | 
-          ⚠️ Warnings: 6
+          ❌ Failed: 0
         </div>
         <div class="health-checks">
 

src/analytics.js

@@ -190,8 +190,7 @@ class SupabaseAnalytics {
       vendor: vendor || 'unknown',
       total_checks: results.totalChecks,
       passed_checks: results.passedChecks,
-      failed_checks: results.failedChecks,
-      warnings: results.warnings
+      failed_checks: results.failedChecks
     });
   }
 

src/cli.js

@@ -189,7 +189,6 @@ class CLI {
       brokerDefault = 'b-1.your-cluster.region.amazonaws.com:9092';
     } else if (vendorAnswer.vendor === 'aiven') {
       brokerMessage = 'kafka-xxxxx.aivencloud.com:12345';
-      brokerDefault = 'superstream-test-superstream-3591.k.aivencloud.com:18848';
     }
 
     const kafkaAnswers = await inquirer.prompt([
@@ -299,6 +298,59 @@ class CLI {
       console.log(chalk.gray(`   Session Timeout: 45000ms`));
       console.log(chalk.gray(`   API Key: ${confluentAnswers.username ? '***' + confluentAnswers.username.slice(-4) : 'NOT SET'}`));
       console.log(chalk.gray(`   API Secret: ${confluentAnswers.password ? '***' + confluentAnswers.password.slice(-4) : 'NOT SET'}`));
+      
+      // Ask for Confluent Cloud API credentials for ACL analysis (optional)
+      const aclAnalysisAnswer = await inquirer.prompt([
+        {
+          type: 'confirm',
+          name: 'enableAclAnalysis',
+          message: 'Enable ACL analysis (requires additional API credentials)?',
+          default: false
+        }
+      ]);
+      
+      if (aclAnalysisAnswer.enableAclAnalysis) {
+        const aclCredentials = await inquirer.prompt([
+          {
+            type: 'input',
+            name: 'resourceApiKey',
+            message: 'Confluent Cloud resource API Key (for ACL analysis):',
+            validate: (input) => {
+              if (!input.trim()) return 'API Key is required for ACL analysis';
+              return true;
+            }
+          },
+          {
+            type: 'password',
+            name: 'resourceApiSecret',
+            message: 'Confluent Cloud resource API Secret (for ACL analysis):',
+            validate: (input) => {
+              if (!input.trim()) return 'API Secret is required for ACL analysis';
+              return true;
+            }
+          },
+          {
+            type: 'input',
+            name: 'confluentClusterId',
+            message: 'Confluent Cloud Cluster ID:',
+            validate: (input) => {
+              if (!input.trim()) return 'Cluster ID is required for ACL analysis';
+              return true;
+            }
+          }
+        ]);
+        
+        // Store ACL analysis credentials in config
+        this.config.confluent = {
+          resourceApiKey: aclCredentials.resourceApiKey,
+          resourceApiSecret: aclCredentials.resourceApiSecret,
+          clusterId: aclCredentials.confluentClusterId
+        }
+
+        console.log(chalk.gray('🔍 ACL Analysis configuration:'));
+        console.log(chalk.gray(`   API Key: ${aclCredentials.resourceApiKey ? '***' + aclCredentials.resourceApiKey.slice(-4) : 'NOT SET'}`));
+        console.log(chalk.gray(`   Cluster ID: ${aclCredentials.confluentClusterId}`));
+      }
     } else if (vendorAnswer.vendor === 'aiven') {
       console.log(chalk.yellow('\n🔐 Aiven Kafka Authentication'));
       const aivenAnswers = await inquirer.prompt([
@@ -683,7 +735,14 @@ class CLI {
         spinner.stop();
 
         // Display summary
-        console.log(chalk.green('\n✅ Analysis completed successfully!'));
+        console.log(chalk.green('\n✅ Analysis completed successfully! \n'));
+
+        if (healthResults.failedChecks === 0) {
+          console.log(chalk.green('🎉 All health checks passed! Your Kafka cluster is healthy. \n'));
+        } else {
+          console.log(chalk.red('🚨 Issues detected. Please review and fix the problems above. \n'));
+        }
+
         console.log(chalk.blue('\n📊 Analysis Summary:'));
         console.log(chalk.gray(`• Total Topics: ${analysisResults.summary.totalTopics}`));
         console.log(chalk.gray(`• Total Partitions: ${analysisResults.summary.totalPartitions}`));
@@ -740,7 +799,7 @@ class CLI {
         vendor: this.config.kafka.vendor,
         topics_count: analysisResults.summary.totalTopics,
         health_checks_count: healthResults ? healthResults.totalChecks : 0,
-        has_issues: healthResults ? (healthResults.failedChecks > 0 || healthResults.warnings > 0) : false
+        has_issues: healthResults ? (healthResults.failedChecks > 0) : false
       }, true); // Include detailed location
 
       // Track health checks with location if available
@@ -750,7 +809,6 @@ class CLI {
           total_checks: healthResults.totalChecks,
           passed_checks: healthResults.passedChecks,
           failed_checks: healthResults.failedChecks,
-          warnings: healthResults.warnings
         }, true); // Include detailed location
       }
 
@@ -778,7 +836,7 @@ class CLI {
         return;
       }
 
-      const healthChecker = new HealthChecker(vendor, this.config.kafka);
+      const healthChecker = new HealthChecker(vendor, this.config.kafka, this.config.confluent);
       const healthResults = await healthChecker.runHealthChecks(clusterInfo, topics, consumerGroups);
 
       // Add health check results to the analysis results for file output