Merge pull request #54 from pierDipi/insecure-skip-verify

Allow setting `InsecureSkipVerify` for prefiller and decoder proxy

Author: lionelvillard

README.md

@@ -171,8 +171,16 @@ Usage of ./bin/llm-d-routing-sidecar:
         The path to the certificate for secure proxy. The certificate and private key files are assumed to be named tls.crt and tls.key, respectively. If not set, and secureProxy is enabled, then a self-signed certificate is used (for testing).
   -connector string
         the P/D connector being used. Either nixl, nixlv2 or lmcache (default "nixl")
+  -decoder-tls-insecure-skip-verify
+        configures the proxy to skip TLS verification for requests to decoder
   -decoder-use-tls
         whether to use TLS when sending requests to the decoder
+  -enable-ssrf-protection
+        enable SSRF protection using InferencePool allowlisting
+  -inference-pool-name string
+        the specific InferencePool name to watch (defaults to INFERENCE_POOL_NAME env var)
+  -inference-pool-namespace string
+        the Kubernetes namespace to watch for InferencePool resources (defaults to INFERENCE_POOL_NAMESPACE env var)
   -log_backtrace_at value
         when logging hits line file:N, emit a stack trace
   -log_dir string
@@ -187,6 +195,8 @@ Usage of ./bin/llm-d-routing-sidecar:
         If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)
   -port string
         the port the sidecar is listening on (default "8000")
+  -prefiller-tls-insecure-skip-verify
+        configures the proxy to skip TLS verification for requests to prefiller
   -prefiller-use-tls
         whether to use TLS when sending requests to prefillers
   -secure-proxy

cmd/llm-d-routing-sidecar/main.go

@@ -33,6 +33,8 @@ func main() {
 	connector := flag.String("connector", "nixl", "the P/D connector being used. Either nixl, nixlv2 or lmcache")
 	prefillerUseTLS := flag.Bool("prefiller-use-tls", false, "whether to use TLS when sending requests to prefillers")
 	decoderUseTLS := flag.Bool("decoder-use-tls", false, "whether to use TLS when sending requests to the decoder")
+	prefillerInsecureSkipVerify := flag.Bool("prefiller-tls-insecure-skip-verify", false, "configures the proxy to skip TLS verification for requests to prefiller")
+	decoderInsecureSkipVerify := flag.Bool("decoder-tls-insecure-skip-verify", false, "configures the proxy to skip TLS verification for requests to decoder")
 	secureProxy := flag.Bool("secure-proxy", true, "Enables secure proxy. Defaults to true.")
 	certPath := flag.String(
 		"cert-path", "", "The path to the certificate for secure proxy. The certificate and private key files "+
@@ -83,10 +85,15 @@ func main() {
 	}
 
 	config := proxy.Config{
-		Connector:       *connector,
-		PrefillerUseTLS: *prefillerUseTLS,
-		SecureProxy:     *secureProxy,
-		CertPath:        *certPath,
+		Connector:                   *connector,
+		PrefillerUseTLS:             *prefillerUseTLS,
+		SecureProxy:                 *secureProxy,
+		CertPath:                    *certPath,
+		PrefillerInsecureSkipVerify: *prefillerInsecureSkipVerify,
+		DecoderInsecureSkipVerify:   *decoderInsecureSkipVerify,
+		EnableSSRFProtection:        *enableSSRFProtection,
+		InferencePoolNamespace:      *inferencePoolNamespace,
+		InferencePoolName:           *inferencePoolName,
 	}
 
 	proxy, err := proxy.NewProxy(*port, targetURL, config)

internal/proxy/proxy.go

@@ -74,14 +74,20 @@ type Config struct {
 	// CertPath is the location of the TLS certificates
 	CertPath string
 
-	// enableSSRFProtection enables SSRF protection.
-	enableSSRFProtection bool
+	// PrefillerInsecureSkipVerify configure the proxy to skip TLS verification for requests to prefiller.
+	PrefillerInsecureSkipVerify bool
 
-	// inferencePoolNamespace InferencePool object namespace.
-	inferencePoolNamespace string
+	// DecoderInsecureSkipVerify configure the proxy to skip TLS verification for requests to decoder.
+	DecoderInsecureSkipVerify bool
 
-	// inferencePoolNamespace InferencePool object name.
-	inferencePoolName string
+	// EnableSSRFProtection enables SSRF protection.
+	EnableSSRFProtection bool
+
+	// InferencePoolNamespace InferencePool object namespace.
+	InferencePoolNamespace string
+
+	// InferencePoolName InferencePool object name.
+	InferencePoolName string
 }
 
 type protocolRunner func(http.ResponseWriter, *http.Request, string)
@@ -107,7 +113,7 @@ func NewProxy(port string, decodeURL *url.URL, config Config) (*Server, error) {
 	cache, _ := lru.New[string, http.Handler](16) // nolint:all
 
 	// Create SSRF protection validator
-	validator, err := NewAllowlistValidator(config.enableSSRFProtection, config.inferencePoolNamespace, config.inferencePoolName)
+	validator, err := NewAllowlistValidator(config.EnableSSRFProtection, config.InferencePoolNamespace, config.InferencePoolName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create SSRF protection validator: %w", err)
 	}
@@ -159,7 +165,13 @@ func (s *Server) Start(ctx context.Context) error {
 	// Configure handlers
 	mux := s.createRoutes()
 
-	server := &http.Server{Handler: mux}
+	server := &http.Server{
+		Handler: mux,
+		// No ReadTimeout/WriteTimeout for LLM inference - can take hours for large contexts
+		IdleTimeout:       300 * time.Second, // 5 minutes for keep-alive connections
+		ReadHeaderTimeout: 30 * time.Second,  // Reasonable for headers only
+		MaxHeaderBytes:    1 << 20,           // 1 MB for headers is sufficient
+	}
 
 	// Create TLS certificates
 	if s.config.SecureProxy {
@@ -175,6 +187,15 @@ func (s *Server) Start(ctx context.Context) error {
 		}
 		server.TLSConfig = &tls.Config{
 			Certificates: []tls.Certificate{cert},
+			MinVersion:   tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+				tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+				tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			},
 		}
 		logger.Info("server TLS configured")
 	}
@@ -223,6 +244,22 @@ func (s *Server) createRoutes() *http.ServeMux {
 
 	// Passthrough decoder handler
 	decoderProxy := httputil.NewSingleHostReverseProxy(s.decoderURL)
+	if s.decoderURL.Scheme == "https" {
+		decoderProxy.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: s.config.DecoderInsecureSkipVerify,
+				MinVersion:         tls.VersionTLS12,
+				CipherSuites: []uint16{
+					tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+					tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+					tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+					tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+					tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+					tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+				},
+			},
+		}
+	}
 	decoderProxy.ErrorHandler = func(res http.ResponseWriter, _ *http.Request, err error) {
 
 		// Log errors from the decoder proxy
@@ -255,8 +292,24 @@ func (s *Server) prefillerProxyHandler(hostPort string) (http.Handler, error) {
 		return nil, err
 	}
 
-	proxy = httputil.NewSingleHostReverseProxy(u)
-	s.prefillerProxies.Add(hostPort, proxy)
+	newProxy := httputil.NewSingleHostReverseProxy(u)
+	if u.Scheme == "https" {
+		newProxy.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: s.config.PrefillerInsecureSkipVerify,
+				MinVersion:         tls.VersionTLS12,
+				CipherSuites: []uint16{
+					tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+					tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+					tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+					tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+					tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+					tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+				},
+			},
+		}
+	}
+	s.prefillerProxies.Add(hostPort, newProxy)
 
-	return proxy, nil
+	return newProxy, nil
 }