restpre some magic

Author: richard-rogers

langkit/count_regexes.py

@@ -14,8 +14,12 @@
 pattern_loader = PatternLoader()
 response_pattern_loader = PatternLoader()
 
+_initialized = False
+
 
 def count_patterns(group, text: str) -> int:
+    if not _initialized:
+        init()
     count = 0
     for expression in group["expressions"]:
         if expression.search(text):
@@ -68,6 +72,8 @@ def init(
     config: Optional[LangKitConfig] = None,
     response_pattern_file_path: Optional[str] = None,
 ):
+    global _initialized
+    _initialized = True
     language = language or ""
     config = deepcopy(config or lang_config)
     if pattern_file_path:

langkit/injections.py

@@ -15,8 +15,11 @@
 _index_embeddings = None
 _transformer_model = None
 
+_initialized = False
 
 def injection(prompt: Union[Dict[str, List], pd.DataFrame]) -> Union[List, pd.Series]:
+    if not _initialized:
+        init()
     global _transformer_model
     global _index_embeddings
     if _transformer_model is None:
@@ -45,6 +48,8 @@ def init(
     version: Optional[str] = None,
     config: Optional[LangKitConfig] = None,
 ):
+    global _initialized
+    _initialized = True
     global _registered
     unregister_udfs(_registered)
     config = config or deepcopy(lang_config)

langkit/input_output.py

@@ -14,10 +14,14 @@
 
 _transformer_model = None
 
+_initialized = False
+
 diagnostic_logger = getLogger(__name__)
 
 
 def prompt_response_similarity(text):
+    if not _initialized:
+        init()
     global _transformer_model
 
     if _transformer_model is None:
@@ -49,6 +53,8 @@ def init(
     custom_encoder: Optional[Callable] = None,
     config: Optional[LangKitConfig] = None,
 ):
+    global _initialized
+    _initialized = True
     global _registered
     unregister_udfs(_registered)
     if transformer_name and custom_encoder:

langkit/nlp_scores.py

@@ -14,11 +14,14 @@
 
 diagnostic_logger = getLogger(__name__)
 
+_initialized = False
 
 _registered: Set[str] = set()
 
 
 def _register_score_udfs():
+    if not _initialized:
+        init()
     global _registered
     unregister_udfs(_registered)
     if _corpus:
@@ -92,6 +95,8 @@ def init(
     rouge_type: str = "",
     config: Optional[LangKitConfig] = None,
 ):
+    global _initialized
+    _initialized = True
     config = config or deepcopy(lang_config)
     global _corpus
     global _scores

langkit/regexes.py

@@ -14,8 +14,12 @@
 pattern_loader = PatternLoader()
 response_pattern_loader = PatternLoader()
 
+_initialized = False
+
 
 def has_patterns(text, regex_groups):
+    if not _initialized:
+        init()
     if regex_groups:
         for group in regex_groups:
             for expression in group["expressions"]:
@@ -35,6 +39,8 @@ def wrappee(text):
 
 
 def _register_udfs(config: Optional[LangKitConfig] = None):
+    global _initialized
+    _initialized = True
     global _registered
     unregister_udfs(_registered)
     if config is None:
@@ -70,6 +76,8 @@ def init(
     config: Optional[LangKitConfig] = None,
     response_pattern_file_path: Optional[str] = None,
 ):
+    global _initialized
+    _initialized = True
     config = deepcopy(config or lang_config)
     if pattern_file_path:
         config.pattern_file_path = pattern_file_path

langkit/sentiment.py

@@ -6,7 +6,24 @@
 from langkit.whylogs.unreg import unregister_udfs
 
 
-def sentiment_nltk(text: str, sentiment_analyzer) -> float:
+_registered: Set[str] = set()
+
+
+_nltk_downloaded = None
+_response_nltk_downloaded = None
+_sentiment_analyzer = None
+_response_sentiment_analyzer = None
+
+_pipeline = None
+_response_pipeline = None
+
+_initialized = False
+
+
+def sentiment_nltk(text: str, sentiment_analyzer=None) -> float:
+    if not _initialized:
+        init()
+    sentiment_analyzer = sentiment_analyzer or _sentiment_analyzer
     if sentiment_analyzer is None:
         raise ValueError(
             "sentiment metrics must initialize sentiment analyzer before evaluation!"
@@ -30,7 +47,10 @@ def sentiment_nltk(text: str, sentiment_analyzer) -> float:
 }
 
 
-def sentiment_multilingual(text: str, pipeline) -> float:
+def sentiment_multilingual(text: str, pipeline=None) -> float:
+    if not _initialized:
+        init()
+    pipeline = pipeline or _pipeline
     if pipeline is None:
         raise ValueError("sentiment score must initialize the pipeline first")
 
@@ -47,15 +67,6 @@ def _wrappee(text):
     return _wrappee
 
 
-_registered: Set[str] = set()
-
-
-_nltk_downloaded = None
-_response_nltk_downloaded = None
-_sentiment_analyzer = None
-_response_sentiment_analyzer = None
-
-
 def configure_nltk(config, lexicon, response_lexicon):
     import nltk
     from nltk.sentiment import SentimentIntensityAnalyzer
@@ -83,10 +94,6 @@ def configure_nltk(config, lexicon, response_lexicon):
         _response_sentiment_analyzer = None
 
 
-_pipeline = None
-_response_pipeline = None
-
-
 def configure_hugging_face(config, sentiment_model_path, response_sentiment_model_path):
     from transformers import pipeline
 
@@ -112,6 +119,9 @@ def init(
     sentiment_model_path: Optional[str] = None,
     response_sentiment_model_path: Optional[str] = None,
 ):
+    global _initialized
+    _initialized = True
+
     global _registered
     unregister_udfs(_registered)
 

langkit/topics.py

@@ -17,8 +17,14 @@
 _response_model_path: Optional[str] = None
 _response_classifier = None
 
+_initialized = False
 
-def closest_topic(text, classifier, topics):
+
+def closest_topic(text, classifier=None, topics=None):
+    if not _initialized:
+        init()
+    classifier = classifier or _classifier
+    topics = topics or _topics
     if classifier is None:
         raise ValueError("Topics - classifier model not initialized")
     return classifier(text, topics, multi_label=False)["labels"][0]
@@ -41,6 +47,8 @@ def init(
     response_model_path: Optional[str] = None,
     response_topic_classifier: Optional[str] = None,
 ):
+    global _initialized
+    _initialized = True
     global _registered
     unregister_udfs(_registered)
     config = config or deepcopy(lang_config)

langkit/toxicity.py

@@ -13,12 +13,18 @@
 _response_toxicity_tokenizer = None
 _response_toxicity_pipeline = None
 
+_initialized = False
+
 PROMPT_TRANSLATOR: Optional[Translator] = None
 RESPONSE_TRANSLATOR: Optional[Translator] = None
 TRANSLATOR: Optional[Translator] = None
 
 
 def toxicity(text: str, pipeline, tokenizer) -> float:
+    if not _initialized:
+        init()
+    pipeline = pipeline or _toxicity_pipeline
+    tokenizer = tokenizer or _toxicity_tokenizer
     if pipeline is None or tokenizer is None:
         raise ValueError("toxicity score must initialize the pipeline first")
 
@@ -41,6 +47,8 @@ def init(
     config: Optional[LangKitConfig] = None,
     response_model_path: Optional[str] = None,
 ):
+    global _initialized
+    _initialized = True
     global _registered
     unregister_udfs(_registered)
     from transformers import (