fix opencv version, generic llm invocations

environments/build/ubuntu-latest/cuda.yml

@@ -12,7 +12,7 @@ dependencies:
   - make=4.3
   - nodejs
   - ffmpeg
-  - opencv
+  - opencv=4.8.1
   - openh264
   - git
   - pip:

environments/linux-cuda-latest.yml

@@ -7,7 +7,7 @@ channels:
 dependencies:
   - cuda-toolkit=12.1.1
   - python=3.10.11
-  - opencv
+  - opencv=4.8.1
   - openh264
   - pip
   - numpy

environments/linux-cuda.yml

@@ -7,7 +7,7 @@ channels:
 dependencies:
   - cuda-toolkit=11.8.0
   - python=3.10.11
-  - opencv
+  - opencv=4.8.1
   - openh264
   - pip
   - numpy

environments/linux-tensorrt.yml

@@ -9,7 +9,7 @@ dependencies:
   - python=3.10.11
   - torchvision=0.14.1
   - pytorch=1.13.1
-  - opencv
+  - opencv=4.8.1
   - openh264
   - pip
   - numpy

src/python/enfugue/__main__.py

@@ -7,8 +7,9 @@
 import logging
 import traceback
 
-from typing import Optional, Any, List
+from typing import Optional, Any, List, Union, Dict, Iterator
 
+from contextlib import contextmanager
 from PIL import Image
 from copy import deepcopy
 
@@ -20,6 +21,63 @@
 
 from enfugue.util import logger, merge_into, get_local_configuration
 
+@contextmanager
+def get_context(debug: bool=False) -> Iterator:
+    """
+    Either blank or debug context manager
+    """
+    if debug:
+        from pibble.util.log import DebugUnifiedLoggingContext
+        with DebugUnifiedLoggingContext():
+            yield
+    else:
+        yield
+
+def get_configuration(
+    config: Optional[str]=None,
+    overrides: Optional[Union[str,Dict[str, Any]]]=None,
+    merge: bool=False,
+    debug: bool=False,
+) -> Dict[str, Any]:
+    """
+    Gets the configuration to use based on passed arguments.
+    """
+    if config is not None:
+        if config.endswith("json"):
+            configuration = load_json(config)
+        elif config.endswith("yml") or config.endswith("yaml"):
+            from pibble.util.files import load_yaml
+            configuration = load_yaml(config)
+        else:
+            raise IOError(f"Unknown format for configuration file {config}")
+
+        if merge:
+            configuration = merge_into(configuration, get_local_configuration())
+    else:
+        configuration = get_local_configuration()
+
+    while "configuration" in configuration:
+        configuration = configuration["configuration"]
+
+
+    if overrides:
+        if isinstance(overrides, str):
+            overrides = json.loads(overrides)
+        merge_into(overrides, configuration)
+
+    if debug:
+        log_overrides = {
+            "logging": {
+                "level": "debug",
+                "handler": "stream",
+                "stream": "stdout",
+                "colored": True
+             }
+        }
+        merge_into(log_overrides, configuration)
+
+    return configuration
+
 @click.group(name="enfugue")
 def main() -> None:
     """
@@ -115,59 +173,93 @@ def dump_config(filename: Optional[str] = None, json: bool = False) -> None:
 
         click.echo(yaml.dump(configuration))
 
+@click.option("-c", "--config", help="An optional path to a configuration file to use instead of the default.")
+@click.option("-r", "--role", help="An optional role of the conversation to use instead of the default.")
+@click.option("-s", "--system", help="An optional system message to send instead of the default for the chosen role (or default.)")
+@click.option("-m", "--merge", is_flag=True, default=False, help="When set, merge the passed configuration with the default configuration instead of replacing it.")
+@click.option("-d", "--debug", help="Enable debug logging.", is_flag=True, default=False)
+@click.option("-u", "--unsafe", help="Disable safety.", is_flag=True, default=False)
+@click.option("-t", "--temperature", help="The temperature (randomness) of the responses.", default=0.7, show_default=True)
+@click.option("-k", "--top-k", help="The number of tokens to limit to when selecting the next token in a response.", default=50, show_default=True)
+@click.option("-p", "--top-p", help="The p-value of tokens to limit from when selecting the next token in a response.", default=0.95, show_default=True)
+@click.option("-f", "--forgetful", help="Enable 'forgetful' mode - i.e. refresh the conversation after each response.", is_flag=True, default=False)
+@main.command(short_help="Starts a chat with an LLM.")
+def chat(
+    config: Optional[str] = None,
+    role: Optional[str] = None,
+    system: Optional[str] = None,
+    merge: bool = False,
+    overrides: str = None,
+    debug: bool = False,
+    unsafe: bool = False,
+    temperature: float=0.7,
+    top_k: int=50,
+    top_p: float=0.95,
+    forgetful: bool = False
+) -> None:
+    """
+    Runs an interactive chat in the command line.
+    """
+    configuration = get_configuration(
+        config,
+        overrides=overrides,
+        merge=merge,
+        debug=debug
+    )
+    from enfugue.diffusion.manager import DiffusionPipelineManager
+    manager = DiffusionPipelineManager(configuration)
+
+    with get_context(debug):
+        click.echo(termcolor.colored("Loading language model. Say 'reset' at any time to start the conversation over. Use Ctrl+D to exit or say 'exit.'", "yellow"))
+        if unsafe:
+            click.echo(termcolor.colored("Safety is disengaged. Responses may be inappropriate or offensive, user discretion is advised.", "red"))
+        with manager.conversation.converse(
+            role=role,
+            system=system,
+            safe=not unsafe,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p
+        ) as chat:
+            try:
+                click.echo(termcolor.colored("[assistant] {0}".format(chat()), "cyan")) # First message
+                while True:
+                    click.echo(
+                        termcolor.colored(
+                            "[assistant] {0}".format(
+                                chat(input(termcolor.colored("[user] ", "green")))
+                            ),
+                            "cyan"
+                        )
+                    )
+                    if forgetful:
+                        chat("RESET")
+            finally:
+                click.echo("Goodbye!")
 
 @click.option("-c", "--config", help="An optional path to a configuration file to use instead of the default.")
 @click.option("-m", "--merge", is_flag=True, default=False, help="When set, merge the passed configuration with the default configuration instead of replacing it.")
 @click.option("-o", "--overrides", help="an optional json object containing override configuration.")
 @click.option("-d", "--debug", help="Enable debug logging.", is_flag=True, default=False)
 @main.command(short_help="Runs the server.")
 def run(
-    config: str = None,
+    config: Optional[str] = None,
     merge: bool = False,
     overrides: str = None,
-    debug: bool = False
+    debug: bool = False,
 ) -> None:
     """
     Runs the server synchronously using cherrypy.
     """
-    from enfugue.server import EnfugueServer
-
-    if config is not None:
-        if config.endswith("json"):
-
-            configuration = json_file(config)
-        elif config.endswith("yml") or config.endswith("yaml"):
-            from pibble.util.files import load_yaml
-
-            configuration = load_yaml(config)
-        else:
-            raise IOError(f"Unknown format for configuration file {config}")
-
-        if merge:
-            configuration = merge_into(configuration, get_local_configuration())
-    else:
-        configuration = get_local_configuration()
-
-    while "configuration" in configuration:
-        configuration = configuration["configuration"]
-
-    if overrides:
-        overrides = json.loads(overrides)
-        merge_into(overrides, configuration)
-    if debug:
-        log_overrides = {
-            "logging": {
-                "level": "debug",
-                "handler": "stream",
-                "stream": "stdout",
-                "colored": True
-             }
-        }
-        merge_into(log_overrides, configuration)
+    configuration = get_configuration(
+        config,
+        overrides=overrides,
+        merge=merge,
+        debug=debug
+    )
 
     # Determine how many servers we need to run
     try:
-
         all_host = configuration["server"]["host"]
         all_port = configuration["server"]["port"]
         all_domain = configuration["server"].get("domain", None)
@@ -243,4 +335,7 @@ def get_for_index(index: int, config_value: Any) -> Any:
     main()
     sys.exit(0)
 except Exception as ex:
+    sys.stderr.write(f"{ex}\r\n")
+    sys.stderr.write(traceback.format_exc())
+    sys.stderr.flush()
     sys.exit(5)

src/python/enfugue/diffusion/manager.py

@@ -55,8 +55,8 @@
         Upscaler,
         IPAdapter,
         BackgroundRemover,
-        CaptionUpsampler,
         Interpolator,
+        Conversation
     )
     from torch import Tensor
 
@@ -152,12 +152,15 @@ class DiffusionPipelineManager:
 
     def __init__(
         self,
-        configuration: Optional[APIConfiguration] = None,
+        configuration: Optional[Union[Dict[str, Any], APIConfiguration]] = None,
         optimize: bool = True
     ) -> None:
         self.configuration = APIConfiguration()
-        if configuration:
-            self.configuration = configuration
+        if configuration is not None:
+            if isinstance(configuration, APIConfiguration):
+                self.configuration = configuration
+            else:
+                self.configuration = APIConfiguration(**configuration)
         if optimize:
             self.optimize_configuration()
         self.apply_patches()
@@ -4198,7 +4201,7 @@ def upscaler(self) -> Upscaler:
                 dtype=self.dtype,
                 offline=self.offline
             )
-            self._upscaler.task_callback = self._task_callback
+            self._upscaler.task_callback = self.task_callback
         return self._upscaler
 
     @property
@@ -4214,7 +4217,7 @@ def control_image_processor(self) -> ControlImageProcessor:
                 dtype=self.dtype,
                 offline=self.offline
             )
-            self._control_image_processor.task_callback = self._task_callback
+            self._control_image_processor.task_callback = self.task_callback
         return self._control_image_processor
 
     @property
@@ -4230,7 +4233,7 @@ def background_remover(self) -> BackgroundRemover:
                 dtype=self.dtype,
                 offline=self.offline
             )
-            self._background_remover.task_callback = self._task_callback
+            self._background_remover.task_callback = self.task_callback
         return self._background_remover
 
     @property
@@ -4246,24 +4249,24 @@ def ip_adapter(self) -> IPAdapter:
                 dtype=self.dtype,
                 offline=self.offline
             )
-            self._ip_adapter.task_callback = self._task_callback
+            self._ip_adapter.task_callback = self.task_callback
         return self._ip_adapter
 
     @property
-    def caption_upsampler(self) -> CaptionUpsampler:
+    def conversation(self) -> Conversation:
         """
-        Gets the caption upsampler.
+        Gets an LLM conversation.
         """
-        if not hasattr(self, "_caption_upsampler"):
-            from enfugue.diffusion.support import CaptionUpsampler
-            self._caption_upsampler = CaptionUpsampler(
+        if not hasattr(self, "_conversation"):
+            from enfugue.diffusion.support import Conversation
+            self._conversation = Conversation(
                 self.engine_language_dir,
                 device=self.device,
                 dtype=self.dtype,
                 offline=self.offline
             )
-            self._caption_upsampler.task_callback = self._task_callback
-        return self._caption_upsampler
+            self._conversation.task_callback = self.task_callback
+        return self._conversation
 
     @property
     def interpolator(self) -> Interpolator:
@@ -4278,7 +4281,7 @@ def interpolator(self) -> Interpolator:
                 dtype=self.dtype,
                 offline=self.offline
             )
-            self._interpolator.task_callback = self._task_callback
+            self._interpolator.task_callback = self.task_callback
         return self._interpolator
 
     def get_svd_pipeline(self, use_xt: bool = True) -> StableVideoDiffusionPipeline:

src/python/enfugue/diffusion/support/__init__.py

@@ -6,9 +6,9 @@
 from enfugue.diffusion.support.upscale import Upscaler
 from enfugue.diffusion.support.background import BackgroundRemover
 from enfugue.diffusion.support.ip import IPAdapter
-from enfugue.diffusion.support.llm import CaptionUpsampler
+from enfugue.diffusion.support.llm import Conversation
 from enfugue.diffusion.support.interpolate import Interpolator
 
-EdgeDetector, LineDetector, DepthDetector, PoseDetector, ControlImageProcessor, Upscaler, BackgroundRemover, IPAdapter, CaptionUpsampler, Interpolator  # Silence importchecker
+EdgeDetector, LineDetector, DepthDetector, PoseDetector, ControlImageProcessor, Upscaler, BackgroundRemover, IPAdapter, Conversation, Interpolator  # Silence importchecker
 
-__all__ = ["EdgeDetector", "LineDetector", "DepthDetector", "PoseDetector", "ControlImageProcessor", "Upscaler", "BackgroundRemover", "IPAdapter", "CaptionUpsampler", "Interpolator"]
+__all__ = ["EdgeDetector", "LineDetector", "DepthDetector", "PoseDetector", "ControlImageProcessor", "Upscaler", "BackgroundRemover", "IPAdapter", "Conversation", "Interpolator"]

src/python/enfugue/diffusion/support/llm/__init__.py

@@ -1,3 +1,3 @@
-from enfugue.diffusion.support.llm.caption import CaptionUpsampler
-CaptionUpsampler # Silence importchecker
-__all__ = ["CaptionUpsampler"]
+from enfugue.diffusion.support.llm.conversation import Conversation
+Conversation # Silence importchecker
+__all__ = ["Conversation"]