[WIP] - SQL Model Data Layer

.gitignore

@@ -2,6 +2,7 @@ build
 dist
 
 *.egg-info
+*.ipynb
 
 .env
 

backend/chainlit/cli/__init__.py

@@ -47,6 +47,15 @@ def cli():
 
 # Define the function to run Chainlit with provided options
 def run_chainlit(target: str):
+    import os
+    import subprocess
+
+    def auto_run_alembic_upgrade():
+        try:
+            subprocess.run(["alembic", "upgrade", "head"], check=True)
+            logger.info("Alembic migrations applied (upgrade to head).")
+        except Exception as e:
+            logger.error(f"Failed to run Alembic migrations: {e}")
     host = os.environ.get("CHAINLIT_HOST", DEFAULT_HOST)
     port = int(os.environ.get("CHAINLIT_PORT", DEFAULT_PORT))
     root_path = os.environ.get("CHAINLIT_ROOT_PATH", DEFAULT_ROOT_PATH)
@@ -76,6 +85,28 @@ def run_chainlit(target: str):
     config.run.module_name = target
     load_module(config.run.module_name)
 
+    # Check if SQLModelDataLayer is used and warn about Alembic migrations
+    data_layer_func = getattr(config.code, "data_layer", None)
+    if data_layer_func:
+        try:
+            dl_instance = data_layer_func()
+            from backend.chainlit.data.sql_data_layer import SQLModelDataLayer
+            if isinstance(dl_instance, SQLModelDataLayer):
+                # Get current version
+                try:
+                    from chainlit.version import __version__
+                except Exception:
+                    __version__ = "unknown"
+                logger.info(f"SQLModelDataLayer detected. Chainlit version: {__version__}.")
+                auto_migrate = os.environ.get("CHAINLIT_AUTO_MIGRATE", "false").lower() in ["true", "1", "yes"]
+                if auto_migrate:
+                    logger.info("Auto-migration enabled. Running Alembic migrations...")
+                    auto_run_alembic_upgrade()
+                else:
+                    logger.info("Auto-migration disabled. Run 'alembic upgrade head' after updating models or upgrading Chainlit.")
+        except Exception as e:
+            logger.warning(f"Could not check data layer type: {e}")
+
     ensure_jwt_secret()
     assert_app()
 

backend/chainlit/data/alembic/README.md

@@ -0,0 +1,39 @@
+# Alembic Migrations for Chainlit SQLModelDataLayer
+
+This directory contains Alembic migration scripts for the SQLModel-based data layer.
+
+## Best Practices
+
+- **Do not use `SQLModel.metadata.create_all()` in production.**
+- Always manage schema changes with Alembic migrations.
+- Keep migration scripts in version control.
+- Run migrations before starting the app, or enable auto-migration with `CHAINLIT_AUTO_MIGRATE=true`.
+
+## Usage
+
+1. **Configure your database URL** in `alembic.ini`:
+   ```ini
+   sqlalchemy.url = <your-database-url>
+   ```
+
+2. **Autogenerate a migration** (after changing models):
+   ```bash
+   alembic revision --autogenerate -m "Initial tables"
+   ```
+
+3. **Apply migrations**:
+   ```bash
+   alembic upgrade head
+   ```
+
+## Initial Migration
+
+The first migration should create all tables defined in `chainlit.models`.
+
+## env.py
+
+Alembic is configured to use `SQLModel.metadata` from `chainlit.models`.
+
+---
+
+For more details, see the [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/).

backend/chainlit/data/alembic/env.py

@@ -0,0 +1,52 @@
+import sys
+import os
+from logging.config import fileConfig
+from sqlalchemy import engine_from_config, pool
+from alembic import context
+
+# Add the parent directory to sys.path to import models
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from chainlit.models import SQLModel
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+target_metadata = SQLModel.metadata
+
+def run_migrations_offline():
+    """
+    Run migrations in 'offline' mode.
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+def run_migrations_online():
+    """
+    Run migrations in 'online' mode.
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

backend/chainlit/data/alembic/versions/0001_create_tables.py

@@ -0,0 +1,50 @@
+"""
+Initial migration: migrate camelCase columns to snake_case for SQLModelDataLayer
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '0001_create_tables'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+def upgrade():
+    # Thread table: rename camelCase columns to snake_case
+    with op.batch_alter_table('thread') as batch_op:
+        batch_op.rename_column('createdAt', 'created_at')
+        batch_op.rename_column('userId', 'user_id')
+        batch_op.rename_column('userIdentifier', 'user_identifier')
+        # tags and metadata are already snake_case or compatible
+    # Repeat for other tables (User, Step, Element, Feedback)
+    with op.batch_alter_table('user') as batch_op:
+        batch_op.rename_column('createdAt', 'created_at')
+    with op.batch_alter_table('step') as batch_op:
+        batch_op.rename_column('threadId', 'thread_id')
+        batch_op.rename_column('parentId', 'parent_id')
+        batch_op.rename_column('createdAt', 'created_at')
+    with op.batch_alter_table('element') as batch_op:
+        batch_op.rename_column('threadId', 'thread_id')
+        batch_op.rename_column('objectKey', 'object_key')
+    with op.batch_alter_table('feedback') as batch_op:
+        batch_op.rename_column('forId', 'for_id')
+    # If tables do not exist, Alembic will error; users should run this only once during migration.
+
+def downgrade():
+    # Reverse the renames for downgrade
+    with op.batch_alter_table('thread') as batch_op:
+        batch_op.rename_column('created_at', 'createdAt')
+        batch_op.rename_column('user_id', 'userId')
+        batch_op.rename_column('user_identifier', 'userIdentifier')
+    with op.batch_alter_table('user') as batch_op:
+        batch_op.rename_column('created_at', 'createdAt')
+    with op.batch_alter_table('step') as batch_op:
+        batch_op.rename_column('thread_id', 'threadId')
+        batch_op.rename_column('parent_id', 'parentId')
+        batch_op.rename_column('created_at', 'createdAt')
+    with op.batch_alter_table('element') as batch_op:
+        batch_op.rename_column('thread_id', 'threadId')
+        batch_op.rename_column('object_key', 'objectKey')
+    with op.batch_alter_table('feedback') as batch_op:
+        batch_op.rename_column('for_id', 'forId')