diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml new file mode 100644 index 0000000..344dff3 --- /dev/null +++ b/.github/workflows/pre-merge.yaml @@ -0,0 +1,27 @@ +name: Pre-Merge Update + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + update-code: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + + - name: Run custom update script + run: | + ./scripts/generate-sample-env.sh ./bhasai ./common + + - name: Commit changes + run: | + git config --global user.name "GitHub Action" + git config --global user.email "action@github.com" + git add . + git commit -m "Update code before merge" || echo "No changes to commit" + git push origin ${{ github.head_ref }} diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..955d5c3 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,9 @@ +import ./common/monitoring/Caddyfile +import ./common/minio/Caddyfile +import ./common/environment/Caddyfile +import ./common/fusionauth/Caddyfile +# The registry doesn't have a auth thus exposing it publicly means anyone can access the images pushed to this registry +# import ./common/registry/Caddyfile +import ./common/uptime/Caddyfile +import ./bhasai/Caddyfile +import ./common/superset/Caddyfile \ No newline at end of file diff --git a/Caddyfile.SDC b/Caddyfile.SDC new file mode 100644 index 0000000..bca8b98 --- /dev/null +++ b/Caddyfile.SDC @@ -0,0 +1,7 @@ +{$DOMAIN_SCHEME}://{$DOMAIN_NAME} { + {$TLS_ENTRY} + + import ./common/monitoring/Caddyfile.SDC + import ./bhasai/Caddyfile.SDC + +} \ No newline at end of file diff --git a/Makefile b/Makefile index 6d2ba16..8129efc 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ SDC := $(if $(filter 1,$(SDC)),-f docker-compose.SDC.yaml,) DOCKER_COMPOSE_COMMAND=docker compose ${SDC} $(REMOVE_ANSI_FLAG) -p bhasai +generate: + @./bhasai/ai_tools/generate.sh + install-docker: @./scripts/install-docker.sh @@ -19,12 +22,15 @@ migrate-volume: setup-webhook: @./scripts/webhook/setup-webhook.sh + +generate-env: + @./scripts/generate-env.sh reload-caddy: @echo "Reloading caddy" $(DOCKER_COMPOSE_COMMAND) exec -w /etc/caddy caddy caddy reload || true -deploy: $(if $(filter 1,$(ENABLE_GIT_PULL)),git-pull,) $(if $(filter 1,$(DISABLE_PULL)),,pull build) reload-caddy +deploy: generate $(if $(filter 1,$(ENABLE_GIT_PULL)),git-pull,) $(if $(filter 1,$(DISABLE_PULL)),,pull build) reload-caddy $(DOCKER_COMPOSE_COMMAND) up -d $(FORCE_RECREATE_FLAG) $(REMOVE_ORPHANS_FLAG) ${services} restart: diff --git a/bhasai/Caddyfile b/bhasai/Caddyfile new file mode 100644 index 0000000..0baee12 --- /dev/null +++ b/bhasai/Caddyfile @@ -0,0 +1,17 @@ +import ./admin/Caddyfile +import ./ai_tools_proxy/Caddyfile +import ./auth_service/Caddyfile +import ./autotune/Caddyfile +import ./bff/Caddyfile +import ./dataset_service/Caddyfile +import ./deployer/Caddyfile +import ./document_service/Caddyfile +import ./inboundjs/Caddyfile +import ./nodehub/Caddyfile +import ./pdf_parser/Caddyfile +import ./telemetry/Caddyfile +import ./transport_socket/Caddyfile +import ./uci_apis/Caddyfile +import ./user_management_service/Caddyfile +import ./yt_parser/Caddyfile +import ./ai_tools/Caddyfile \ No newline at end of file diff --git a/bhasai/Caddyfile.SDC b/bhasai/Caddyfile.SDC new file mode 100644 index 0000000..b1a1191 --- /dev/null +++ b/bhasai/Caddyfile.SDC @@ -0,0 +1,20 @@ +import ./admin/Caddyfile.SDC +import ./ai_tools_proxy/Caddyfile.SDC +import ./auth_service/Caddyfile.SDC +import ./autotune/Caddyfile.SDC +import ./bff/Caddyfile.SDC +import ./dataset_service/Caddyfile.SDC +import ./deployer/Caddyfile.SDC +import ./document_service/Caddyfile.SDC +import ./inboundjs/Caddyfile.SDC +import ./nodehub/Caddyfile.SDC +import ./pdf_parser/Caddyfile.SDC +import ./telemetry/Caddyfile.SDC +import ./transport_socket/Caddyfile.SDC +import ./uci_apis/Caddyfile.SDC +import ./user_management_service/Caddyfile.SDC +import ./yt_parser/Caddyfile.SDC +import ./flowise/Caddyfile.SDC +import ./app/Caddyfile.SDC +import ./user_service/Caddyfile.SDC +import ./ai_tools/Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/admin/Caddyfile b/bhasai/admin/Caddyfile new file mode 100644 index 0000000..9b962a1 --- /dev/null +++ b/bhasai/admin/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://admin.{$DOMAIN_NAME} { + reverse_proxy admin:3000 +} \ No newline at end of file diff --git a/bhasai/admin/Caddyfile.SDC b/bhasai/admin/Caddyfile.SDC new file mode 100644 index 0000000..661275a --- /dev/null +++ b/bhasai/admin/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /admin* { + reverse_proxy admin:3000 +} diff --git a/bhasai/admin/docker-compose.SDC.yaml b/bhasai/admin/docker-compose.SDC.yaml new file mode 100644 index 0000000..b648051 --- /dev/null +++ b/bhasai/admin/docker-compose.SDC.yaml @@ -0,0 +1,17 @@ +services: + admin: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/admin.git#${ADMIN_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_UCI_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/uci-apis + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/bff + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://{DOMAIN_NAME}/transport-socket + NEXT_PUBLIC_AUTH_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/auth-service + NEXT_PUBLIC_DATASET_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/dataset-service + NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service + NEXT_PUBLIC_PWA_DEPLOYER_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer + NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/telemetry + NEXT_PUBLIC_FLOWISE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/flowise + NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/autotune + SERVE_PATH: "/admin" + restart: always diff --git a/bhasai/admin/docker-compose.yaml b/bhasai/admin/docker-compose.yaml new file mode 100644 index 0000000..27bde39 --- /dev/null +++ b/bhasai/admin/docker-compose.yaml @@ -0,0 +1,17 @@ +services: + admin: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/admin.git#${ADMIN_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_UCI_API_URL: ${DOMAIN_SCHEME:-https}://uci-apis.${DOMAIN_NAME} + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://bff.${DOMAIN_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} + NEXT_PUBLIC_AUTH_SERVICE_URL: ${DOMAIN_SCHEME:-https}://auth-service.${DOMAIN_NAME} + NEXT_PUBLIC_DATASET_SERVICE_URL: ${DOMAIN_SCHEME:-https}://dataset-service.${DOMAIN_NAME} + NEXT_PUBLIC_DOC_SERVICE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} + NEXT_PUBLIC_PWA_DEPLOYER_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME} + NEXT_PUBLIC_TELEMETRY_API_URL: ${DOMAIN_SCHEME:-https}://telemetry.${DOMAIN_NAME} + NEXT_PUBLIC_FLOWISE_URL: ${DOMAIN_SCHEME:-https}://flowise.${DOMAIN_NAME} + NEXT_PUBLIC_AUTOTUNE_API_URL: ${DOMAIN_SCHEME:-https}://autotune.${DOMAIN_NAME} + SERVE_PATH: "" + restart: always diff --git a/bhasai/admin/sample.env b/bhasai/admin/sample.env new file mode 100644 index 0000000..99e423a --- /dev/null +++ b/bhasai/admin/sample.env @@ -0,0 +1,3 @@ +#admin +DOMAIN_SCHEME= +DOMAIN_NAME= \ No newline at end of file diff --git a/bhasai/ai_tools/.gitignore b/bhasai/ai_tools/.gitignore new file mode 100644 index 0000000..8f63f43 --- /dev/null +++ b/bhasai/ai_tools/.gitignore @@ -0,0 +1,3 @@ +docker-compose.yaml +Caddyfile +Caddyfile.SDC \ No newline at end of file diff --git a/bhasai/ai_tools/generate.sh b/bhasai/ai_tools/generate.sh new file mode 100755 index 0000000..461fac7 --- /dev/null +++ b/bhasai/ai_tools/generate.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +################################################################################################## +# Description: This script generates Docker Compose and Caddy configuration # +# files based on a AI_SERVICES environment variable # +# # +# Required Environment Variables: # +# - org: GitHub organization # +# - DOCKER_REGISTRY_URL: Docker registry to use for the container images (default: 'ghcr.io') # +################################################################################################## + + +source .env + +required_env=("org" "DOCKER_REGISTRY_URL" "AI_SERVICES") + +# Check if required environment variables are set +for var in "${required_env[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: '$var' environment variable is not set." + exit 1 + fi +done + +directory="bhasai/ai_tools" + +GITHUB_REPOSITORY_URL="${org}/ai-tools" +GITHUB_BRANCH="${AI_TOOLS_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH?DEFAULT_GITHUB_BRANCH is not set}}" + +command -v jq >/dev/null || { echo "Error: jq is required but not found. Please install it."; exit 1; } + +# Get the number of models from the config.json file +count=$(echo "$AI_SERVICES" | jq -r '.models | length') +ComposeFile="${directory}/docker-compose.yaml" +Caddyfile="${directory}/Caddyfile" +CaddyfileSDC="${directory}/Caddyfile.SDC" + +# Generate docker-compose.yaml file +printf "services:\n" > $ComposeFile + +printf '{$DOMAIN_SCHEME}://ai-tools.{$DOMAIN_NAME} {' > $Caddyfile +printf "" > $CaddyfileSDC + +# Loop through each model +for ((i=0; i<$count; i++)); do + # Get model details from config.json + serviceName=$(echo "$AI_SERVICES" | jq -r ".models[$i].serviceName") + apiBasePath=$(echo "$AI_SERVICES" | jq -r ".models[$i].apiBasePath") + containerPort=$(echo "$AI_SERVICES" | jq -r ".models[$i].containerPort") + gpu=$(echo "$AI_SERVICES" | jq ".models[$i].gpu") + + containerImage="${DOCKER_REGISTRY_URL}/${GITHUB_REPOSITORY_URL}/${serviceName}:\${AI_TOOLS_IMAGE_TAG:-\${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}}" + + echo "handle_path /ai-tools/${apiBasePath}* { + reverse_proxy ${serviceName}:${containerPort} +}" >> $CaddyfileSDC + + echo " + handle_path /${apiBasePath}* { + reverse_proxy ${serviceName}:${containerPort} + } + " >> $Caddyfile + + + # Get environment variables for the model + environment=($(echo "$AI_SERVICES" | jq -r ".models[$i].environment | keys[]")) + + # Add service details to docker-compose.yaml + + printf " ${serviceName}:\n image: $containerImage\n restart: always\n" >> $ComposeFile + # Add environment variables to docker-compose.yaml + if [[ ${#environment[@]} -gt 0 ]]; then + printf " environment:\n" >> $ComposeFile + fi + for key in "${environment[@]}"; do + printf " - ${key}=\${${key}}\n" >> $ComposeFile + done + + if [[ "$gpu" == 'true' ]] ;then + + printf " deploy:\n resources:\n reservations:\n devices:\n - driver: nvidia\n count: 1\n capabilities: [gpu]\n" >> $ComposeFile + + fi + printf "\n" >> $ComposeFile + +done +printf "}" >> $Caddyfile diff --git a/bhasai/ai_tools_proxy/Caddyfile b/bhasai/ai_tools_proxy/Caddyfile new file mode 100644 index 0000000..644a240 --- /dev/null +++ b/bhasai/ai_tools_proxy/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://ai-tools-proxy.{$DOMAIN_NAME} { + reverse_proxy ai_tools_proxy:3000 +} diff --git a/bhasai/ai_tools_proxy/Caddyfile.SDC b/bhasai/ai_tools_proxy/Caddyfile.SDC new file mode 100644 index 0000000..98f18bc --- /dev/null +++ b/bhasai/ai_tools_proxy/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /ai-tools-proxy* { + reverse_proxy ai_tools_proxy:3000 +} diff --git a/bhasai/ai_tools_proxy/docker-compose.SDC.yaml b/bhasai/ai_tools_proxy/docker-compose.SDC.yaml new file mode 100644 index 0000000..08e95b3 --- /dev/null +++ b/bhasai/ai_tools_proxy/docker-compose.SDC.yaml @@ -0,0 +1,22 @@ + services: + ai_tools_proxy: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/ai-tools-proxy:${AI_TOOLS_PROXY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AI_TOOLS_PROXY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AI_TOOLS_PROXY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/ai_tools_proxy" + DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT:-3} + DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT:-30000} + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: ai-tools-proxy + ENVIRONMENT: ${env:-dev} + TELEMETRY_URL: http://telemetry:3000 + OPENAI_API_KEY: ${OPENAI_API_KEY} diff --git a/bhasai/ai_tools_proxy/docker-compose.yaml b/bhasai/ai_tools_proxy/docker-compose.yaml new file mode 100644 index 0000000..08e95b3 --- /dev/null +++ b/bhasai/ai_tools_proxy/docker-compose.yaml @@ -0,0 +1,22 @@ + services: + ai_tools_proxy: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/ai-tools-proxy:${AI_TOOLS_PROXY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AI_TOOLS_PROXY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AI_TOOLS_PROXY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/ai_tools_proxy" + DEFAULT_RETRY_COUNT: ${DEFAULT_RETRY_COUNT:-3} + DEFAULT_TIMEOUT: ${DEFAULT_TIMEOUT:-30000} + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: ai-tools-proxy + ENVIRONMENT: ${env:-dev} + TELEMETRY_URL: http://telemetry:3000 + OPENAI_API_KEY: ${OPENAI_API_KEY} diff --git a/bhasai/ai_tools_proxy/sample.env b/bhasai/ai_tools_proxy/sample.env new file mode 100644 index 0000000..c59d1a5 --- /dev/null +++ b/bhasai/ai_tools_proxy/sample.env @@ -0,0 +1,15 @@ +#ai_tools_proxy +POSTGRES_USER= +POSTGRES_PASSWORD= +DEFAULT_RETRY_COUNT= +DEFAULT_TIMEOUT= +ULCA_API_KEY= +ULCA_CONFIG_URL= +ULCA_USER_ID= +DOMAIN_SCHEME= +DOMAIN_NAME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +env= +OPENAI_API_KEY= + diff --git a/bhasai/app/Caddyfile b/bhasai/app/Caddyfile new file mode 100644 index 0000000..bdb3acd --- /dev/null +++ b/bhasai/app/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://app.{$DOMAIN_NAME} { + reverse_proxy app:3000 +} \ No newline at end of file diff --git a/bhasai/app/Caddyfile.SDC b/bhasai/app/Caddyfile.SDC new file mode 100644 index 0000000..7628748 --- /dev/null +++ b/bhasai/app/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle /* { + reverse_proxy app:3000 +} diff --git a/bhasai/app/docker-compose.SDC.yaml b/bhasai/app/docker-compose.SDC.yaml new file mode 100644 index 0000000..fd2f6ab --- /dev/null +++ b/bhasai/app/docker-compose.SDC.yaml @@ -0,0 +1,27 @@ +services: + app: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/kisai-bot.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_BOT_ID: ${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_CONFIG_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer/deployment/config/${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_BOT_NAME: ${NEXT_PUBLIC_BOT_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME} + NEXT_PUBLIC_TELEMETRY_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/telemetry + NEXT_PUBLIC_AI_TOOLS_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/ai-tools-proxy + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/bff + NEXT_PUBLIC_USER_SERVICE_APP_ID: ${NEXT_PUBLIC_USER_SERVICE_APP_ID} + NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' + NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/user-service + NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} + NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG:-false} + NEXT_PUBLIC_SOCKET_PATH: '/transport-socket/socket.io' + NEXT_PUBLIC_GOOGLE_KEY: ${NEXT_PUBLIC_GOOGLE_KEY} + FUSIONAUTH_URL: http://fusionauth:9011/ + FUSIONAUTH_KEY: ${FUSIONAUTH_KEY} + NEXT_PUBLIC_DATASET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/dataset-service + NEXT_PUBLIC_ENTITY_DATASET_ID: ${NEXT_PUBLIC_ENTITY_DATASET_ID} + NEXT_PUBLIC_CLARITY_PROJECT_ID: ${NEXT_PUBLIC_CLARITY_PROJECT_ID} + NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING:-false} + NEXT_PUBLIC_WEATHER_API: ${NEXT_PUBLIC_WEATHER_API} + \ No newline at end of file diff --git a/bhasai/app/docker-compose.yaml b/bhasai/app/docker-compose.yaml new file mode 100644 index 0000000..e584a81 --- /dev/null +++ b/bhasai/app/docker-compose.yaml @@ -0,0 +1,27 @@ +services: + app: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/kisai-bot.git#${APP_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + NEXT_PUBLIC_BOT_ID: ${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_CONFIG_BASE_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME}/deployment/config/${NEXT_PUBLIC_BOT_ID} + NEXT_PUBLIC_BOT_NAME: ${NEXT_PUBLIC_BOT_NAME} + NEXT_PUBLIC_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} + NEXT_PUBLIC_TELEMETRY_API: ${DOMAIN_SCHEME:-https}://telemetry.${DOMAIN_NAME} + NEXT_PUBLIC_AI_TOOLS_API: ${DOMAIN_SCHEME:-https}://ai-tools-proxy.${DOMAIN_NAME} + NEXT_PUBLIC_BFF_API_URL: ${DOMAIN_SCHEME:-https}://bff.${DOMAIN_NAME} + NEXT_PUBLIC_USER_SERVICE_APP_ID: ${NEXT_PUBLIC_USER_SERVICE_APP_ID} + NEXT_PUBLIC_JWKS_URI: 'http://fusionauth:9011/.well-known/jwks.json' + NEXT_PUBLIC_USER_SERVICE_URL: ${DOMAIN_SCHEME:-https}://user-service.${DOMAIN_NAME} + NEXT_PUBLIC_ORG_ID: ${NEXT_PUBLIC_ORG_ID} + NEXT_PUBLIC_DEBUG: ${NEXT_PUBLIC_DEBUG:-false} + NEXT_PUBLIC_SOCKET_PATH: '' + NEXT_PUBLIC_GOOGLE_KEY: ${NEXT_PUBLIC_GOOGLE_KEY} + FUSIONAUTH_URL: http://fusionauth:9011/ + FUSIONAUTH_KEY: ${FUSIONAUTH_KEY} + NEXT_PUBLIC_DATASET_URL: ${DOMAIN_SCHEME:-https}://dataset-service.${DOMAIN_NAME} + NEXT_PUBLIC_ENTITY_DATASET_ID: ${NEXT_PUBLIC_ENTITY_DATASET_ID} + NEXT_PUBLIC_CLARITY_PROJECT_ID: ${NEXT_PUBLIC_CLARITY_PROJECT_ID} + NEXT_PUBLIC_SHOW_ONBOARDING: ${NEXT_PUBLIC_SHOW_ONBOARDING:-false} + NEXT_PUBLIC_WEATHER_API: ${NEXT_PUBLIC_WEATHER_API} + \ No newline at end of file diff --git a/bhasai/app/sample.env b/bhasai/app/sample.env new file mode 100644 index 0000000..6d93593 --- /dev/null +++ b/bhasai/app/sample.env @@ -0,0 +1,12 @@ +#app +NEXT_PUBLIC_BOT_ID= +NEXT_PUBLIC_ORG_ID= +NEXT_PUBLIC_GOOGLE_KEY= +NEXT_PUBLIC_BOT_NAME= +FUSIONAUTH_KEY= +NEXT_PUBLIC_ENTITY_DATASET_ID= +NEXT_PUBLIC_CLARITY_PROJECT_ID= +NEXT_PUBLIC_USER_SERVICE_APP_ID= +NEXT_PUBLIC_DEBUG= +NEXT_PUBLIC_SHOW_ONBOARDING= +NEXT_PUBLIC_WEATHER_API= \ No newline at end of file diff --git a/bhasai/auth_service/Caddyfile b/bhasai/auth_service/Caddyfile new file mode 100644 index 0000000..f02ce04 --- /dev/null +++ b/bhasai/auth_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://auth-service.{$DOMAIN_NAME} { + reverse_proxy auth_service:3000 +} \ No newline at end of file diff --git a/bhasai/auth_service/Caddyfile.SDC b/bhasai/auth_service/Caddyfile.SDC new file mode 100644 index 0000000..7315e70 --- /dev/null +++ b/bhasai/auth_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /auth-service* { + reverse_proxy auth_service:3000 +} diff --git a/bhasai/auth_service/docker-compose.SDC.yaml b/bhasai/auth_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..f52215b --- /dev/null +++ b/bhasai/auth_service/docker-compose.SDC.yaml @@ -0,0 +1,9 @@ +services: + auth_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/auth-service:${AUTH_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTH_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTH_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} \ No newline at end of file diff --git a/bhasai/auth_service/docker-compose.yaml b/bhasai/auth_service/docker-compose.yaml new file mode 100644 index 0000000..f52215b --- /dev/null +++ b/bhasai/auth_service/docker-compose.yaml @@ -0,0 +1,9 @@ +services: + auth_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/auth-service:${AUTH_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTH_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTH_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} \ No newline at end of file diff --git a/bhasai/auth_service/sample.env b/bhasai/auth_service/sample.env new file mode 100644 index 0000000..04cf495 --- /dev/null +++ b/bhasai/auth_service/sample.env @@ -0,0 +1,2 @@ +#auth_service +FUSIONAUTH_API_KEY= \ No newline at end of file diff --git a/bhasai/autotune/Caddyfile b/bhasai/autotune/Caddyfile new file mode 100644 index 0000000..280f9dd --- /dev/null +++ b/bhasai/autotune/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://autotune.{$DOMAIN_NAME} { + reverse_proxy autotune:8000 +} diff --git a/bhasai/autotune/Caddyfile.SDC b/bhasai/autotune/Caddyfile.SDC new file mode 100644 index 0000000..7c636cf --- /dev/null +++ b/bhasai/autotune/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /autotune* { + reverse_proxy autotune:8000 +} diff --git a/bhasai/autotune/docker-compose.SDC.yaml b/bhasai/autotune/docker-compose.SDC.yaml new file mode 100644 index 0000000..960879b --- /dev/null +++ b/bhasai/autotune/docker-compose.SDC.yaml @@ -0,0 +1,49 @@ + services: + autotune: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && python manage.py database && python manage.py runserver 0.0.0.0:8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} + + autotune_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && celery -A autotune worker --loglevel=info -P gevent" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} diff --git a/bhasai/autotune/docker-compose.yaml b/bhasai/autotune/docker-compose.yaml new file mode 100644 index 0000000..960879b --- /dev/null +++ b/bhasai/autotune/docker-compose.yaml @@ -0,0 +1,49 @@ + services: + autotune: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && python manage.py database && python manage.py runserver 0.0.0.0:8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} + + autotune_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/autotune:${AUTOTUNE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${AUTOTUNE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${AUTOTUNE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "poetry install && celery -A autotune worker --loglevel=info -P gevent" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/autotune" + REDIS_URL: redis://redis:6379/7 + OPENAI_API_KEY: ${OPENAI_API_KEY} + MINIO_BASE_URL: "${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME}" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: autotune + MINIO_SECURE_CONN: "True" + CELERY_MAX_RETRIES: ${CELERY_MAX_RETRIES:-3} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY} + HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME} + HUGGING_FACE_TOKEN: ${HUGGING_FACE_TOKEN} + GITHUB_PAT: ${GITHUB_PAT} diff --git a/bhasai/autotune/sample.env b/bhasai/autotune/sample.env new file mode 100644 index 0000000..dfff2ae --- /dev/null +++ b/bhasai/autotune/sample.env @@ -0,0 +1,27 @@ +#autotune +POSTGRES_USER= +POSTGRES_PASSWORD= +OPENAI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +CELERY_MAX_RETRIES= +DJANGO_SECRET_KEY= +HUGGING_FACE_USERNAME= +HUGGING_FACE_TOKEN= +GITHUB_PAT= + +#autotune_celery_worker +POSTGRES_USER= +POSTGRES_PASSWORD= +OPENAI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +CELERY_MAX_RETRIES= +DJANGO_SECRET_KEY= +HUGGING_FACE_USERNAME= +HUGGING_FACE_TOKEN= +GITHUB_PAT= \ No newline at end of file diff --git a/bhasai/bff/Caddyfile b/bhasai/bff/Caddyfile new file mode 100644 index 0000000..999bb55 --- /dev/null +++ b/bhasai/bff/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://bff.{$DOMAIN_NAME} { + reverse_proxy bff:3000 +} \ No newline at end of file diff --git a/bhasai/bff/Caddyfile.SDC b/bhasai/bff/Caddyfile.SDC new file mode 100644 index 0000000..02525b9 --- /dev/null +++ b/bhasai/bff/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /bff* { + reverse_proxy bff:3000 +} diff --git a/bhasai/bff/docker-compose.SDC.yaml b/bhasai/bff/docker-compose.SDC.yaml new file mode 100644 index 0000000..7a92e34 --- /dev/null +++ b/bhasai/bff/docker-compose.SDC.yaml @@ -0,0 +1,39 @@ +services: + bff: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/bff:${BFF_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${BFF_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${BFF_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + environment: + BFF_BASE_URL: http://bff:3000 + EXCEL_PARSER_URL: http://excel_parser:8000 + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/bff?sslmode=disable&connect_timeout=100" + MINIO_PDF_BUCKET: bff + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/pdf-parser + PDF_SPLIT: '50' + REDIS_DB: 8 + REDIS_HOST: redis + REDIS_PORT: 6379 + OUTBOUND_BASE_URL: http://inboundjs:3000/outbound + TRANSPORT_SOCKET_BASE_URL: http://transport_socket:3000 + UCI_APIS_BASE_URL: http://uci_apis:3000 + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + YT_PARSER_URL: http://yt_parser:8000 + REGISTRY_URL: https://github.com/BharatSahAIyak/packages/blob/main/packages/transformers/registry.json + INBOUNDJS_BASE_URL: http://inboundjs:3000 + DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/document-service + AZURE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/ai-tools/text_translation/azure_dict/remote/ + USER_SERVICE_ADMIN_API_KEY: ${ADMIN_API_KEY} \ No newline at end of file diff --git a/bhasai/bff/docker-compose.yaml b/bhasai/bff/docker-compose.yaml new file mode 100644 index 0000000..7998e43 --- /dev/null +++ b/bhasai/bff/docker-compose.yaml @@ -0,0 +1,39 @@ +services: + bff: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/bff:${BFF_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${BFF_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${BFF_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + environment: + BFF_BASE_URL: http://bff:3000 + EXCEL_PARSER_URL: http://excel_parser:8000 + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/bff?sslmode=disable&connect_timeout=100" + MINIO_PDF_BUCKET: bff + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_PORT: 443 + MINIO_SECURE_CONN: "True" + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_BASE_URL: ${DOMAIN_SCHEME:-https}://pdf-parser.${DOMAIN_NAME} + PDF_SPLIT: '50' + REDIS_DB: 8 + REDIS_HOST: redis + REDIS_PORT: 6379 + OUTBOUND_BASE_URL: http://inboundjs:3000/outbound + TRANSPORT_SOCKET_BASE_URL: http://transport_socket:3000 + UCI_APIS_BASE_URL: http://uci_apis:3000 + ULCA_API_KEY: ${ULCA_API_KEY} + ULCA_CONFIG_URL: ${ULCA_CONFIG_URL} + ULCA_USER_ID: ${ULCA_USER_ID} + YT_PARSER_URL: http://yt_parser:8000 + REGISTRY_URL: https://github.com/BharatSahAIyak/packages/blob/main/packages/transformers/registry.json + INBOUNDJS_BASE_URL: http://inboundjs:3000 + DOCUMENT_SERVICE_BASE_URL: ${DOMAIN_SCHEME:-https}://document-service.${DOMAIN_NAME} + AZURE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME}/text_translation/azure_dict/remote/ + USER_SERVICE_ADMIN_API_KEY: ${ADMIN_API_KEY} diff --git a/bhasai/bff/sample.env b/bhasai/bff/sample.env new file mode 100644 index 0000000..09dd326 --- /dev/null +++ b/bhasai/bff/sample.env @@ -0,0 +1,12 @@ +#bff +POSTGRES_USER= +POSTGRES_PASSWORD= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +OPENAI_API_KEY= +ULCA_API_KEY= +ULCA_CONFIG_URL= +ULCA_USER_ID= +ADMIN_API_KEY= diff --git a/bhasai/dataset_service/Caddyfile b/bhasai/dataset_service/Caddyfile new file mode 100644 index 0000000..a80670c --- /dev/null +++ b/bhasai/dataset_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://dataset-service.{$DOMAIN_NAME} { + reverse_proxy dataset_service:3000 +} diff --git a/bhasai/dataset_service/Caddyfile.SDC b/bhasai/dataset_service/Caddyfile.SDC new file mode 100644 index 0000000..5f6b529 --- /dev/null +++ b/bhasai/dataset_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /dataset-service* { + reverse_proxy dataset_service:3000 +} diff --git a/bhasai/dataset_service/docker-compose.SDC.yaml b/bhasai/dataset_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..c075712 --- /dev/null +++ b/bhasai/dataset_service/docker-compose.SDC.yaml @@ -0,0 +1,19 @@ +services: + dataset_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/dataset-service:${DATASET_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DATASET_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DATASET_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/dataset_service" + MINIO_ENDPOINT: "${DOMAIN_SCHEME}://cdn-api.${DOMAIN_NAME}" + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: dataset-service + OPENAI_API_KEY: ${OPENAI_API_KEY} + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 9 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/dataset_service/docker-compose.yaml b/bhasai/dataset_service/docker-compose.yaml new file mode 100644 index 0000000..c075712 --- /dev/null +++ b/bhasai/dataset_service/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + dataset_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/dataset-service:${DATASET_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DATASET_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DATASET_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/dataset_service" + MINIO_ENDPOINT: "${DOMAIN_SCHEME}://cdn-api.${DOMAIN_NAME}" + MINIO_SECURE_CONN: "True" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: dataset-service + OPENAI_API_KEY: ${OPENAI_API_KEY} + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 9 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/dataset_service/sample.env b/bhasai/dataset_service/sample.env new file mode 100644 index 0000000..2e0b620 --- /dev/null +++ b/bhasai/dataset_service/sample.env @@ -0,0 +1,9 @@ +#dataset_service +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +POSTGRES_USER= +POSTGRES_PASSWORD= +AI_TOOLS_EMBEDDING_BATCH_SIZE= +OPENAI_API_KEY= diff --git a/bhasai/deployer/Caddyfile b/bhasai/deployer/Caddyfile new file mode 100644 index 0000000..408c0ee --- /dev/null +++ b/bhasai/deployer/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://deployer.{$DOMAIN_NAME} { + reverse_proxy deployer:3000 +} \ No newline at end of file diff --git a/bhasai/deployer/Caddyfile.SDC b/bhasai/deployer/Caddyfile.SDC new file mode 100644 index 0000000..2f3bd9c --- /dev/null +++ b/bhasai/deployer/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /deployer* { + reverse_proxy deployer:3000 +} diff --git a/bhasai/deployer/docker-compose.SDC.yaml b/bhasai/deployer/docker-compose.SDC.yaml new file mode 100644 index 0000000..dc7b01d --- /dev/null +++ b/bhasai/deployer/docker-compose.SDC.yaml @@ -0,0 +1,20 @@ +services: + deployer: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/deployer:${DEPLOYER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DEPLOYER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DEPLOYER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/deployer?sslmode=disable&connect_timeout=100" + VERCEL_TOKEN: ${DEPLOYER_VERCEL_TOKEN} + PWA_REPO: ${DEPLOYER_PWA_REPO} + APP_ENV: ${DEPLOYER_APP_ENV} + SELF_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/deployer + MINIO_BUCKETNAME: deployer + STORAGE_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + STORAGE_PORT: 443 + STORAGE_USE_SSL: 'true' + STORAGE_ACCESS_KEY: ${MINIO_ACCESS_KEY} + STORAGE_SECRET_KEY: ${MINIO_SECRET_KEY} + STORAGE_MODE: "minio" + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/deployer/docker-compose.yaml b/bhasai/deployer/docker-compose.yaml new file mode 100644 index 0000000..4e9cada --- /dev/null +++ b/bhasai/deployer/docker-compose.yaml @@ -0,0 +1,20 @@ +services: + deployer: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/deployer:${DEPLOYER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DEPLOYER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DEPLOYER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/deployer?sslmode=disable&connect_timeout=100" + VERCEL_TOKEN: ${DEPLOYER_VERCEL_TOKEN} + PWA_REPO: ${DEPLOYER_PWA_REPO} + APP_ENV: ${DEPLOYER_APP_ENV} + SELF_BASE_URL: ${DOMAIN_SCHEME:-https}://deployer.${DOMAIN_NAME} + MINIO_BUCKETNAME: deployer + STORAGE_ENDPOINT: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + STORAGE_PORT: 443 + STORAGE_USE_SSL: 'true' + STORAGE_ACCESS_KEY: ${MINIO_ACCESS_KEY} + STORAGE_SECRET_KEY: ${MINIO_SECRET_KEY} + STORAGE_MODE: "minio" + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/deployer/sample.env b/bhasai/deployer/sample.env new file mode 100644 index 0000000..a54ee01 --- /dev/null +++ b/bhasai/deployer/sample.env @@ -0,0 +1,11 @@ +#deployer +POSTGRES_USER= +POSTGRES_PASSWORD= +DEPLOYER_VERCEL_TOKEN= +DEPLOYER_PWA_REPO= +DEPLOYER_APP_ENV= +DOMAIN_SCHEME= +DOMAIN_NAME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +VERCEL_ENV_CONFIG= diff --git a/bhasai/docker-compose.SDC.yaml b/bhasai/docker-compose.SDC.yaml new file mode 100644 index 0000000..763559b --- /dev/null +++ b/bhasai/docker-compose.SDC.yaml @@ -0,0 +1,23 @@ +include: + - ./admin/docker-compose.SDC.yaml + - ./ai_tools_proxy/docker-compose.SDC.yaml + - ./auth_service/docker-compose.SDC.yaml + - ./autotune/docker-compose.SDC.yaml + - ./bff/docker-compose.SDC.yaml + - ./dataset_service/docker-compose.SDC.yaml + - ./deployer/docker-compose.SDC.yaml + - ./document_service/docker-compose.SDC.yaml + - ./inboundjs/docker-compose.SDC.yaml + - ./nodehub/docker-compose.SDC.yaml + - ./orchestrator/docker-compose.SDC.yaml + - ./pdf_parser/docker-compose.SDC.yaml + - ./telemetry/docker-compose.SDC.yaml + - ./transport_socket/docker-compose.SDC.yaml + - ./uci_apis/docker-compose.SDC.yaml + - ./user_management_service/docker-compose.SDC.yaml + - ./vault/docker-compose.yaml + - ./yt_parser/docker-compose.SDC.yaml + - ./flowise/docker-compose.SDC.yaml + - ./app/docker-compose.SDC.yaml + - ./user_service/docker-compose.SDC.yaml + - ./ai_tools/docker-compose.yaml \ No newline at end of file diff --git a/bhasai/docker-compose.yaml b/bhasai/docker-compose.yaml new file mode 100644 index 0000000..69c44a2 --- /dev/null +++ b/bhasai/docker-compose.yaml @@ -0,0 +1,20 @@ +include: + - ./admin/docker-compose.yaml + - ./ai_tools_proxy/docker-compose.yaml + - ./auth_service/docker-compose.yaml + - ./autotune/docker-compose.yaml + - ./bff/docker-compose.yaml + - ./dataset_service/docker-compose.yaml + - ./deployer/docker-compose.yaml + - ./document_service/docker-compose.yaml + - ./inboundjs/docker-compose.yaml + - ./nodehub/docker-compose.yaml + - ./orchestrator/docker-compose.yaml + - ./pdf_parser/docker-compose.yaml + - ./telemetry/docker-compose.yaml + - ./transport_socket/docker-compose.yaml + - ./uci_apis/docker-compose.yaml + - ./user_management_service/docker-compose.yaml + - ./vault/docker-compose.yaml + - ./yt_parser/docker-compose.yaml + - ./ai_tools/docker-compose.yaml diff --git a/bhasai/document_service/Caddyfile b/bhasai/document_service/Caddyfile new file mode 100644 index 0000000..de8c4df --- /dev/null +++ b/bhasai/document_service/Caddyfile @@ -0,0 +1,4 @@ +{$DOMAIN_SCHEME}://document-service.{$DOMAIN_NAME} { + reverse_proxy document_service:3000 +} + diff --git a/bhasai/document_service/Caddyfile.SDC b/bhasai/document_service/Caddyfile.SDC new file mode 100644 index 0000000..71c6ca0 --- /dev/null +++ b/bhasai/document_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /document-service* { + reverse_proxy document_service:3000 +} diff --git a/bhasai/document_service/docker-compose.SDC.yaml b/bhasai/document_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..f5554cc --- /dev/null +++ b/bhasai/document_service/docker-compose.SDC.yaml @@ -0,0 +1,22 @@ +services: + document_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/document-service:${DOCUMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DOCUMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DOCUMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/document_service" + MINIO_ENDPOINT: "minio" + MINIO_PORT: "9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: document-service + MINIO_SECURE_CONN: "False" + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_URL: http://pdf_parser:8000 + YT_PARSER_URL: http://yt_parser:8000 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 4 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/document_service/docker-compose.yaml b/bhasai/document_service/docker-compose.yaml new file mode 100644 index 0000000..f5554cc --- /dev/null +++ b/bhasai/document_service/docker-compose.yaml @@ -0,0 +1,22 @@ +services: + document_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/document-service:${DOCUMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${DOCUMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${DOCUMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/document_service" + MINIO_ENDPOINT: "minio" + MINIO_PORT: "9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: document-service + MINIO_SECURE_CONN: "False" + OPENAI_API_KEY: ${OPENAI_API_KEY} + PDF_PARSER_URL: http://pdf_parser:8000 + YT_PARSER_URL: http://yt_parser:8000 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 4 + AI_TOOLS_EMBEDDING_BASE_URL: ${DOMAIN_SCHEME:-https}://ai-tools.${DOMAIN_NAME} + AI_TOOLS_BATCH_SIZE: ${AI_TOOLS_EMBEDDING_BATCH_SIZE:-100} diff --git a/bhasai/document_service/sample.env b/bhasai/document_service/sample.env new file mode 100644 index 0000000..3dbba5a --- /dev/null +++ b/bhasai/document_service/sample.env @@ -0,0 +1,8 @@ +#document_service +POSTGRES_USER= +POSTGRES_PASSWORD= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +OPENAI_API_KEY= +AI_TOOLS_EMBEDDING_BATCH_SIZE= +DOMAIN_NAME= diff --git a/bhasai/flowise/Caddyfile b/bhasai/flowise/Caddyfile new file mode 100644 index 0000000..8c7269e --- /dev/null +++ b/bhasai/flowise/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://flowise.{$DOMAIN_NAME} { + reverse_proxy flowise:3000 +} \ No newline at end of file diff --git a/bhasai/flowise/Caddyfile.SDC b/bhasai/flowise/Caddyfile.SDC new file mode 100644 index 0000000..ee05ab9 --- /dev/null +++ b/bhasai/flowise/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle /flowise* { + reverse_proxy flowise:3000 +} diff --git a/bhasai/flowise/docker-compose.SDC.yaml b/bhasai/flowise/docker-compose.SDC.yaml new file mode 100644 index 0000000..6be40a4 --- /dev/null +++ b/bhasai/flowise/docker-compose.SDC.yaml @@ -0,0 +1,10 @@ +services: + flowise: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${FLOWISE_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + REACT_APP_UCI_API: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/uci-apis + REACT_APP_SOCKET_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/transport-socket + REACT_APP_REGISTRY_BASE_URL: ${DOMAIN_SCHEME:-https}://${DOMAIN_NAME}/nodehub + SERVE_PATH: "/flowise" + restart: always diff --git a/bhasai/flowise/docker-compose.yaml b/bhasai/flowise/docker-compose.yaml new file mode 100644 index 0000000..0cd73f2 --- /dev/null +++ b/bhasai/flowise/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + flowise: + build: + context: https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${org}/flowise.git#${FLOWISE_GITHUB_BRANCH:-${DEFAULT_GITHUB_BRANCH:?DEFAULT_GITHUB_BRANCH is not set}} + args: + REACT_APP_UCI_API: ${DOMAIN_SCHEME:-https}://uci-apis.${DOMAIN_NAME} + REACT_APP_SOCKET_URL: ${DOMAIN_SCHEME:-https}://transport-socket.${DOMAIN_NAME} + REACT_APP_REGISTRY_BASE_URL: ${DOMAIN_SCHEME:-https}://nodehub.${DOMAIN_NAME} + SERVE_PATH: "" + restart: always diff --git a/bhasai/flowise/sample.env b/bhasai/flowise/sample.env new file mode 100644 index 0000000..2389c05 --- /dev/null +++ b/bhasai/flowise/sample.env @@ -0,0 +1,3 @@ +#flowise +DOMAIN_NAME= +DOMAIN_SCHEME= \ No newline at end of file diff --git a/bhasai/inboundjs/Caddyfile b/bhasai/inboundjs/Caddyfile new file mode 100644 index 0000000..340d8f9 --- /dev/null +++ b/bhasai/inboundjs/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://inboundjs.{$DOMAIN_NAME} { + reverse_proxy inboundjs:3000 +} \ No newline at end of file diff --git a/bhasai/inboundjs/Caddyfile.SDC b/bhasai/inboundjs/Caddyfile.SDC new file mode 100644 index 0000000..47b576e --- /dev/null +++ b/bhasai/inboundjs/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /inboundjs* { + reverse_proxy inboundjs:3000 +} diff --git a/bhasai/inboundjs/docker-compose.SDC.yaml b/bhasai/inboundjs/docker-compose.SDC.yaml new file mode 100644 index 0000000..5f661d1 --- /dev/null +++ b/bhasai/inboundjs/docker-compose.SDC.yaml @@ -0,0 +1,13 @@ +services: + inboundjs: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/inbound-js:${INBOUNDJS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${INBOUNDJS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${INBOUNDJS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/inboundjs" + ORCHESTRATOR_API_ENDPOINT: "http://orchestrator:3000" + UCI_API_BASE_URL: http://uci-apis:3000 + UCI_API_TOKEN: ${UCI_ADMIN_TOKEN} + TELEMETRY_SERVICE_HOST: http://telemetry:3000 + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/inboundjs/docker-compose.yaml b/bhasai/inboundjs/docker-compose.yaml new file mode 100644 index 0000000..5f661d1 --- /dev/null +++ b/bhasai/inboundjs/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + inboundjs: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/inbound-js:${INBOUNDJS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${INBOUNDJS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${INBOUNDJS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/inboundjs" + ORCHESTRATOR_API_ENDPOINT: "http://orchestrator:3000" + UCI_API_BASE_URL: http://uci-apis:3000 + UCI_API_TOKEN: ${UCI_ADMIN_TOKEN} + TELEMETRY_SERVICE_HOST: http://telemetry:3000 + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/inboundjs/sample.env b/bhasai/inboundjs/sample.env new file mode 100644 index 0000000..3a8f5c4 --- /dev/null +++ b/bhasai/inboundjs/sample.env @@ -0,0 +1,5 @@ +#inboundjs +env= +UCI_ADMIN_TOKEN= +POSTGRES_USER= +POSTGRES_PASSWORD= \ No newline at end of file diff --git a/bhasai/nodehub/Caddyfile b/bhasai/nodehub/Caddyfile new file mode 100644 index 0000000..8bd6763 --- /dev/null +++ b/bhasai/nodehub/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://nodehub.{$DOMAIN_NAME} { + reverse_proxy nodehub:3000 +} \ No newline at end of file diff --git a/bhasai/nodehub/Caddyfile.SDC b/bhasai/nodehub/Caddyfile.SDC new file mode 100644 index 0000000..63479ea --- /dev/null +++ b/bhasai/nodehub/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /nodehub* { + reverse_proxy nodehub:3000 +} diff --git a/bhasai/nodehub/docker-compose.SDC.yaml b/bhasai/nodehub/docker-compose.SDC.yaml new file mode 100644 index 0000000..e619d1d --- /dev/null +++ b/bhasai/nodehub/docker-compose.SDC.yaml @@ -0,0 +1,11 @@ +services: + nodehub: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/nodehub:${NODEHUB_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${NODEHUB_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${NODEHUB_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/nodehub" + AUTH_SERVICE_URL: "http://auth_service:3000" + REGISTRY_BASE_URL: https://raw.githubusercontent.com/BharatSahAIyak/packages/main/packages/transformers/registry.json + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/nodehub/docker-compose.yaml b/bhasai/nodehub/docker-compose.yaml new file mode 100644 index 0000000..e619d1d --- /dev/null +++ b/bhasai/nodehub/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + nodehub: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/nodehub:${NODEHUB_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${NODEHUB_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${NODEHUB_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/nodehub" + AUTH_SERVICE_URL: "http://auth_service:3000" + REGISTRY_BASE_URL: https://raw.githubusercontent.com/BharatSahAIyak/packages/main/packages/transformers/registry.json + ENVIRONMENT: ${env:-dev} \ No newline at end of file diff --git a/bhasai/nodehub/sample.env b/bhasai/nodehub/sample.env new file mode 100644 index 0000000..973cdcb --- /dev/null +++ b/bhasai/nodehub/sample.env @@ -0,0 +1,2 @@ +#nodehub +env= \ No newline at end of file diff --git a/bhasai/orchestrator/docker-compose.SDC.yaml b/bhasai/orchestrator/docker-compose.SDC.yaml new file mode 100644 index 0000000..820a43b --- /dev/null +++ b/bhasai/orchestrator/docker-compose.SDC.yaml @@ -0,0 +1,9 @@ +services: + orchestrator: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/orchestrator:${ORCHESTRATOR_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${ORCHESTRATOR_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${ORCHESTRATOR_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + OUTBOUND_SERVICE_BASE_URL: "http://inboundjs:3000/outbound" + ENVIRONMENT: ${env:-dev} diff --git a/bhasai/orchestrator/docker-compose.yaml b/bhasai/orchestrator/docker-compose.yaml new file mode 100644 index 0000000..820a43b --- /dev/null +++ b/bhasai/orchestrator/docker-compose.yaml @@ -0,0 +1,9 @@ +services: + orchestrator: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/orchestrator:${ORCHESTRATOR_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${ORCHESTRATOR_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${ORCHESTRATOR_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + restart: always + environment: + OUTBOUND_SERVICE_BASE_URL: "http://inboundjs:3000/outbound" + ENVIRONMENT: ${env:-dev} diff --git a/bhasai/orchestrator/sample.env b/bhasai/orchestrator/sample.env new file mode 100644 index 0000000..1cc8eed --- /dev/null +++ b/bhasai/orchestrator/sample.env @@ -0,0 +1,2 @@ +#orchestrator +env= \ No newline at end of file diff --git a/bhasai/pdf_parser/Caddyfile b/bhasai/pdf_parser/Caddyfile new file mode 100644 index 0000000..43b8549 --- /dev/null +++ b/bhasai/pdf_parser/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://pdf-parser.{$DOMAIN_NAME} { + reverse_proxy pdf_parser:8000 +} diff --git a/bhasai/pdf_parser/Caddyfile.SDC b/bhasai/pdf_parser/Caddyfile.SDC new file mode 100644 index 0000000..71aae38 --- /dev/null +++ b/bhasai/pdf_parser/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /pdf-parser* { + reverse_proxy pdf_parser:8000 +} diff --git a/bhasai/pdf_parser/docker-compose.SDC.yaml b/bhasai/pdf_parser/docker-compose.SDC.yaml new file mode 100644 index 0000000..6d8d664 --- /dev/null +++ b/bhasai/pdf_parser/docker-compose.SDC.yaml @@ -0,0 +1,78 @@ +services: + pdf_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${PDF_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "./setup.sh && python manage.py runserver 0.0.0.0:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: always + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} + depends_on: + - redis + + pdf_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: bash -c "./setup.sh && celery -A pdf_parser worker -P solo -l info" + restart: always + mem_limit: ${PDF_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} + depends_on: + - redis \ No newline at end of file diff --git a/bhasai/pdf_parser/docker-compose.yaml b/bhasai/pdf_parser/docker-compose.yaml new file mode 100644 index 0000000..6d8d664 --- /dev/null +++ b/bhasai/pdf_parser/docker-compose.yaml @@ -0,0 +1,78 @@ +services: + pdf_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + mem_limit: ${PDF_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + command: bash -c "./setup.sh && python manage.py runserver 0.0.0.0:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: always + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} + depends_on: + - redis + + pdf_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/pdf-parser:${PDF_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: bash -c "./setup.sh && celery -A pdf_parser worker -P solo -l info" + restart: always + mem_limit: ${PDF_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${PDF_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + OPENAI_API_KEY: ${OPENAI_API_KEY} + BHASHINI_API_KEY: ${BHASHINI_API_KEY} + TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" + MINIO_BASE_URL: ${DOMAIN_SCHEME:-https}://cdn-api.${DOMAIN_NAME} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: pdf_parser + MINIO_SECURE_CONN: "True" + ENABLE_TRANSLATE: ${ENABLE_TRANSLATE} + ENABLE_SUMMARIZE: ${ENABLE_SUMMARIZE} + ENABLE_IMAGE_PARSING: ${ENABLE_IMAGE_PARSING} + SPLIT: ${SPLIT:-5} + PARALLEL_FACTOR: ${PARALLEL_FACTOR:-2} + REDIS_CONN: redis://redis:6379/0 + TORCH_DEVICE: ${TORCH_DEVICE:-cuda} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + PDF_PARSER_MAX_RETRIES : ${PDF_PARSER_MAX_RETRIES:-3} + PDF_PARSER_SECRET_KEY : ${PDF_PARSER_SECRET_KEY} + AZURE_TRANSLATE_KEY : ${AZURE_TRANSLATE_KEY} + TRANSLATION_SERVICES : ${TRANSLATION_SERVICES:-["azure", "bhasini"]} + depends_on: + - redis \ No newline at end of file diff --git a/bhasai/pdf_parser/sample.env b/bhasai/pdf_parser/sample.env new file mode 100644 index 0000000..7facf66 --- /dev/null +++ b/bhasai/pdf_parser/sample.env @@ -0,0 +1,37 @@ +#pdf_parser +OPENAI_API_KEY= +BHASHINI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +ENABLE_TRANSLATE= +ENABLE_SUMMARIZE= +ENABLE_IMAGE_PARSING= +SPLIT= +PARALLEL_FACTOR= +TORCH_DEVICE= +DISCORD_WEBHOOK_URL= +PDF_PARSER_MAX_RETRIES= +PDF_PARSER_SECRET_KEY= +AZURE_TRANSLATE_KEY= +TRANSLATION_SERVICES= + +#pdf_parser_celery_worker +OPENAI_API_KEY= +BHASHINI_API_KEY= +DOMAIN_NAME= +DOMAIN_SCHEME= +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +ENABLE_TRANSLATE= +ENABLE_SUMMARIZE= +ENABLE_IMAGE_PARSING= +SPLIT= +PARALLEL_FACTOR= +TORCH_DEVICE= +DISCORD_WEBHOOK_URL= +PDF_PARSER_MAX_RETRIES= +PDF_PARSER_SECRET_KEY= +AZURE_TRANSLATE_KEY= +TRANSLATION_SERVICES= diff --git a/bhasai/telemetry/Caddyfile b/bhasai/telemetry/Caddyfile new file mode 100644 index 0000000..951099a --- /dev/null +++ b/bhasai/telemetry/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://telemetry.{$DOMAIN_NAME} { + reverse_proxy telemetry:3000 +} \ No newline at end of file diff --git a/bhasai/telemetry/Caddyfile.SDC b/bhasai/telemetry/Caddyfile.SDC new file mode 100644 index 0000000..dbc64b5 --- /dev/null +++ b/bhasai/telemetry/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /telemetry* { + reverse_proxy telemetry:3000 +} diff --git a/bhasai/telemetry/docker-compose.SDC.yaml b/bhasai/telemetry/docker-compose.SDC.yaml new file mode 100644 index 0000000..3c21638 --- /dev/null +++ b/bhasai/telemetry/docker-compose.SDC.yaml @@ -0,0 +1,13 @@ +services: + telemetry: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/telemetry:${TELEMETRY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TELEMETRY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TELEMETRY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/telemetry" + CLICKHOUSE_HOST: http://clickhouse:8123 + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-default} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} + AUTH_SERVICE_URL: http://auth_service:3000 diff --git a/bhasai/telemetry/docker-compose.yaml b/bhasai/telemetry/docker-compose.yaml new file mode 100644 index 0000000..3c21638 --- /dev/null +++ b/bhasai/telemetry/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + telemetry: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/telemetry:${TELEMETRY_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TELEMETRY_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TELEMETRY_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/telemetry" + CLICKHOUSE_HOST: http://clickhouse:8123 + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-default} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} + AUTH_SERVICE_URL: http://auth_service:3000 diff --git a/bhasai/telemetry/sample.env b/bhasai/telemetry/sample.env new file mode 100644 index 0000000..774e9cb --- /dev/null +++ b/bhasai/telemetry/sample.env @@ -0,0 +1,6 @@ +#telemetry +POSTGRES_USER= +POSTGRES_PASSWORD= +CLICKHOUSE_DB= +CLICKHOUSE_USER= +CLICKHOUSE_PASSWORD= \ No newline at end of file diff --git a/bhasai/transport_socket/Caddyfile b/bhasai/transport_socket/Caddyfile new file mode 100644 index 0000000..311c310 --- /dev/null +++ b/bhasai/transport_socket/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://transport-socket.{$DOMAIN_NAME} { + reverse_proxy transport_socket:3000 +} diff --git a/bhasai/transport_socket/Caddyfile.SDC b/bhasai/transport_socket/Caddyfile.SDC new file mode 100644 index 0000000..41340bd --- /dev/null +++ b/bhasai/transport_socket/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /transport-socket* { + reverse_proxy transport_socket:3000 +} diff --git a/bhasai/transport_socket/docker-compose.SDC.yaml b/bhasai/transport_socket/docker-compose.SDC.yaml new file mode 100644 index 0000000..31b51eb --- /dev/null +++ b/bhasai/transport_socket/docker-compose.SDC.yaml @@ -0,0 +1,12 @@ +services: + transport_socket: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/transport-socket:${TRANSPORT_SOCKET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TRANSPORT_SOCKET_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TRANSPORT_SOCKET_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + JWKS_URI: http://fusionauth:9011/.well-known/jwks.json + REDIS_URL: redis://redis:6379/1 + ADAPTER_URL: http://inboundjs:3000/inbound/bot + DEBUG: ${TRANSPORT_SOCKET_DEBUG} + SERVER_PORT: 3000 \ No newline at end of file diff --git a/bhasai/transport_socket/docker-compose.yaml b/bhasai/transport_socket/docker-compose.yaml new file mode 100644 index 0000000..31b51eb --- /dev/null +++ b/bhasai/transport_socket/docker-compose.yaml @@ -0,0 +1,12 @@ +services: + transport_socket: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/transport-socket:${TRANSPORT_SOCKET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${TRANSPORT_SOCKET_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${TRANSPORT_SOCKET_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + JWKS_URI: http://fusionauth:9011/.well-known/jwks.json + REDIS_URL: redis://redis:6379/1 + ADAPTER_URL: http://inboundjs:3000/inbound/bot + DEBUG: ${TRANSPORT_SOCKET_DEBUG} + SERVER_PORT: 3000 \ No newline at end of file diff --git a/bhasai/transport_socket/sample.env b/bhasai/transport_socket/sample.env new file mode 100644 index 0000000..bd1ae4b --- /dev/null +++ b/bhasai/transport_socket/sample.env @@ -0,0 +1,2 @@ +#transport_socket +TRANSPORT_SOCKET_DEBUG= \ No newline at end of file diff --git a/bhasai/uci_apis/Caddyfile b/bhasai/uci_apis/Caddyfile new file mode 100644 index 0000000..69ef4c0 --- /dev/null +++ b/bhasai/uci_apis/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://uci-apis.{$DOMAIN_NAME} { + reverse_proxy uci_apis:3000 +} \ No newline at end of file diff --git a/bhasai/uci_apis/Caddyfile.SDC b/bhasai/uci_apis/Caddyfile.SDC new file mode 100644 index 0000000..03f273b --- /dev/null +++ b/bhasai/uci_apis/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /uci-apis* { + reverse_proxy uci_apis:3000 +} diff --git a/bhasai/uci_apis/docker-compose.SDC.yaml b/bhasai/uci_apis/docker-compose.SDC.yaml new file mode 100644 index 0000000..2f1443b --- /dev/null +++ b/bhasai/uci_apis/docker-compose.SDC.yaml @@ -0,0 +1,19 @@ +services: + uci_apis: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/uci-apis:${UCI_APIS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${UCI_APIS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${UCI_APIS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + UCI_API_PSQL_DB_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/uci_apis" + REDIS_URL: redis://redis:6379/2 + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} + ADMIN_TOKEN: ${UCI_ADMIN_TOKEN} + VAULT_ADDR: http://vault:8200 + VAULT_USERNAME: ${VAULT_USERNAME} + VAULT_PASSWORD: ${VAULT_PASSWORD} + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/uci_apis/docker-compose.yaml b/bhasai/uci_apis/docker-compose.yaml new file mode 100644 index 0000000..2f1443b --- /dev/null +++ b/bhasai/uci_apis/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + uci_apis: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/uci-apis:${UCI_APIS_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${UCI_APIS_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${UCI_APIS_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + UCI_API_PSQL_DB_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/uci_apis" + REDIS_URL: redis://redis:6379/2 + FUSIONAUTH_URL: http://fusionauth:9011 + FUSIONAUTH_KEY: ${FUSIONAUTH_API_KEY} + ADMIN_TOKEN: ${UCI_ADMIN_TOKEN} + VAULT_ADDR: http://vault:8200 + VAULT_USERNAME: ${VAULT_USERNAME} + VAULT_PASSWORD: ${VAULT_PASSWORD} + AUTH_SERVICE_URL: http://auth_service:3000 \ No newline at end of file diff --git a/bhasai/uci_apis/sample.env b/bhasai/uci_apis/sample.env new file mode 100644 index 0000000..9810c0d --- /dev/null +++ b/bhasai/uci_apis/sample.env @@ -0,0 +1,8 @@ + +#uci_apis +UCI_ADMIN_TOKEN= +VAULT_USERNAME= +VAULT_PASSWORD= +POSTGRES_USER= +POSTGRES_PASSWORD= +FUSIONAUTH_API_KEY= \ No newline at end of file diff --git a/bhasai/user_management_service/Caddyfile b/bhasai/user_management_service/Caddyfile new file mode 100644 index 0000000..05a232c --- /dev/null +++ b/bhasai/user_management_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://user-management-service.{$DOMAIN_NAME} { + reverse_proxy user_management_service:3000 +} \ No newline at end of file diff --git a/bhasai/user_management_service/Caddyfile.SDC b/bhasai/user_management_service/Caddyfile.SDC new file mode 100644 index 0000000..6953d3f --- /dev/null +++ b/bhasai/user_management_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /user-management-service* { + reverse_proxy user_management_service:3000 +} diff --git a/bhasai/user_management_service/docker-compose.SDC.yaml b/bhasai/user_management_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..b3e0afa --- /dev/null +++ b/bhasai/user_management_service/docker-compose.SDC.yaml @@ -0,0 +1,11 @@ +services: + user_management_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/user-management-service:${USER_MANAGEMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${USER_MANAGEMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_MANAGEMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/user_management_service" diff --git a/bhasai/user_management_service/docker-compose.yaml b/bhasai/user_management_service/docker-compose.yaml new file mode 100644 index 0000000..b3e0afa --- /dev/null +++ b/bhasai/user_management_service/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + user_management_service: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/user-management-service:${USER_MANAGEMENT_SERVICE_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + restart: always + mem_limit: ${USER_MANAGEMENT_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_MANAGEMENT_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/user_management_service" diff --git a/bhasai/user_management_service/sample.env b/bhasai/user_management_service/sample.env new file mode 100644 index 0000000..bc99394 --- /dev/null +++ b/bhasai/user_management_service/sample.env @@ -0,0 +1,4 @@ + +#user_management_service +POSTGRES_USER= +POSTGRES_PASSWORD= \ No newline at end of file diff --git a/bhasai/user_service/Caddyfile b/bhasai/user_service/Caddyfile new file mode 100644 index 0000000..06d71c6 --- /dev/null +++ b/bhasai/user_service/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://user-service.{$DOMAIN_NAME} { + reverse_proxy user_service:3000 +} \ No newline at end of file diff --git a/bhasai/user_service/Caddyfile.SDC b/bhasai/user_service/Caddyfile.SDC new file mode 100644 index 0000000..ea97fb7 --- /dev/null +++ b/bhasai/user_service/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /user-service* { + reverse_proxy user_service:3000 +} diff --git a/bhasai/user_service/docker-compose.SDC.yaml b/bhasai/user_service/docker-compose.SDC.yaml new file mode 100644 index 0000000..8d0d4df --- /dev/null +++ b/bhasai/user_service/docker-compose.SDC.yaml @@ -0,0 +1,20 @@ +services: + user_service: + image: samagragovernance/esamwad-user-service:latest + restart: always + mem_limit: ${USER_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + - ADMIN_API_KEY=${ADMIN_API_KEY} + - ALLOW_DEFAULT_OTP=${ALLOW_DEFAULT_OTP:-false} + - DEFAULT_OTP=${DEFAULT_OTP:-1234} + - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS:-[]} + - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD} + - DEFAULT_USER_PASSWORD="http://fusionauth:9011" + - FUSIONAUTH_OLD_BASE_URL"="http://fusionauth:9011" + - GUPSHUP_BASEURL="http://enterprise.smsgupshup.com/GatewayAPI/rest" + - GUPSHUP_OTP_TEMPLATE=${GUPSHUP_OTP_TEMPLATE} + - GUPSHUP_PASSWORD=${GUPSHUP_PASSWORD} + - GUPSHUP_USERNAME=${GUPSHUP_USERNAME} + - OTP_EXPIRY=${OTP_EXPIRY:-30} + - APP_${FUSIONAUTH_APP_ID}="{"host":"http://fusionauth:9011","salt":"${FUSIONAUTH_SALT}","apiKey":"${FUSIONAUTH_API_KEY}","encryption":{"enabled":false}}" diff --git a/bhasai/user_service/docker-compose.yaml b/bhasai/user_service/docker-compose.yaml new file mode 100644 index 0000000..8d0d4df --- /dev/null +++ b/bhasai/user_service/docker-compose.yaml @@ -0,0 +1,20 @@ +services: + user_service: + image: samagragovernance/esamwad-user-service:latest + restart: always + mem_limit: ${USER_SERVICE_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${USER_SERVICE_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + - ADMIN_API_KEY=${ADMIN_API_KEY} + - ALLOW_DEFAULT_OTP=${ALLOW_DEFAULT_OTP:-false} + - DEFAULT_OTP=${DEFAULT_OTP:-1234} + - DEFAULT_OTP_USERS=${DEFAULT_OTP_USERS:-[]} + - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD} + - DEFAULT_USER_PASSWORD="http://fusionauth:9011" + - FUSIONAUTH_OLD_BASE_URL"="http://fusionauth:9011" + - GUPSHUP_BASEURL="http://enterprise.smsgupshup.com/GatewayAPI/rest" + - GUPSHUP_OTP_TEMPLATE=${GUPSHUP_OTP_TEMPLATE} + - GUPSHUP_PASSWORD=${GUPSHUP_PASSWORD} + - GUPSHUP_USERNAME=${GUPSHUP_USERNAME} + - OTP_EXPIRY=${OTP_EXPIRY:-30} + - APP_${FUSIONAUTH_APP_ID}="{"host":"http://fusionauth:9011","salt":"${FUSIONAUTH_SALT}","apiKey":"${FUSIONAUTH_API_KEY}","encryption":{"enabled":false}}" diff --git a/bhasai/user_service/sample.env b/bhasai/user_service/sample.env new file mode 100644 index 0000000..a4922af --- /dev/null +++ b/bhasai/user_service/sample.env @@ -0,0 +1,14 @@ +ADMIN_API_KEY= +ALLOW_DEFAULT_OTP= +DEFAULT_OTP= +DEFAULT_OTP_USERS= +DEFAULT_USER_PASSWORD= +GUPSHUP_OTP_TEMPLATE= +GUPSHUP_PASSWORD= +GUPSHUP_USERNAME= +OTP_EXPIRY= +#fusionauth application id converted to snake case +FUSIONAUTH_APP_ID= +#random string +FUSIONAUTH_SALT= +FUSIONAUTH_API_KEY= \ No newline at end of file diff --git a/bhasai/utility/README.md b/bhasai/utility/README.md new file mode 100644 index 0000000..3b903b3 --- /dev/null +++ b/bhasai/utility/README.md @@ -0,0 +1 @@ +# This folder has fake yamls which allows env processor to generate env which are skipped \ No newline at end of file diff --git a/bhasai/utility/docker-compose.yaml b/bhasai/utility/docker-compose.yaml new file mode 100644 index 0000000..fb3d006 --- /dev/null +++ b/bhasai/utility/docker-compose.yaml @@ -0,0 +1,8 @@ +services: + fake: + environment: + DEFAULT_GITHUB_BRANCH: ${DEFAULT_GITHUB_BRANCH:-dev} + DEFAULT_IMAGE_TAG: ${DEFAULT_IMAGE_TAG:-dev} + org: ${org:-bharatsahaiyak} + AI_SERVICES: ${AI_SERVICES:-'[]'} + GRAFANA_DISCORD_WEBHOOK_URL: ${GRAFANA_DISCORD_WEBHOOK_URL:-https://} \ No newline at end of file diff --git a/bhasai/vault/Dockerfile b/bhasai/vault/Dockerfile new file mode 100644 index 0000000..07b9f34 --- /dev/null +++ b/bhasai/vault/Dockerfile @@ -0,0 +1,2 @@ +FROM vault:1.13.3 +RUN apk add bash jq \ No newline at end of file diff --git a/bhasai/vault/admin-policy.hcl b/bhasai/vault/admin-policy.hcl new file mode 100644 index 0000000..3609b94 --- /dev/null +++ b/bhasai/vault/admin-policy.hcl @@ -0,0 +1,59 @@ +# Read system health check +path "sys/health" +{ + capabilities = ["read", "sudo"] +} + +# Create and manage ACL policies broadly across Vault + +# List existing policies +path "sys/policies/acl" +{ + capabilities = ["list"] +} + +# Create and manage ACL policies +path "sys/policies/acl/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Enable and manage authentication methods broadly across Vault + +# Manage auth methods broadly across Vault +path "auth/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Create, update, and delete auth methods +path "sys/auth/*" +{ + capabilities = ["create", "update", "delete", "sudo"] +} + +# List auth methods +path "sys/auth" +{ + capabilities = ["read"] +} + +# Enable and manage the key/value secrets engine at `secret/` path + +# List, create, update, and delete key/value secrets +path "kv/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Manage secrets engines +path "sys/mounts/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# List existing secrets engines. +path "sys/mounts" +{ + capabilities = ["read"] +} \ No newline at end of file diff --git a/bhasai/vault/docker-compose.yaml b/bhasai/vault/docker-compose.yaml new file mode 100644 index 0000000..bed34f7 --- /dev/null +++ b/bhasai/vault/docker-compose.yaml @@ -0,0 +1,22 @@ +volumes: + vault: + +services: + vault: + build: + context: . + dockerfile: Dockerfile + restart: always + environment: + VAULT_USERNAME: ${VAULT_USERNAME:?VAULT_USERNAME is not set} + VAULT_PASSWORD: ${VAULT_PASSWORD:?VAULT_PASSWORD is not set} + volumes: + - vault:/vault/file + - ./vault.json:/vault/config/vault.json + - ./admin-policy.hcl:/vault/policies/admin-policy.hcl + - ./vault-init.sh:/vault-init.sh + - ../../.env:/bhasai/.env + - ./:/bhasai/ + cap_add: + - IPC_LOCK + command: sh -c "bash vault-init.sh" diff --git a/bhasai/vault/vault-init.sh b/bhasai/vault/vault-init.sh new file mode 100644 index 0000000..f268257 --- /dev/null +++ b/bhasai/vault/vault-init.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -me + +if [[ -z $VAULT_USERNAME && -z $VAULT_PASSWORD ]] ; then + echo "VAULT_USERNAME and VAULT_PASSWORD are not set" + exit 1 +fi + +vault server -config=/vault/config/vault.json & + +sleep 2 + +export VAULT_ADDR="http://127.0.0.1:8200" +VAULT_HEALTH_URL="$VAULT_ADDR/v1/sys/seal-status" +while true; do + response=$(wget -qO- "$VAULT_HEALTH_URL") + + # Check the exit status of wget to handle errors + if [ $? -ne 0 ]; then + echo "Waiting for Vault to be ready" + else + initialized=$(echo "$response" | jq -r '.initialized') + sealed=$(echo "$response" | jq -r '.sealed') + echo "Vault Status - Initialized: $initialized, Sealed: $sealed" + break + fi + + sleep 2 +done + +if [ "$initialized" = false ]; then +vault operator init > /vault/file/generated_keys.txt +fi +if [ "$sealed" = true ]; then +# Parse unsealed keys +while IFS= read -r line; do + key=$(echo "$line" | cut -c15-) + if [ -n "$key" ]; then + vault operator unseal "$key" + fi +done < <(grep "Unseal Key " /vault/file/generated_keys.txt) +fi + + +# Get root token +VAULT_TOKEN=$(grep "Initial Root Token: " < /vault/file/generated_keys.txt | cut -c21- ) +export VAULT_TOKEN + +if vault secrets list | grep -q "kv/"; then + echo "KV version 2 secrets engine is already enabled." +else + vault secrets enable -version=2 kv +fi + +if ! vault auth list | grep -q "userpass/"; then + vault auth enable userpass +else + echo "userpass authentication method is already enabled." +fi + +if ! vault policy list | grep -q "admin-policy"; then + + vault policy write admin-policy /vault/policies/admin-policy.hcl +else + echo "admin-policy already exists." +fi + + +if ! vault read auth/userpass/users/"$VAULT_USERNAME" > /dev/null 2>&1; then + vault write auth/userpass/users/"$VAULT_USERNAME" \ + password="$VAULT_PASSWORD" \ + policies=admin-policy +else + echo "User $VAULT_USERNAME already exists." +fi + +fg %1 diff --git a/bhasai/vault/vault.json b/bhasai/vault/vault.json new file mode 100644 index 0000000..0280b2a --- /dev/null +++ b/bhasai/vault/vault.json @@ -0,0 +1,14 @@ +{ +"backend": { + "file": { + "path": "/vault/file" + } + }, + "listener": { + "tcp": { + "address": "0.0.0.0:8200", + "tls_disable": 1 + } + }, + "ui": true +} \ No newline at end of file diff --git a/bhasai/yt_parser/Caddyfile b/bhasai/yt_parser/Caddyfile new file mode 100644 index 0000000..9667bfc --- /dev/null +++ b/bhasai/yt_parser/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_NAME}://yt-parser.{$DOMAIN_NAME} { + reverse_proxy yt_parser:8000 +} \ No newline at end of file diff --git a/bhasai/yt_parser/Caddyfile.SDC b/bhasai/yt_parser/Caddyfile.SDC new file mode 100644 index 0000000..6d0de12 --- /dev/null +++ b/bhasai/yt_parser/Caddyfile.SDC @@ -0,0 +1,3 @@ +handle_path /yt-parser* { + reverse_proxy yt_parser:8000 +} diff --git a/bhasai/yt_parser/docker-compose.SDC.yaml b/bhasai/yt_parser/docker-compose.SDC.yaml new file mode 100644 index 0000000..19a02f1 --- /dev/null +++ b/bhasai/yt_parser/docker-compose.SDC.yaml @@ -0,0 +1,51 @@ +services: + yt_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: uvicorn main:app --host 0.0.0.0 --port 8000 + restart: always + mem_limit: ${YT_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + + yt_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: celery -A worker worker -P gevent --loglevel=info + restart: always + mem_limit: ${YT_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + \ No newline at end of file diff --git a/bhasai/yt_parser/docker-compose.yaml b/bhasai/yt_parser/docker-compose.yaml new file mode 100644 index 0000000..19a02f1 --- /dev/null +++ b/bhasai/yt_parser/docker-compose.yaml @@ -0,0 +1,51 @@ +services: + yt_parser: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: uvicorn main:app --host 0.0.0.0 --port 8000 + restart: always + mem_limit: ${YT_PARSER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + + yt_parser_celery_worker: + image: ${DOCKER_REGISTRY_URL:-ghcr.io}/${org}/yt-parser:${YT_PARSER_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: celery -A worker worker -P gevent --loglevel=info + restart: always + mem_limit: ${YT_PARSER_CELERY_WORKER_MEM_LIMIT:-${DEFAULT_MEM_LIMIT:-256m}} + cpus: ${YT_PARSER_CELERY_WORKER_CPU_LIMIT:-${DEFAULT_CPU_LIMIT:-0.5}} + environment: + CELERY_BROKER_URL: redis://redis:6379/3 + CELERY_RESULT_BACKEND: redis://redis:6379/3 + WEBHOOK_URL: http://bff:3000/document/processed-webhook/youtube + MINIO_BASE_URL: "minio:9000" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_BUCKET: yt_parser + MINIO_SECURE_CONN: "False" + REDIS_CONN: redis://redis:6379/3 + BHASHINI_USERID: ${ULCA_USER_ID} + BHASHINI_ULCA_API_KEY: ${ULCA_API_KEY} + AZURE_SUBSCRIPTION_KEY: ${AZURE_SUBSCRIPTION_KEY} + DISCORD_WEBHOOK_URL: ${DISCORD_WEBHOOK_URL} + ASR_SERVICE_PRIORITY: ${ASR_SERVICE_PRIORITY} + YOUTUBE_DATA_API_KEY: ${YOUTUBE_DATA_API_KEY} + depends_on: + - redis + \ No newline at end of file diff --git a/bhasai/yt_parser/sample.env b/bhasai/yt_parser/sample.env new file mode 100644 index 0000000..02f029f --- /dev/null +++ b/bhasai/yt_parser/sample.env @@ -0,0 +1,19 @@ +#yt_parser +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +ULCA_API_KEY= +ULCA_USER_ID= +AZURE_SUBSCRIPTION_KEY= +DISCORD_WEBHOOK_URL= +ASR_SERVICE_PRIORITY= +YOUTUBE_DATA_API_KEY= + +#yt_parser_celery_worker +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +ULCA_USER_ID= +ULCA_API_KEY= +AZURE_SUBSCRIPTION_KEY= +DISCORD_WEBHOOK_URL= +ASR_SERVICE_PRIORITY= +YOUTUBE_DATA_API_KEY= diff --git a/docker-compose.SDC.yaml b/docker-compose.SDC.yaml new file mode 100644 index 0000000..6cfd58e --- /dev/null +++ b/docker-compose.SDC.yaml @@ -0,0 +1,28 @@ +include: + - ./common/monitoring/docker-compose.SDC.yaml + - ./common/minio/docker-compose.yaml +# - ./common/environment/docker-compose.yaml + - ./common/fusionauth/docker-compose.yaml + - ./common/uptime/docker-compose.yaml + - ./common/clickhouse/docker-compose.yaml + - ./common/redis/docker-compose.yaml + - ./common/db/docker-compose.yaml +# - ./common/registry/docker-compose.yaml + - ./bhasai/docker-compose.SDC.yaml + - ./common/superset/docker-compose.yaml + + + +services: + caddy: + extends: + file: common/caddy/docker-compose.SDC.yaml + service: caddy + volumes: + - ./bhasai:/etc/caddy/bhasai +# - ./certs:/certs + + +volumes: + caddy_data: + caddy_config: diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..7c30728 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,28 @@ +include: + - ./common/monitoring/docker-compose.yaml + - ./common/minio/docker-compose.yaml + - ./common/environment/docker-compose.yaml + - ./common/fusionauth/docker-compose.yaml + - ./common/uptime/docker-compose.yaml + - ./common/clickhouse/docker-compose.yaml + - ./common/redis/docker-compose.yaml + - ./common/db/docker-compose.yaml +# - ./common/registry/docker-compose.yaml + - ./bhasai/docker-compose.yaml + - ./common/superset/docker-compose.yaml + + + +services: + caddy: + extends: + file: common/caddy/docker-compose.yaml + service: caddy + volumes: + - ./bhasai:/etc/caddy/bhasai +# - project-specific-volume-mappings:/etc/caddy/project-name + + +volumes: + caddy_data: + caddy_config: diff --git a/docs/no-internet-deployment.md b/docs/no-internet-deployment.md new file mode 100644 index 0000000..16189fd --- /dev/null +++ b/docs/no-internet-deployment.md @@ -0,0 +1,274 @@ +## For installing yq : + +On machine with internet access(say Machine1): + +1. `sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64` +2. `scp -i "key" yq_linux_amd64 ubuntu@:/tmp/yq_linux_amd64` + +On machine with no internet access(say Machine2): + +1. `sudo mv /tmp/yq_linux_amd64 /usr/bin/yq` +2. `sudo chmod +x /usr/bin/yq` +3. `yq --version` + +## For jq installation: (Not tested yet): +1. `sudo wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64` +2. `cp -i "key" jq_linux_amd64 ubuntu@:/tmp/jq_linux_amd64` + +On machine with no internet access(say Machine2): +1. `sudo mv /tmp/jq_linux_amd64 /usr/bin/jq` +2. `sudo chmod +x /usr/bin/jq` +3. `jq --version` + +## Installing build-essential : + +**Step 1: Download the Packages on a Machine with Internet Access** +1. On a machine with internet access, update your package list: `sudo apt-get update` +2. Download the `build-essential` package and its dependencies using `apt-get` with the `--download-only` option: + `sudo apt-get install --download-only build-essential + ` + This will download all the necessary `.deb` files to your system's cache (usually in `/var/cache/apt/archives`). +3. Collect the downloaded `.deb` files: + * `mkdir -p ~/build-essential-packages` + * `sudo cp /var/cache/apt/archives/*.deb ~/build-essential-packages/` + This will copy all the `.deb` files into the `~/build-essential-packages` directory. +4. Compress the folder containing the `.deb` files to make it easier to transfer: + `tar -czvf build-essential-packages.tar.gz -C ~/build-essential-packages . + ` + +**Step 2: Transfer the Packages to the Server** +1. Use `scp` to transfer the compressed file to your server (replace `user@server_ip:/path/to/destination` with your server details): + `scp build-essential-packages.tar.gz user@server_ip:/path/to/destination` +2. SSH into your server and navigate to the destination directory: + * `ssh user@server_ip` + * `cd /path/to/destination` +3. Extract the compressed file: + `tar -xzvf build-essential-packages.tar.gz` + +**Step 3: Install the Packages on the Server** +1. Navigate to the directory containing the `.deb` files: + `cd build-essential-packages` +2. Install all the packages using `dpkg`: + `sudo dpkg -i *.deb` +3. If there are any missing dependencies, fix them by running: + `sudo apt-get install -f` + +If your system does not have internet access and you run apt-get install -f, it might not be able to fix the broken dependencies unless you have already downloaded the necessary packages and dependencies to your local cache or have them available in a local repository. In such cases, you would need to manually download the required packages and install them. + +4. Add the current user in docker group: +`usermod -aG docker ${USER}` + +## Docker Package Installation + +**Overview :** +- **Download**: Use a machine with internet access to download Docker and its dependencies. +- **Transfer**: Move the files to the server using `scp`. +- **Install**: Use `dpkg` on the server to install the packages. + +### Step 1: Download the Packages on a Machine with Internet Access** + +1. **Set up Docker's apt repository:** + + On a machine with internet access, update your package list and set up Docker's apt repository: + + `sudo apt-get update` + `sudo apt-get install -y ca-certificates curl` + `sudo install -m 0755 -d /etc/apt/keyrings` + `sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc` + `sudo chmod a+r /etc/apt/keyrings/docker.asc` + +2. **Add Docker's repository to your apt sources:** + ```bash + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + ``` +3. **Download the Docker packages and their dependencies:** + `mkdir -p ~/docker-packages` + `cd ~/docker-packages` + `sudo apt-get download docker-ce docker-ce-cli containerd.io` + `docker-buildx-plugin` + `docker-compose-plugin` + + This will download all the necessary `.deb` files into the `~/docker-packages` directory. +4. **Compress the folder containing the `.deb` files for easy transfer:** + `tar -czvf docker-packages.tar.gz -C ~/ docker-packages` + +### Step 2: Transfer the Packages to the Server + +1. **Use `scp` to transfer the compressed file to your server:** + Replace `@:~/path/to/destination` with your server details: + `scp -i "/path/to/key" docker-packages.tar.gz @:~/path/to/destination` +2. **SSH into your server and navigate to the destination directory:** + `ssh -i "/path/to/key" @` + `cd ~/path/to/destination` +3. **Extract the compressed file:** + `tar -xzvf docker-packages.tar.gz` + +### Step 3: Install the Docker Packages on the Server + +1. **Navigate to the directory containing the `.deb` files:** + `cd docker-packages` +2. **Install all the packages using `dpkg`:** + `sudo dpkg -i *.deb` + 3. **Fix any missing dependencies:** (Requires Internet) + `sudo apt-get install -f` + + This will install any missing dependencies from the `.deb` files that are already present in the directory. + + +## How to Use a Docker Image Offline. + +## Overview : +To use a Docker image offline, pull the `hello-world` image on a connected machine and save it as `hello-world.tar`. Transfer this file to the offline server, load it with `docker load -i hello-world.tar`, and then run it using `docker run hello-world`. + +**1. Pre-download the `hello-world` Image** +* On a machine with internet access, pull the `hello-world` Docker image: +`docker pull hello-world` + +**2. Save the Image** +Save the pulled `hello-world` image as a tar file: +* `docker save -o hello-world.tar hello-world` + +**3. Transfer the Image to the Offline Server** +* `scp hello-world.tar user@your-server:/path/to/save` + +**4. Load the Image on the Offline Server** +* `docker load -i /path/to/save/hello-world.tar` + +**5. Verify the Image** +* `docker images` +You should see the `hello-world` image listed. + +**6. Use the Image** +* `docker run hello-world` + + +## Installing GPU packages: +To download all the necessary NVIDIA CUDA drivers, toolkit, and Docker-related packages into a separate folder named `nvidia_packages` on the machine with internet access, you can use the following script: + +### Step 1: Download the Packages to `nvidia_packages` Folder + +```bash +#!/bin/bash + +# Create a directory to store all downloaded packages +mkdir -p ~/nvidia_packages +cd ~/nvidia_packages + +# Check if an NVIDIA GPU is present +if lspci | grep -i nvidia &>/dev/null; then + echo "NVIDIA GPU found." + echo "Downloading NVIDIA CUDA drivers and Toolkit" + export distro="$(lsb_release -is | tr '[:upper:]' '[:lower:]')$(lsb_release -rs | tr -d '.')" + export arch=$(uname -m) + + # Download the CUDA keyring + wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.1-1_all.deb + + # Add NVIDIA repo and download CUDA drivers + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt-get -qq update + apt-get download $(apt-cache depends cuda-drivers | grep Depends | sed "s/.*ends:\ //" | tr '\n' ' ') + + echo "Downloading NVIDIA container toolkit" + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + + sudo apt-get -qq update + apt-get download nvidia-container-toolkit nvidia-docker2 + + echo "Downloading Docker-related packages" + apt-get download jq docker.io + + echo "All NVIDIA and Docker packages downloaded to ~/nvidia_packages" +else + echo "No NVIDIA GPU found. Skipping NVIDIA package download." +fi + +``` + +### Step 2: Transfer the `nvidia_packages` Folder + +1. **Transfer the `nvidia_packages` Folder via `scp`:** + + + `scp -i "key" -r ~/nvidia_packages user@offline-machine:/path/to/destination` + +### Step 3: Install the Packages on the Offline Machine + +1. **Install the Packages on the Offline Machine:** + + On the offline machine, navigate to the directory where you transferred the `nvidia_packages` folder and install all the `.deb` packages: + + `cd /path/to/destination/nvidia_packages` + `sudo dpkg -i *.deb` + `sudo apt-get install -f` + +2. **Restart the machine** + + +## Setting up Webhook : + +Given that you already have the necessary template files (`hooks.json.template`, `webhook.service.template`) and the `.env` file on the offline machine, the primary task is to install the `webhook` binary and then run your setup script. Here’s how to proceed: + +### Step 1: Prepare the `webhook` Binary on the Internet-Connected Machine + +1. **Download the Webhook Binary:** + + On the machine with internet access, download the `webhook` binary: + + ```bash + mkdir -p ~/webhook_setup + cd ~/webhook_setup + curl -sSL https://github.com/adnanh/webhook/releases/download/2.8.1/webhook-linux-amd64.tar.gz -o webhook-linux-amd64.tar.gz + tar -zxvf webhook-linux-amd64.tar.gz --strip-components=1 --directory . + ``` + +2. **Transfer the Binary to the Offline Machine:** + + Transfer the `webhook` binary to the offline machine using `scp`: + + ```bash + scp ~/webhook_setup/webhook user@offline-machine:/path/to/destination + ``` + +### Step 2: Install the `webhook` Binary on the Offline Machine + +1. **Move the Binary to `/usr/local/bin`:** + + On the offline machine, move the `webhook` binary to `/usr/local/bin` and make it executable: + + ```bash + sudo mv /path/to/destination/webhook /usr/local/bin/ + sudo chmod +x /usr/local/bin/webhook + ``` + +### Step 3: Run the Setup Script on the Offline Machine + +1. **Ensure the `.env` File is Correct:** + + Verify that your `.env` file has the correct values for `WEBHOOK_PASSWORD`, `WEBHOOK_USER`, and `WEBHOOK_GROUP`. + +2. **Run the Setup Script:** + + Navigate to the directory where your setup script is located and run it: + + ```bash + cd /path/to/setup_script + ./setup_webhook.sh + ``` + +### Step 4: Verify the Setup + +1. **Check the Webhook Service Status:** + + After running the setup script, ensure that the webhook service is running: + + ```bash + sudo systemctl status webhook.service + ``` \ No newline at end of file diff --git a/sample.env b/sample.env new file mode 100644 index 0000000..5e972ce --- /dev/null +++ b/sample.env @@ -0,0 +1,91 @@ +AI_SERVICES=''[]'' +AI_TOOLS_EMBEDDING_BATCH_SIZE='100' +ALLOW_DEFAULT_OTP='false' +CELERY_MAX_RETRIES='3' +CLICKHOUSE_DB='default' +CLICKHOUSE_PASSWORD='' +CLICKHOUSE_USER='default' +DEFAULT_OTP='1234' +DEFAULT_OTP_USERS='[]' +DEFAULT_RETRY_COUNT='3' +DEFAULT_TIMEOUT='30000' +DOCKER_REGISTRY_URL='ghcr.io' +DOMAIN_NAME='localdev.me' +DOMAIN_SCHEME='http' +DOMAIN_SCHEME='https' +ES_JAVA_OPTS='-Xms2048m -Xmx2048m -XX:InitiatingHeapOccupancyPercent=75 -Djava.awt.headless=true' +FUSIONAUTH_APP_RUNTIME_MODE='development' +GF_SECURITY_ADMIN_USER='admin' +GF_SERVER_SERVE_FROM_SUB_PATH='false' +GRAFANA_DISCORD_WEBHOOK_URL='https://' +LOKI_TABLE_INDEX_PERIOD='24h' +NEXT_PUBLIC_DEBUG='false' +NEXT_PUBLIC_SHOW_ONBOARDING='false' +OTP_EXPIRY='30' +PARALLEL_FACTOR='2' +PDF_PARSER_MAX_RETRIES='3' +SPLIT='5' +TORCH_DEVICE='cuda' +TRANSLATION_SERVICES='["azure", "bhasini"]' +env='dev' +org='bharatsahaiyak' +ADMIN_API_KEY= +ASR_SERVICE_PRIORITY= +AZURE_SUBSCRIPTION_KEY= +AZURE_TRANSLATE_KEY= +BHASHINI_API_KEY= +DEFAULT_USER_PASSWORD= +DEPLOYER_APP_ENV= +DEPLOYER_PWA_REPO= +DEPLOYER_VERCEL_TOKEN= +DISCORD_WEBHOOK_URL= +DJANGO_SECRET_KEY= +ENABLE_IMAGE_PARSING= +ENABLE_SUMMARIZE= +ENABLE_TRANSLATE= +ENVIRONMENT_PASSWORD= +ENVIRONMENT_USERNAME= +FUSIONAUTH_ADMIN_EMAIL= +FUSIONAUTH_ADMIN_PASSWORD= +FUSIONAUTH_API_KEY= +FUSIONAUTH_APP_ID= +FUSIONAUTH_KEY= +FUSIONAUTH_POSTGRES_PASSWORD= +FUSIONAUTH_POSTGRES_ROOT_PASSWORD= +FUSIONAUTH_POSTGRES_ROOT_USERNAME= +FUSIONAUTH_POSTGRES_USERNAME= +FUSIONAUTH_SALT= +GF_SECURITY_ADMIN_PASSWORD= +GITHUB_PAT= +GIT_PASSWORD= +GIT_USERNAME= +GUPSHUP_OTP_TEMPLATE= +GUPSHUP_PASSWORD= +GUPSHUP_USERNAME= +HUGGING_FACE_TOKEN= +HUGGING_FACE_USERNAME= +MINIO_ACCESS_KEY= +MINIO_ROOT_PASSWORD= +MINIO_ROOT_USER= +MINIO_SECRET_KEY= +NEXT_PUBLIC_BOT_ID= +NEXT_PUBLIC_BOT_NAME= +NEXT_PUBLIC_CLARITY_PROJECT_ID= +NEXT_PUBLIC_ENTITY_DATASET_ID= +NEXT_PUBLIC_GOOGLE_KEY= +NEXT_PUBLIC_ORG_ID= +NEXT_PUBLIC_USER_SERVICE_APP_ID= +NEXT_PUBLIC_WEATHER_API= +OPENAI_API_KEY= +PDF_PARSER_SECRET_KEY= +POSTGRES_PASSWORD= +POSTGRES_USER= +TLS_ENTRY= +TRANSPORT_SOCKET_DEBUG= +UCI_ADMIN_TOKEN= +ULCA_API_KEY= +ULCA_CONFIG_URL= +ULCA_USER_ID= +VAULT_PASSWORD= +VAULT_USERNAME= +YOUTUBE_DATA_API_KEY= diff --git a/scripts/generate-env.sh b/scripts/generate-env.sh new file mode 100755 index 0000000..a855e80 --- /dev/null +++ b/scripts/generate-env.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Path to the sample.env file +sample_env_file="sample.env" + +# Path to the generated .env file +generated_env_file=".env" + +# Check if the .env file already exists +if [ -f "$generated_env_file" ]; then + echo ".env file already exists. Generation aborted." + exit 1 +fi + +# Read the sample.env file line by line +while IFS= read -r line; do + # Check if the line contains the pattern "random-generate-" + if [[ $line == *"random-generate-lower"* ]]; then + # Extract the variable name + variable_name=$(echo "$line" | cut -d '=' -f 1) + + # Generate a random value with the specified number of digits + random_value=$(tr -dc a-z0-9 > "$OUTPUT_FILE" +} + + +# Iterate over each base directory +for BASE_DIRECTORY in "${BASE_DIRECTORIES[@]}"; do + # Check if the base directory exists + if [[ ! -d "$BASE_DIRECTORY" ]]; then + echo "Directory $BASE_DIRECTORY does not exist." + continue + fi + + # Find all matching files in the base directory and its subdirectories + find "$BASE_DIRECTORY" -type f -name 'docker-compose*yaml' | while read -r file; do + extract_env_vars "$file" + done +done + +# Check if OUTPUT_FILE has content +if [[ -s "$OUTPUT_FILE" ]]; then + # Remove duplicate entries and sort + sort -u "$OUTPUT_FILE" -o "$OUTPUT_FILE" + + # Sort lines with '=' to the top and lines without '=' to the bottom +(grep '=' "$OUTPUT_FILE"; grep -v '=' "$OUTPUT_FILE") > temp_output && mv temp_output "$OUTPUT_FILE" + + echo "Environment variables have been extracted to $OUTPUT_FILE" +else + echo "No environment variables found in the specified files." +fi + +sed -E -i "s/^([A-Za-z0-9_]+)=([^\n]*)$/\1='\2'/; s/^([A-Za-z0-9_]+)$/\1=/" $OUTPUT_FILE + +awk -F= '!seen[$1]++ || $2 != ""' $OUTPUT_FILE > temp_file && mv temp_file $OUTPUT_FILE