diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42ae49fb9..4b5caec45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,18 +6,21 @@ on: workflow_dispatch: + schedule: + - cron: '0 7 * * 0' # Runs every Sunday at 7:00 AM UTC (which is 9:00 AM TLV) + jobs: test_main: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Use Node.js 16 + - name: Use Node.js 20 uses: actions/setup-node@v2 with: - node-version: lts/* + node-version: 20 - name: Restore Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: | node_modules @@ -39,13 +42,12 @@ jobs: matrix: database: [ "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9", - "spanner", "mysql", "mysql5", - "mssql", "mssql17", + "mssql", + spanner, "mongo", "mongo4", - "firestore", "dynamodb", - "google-sheets" + "firestore" ] env: @@ -54,19 +56,19 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js 16 + - name: Use Node.js 20 uses: actions/setup-node@v2 with: - node-version: lts/* + node-version: 20 - name: Restore Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: | node_modules */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - + - name: Downloading Image if: ${{ ( matrix.database != 'google-sheets') }} run: | diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5b01d8994..013182920 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -6,22 +6,36 @@ on: gcp: type: boolean description: GCP container registry - default: true + default: false aws: type: boolean description: AWS ECR container registry - default: true + default: false dockerHub: type: boolean description: Docker container registry - default: true + default: false + + image-tag: + type: string + description: Image tag + default: latest + + container-registry-folder: + type: choice + description: Container registry folder + options: + - velo-manged-external-db + - velo-manged-external-db-test + - velo-external-db + - velo-external-db-testing jobs: Build-and-push: runs-on: ubuntu-latest - + env: ECR_REPOSITORY: ${{ secrets.AWS_REPO_NAME }} PUBLIC_ECR_URL: ${{ secrets.AWS_REPO_URL }} @@ -29,12 +43,12 @@ jobs: DOCKERHUB_REPOSITORY: ${{ secrets.DOCKERHUB_REPOSITORY }} GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - CONTAINER_REGISTRY_FOLDER: ${{ secrets.GCP_CONTAINER_REGISTRY_FOLDER }} - IMAGE_TAG: latest + IMAGE_TAG: ${{ github.event.inputs.image-tag }} + CONTAINER_REGISTRY_FOLDER: ${{ github.event.inputs.container-registry-folder }} steps: - name: Git checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Build docker image - name: Build Docker image @@ -56,10 +70,20 @@ jobs: if: github.event.inputs.aws == 'true' id: aws-push-image run: | - docker tag $IMAGE_NAME:$IMAGE_TAG $PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG + # Setting image tag to image tag input + docker tag $IMAGE_NAME $PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG + # Setting image to with github branch name + docker tag $IMAGE_NAME $PUBLIC_ECR_URL/$IMAGE_NAME:branch-${GITHUB_REF##*/} + # Setting image to with github sha + docker tag $IMAGE_NAME $PUBLIC_ECR_URL/$IMAGE_NAME:sha-${GITHUB_SHA} + echo "Pushing image to ECR..." - docker push $PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG + docker push $PUBLIC_ECR_URL/$IMAGE_NAME --all-tags + + echo "Pushing image to ECR with following tags:" echo "::set-output name=image::$PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG" + echo "::set-output name=image::$PUBLIC_ECR_URL/$IMAGE_NAME:branch-${GITHUB_REF##*/}" + echo "::set-output name=image::$PUBLIC_ECR_URL/$IMAGE_NAME:sha-${GITHUB_SHA}" - name: Login to Docker Hub if: github.event.inputs.dockerHub == 'true' @@ -67,35 +91,63 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Tag and push the image to Docker Hub if: github.event.inputs.dockerHub == 'true' id: dockerhub-push-image run: | + # Setting image tag to image tag input docker tag $IMAGE_NAME $DOCKERHUB_REPOSITORY/$IMAGE_NAME:$IMAGE_TAG + # Setting image to with github branch name + docker tag $IMAGE_NAME $DOCKERHUB_REPOSITORY/$IMAGE_NAME:branch-${GITHUB_REF##*/} + # Setting image to with github sha + docker tag $IMAGE_NAME $DOCKERHUB_REPOSITORY/$IMAGE_NAME:sha-${GITHUB_SHA} + echo "Pushing image to Docker Hub..." - docker push $DOCKERHUB_REPOSITORY/$IMAGE_NAME:$IMAGE_TAG + docker push $DOCKERHUB_REPOSITORY/$IMAGE_NAME --all-tags + + echo "Pushing image to Docker Hub with following tags:" echo "::set-output name=image::$DOCKERHUB_REPOSITORY/$IMAGE_NAME:$IMAGE_TAG" + echo "::set-output name=image::$DOCKERHUB_REPOSITORY/$IMAGE_NAME:branch-${GITHUB_REF##*/}" + echo "::set-output name=image::$DOCKERHUB_REPOSITORY/$IMAGE_NAME:sha-${GITHUB_SHA}" # Authenticate to Google Cloud - id: auth name: Authenticate to Google Cloud if: github.event.inputs.gcp == 'true' - uses: 'google-github-actions/auth@v0' + uses: 'google-github-actions/auth@v2' with: credentials_json: '${{ secrets.GCP_SA_KEY }}' # Set up Cloud SDK - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v2 if: github.event.inputs.gcp == 'true' with: project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Docker Auth + if: github.event.inputs.gcp == 'true' + run: |- + gcloud auth configure-docker us-docker.pkg.dev --quiet + # Push image to Google Container Registry - name: Tag and push image to GCP container registry if: github.event.inputs.gcp == 'true' run: | gcloud auth configure-docker -q - docker tag $IMAGE_NAME:$IMAGE_TAG gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER - docker push gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:$IMAGE_TAG + + # Setting image tag to image tag input + docker tag $IMAGE_NAME gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:$IMAGE_TAG + # Setting image to with github branch name + docker tag $IMAGE_NAME gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:branch-${GITHUB_REF##*/} + # Setting image to with github sha + docker tag $IMAGE_NAME gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:sha-${GITHUB_SHA} + + echo "Pushing image to GCP container registry..." + docker push gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER --all-tags + + echo "Pushing image to GCP container registry with following tags:" + echo "::set-output name=image::gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:$IMAGE_TAG" + echo "::set-output name=image::gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:branch-${GITHUB_REF##*/}" + echo "::set-output name=image::gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:sha-${GITHUB_SHA}" diff --git a/Dockerfile b/Dockerfile index c9120d0da..fa188d10c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Compile stage -FROM node:lts-slim +FROM node:20-slim WORKDIR /usr/lib/app # Copy the project dir @@ -13,7 +13,7 @@ RUN npm install RUN nx run-many --target=build --all -FROM node:lts-alpine +FROM node:20-alpine WORKDIR /usr/lib/app diff --git a/apps/velo-external-db/test/resources/docker-compose.yaml b/apps/velo-external-db/test/resources/docker-compose.yaml index f472f6355..bbeca596b 100644 --- a/apps/velo-external-db/test/resources/docker-compose.yaml +++ b/apps/velo-external-db/test/resources/docker-compose.yaml @@ -126,7 +126,7 @@ services: POSTGRES_DB: test-db mysql: - image: mysql:8 + image: mysql:8.0 command: --default-authentication-plugin=mysql_native_password restart: always environment: diff --git a/libs/external-db-mysql/tests/e2e-testkit/docker-compose.yaml b/libs/external-db-mysql/tests/e2e-testkit/docker-compose.yaml index 558361968..480fc57b0 100644 --- a/libs/external-db-mysql/tests/e2e-testkit/docker-compose.yaml +++ b/libs/external-db-mysql/tests/e2e-testkit/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: mysql: - image: mysql:8 + image: mysql:8.0 command: --default-authentication-plugin=mysql_native_password restart: always environment: diff --git a/libs/velo-external-db-core/src/converters/data_utils.ts b/libs/velo-external-db-core/src/converters/data_utils.ts index cef555b7c..87eb6b55c 100644 --- a/libs/velo-external-db-core/src/converters/data_utils.ts +++ b/libs/velo-external-db-core/src/converters/data_utils.ts @@ -6,12 +6,19 @@ export const asWixData = (item: Item) => { return generateIdsIfNeeded(packDates(item)) } +const replaceNonAlphanumeric = (str: string) => { + // Replace non-alphanumeric characters with dashes + return str.replace(/[^a-zA-Z0-9]/g, '-') +} + export const generateIdsIfNeeded = (item: Item): ItemWithId => { if ('_id' in item) return item as ItemWithId const sha = crypto.createHash('sha1') const fieldsConcat = Object.values(item).join('') - return { ...item, _id: sha.update(fieldsConcat).digest('base64') } + const base64Digest = sha.update(fieldsConcat).digest('base64') + const validId = replaceNonAlphanumeric(base64Digest) + return { ...item, _id: validId } } const packDates = (item: Item) => Object.entries(item) diff --git a/libs/velo-external-db-core/src/router.ts b/libs/velo-external-db-core/src/router.ts index b52e2f5df..dca5ad39f 100644 --- a/libs/velo-external-db-core/src/router.ts +++ b/libs/velo-external-db-core/src/router.ts @@ -79,11 +79,23 @@ const executeHook = async(hooks: DataHooks | SchemaHooks, _actionName: string, p export const createRouter = () => { const router = express.Router() - router.use(express.json()) + router.use(express.json({ limit: '4mb' })) + router.use(express.urlencoded({ limit: '4mb', extended: true })) router.use(compression()) router.use('/assets', express.static(path.join(__dirname, 'assets'))) router.use(unless(['/', '/provision', '/favicon.ico'], secretKeyAuthMiddleware({ secretKey: cfg.secretKey }))) + //set timeout of 1.5 minutes per request + router.use((req, res, next) => { + const NinetySecondsInMs = 90 * 1000 + res.setTimeout(NinetySecondsInMs, () => { + console.warn(`Request has timed out - ${req.method} ${req.url}`) + console.dir({ body: req.body }, { depth: 3 }) + res.status(408).send('Request Timeout') + }) + next() + }) + config.forEach(({ pathPrefix, roles }) => router.use(includes([pathPrefix], authRoleMiddleware({ roles })))) // *************** INFO ********************** @@ -201,7 +213,7 @@ export const createRouter = () => { const { collectionName } = req.body const customContext = {} const { items } = await executeDataHooksFor(DataActions.BeforeBulkUpdate, dataPayloadFor(BULK_UPDATE, req.body), requestContextFor(BULK_UPDATE, req.body), customContext) - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) + roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) const data = await schemaAwareDataService.bulkUpdate(collectionName, items) const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkUpdate, data, requestContextFor(BULK_UPDATE, req.body), customContext) diff --git a/package.json b/package.json index 9e513ed47..74431510e 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "google-spreadsheet": "^3.3.0", "moment": "^2.29.3", "mongodb": "^4.6.0", - "mssql": "^8.1.0", + "mssql": "^11.0.1", "mysql": "^2.18.1", "nested-property": "^4.0.0", "node-cache": "^5.1.2", @@ -78,13 +78,13 @@ "@types/chance": "^1.1.3", "@types/compression": "^1.7.2", "@types/ejs": "^3.1.1", - "@types/express": "4.17.13", + "@types/express": "^5.0.1", "@types/google-spreadsheet": "^3.3.0", "@types/jest": "^27.4.1", "@types/jest-when": "^3.5.0", "@types/mssql": "^8.0.2", "@types/mysql": "^2.15.21", - "@types/node": "^16.11.7", + "@types/node": "^22.14.1", "@types/pg": "8.6.5", "@types/rewire": "^2.5.28", "@types/sqlstring": "^2.3.0", @@ -107,8 +107,8 @@ "prettier": "^2.5.1", "rewire": "^6.0.0", "ts-jest": "27.1.4", - "ts-node": "9.1.1", - "typescript": "~4.6.2", + "ts-node": "^10.9.2", + "typescript": "^4.9.5", "uuid-validate": "^0.0.3" }, "lint-staged": {