diff --git a/src/server/api/constants/Constants.js b/src/server/api/constants/Constants.js index f8edff0..d8fc48e 100644 --- a/src/server/api/constants/Constants.js +++ b/src/server/api/constants/Constants.js @@ -1,6 +1,11 @@ const CC_SERVER_URL = 'https://index.commoncrawl.org/'; const RABBIT_MQ_URL = 'amqp://localhost:5672'; +const ELASTIC_LOCAL_URL = 'https://localhost:9200'; +const ELASTIC_HTTP_CRED = '/home/priyanshu/Downloads/ELK/primary-node/config/certs/http_ca.crt'; +const INDEX_NAME = 'openapi_definition'; const QUEUE_NAME = 'index-files-jobs'; +const ELASTIC_USERNAME = 'elastic'; +const ELASTIC_PASSWORD = 'pxnS=+fLwN-0j2z=iPgX'; const FILE_PATH = `${__dirname}/../../dist/data/output.txt`; const RESULTS_FILE_PATH = `${__dirname}/../../dist/results/results.txt`; const DIST_PATH = `${__dirname}/../../dist`; @@ -12,4 +17,5 @@ const SUB_DOMAIN_REGEX = /^(?:https?:\/\/)?api\.[^\/\s]+\.[^\/\s]+(?:\/[^\/\s]*) const KEYWORD_REGEX = /^(?:https?:\/\/)?[^\/\s]+(?:\/[^\/\s]+)*(?:\/(openapi|swagger))(?:\/[^\/\s]*)*$/gi; const BASE_URL = 'http://localhost:1337'; -module.exports = { CC_SERVER_URL, RABBIT_MQ_URL, QUEUE_NAME, FILE_PATH, CHUNK_SIZE, PROGRESS_BAR_WIDTH, URL_REGEX, API_DEFINITION_REGEX, SUB_DOMAIN_REGEX, KEYWORD_REGEX, RESULTS_FILE_PATH, DIST_PATH, BASE_URL }; +module.exports = { CC_SERVER_URL, RABBIT_MQ_URL, QUEUE_NAME, FILE_PATH, CHUNK_SIZE, PROGRESS_BAR_WIDTH, URL_REGEX, API_DEFINITION_REGEX, SUB_DOMAIN_REGEX, KEYWORD_REGEX, RESULTS_FILE_PATH, DIST_PATH, BASE_URL, ELASTIC_LOCAL_URL, + ELASTIC_HTTP_CRED, INDEX_NAME, ELASTIC_USERNAME, ELASTIC_PASSWORD }; diff --git a/src/server/api/controllers/IndexingController.js b/src/server/api/controllers/IndexingController.js new file mode 100644 index 0000000..a710801 --- /dev/null +++ b/src/server/api/controllers/IndexingController.js @@ -0,0 +1,66 @@ +const SwaggerParser = require('swagger-parser'); +const { fetchDefinitionsFromDBUtils } = require('../utils/DBUtils'); +const { + transformingDefinitionForIndexing, + checkForIndex, + removeIndex, +} = require('../utils/ElasticsearchUtils'); +const { INDEX_NAME } = require('../constants/Constants'); + +module.exports = { + /** + * Starts the process of indexing validated OpenAPI definitions. + * + * @function startIndexing + * @async + * @param {Object} req - The Express request object. + * @param {Object} res - The Express response object. + * @returns {Object} The response indicating the indexing status. + * @throws {400} If there are no OpenAPI definitions to fetch. + * @throws {500} If an internal server error occurs during indexing. + * + * @example + * // Request: + * // POST /start-indexing + * // + * // Response: + * // 200 OK + * // 'OpenAPI definitions have been indexed!' + */ + startIndexing: async function (req, res) { + try { + const client = sails.client; + const transformedResults = []; + const validatedAPIDefinitions = await fetchDefinitionsFromDBUtils(); + + if (validatedAPIDefinitions.length === 0) { + return res.badRequest('There are no OpenAPI definitions to fetch!'); + } + + for (const { url } of validatedAPIDefinitions) { + const document = await SwaggerParser.parse(url); + transformedResults.push( + transformingDefinitionForIndexing(document, url) + ); + } + + const isExists = await checkForIndex(client); + if (isExists) { + await removeIndex(client); + } + + await client.helpers.bulk({ + datasource: transformedResults, + onDocument() { + return { + index: { _index: INDEX_NAME }, + }; + }, + }); + + return res.send('OpenAPI definitions has been indexed!'); + } catch (error) { + res.serverError(error); + } + }, +}; diff --git a/src/server/api/controllers/SearchController.js b/src/server/api/controllers/SearchController.js new file mode 100644 index 0000000..505c032 --- /dev/null +++ b/src/server/api/controllers/SearchController.js @@ -0,0 +1,51 @@ +const { searchForQuery } = require('../utils/ElasticsearchUtils'); + +module.exports = { + /** + * Starts a search operation based on provided keywords. + * + * @function startSearching + * @async + * @param {Object} req - The Express request object. + * @param {Object} res - The Express response object. + * @returns {Object} The JSON response containing search results. + * @throws {400} If the query parameter is missing. + * @throws {500} If an internal server error occurs. + * + * @example + * // Request: + * // GET /search?q=authentication + * // + * // Response: + * // 200 OK + * // [ + * // { + * // '_id': '123', + * // '_score': 1.0, + * // '_source': { + * // 'url': 'https://api.example.com', + * // 'content': { + * // /* ... * / + * // } + * // } + * // }, + * // /* ... * / + * // ] + */ + startSearching: async function (req, res) { + try { + const client = sails.client; + const keywords = req.query.q; + + if (keywords === undefined) { + return res.badRequest('Query not found!'); + } + + const searchResults = await searchForQuery(client, keywords); + + return res.json(searchResults); + } catch (error) { + return res.serverError(error.message); + } + }, +}; diff --git a/src/server/api/utils/ConnectElasticSearchUtils.js b/src/server/api/utils/ConnectElasticSearchUtils.js new file mode 100644 index 0000000..c506626 --- /dev/null +++ b/src/server/api/utils/ConnectElasticSearchUtils.js @@ -0,0 +1,22 @@ +const fs = require('fs'); +const { Client } = require('@elastic/elasticsearch'); +const { ELASTIC_LOCAL_URL, ELASTIC_HTTP_CRED, ELASTIC_USERNAME, ELASTIC_PASSWORD } = require('../constants/Constants'); + +const client = new Client({ + node: ELASTIC_LOCAL_URL, + log: 'error', + auth: { + username: ELASTIC_USERNAME, + password: ELASTIC_PASSWORD + }, + tls: { + ca: fs.readFileSync(ELASTIC_HTTP_CRED), + rejectUnauthorized: false + } +}); + +module.exports = { + getClient: function() { + return client; + }, +}; diff --git a/src/server/api/utils/DBUtils.js b/src/server/api/utils/DBUtils.js index a2d58fc..4e83ad1 100644 --- a/src/server/api/utils/DBUtils.js +++ b/src/server/api/utils/DBUtils.js @@ -7,7 +7,7 @@ module.exports = { * @function cleanDB * @param {Object} model - The database model to clean (e.g., Waterline model). * @throws {Error} If there is an error during the database cleaning process, it will be thrown. - */ + */ cleanDB: async function (model) { try { await model.destroy({}); @@ -28,23 +28,23 @@ module.exports = { * Use 'aes' for ascending and 'des' for descending. * @returns {Promise} A Promise that resolves with an array of index files fetched from the database. * @throws {Error} If there is an error during the database query, it will be thrown. - */ + */ fetchIndexFilesFromDBUtils: async function (filter) { try { const orderMap = { - 'aes': 1, - 'des': -1 + aes: 1, + des: -1, }; const skp = filter.skip === undefined ? 0 : parseInt(filter.skip); - const lmt = filter.limit === undefined ? Infinity : parseInt(filter.limit); + const lmt = + filter.limit === undefined ? Infinity : parseInt(filter.limit); const srt = filter.sort === undefined ? 1 : orderMap[filter.sort]; return await IndexFilesModel.find({}) .sort({ URL: srt }) .skip(skp) .limit(lmt); - } catch (error) { throw error; } @@ -57,11 +57,11 @@ module.exports = { * @param {Array} indexFiles - An array of index file URLs to be saved to the database. * @returns {Promise} A Promise that resolves when all index file URLs have been saved to the database. * @throws {Error} If there is an error during the database insertion process, it will be thrown. - */ + */ saveIndexFiles: async function (indexFiles) { try { - indexFiles.forEach(async url => { - await IndexFilesModel.create({URL: url}); + indexFiles.forEach(async (url) => { + await IndexFilesModel.create({ URL: url }); }); } catch (error) { throw error; @@ -75,18 +75,36 @@ module.exports = { * @param {string[]} definitions - An array of API definitions (URLs) to be saved. * @throws {Error} If there's an error while saving the definitions to the file or the database. * @returns {Promise} A Promise that resolves once all definitions have been saved. - */ - saveAPIsDefinitions: async function(definitions) { + */ + saveAPIsDefinitions: async function (definitions) { try { - definitions.forEach(async url => { - validatingService(url).then(async () => { - await APIsDefinitionsModel.create({url}); - }).catch(error => { - console.log('Invalid definition!',error.message); - }); + definitions.forEach(async (url) => { + validatingService(url) + .then(async () => { + await APIsDefinitionsModel.create({ url }); + }) + .catch((error) => { + console.log(`\nInvalid definition! ${error.message}\n`); + }); }); - } catch(error) { + } catch (error) { throw error; } - } -} + }, + + /** + * Fetches API definitions from the database using utility functions. + * + * @async + * @function fetchDefinitionsFromDBUtils + * @returns {Promise} A promise that resolves to an array of API definitions. + * @throws {Error} If an error occurs while fetching the API definitions from the database. + */ + fetchDefinitionsFromDBUtils: async function () { + try { + return await APIsDefinitionsModel.find({}); + } catch (error) { + throw error; + } + }, +}; diff --git a/src/server/api/utils/ElasticsearchUtils.js b/src/server/api/utils/ElasticsearchUtils.js new file mode 100644 index 0000000..d2f714f --- /dev/null +++ b/src/server/api/utils/ElasticsearchUtils.js @@ -0,0 +1,78 @@ +const { INDEX_NAME } = require('../constants/Constants'); + +module.exports = { + /** + * Transforms an API definition document into an indexable format for search purposes. + * + * @function transformingDefinitionForIndexing + * @param {Object} document - The API definition document to be transformed. + * @param {string} url - The URL associated with the API definition. + * @returns {Object} An indexable representation of the API definition. + * @throws {Error} If an error occurs during the transformation process. + */ + transformingDefinitionForIndexing: function (document, url) { + try { + const indexableInfo = { + url, + info: document.info, + tags: document.tags !== undefined ? document.tags : null, + servers: document.servers !== undefined ? document.servers : null, + }; + + return { + url: url, + content: indexableInfo, + }; + } catch (error) { + throw error; + } + }, + + checkForIndex: async function (client) { + try { + return await client.indices.exists({ index: INDEX_NAME }); + } catch (error) { + throw error; + } + }, + + removeIndex: async function (client) { + try { + await client.indices.delete({ index: INDEX_NAME }); + } catch (error) { + throw error; + } + }, + + /** + * Searches for API definitions in an Elasticsearch index based on specified keywords. + * + * @async + * @function searchForQuery + * @param {Object} client - The Elasticsearch client used for performing the search. + * @param {string} keywords - The keywords to search for in the API definitions. + * @returns {Promise>} A promise that resolves to an array of matched API definition hits. + * @throws {Error} If an error occurs during the search process. + */ + searchForQuery: async function (client, keywords) { + try { + const response = await client.search({ + index: INDEX_NAME, + query: { + multi_match: { + query: keywords, + fields: [ + 'content.info.title', + 'content.info.description', + 'content.tags.name', + ], + }, + }, + }); + + return response.hits.hits; + } catch (error) { + throw error; + } + }, +}; diff --git a/src/server/config/bootstrap.js b/src/server/config/bootstrap.js index 75ccad0..b3987f8 100644 --- a/src/server/config/bootstrap.js +++ b/src/server/config/bootstrap.js @@ -1,5 +1,8 @@ const childProcess = require('child_process'); +const ElasticSearchConnection = require('../api/utils/ConnectElasticSearchUtils'); const mongoose = require('mongoose'); +const { INDEX_NAME } = require('../api/constants/Constants'); +const { checkForIndex, removeIndex } = require('../api/utils/ElasticsearchUtils'); /** * Seed Function * (sails.config.bootstrap) @@ -19,32 +22,46 @@ module.exports.bootstrap = async function () { // Handle any errors from the worker process workerProcess.on('error', (error) => { - console.error('Worker process error:', error); + sails.log.error('Worker process error:', error); }); // Handle the exit of the worker process workerProcess.on('exit', (code) => { - console.log('Worker process exited with code:', code); + sails.log.info('Worker process exited with code:', code); }); // Handle a graceful shutdown of the worker process on Sails.js app termination process.on('SIGINT', () => { - console.log('Terminating worker process...'); + sails.log.error('Terminating worker process...'); workerProcess.kill('SIGINT'); process.exit(); }); - try { await mongoose.connect(sails.config.datastores.default.url, { useNewUrlParser: true, useUnifiedTopology: true, }); - console.log('MongoDB connected successfully!'); + sails.log.info('MongoDB connected successfully!'); } catch (err) { - console.error('Error connecting to MongoDB:', err.message); + sails.log.error('Error connecting to MongoDB:', err.message); throw error; } + const client = ElasticSearchConnection.getClient(); + + try { + const isExists = await checkForIndex(client); + if (isExists) { + await removeIndex(client); + sails.log.info( + `Removing already exist index in order to prevent conflict: Index ${INDEX_NAME} deleted.` + ); + } + + sails.client = client; + } catch (error) { + sails.log.error('Error creating Elasticsearch index:', error.message); + } }; diff --git a/src/server/config/env/production.js b/src/server/config/env/production.js index 2e9fde6..33b555a 100644 --- a/src/server/config/env/production.js +++ b/src/server/config/env/production.js @@ -1,412 +1,43 @@ -/** - * Production environment settings - * (sails.config.*) - * - * What you see below is a quick outline of the built-in settings you need - * to configure your Sails app for production. The configuration in this file - * is only used in your production environment, i.e. when you lift your app using: - * - * ``` - * NODE_ENV=production node app - * ``` - * - * > If you're using git as a version control solution for your Sails app, - * > this file WILL BE COMMITTED to your repository by default, unless you add - * > it to your .gitignore file. If your repository will be publicly viewable, - * > don't add private/sensitive data (like API secrets / db passwords) to this file! - * - * For more best practices and tips, see: - * https://sailsjs.com/docs/concepts/deployment - */ - module.exports = { - - - /************************************************************************** - * * - * Tell Sails what database(s) it should use in production. * - * * - * (https://sailsjs.com/config/datastores) * - * * - **************************************************************************/ datastores: { - - /*************************************************************************** - * * - * Configure your default production database. * - * * - * 1. Choose an adapter: * - * https://sailsjs.com/plugins/databases * - * * - * 2. Install it as a dependency of your Sails app. * - * (For example: npm install sails-mysql --save) * - * * - * 3. Then set it here (`adapter`), along with a connection URL (`url`) * - * and any other, adapter-specific customizations. * - * (See https://sailsjs.com/config/datastores for help.) * - * * - ***************************************************************************/ default: { - // adapter: 'sails-mysql', - // url: 'mysql://user:password@host:port/database', - //-------------------------------------------------------------------------- - // /\ To avoid checking it in to version control, you might opt to set - // || sensitive credentials like `url` using an environment variable. - // - // For example: - // ``` - // sails_datastores__default__url=mysql://admin:myc00lpAssw2D@db.example.com:3306/my_prod_db - // ``` - //-------------------------------------------------------------------------- - - /**************************************************************************** - * * - * More adapter-specific options * - * * - * > For example, for some hosted PostgreSQL providers (like Heroku), the * - * > extra `ssl` object with a `rejectUnauthorized` option must be provided. * - * * - * More info: * - * https://sailsjs.com/config/datastores * - * * - ****************************************************************************/ - // ssl: { rejectUnauthorized: true }, }, - }, - - models: { - - /*************************************************************************** - * * - * To help avoid accidents, Sails automatically sets the automigration * - * strategy to "safe" when your app lifts in production mode. * - * (This is just here as a reminder.) * - * * - * More info: * - * https://sailsjs.com/docs/concepts/models-and-orm/model-settings#?migrate * - * * - ***************************************************************************/ migrate: 'safe', - - /*************************************************************************** - * * - * If, in production, this app has access to physical-layer CASCADE * - * constraints (e.g. PostgreSQL or MySQL), then set those up in the * - * database and uncomment this to disable Waterline's `cascadeOnDestroy` * - * polyfill. (Otherwise, if you are using a databse like Mongo, you might * - * choose to keep this enabled.) * - * * - ***************************************************************************/ - // cascadeOnDestroy: false, - }, - - - /************************************************************************** - * * - * Always disable "shortcut" blueprint routes. * - * * - * > You'll also want to disable any other blueprint routes if you are not * - * > actually using them (e.g. "actions" and "rest") -- but you can do * - * > that in `config/blueprints.js`, since you'll want to disable them in * - * > all environments (not just in production.) * - * * - ***************************************************************************/ blueprints: { shortcuts: false, }, - - - /*************************************************************************** - * * - * Configure your security settings for production. * - * * - * IMPORTANT: * - * If web browsers will be communicating with your app, be sure that * - * you have CSRF protection enabled. To do that, set `csrf: true` over * - * in the `config/security.js` file (not here), so that CSRF app can be * - * tested with CSRF protection turned on in development mode too. * - * * - ***************************************************************************/ security: { - - /*************************************************************************** - * * - * If this app has CORS enabled (see `config/security.js`) with the * - * `allowCredentials` setting enabled, then you should uncomment the * - * `allowOrigins` whitelist below. This sets which "origins" are allowed * - * to send cross-domain (CORS) requests to your Sails app. * - * * - * > Replace "https://example.com" with the URL of your production server. * - * > Be sure to use the right protocol! ("http://" vs. "https://") * - * * - ***************************************************************************/ cors: { - // allowOrigins: [ - // 'https://example.com', - // ] - }, + }, }, - - - /*************************************************************************** - * * - * Configure how your app handles sessions in production. * - * * - * (https://sailsjs.com/config/session) * - * * - * > If you have disabled the "session" hook, then you can safely remove * - * > this section from your `config/env/production.js` file. * - * * - ***************************************************************************/ session: { - - /*************************************************************************** - * * - * Production session store configuration. * - * * - * Uncomment the following lines to finish setting up a package called * - * "@sailshq/connect-redis" that will use Redis to handle session data. * - * This makes your app more scalable by allowing you to share sessions * - * across a cluster of multiple Sails/Node.js servers and/or processes. * - * (See http://bit.ly/redis-session-config for more info.) * - * * - * > While @sailshq/connect-redis is a popular choice for Sails apps, many * - * > other compatible packages (like "connect-mongo") are available on NPM. * - * > (For a full list, see https://sailsjs.com/plugins/sessions) * - * * - ***************************************************************************/ - // adapter: '@sailshq/connect-redis', - // url: 'redis://user:password@localhost:6379/databasenumber', - //-------------------------------------------------------------------------- - // /\ OR, to avoid checking it in to version control, you might opt to - // || set sensitive credentials like this using an environment variable. - // - // For example: - // ``` - // sails_session__url=redis://admin:myc00lpAssw2D@bigsquid.redistogo.com:9562/0 - // ``` - // - //-------------------------------------------------------------------------- - - /*************************************************************************** - * * - * Production configuration for the session ID cookie name. * - * * - * We reccomend prefixing your session cookie with `__Host-`, this limits * - * the scope of your cookie to a single origin to protect against same-site * - * attacks. * - * * - * Note that with the `__Host-` prefix, session cookies will _not_ be sent * - * unless `sails.config.cookie.secure` is set to `true`. * - * * - * Read more: * - * https://sailsjs.com/config/session#?the-session-id-cookie * - * * - ***************************************************************************/ - // name: '__Host-sails.sid', - - /*************************************************************************** - * * - * Production configuration for the session ID cookie. * - * * - * Tell browsers (or other user agents) to ensure that session ID cookies * - * are always transmitted via HTTPS, and that they expire 24 hours after * - * they are set. * - * * - * Note that with `secure: true` set, session cookies will _not_ be * - * transmitted over unsecured (HTTP) connections. Also, for apps behind * - * proxies (like Heroku), the `trustProxy` setting under `http` must be * - * configured in order for `secure: true` to work. * - * * - * > While you might want to increase or decrease the `maxAge` or provide * - * > other options, you should always set `secure: true` in production * - * > if the app is being served over HTTPS. * - * * - * Read more: * - * https://sailsjs.com/config/session#?the-session-id-cookie * - * * - ***************************************************************************/ cookie: { - // secure: true, maxAge: 24 * 60 * 60 * 1000, // 24 hours }, - }, - - - /************************************************************************** - * * - * Set up Socket.io for your production environment. * - * * - * (https://sailsjs.com/config/sockets) * - * * - * > If you have disabled the "sockets" hook, then you can safely remove * - * > this section from your `config/env/production.js` file. * - * * - ***************************************************************************/ sockets: { - - /*************************************************************************** - * * - * Uncomment the `onlyAllowOrigins` whitelist below to configure which * - * "origins" are allowed to open socket connections to your Sails app. * - * * - * > Replace "https://example.com" etc. with the URL(s) of your app. * - * > Be sure to use the right protocol! ("http://" vs. "https://") * - * * - ***************************************************************************/ - // onlyAllowOrigins: [ - // 'https://example.com', - // 'https://staging.example.com', - // ], - - - /*************************************************************************** - * * - * If you are deploying a cluster of multiple servers and/or processes, * - * then uncomment the following lines. This tells Socket.io about a Redis * - * server it can use to help it deliver broadcasted socket messages. * - * * - * > Be sure a compatible version of @sailshq/socket.io-redis is installed! * - * > (See https://sailsjs.com/config/sockets for the latest version info) * - * * - * (https://sailsjs.com/docs/concepts/deployment/scaling) * - * * - ***************************************************************************/ - // adapter: '@sailshq/socket.io-redis', - // url: 'redis://user:password@bigsquid.redistogo.com:9562/databasenumber', - //-------------------------------------------------------------------------- - // /\ OR, to avoid checking it in to version control, you might opt to - // || set sensitive credentials like this using an environment variable. - // - // For example: - // ``` - // sails_sockets__url=redis://admin:myc00lpAssw2D@bigsquid.redistogo.com:9562/0 - // ``` - //-------------------------------------------------------------------------- - + onlyAllowOrigins: ["https://mydomainname.com"] }, - - - /************************************************************************** - * * - * Set the production log level. * - * * - * (https://sailsjs.com/config/log) * - * * - ***************************************************************************/ log: { level: 'debug' }, - - http: { - - /*************************************************************************** - * * - * The number of milliseconds to cache static assets in production. * - * (the "max-age" to include in the "Cache-Control" response header) * - * * - * If you are caching assets with a tool like Cloudflare, you may want to * - * reduce this considerably to allow more flexibility in purging the cache. * - * * - ***************************************************************************/ cache: 365.25 * 24 * 60 * 60 * 1000, // One year - - /*************************************************************************** - * * - * Proxy settings * - * * - * If your app will be deployed behind a proxy/load balancer - for example, * - * on a PaaS like Heroku - then uncomment the `trustProxy` setting below. * - * This tells Sails/Express how to interpret X-Forwarded headers. * - * * - * This setting is especially important if you are using secure cookies * - * (see the `cookies: secure` setting under `session` above) or if your app * - * relies on knowing the original IP address that a request came from. * - * * - * (https://sailsjs.com/config/http) * - * * - ***************************************************************************/ - // trustProxy: true, - }, - - - /************************************************************************** - * * - * Lift the server on port 80. * - * (if deploying behind a proxy, or to a PaaS like Heroku or Deis, you * - * probably don't need to set a port here, because it is oftentimes * - * handled for you automatically. If you are not sure if you need to set * - * this, just try deploying without setting it and see if it works.) * - * * - ***************************************************************************/ - // port: 80, - - - - /************************************************************************** - * * - * Configure an SSL certificate * - * * - * For the safety of your users' data, you should use SSL in production. * - * ...But in many cases, you may not actually want to set it up _here_. * - * * - * Normally, this setting is only relevant when running a single-process * - * deployment, with no proxy/load balancer in the mix. But if, on the * - * other hand, you are using a PaaS like Heroku, you'll want to set up * - * SSL in your load balancer settings (usually somewhere in your hosting * - * provider's dashboard-- not here.) * - * * - * > For more information about configuring SSL in Sails, see: * - * > https://sailsjs.com/config/*#?sailsconfigssl * - * * - **************************************************************************/ - // ssl: undefined, - - - - /************************************************************************** - * * - * Production overrides for any custom settings specific to your app. * - * (for example, production credentials for 3rd party APIs like Stripe) * - * * - * > See config/custom.js for more info on how to configure these options. * - * * - ***************************************************************************/ custom: { - baseUrl: 'https://example.com', - internalEmailAddress: 'support@example.com', - - // sendgridSecret: 'SG.fake.3e0Bn0qSQVnwb1E4qNPz9JZP5vLZYqjh7sn8S93oSHU', - // stripeSecret: 'sk_prod__fake_Nfgh82401348jaDa3lkZ0d9Hm', - //-------------------------------------------------------------------------- - // /\ OR, to avoid checking them in to version control, you might opt to - // || set sensitive credentials like these using environment variables. - // - // For example: - // ``` - // sendgridSecret=SG.fake.3e0Bn0qSQVnwb1E4qNPz9JZP5vLZYqjh7sn8S93oSHU - // sails_custom__stripeSecret=sk_prod__fake_Nfgh82401348jaDa3lkZ0d9Hm - // ``` - //-------------------------------------------------------------------------- }, - - - }; diff --git a/src/server/config/routes.js b/src/server/config/routes.js index f0af0f3..5920958 100644 --- a/src/server/config/routes.js +++ b/src/server/config/routes.js @@ -11,6 +11,8 @@ module.exports.routes = { 'POST /api/v1/run/crawler': 'CrawlingController.startCrawling', - 'GET /api/v1/process/index-files': 'DownloadAndProcessIndexFilesController.startDownloadAndProcessIndexFiles' + 'GET /api/v1/process/index-files': 'DownloadAndProcessIndexFilesController.startDownloadAndProcessIndexFiles', + 'GET /api/v1/indexing': 'IndexingController.startIndexing', + 'GET /api/v1/search': 'SearchController.startSearching' }; diff --git a/src/server/package-lock.json b/src/server/package-lock.json index ec951cf..95d4acf 100644 --- a/src/server/package-lock.json +++ b/src/server/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "license": "MIT", "dependencies": { + "@elastic/elasticsearch": "^8.9.0", "@sailshq/lodash": "^3.10.3", "amqplib": "^0.10.3", "axios": "^1.4.0", @@ -167,6 +168,70 @@ "node": ">=6.9.0" } }, + "node_modules/@elastic/elasticsearch": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.9.0.tgz", + "integrity": "sha512-UyolnzjOYTRL2966TYS3IoJP4tQbvak/pmYmbP3JdphD53RjkyVDdxMpTBv+2LcNBRrvYPTzxQbpRW/nGSXA9g==", + "dependencies": { + "@elastic/transport": "^8.3.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@elastic/elasticsearch/node_modules/tslib": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.1.tgz", + "integrity": "sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig==" + }, + "node_modules/@elastic/transport": { + "version": "8.3.3", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.3.3.tgz", + "integrity": "sha512-g5nc//dq/RQUTMkJUB8Ui8KJa/WflWmUa7yLl4SRZd67PPxIp3cn+OvGMNIhpiLRcfz1upanzgZHb/7Po2eEdQ==", + "dependencies": { + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^5.22.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@elastic/transport/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@elastic/transport/node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/@elastic/transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/@elastic/transport/node_modules/tslib": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.1.tgz", + "integrity": "sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig==" + }, "node_modules/@jsdevtools/ono": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", @@ -696,6 +761,17 @@ "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -2500,6 +2576,14 @@ "node": ">=8" } }, + "node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "engines": { + "node": ">=14" + } + }, "node_modules/htmlparser2": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", @@ -5008,6 +5092,11 @@ "node": ">=6" } }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" + }, "node_modules/semver": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", @@ -5550,6 +5639,14 @@ "node": ">=0.10" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -5953,6 +6050,17 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" }, + "node_modules/undici": { + "version": "5.23.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.23.0.tgz", + "integrity": "sha512-1D7w+fvRsqlQ9GscLBwcAJinqcZGHUKjbOmXdlE/v8BvEGXjeWAax+341q44EuTcHXXnfyKNbKRq4Lg7OzhMmg==", + "dependencies": { + "busboy": "^1.6.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/src/server/package.json b/src/server/package.json index dafabb1..b38d141 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -19,6 +19,7 @@ "kibana" ], "dependencies": { + "@elastic/elasticsearch": "^8.9.0", "@sailshq/lodash": "^3.10.3", "amqplib": "^0.10.3", "axios": "^1.4.0",