Merge branch 'development' into 3.2.0

Author: jvwong

src/client/features/enrichment/index.js

@@ -48,6 +48,10 @@ class Enrichment extends React.Component {
         let { pathways } = await ServerAPI.enrichmentAPI({ query: sources}, 'analysis');
         let enrichmentNetwork = await ServerAPI.enrichmentAPI({ pathways }, 'visualization');
         let networkHasZeroNodes = enrichmentNetwork.graph.elements.nodes.length === 0;
+        enrichmentNetwork.graph.elements.nodes.forEach( node => {
+          const intersection = _.intersection( node.data.geneSet, sources );
+          _.assign( node.data, { intersection } );
+        });
 
         cy.remove('*');
         cy.add({

src/server/external-services/gprofiler/gconvert/index.js

@@ -1,14 +1,12 @@
 const _ = require('lodash');
-const qs = require('query-string');
 const QuickLRU = require('quick-lru');
 const logger = require('../../../logger');
 
 const { fetch } = require('../../../../util');
-const cleanUpEntrez = require('../clean-up-entrez');
 const InvalidParamError = require('../../../../server/errors/invalid-param');
 const { GPROFILER_URL, PC_CACHE_MAX_SIZE, NS_HGNC, NS_HGNC_SYMBOL, NS_UNIPROT, NS_NCBI_GENE, NS_ENSEMBL } = require('../../../../config');
 const { cachePromise } = require('../../../cache');
-const GCONVERT_URL = GPROFILER_URL + 'gconvert.cgi';
+const GCONVERT_URL = GPROFILER_URL + 'api/convert/convert/';
 
 const GPROFILER_NS_MAP = new Map([
   [NS_HGNC, 'HGNC_ACC'],
@@ -18,14 +16,16 @@ const GPROFILER_NS_MAP = new Map([
   [NS_ENSEMBL, 'ENSG']
 ]);
 
+const toJSON = res => res.json();
+
 // create gconvert opts for a gconvert request
 // validates params
 const createGConvertOpts = opts => {
+  // see https://biit.cs.ut.ee/gprofiler/page/apis
   const defaults = {
-    output: 'mini',
     organism: 'hsapiens',
     target: NS_HGNC,
-    prefix: 'ENTREZGENE_ACC'
+    numeric_ns: 'ENTREZGENE_ACC'
   };
 
   let gConvertOpts = _.assign({}, defaults, opts);
@@ -34,34 +34,30 @@ const createGConvertOpts = opts => {
   if( !Array.isArray( query ) ){
     throw new InvalidParamError( `Error creating gconvert request - expected an array of strings for "query", got ${query}`);
   }
-  let gconvertQuery = query.join(' ');
   let gconvertTarget = GPROFILER_NS_MAP.get( target );
 
   if( gconvertTarget == null ){
     throw new InvalidParamError( `Error creating gconvert request - expected a valid "targetDb", got ${target}`);
   }
 
-  gConvertOpts.query = gconvertQuery;
   gConvertOpts.target = gconvertTarget;
 
   return gConvertOpts;
 };
 
-const gConvertResponseHandler = body =>  {
-  let entityInfoList = body.split('\n').map( ele => ele.split('\t') ).filter( ele => ele != '');
+const gConvertResponseHandler = json =>  {
+  let entityInfoList = _.get( json, ['result'] );
   let unrecognized = new Set();
   let duplicate = {};
   let entityMap = new Map();
   let alias = {};
-  const INITIAL_ALIAS_INDEX = 1;
-  const CONVERTED_ALIAS_INDEX = 3;
 
   entityInfoList.forEach( entityInfo => {
-    let convertedAlias = entityInfo[CONVERTED_ALIAS_INDEX];
-    let initialAlias = cleanUpEntrez(entityInfo[INITIAL_ALIAS_INDEX]);
+    let convertedAlias = entityInfo.converted === 'None' ? null : entityInfo.converted;
+    let initialAlias = entityInfo.incoming;
 
-    if( convertedAlias === 'N/A' ){
-      unrecognized.add(initialAlias);
+    if( _.isNull( convertedAlias ) ){
+      unrecognized.add( initialAlias );
       return;
     }
 
@@ -100,9 +96,10 @@ const rawValidatorGconvert = ( query, opts = {} ) => {
     .then( () => createGConvertOpts( _.assign(opts, { query }) ) )
     .then( gconvertOpts => fetch( GCONVERT_URL, {
       method: 'post',
-      body: qs.stringify( gconvertOpts )
+      body: JSON.stringify( gconvertOpts ),
+      headers: { 'Content-Type': 'application/json' }
     }))
-    .then( response => response.text() )
+    .then( toJSON )
     .then( gConvertResponseHandler )
     .catch( err => {
       logger.error(`Error in validatorGconvert - ${err.message}`);

src/server/external-services/gprofiler/gprofiler.js

@@ -1,126 +1,102 @@
 const { fetch } = require('../../../util');
 const _ = require('lodash');
-const qs = require('query-string');
 const QuickLRU = require('quick-lru');
 
-const cleanUpEntrez = require('./clean-up-entrez');
 const { GPROFILER_URL } = require('../../../config');
 const logger = require('../../logger');
 const { cachePromise } = require('../../cache');
 
-const GPROFILER_GOST_URL = GPROFILER_URL + 'index.cgi';
+const toJSON = res => res.json();
+
+// See https://biit.cs.ut.ee/gprofiler/page/apis
+const GPROFILER_GOST_URL = GPROFILER_URL + 'api/gost/profile/';
 const GPROFILER_DEFAULT_OPTS = {
-  output: 'mini',
   organism: 'hsapiens',
-  significant: 0,
-  sort_by_structure: 1,
-  ordered_query: 0,
-  as_ranges: 0,
-  no_iea: 1,
-  underrep: 0,
-  hierfiltering: 'none',
-  user_thr: 0.05,
-  min_set_size: 5,
-  max_set_size: 200,
-  threshold_algo: 'analytical',
-  domain_size_type: 'annotated',
-  custbg: [],
-  'sf_GO:BP': 1,
-  sf_REAC: 1,
-  prefix: 'ENTREZGENE_ACC'
+	sources: ['GO:BP', 'REAC'],
+	user_threshold: 0.05,
+	all_results: false,
+	ordered: false,
+	combined: false,
+	measure_underrepresentation: false,
+	no_iea: true,
+	domain_scope: 'annotated',
+	numeric_ns: 'ENTREZGENE_ACC',
+	significance_threshold_method: 'g_SCS',
+  background: [],
+  no_evidences: true
 };
 
-// parseGProfilerResponse(gProfilerResponse) takes the text response
-// from gProfiler gProfilerResponse and parses it into JSON format
-const parseGProfilerResponse = gProfilerResponse => {
-  let lines = gProfilerResponse.split('\n').map( line => {
-    if( line.substring(0, 1) === '#' ){ return ''; }
-    return line;
-  });
-
-  let elements = _.compact(lines).map( line => line.split('\t') );
+const DEFAULT_FILTER_OPTS = {
+  minSetSize: 5,
+  maxSetSize: 200
+};
 
+const parseGProfilerResponse = ( json, opts ) => {
   let pathways = [];
-  let P_VALUE_INDEX = 2;
-  let PATHWAY_ID_INDEX = 8;
-  let DESCRIPTION_INDEX = 11;
-  let GENE_INTERSECTION_LIST_INDEX = 13;
-
-
-  elements.forEach( ele => {
-    let pathwayId = ele[PATHWAY_ID_INDEX];
-    let pValue = ele[P_VALUE_INDEX];
-    let description = ele[DESCRIPTION_INDEX].trim();
-    let geneIntersectionList = ele[GENE_INTERSECTION_LIST_INDEX].split(',').map( gene => cleanUpEntrez( gene ) );
-
+  const { 
+    minSetSize = DEFAULT_FILTER_OPTS.minSetSize, 
+    maxSetSize = DEFAULT_FILTER_OPTS.maxSetSize
+  } = opts;
+  const pathwayInfoList = _.get( json, ['result'] );  
+  pathwayInfoList.forEach( pathwayInfo => {
+    let { native: id, name, p_value, term_size } = pathwayInfo;
+    if( term_size < minSetSize || term_size > maxSetSize ) return;
     pathways.push({
-      id: pathwayId,
-      data: {
-        name: description,
-        p_value: pValue,
-        intersection: geneIntersectionList
-      }
+      id, 
+      data: { name, p_value }
     });
   });
 
   return { pathways };
 };
 
 
+const validateParams = ( query, opts ) => {
+  let error = null,
+  message = '';
+  const { minSetSize, maxSetSize } = opts;
+
+  if( !Array.isArray( query ) ) message = 'ERROR: query must be an array'; 
+  if( minSetSize && (typeof( minSetSize ) != 'number' || minSetSize < 0) ) message = 'ERROR: minSetSize must be a positive number';
+  if( maxSetSize && (typeof( maxSetSize ) != 'number' || maxSetSize < 0) ) message = 'ERROR: maxSetSize must be a positive number';
+  if( (minSetSize && maxSetSize) && (maxSetSize < minSetSize) ) message = 'ERROR: minSetSize must be less than maxSetSize';
+  
+  if ( !_.isEmpty( message ) ) error = new Error( message );
+  return error;
+};
+
+const getGProfilerOpts = query => _.assign( {}, GPROFILER_DEFAULT_OPTS, { query } );
+
 // enrichmemt(query, opts) takes a list of gene identifiers query
 // and an object of user settings opts
 // and extracts enrichment information
 // from g:Profiler for the query list based on userSetting
-const rawEnrichment = (query, opts) => {
-  return new Promise((resolve, reject) => {
-    let {
-      minSetSize = GPROFILER_DEFAULT_OPTS.min_set_size,
-      maxSetSize = GPROFILER_DEFAULT_OPTS.max_set_size,
-      background = []
-    } = opts;
-
-    if (!Array.isArray(query)) {
-      reject(new Error('ERROR: genes should be an array'));
-    }
-    if( typeof(minSetSize) != 'number' ) {
-      reject(new Error('ERROR: minSetSize should be a number'));
-    }
-    if( minSetSize < 0 ){
-      reject(new Error('ERROR: minSetSize should be >= 0'));
-    }
-    if( typeof(maxSetSize) != 'number' ){
-      reject(new Error('ERROR: maxSetSize should be a number'));
-    }
-    if( maxSetSize < minSetSize ){
-      reject(new Error('ERROR: maxSetSize should be >= minSetSize'));
-    }
-    if( !Array.isArray(background) ){
-      reject(new Error('ERROR: backgroundGenes should be an array'));
-    }
-
-    let gProfilerOpts = _.assign( {}, GPROFILER_DEFAULT_OPTS, {
-      query: query.sort().join(' '),
-      min_set_size: minSetSize,
-      max_set_size: maxSetSize,
-      custbg: background.join(' ')
-    } );
-
-
-    fetch(GPROFILER_GOST_URL, { method: 'post', body: qs.stringify(gProfilerOpts)})
-      .then( res => res.text() )
-      .then( gprofilerRes =>  parseGProfilerResponse( gprofilerRes ) )
-      .then( pathwayInfo => resolve( pathwayInfo ) )
-      .catch( err =>{
-        logger.error(`Error in rawEnrichment - ${err.message}`);
-        throw err;
-      });
-  });
+const rawEnrichment = ( query, opts ) => {
+  const gProfilerOpts = getGProfilerOpts( query, opts );
+
+  return fetch( GPROFILER_GOST_URL, { 
+    method: 'post',
+    body: JSON.stringify( gProfilerOpts ),
+    headers: { 'Content-Type': 'application/json' }
+  })
+  .then( toJSON )
+  .then( gprofilerRes => parseGProfilerResponse( gprofilerRes, opts ) )
+  .catch( err =>{
+    logger.error(`Error in rawEnrichment - ${err.message}`);
+    throw err;
+  });  
 };
 
+
 const lruCache = new QuickLRU({ maxSize: 100 });
 
-const enrichment = cachePromise(rawEnrichment, lruCache);
+const enrichmentWrapper = cachePromise( rawEnrichment, lruCache );
 
+const enrichment = async ( query, opts ) => {
+  const paramError = validateParams( query, opts );
+  if( paramError ) throw paramError;
 
+  return enrichmentWrapper( query.sort(), opts );
+};
 
 module.exports = { enrichment, parseGProfilerResponse };

src/server/routes/enrichment/index.js

@@ -122,9 +122,10 @@ enrichmentRouter.post('/validation', (req, res, next) => {
  *         schema:
  *           "$ref": "#/definitions/success/analysisSuccess"
  *       '400':
- *         description: Invalid input (minSetSize, maxSetSize, backgroundGenes or JSON format)
+ *         description: Invalid input (minSetSize, maxSetSize, or JSON format)
  *         schema:
  *           "$ref": "#/definitions/error/analysisError"
+ * 
 */
 // expose a rest endpoint for enrichment service
 enrichmentRouter.post('/analysis', (req, res, next) => {
@@ -222,7 +223,7 @@ enrichmentRouter.post('/visualization', (req, res, next) => {
  *     analysisObj:
  *       type: object
  *       required:
- *       - genes
+ *       - query
  *       properties:
  *         query:
  *           type: array
@@ -239,14 +240,7 @@ enrichmentRouter.post('/visualization', (req, res, next) => {
  *           type: number
  *           description: "maximum size of functional category, larger categories are
  *             excluded \n default: 200"
- *           example: 400
- *         background:
- *           type: array
- *           description: "Biological identifiers used
- *             as a custom statistical background \n default: []"
- *           example: []
- *           items:
- *             type: string
+ *           example: 400        
  *     visualizationObj:
  *       type: object
  *       required:
@@ -328,11 +322,6 @@ enrichmentRouter.post('/visualization', (req, res, next) => {
  *                   name:
  *                     type: string
  *                     example: DNA-templated transcription, elongation
- *                   intersection:
- *                     type: array
- *                     items:
- *                       type: string
- *                       example: PAF1
  *     visualizationSuccess:
  *       type: object
  *       required:

test/server/enrichment/analysis/analysis-test.js

@@ -8,8 +8,8 @@ const PARSED_GPROFILER = require('./parsed-gprofiler.json');
 describe ('Enrichment service: analysis', function () {
   describe ('Test parseGProfilerResponse()', () => {
     it ('should return to a correct object', () => {
-      const gGostbodytxt = fs.readFileSync( path.resolve( __dirname, 'ggost-body.txt' ), 'utf-8' );
-      const result = parseGProfilerResponse( gGostbodytxt );
+      const gGostbodyJSON = require('./ggost-body.json');
+      const result = parseGProfilerResponse( gGostbodyJSON, {} );
       expect ( result ).to.deep.equal ( PARSED_GPROFILER );
     });
   });