diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java index 97460d5a71..7e9d64edd8 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java @@ -411,14 +411,14 @@ private void loadVariationData() throws NoSuchMethodException, InterruptedExcept if (dbSnpFilePath.toFile().exists()) { if (variationPath.resolve(DBSNP_VERSION_FILENAME).toFile().exists()) { logger.info("Loading dbSNP file '{}'", dbSnpFilePath); - loadRunner.load(dbSnpFilePath, SNP_COLLECTION_NAME, dataRelease); + loadRunner.load(dbSnpFilePath, SNP_DATA, dataRelease); // Create index - createIndex(SNP_COLLECTION_NAME); + createIndex(SNP_DATA); // Update release (collection and sources) List sources = Collections.singletonList(variationPath.resolve(DBSNP_VERSION_FILENAME)); - dataReleaseManager.update(dataRelease, SNP_COLLECTION_NAME, EtlCommons.VARIATION_DATA, sources); + dataReleaseManager.update(dataRelease, SNP_DATA, EtlCommons.VARIATION_DATA, sources); } else { logger.warn("In order to load the dbSNP file you need the version file {} within the folder '{}'", DBSNP_VERSION_FILENAME, variationPath); diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java index 612e8d6a38..611c9eb939 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/ValidationCommandExecutor.java @@ -66,7 +66,7 @@ public ValidationCommandExecutor(AdminCliOptionsParser.ValidationCommandOptions } @Override - public void execute() { + public void execute() throws CellBaseException { checkFilesExist(); VariantAnnotationCalculator variantAnnotationCalculator; diff --git a/cellbase-core/src/main/resources/configuration.yml b/cellbase-core/src/main/resources/configuration.yml index 6e651f00d3..84378507b4 100644 --- a/cellbase-core/src/main/resources/configuration.yml +++ b/cellbase-core/src/main/resources/configuration.yml @@ -106,13 +106,6 @@ download: clinvar: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2021-07.xml.gz # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-02.xml.gz -<<<<<<< HEAD - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz - clinvarVariation: -# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2021-07.xml.gz -# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-02.xml.gz - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz -======= # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/RCV_xml_old_format/ClinVarFullRelease_2024-05.xml.gz version: 2024-05 @@ -122,7 +115,6 @@ download: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/VCV_xml_old_format/ClinVarVariationRelease_2024-05.xml.gz version: 2024-05 ->>>>>>> release-6.2.x clinvarSummary: host: http://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz clinvarVariationAllele: @@ -147,15 +139,10 @@ download: genomicSuperDups: host: http://hgdownload.cse.ucsc.edu/goldenPath gwasCatalog: -<<<<<<< HEAD - host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv - version: "1.0.2 associations_e106_r2022-05-17" -======= #host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv host: "https://ftp.ebi.ac.uk/pub/databases/gwas/releases/2024/05/20/gwas-catalog-associations_ontology-annotated.tsv" #version: "1.0.2 associations_e106_r2022-05-17" version: "2024-05-20" ->>>>>>> release-6.2.x hpo: host: https://ci.monarchinitiative.org/view/hpo/job/hpo.annotations/lastSuccessfulBuild/artifact/rare-diseases/util/annotation/phenotype_to_genes.txt disgenet: diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java index d09291bc3e..c2ded07a26 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java @@ -38,6 +38,7 @@ public class EtlCommons { public static final String HOMO_SAPIENS_NAME ="Homo sapiens"; public static final String GENOME_DATA = "genome"; + public static final String GENOME_SEQUENCE_DATA = "genome_sequence"; public static final String GENE_DATA = "gene"; public static final String REFSEQ_DATA = "refseq"; public static final String GENE_DISEASE_ASSOCIATION_DATA = "gene_disease_association"; @@ -69,7 +70,7 @@ public class EtlCommons { public static final String DBSNP_FILE = "GCF_000001405.40.gz"; public static final String DBSNP_NAME = "dbSNP"; public static final String DBSNP_VERSION_FILENAME = DBSNP_NAME + "Version.json"; - public static final String SNP_COLLECTION_NAME = "snp"; + public static final String SNP_DATA = "snp"; public static final String STRUCTURAL_VARIANTS_DATA = "svs"; public static final String REPEATS_DATA = "repeats"; @@ -79,6 +80,8 @@ public class EtlCommons { public static final String DOID_FILE = "doid.obo"; public static final String PFM_DATA = "regulatory_pfm"; + public static final String REGULATORY_REGION_DATA = "regulatory_region"; + // Build specific data options public static final String GENOME_INFO_DATA = "genome_info"; public static final String DISGENET_DATA = "disgenet"; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java index afbcefc162..f25f937216 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/CellBaseDBAdaptor.java @@ -16,76 +16,25 @@ package org.opencb.cellbase.lib.impl.core; -import org.apache.commons.collections4.CollectionUtils; import org.opencb.cellbase.core.exception.CellBaseException; -import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDataStore; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - public class CellBaseDBAdaptor extends MongoDBAdaptor { - protected List dataReleases; - protected Map mongoDBCollectionByRelease; - public static final String DATA_RELEASE_SEPARATOR = "__v"; - public static String buildCollectionName(String data, int release) { - String name = data + DATA_RELEASE_SEPARATOR + release; - return name; - } - - public Map buildCollectionByReleaseMap(String data) { - Map collectionMap = new HashMap<>(); - if (CollectionUtils.isNotEmpty(dataReleases)) { - for (DataRelease dataRelease : dataReleases) { - if (dataRelease.getCollections().containsKey(data)) { - String collectionName = dataRelease.getCollections().get(data); - collectionMap.put(dataRelease.getRelease(), mongoDataStore.getCollection(collectionName)); - } - } - } else { - // For backward compatibility (i.e., in case data_release collection is missing) - collectionMap.put(0, mongoDataStore.getCollection(data)); - } - - return collectionMap; - } - - public MongoDBCollection getCollectionByRelease(Map collectionMap, Integer dataRelease) - throws CellBaseException { - int release = dataRelease == null ? 0 : dataRelease; - if (!collectionMap.containsKey(release)) { - // If the data release is invalid, throw an exception - String msg = "Data not found in release " + release + ". " + collectionMap.toString(); - logger.error(msg); - throw new CellBaseException(msg); - } - return collectionMap.get(release); - } - public CellBaseDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - this.dataReleases = new ReleaseMongoDBAdaptor(mongoDataStore).getAll().getResults(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("CellBaseDBAdaptor{"); - sb.append("dataRelease=").append(dataReleases); - sb.append('}'); - return sb.toString(); } - public List getDataReleases() { - return dataReleases; + public static String buildCollectionName(String data, int release) { + String name = data + DATA_RELEASE_SEPARATOR + release; + return name; } - public CellBaseDBAdaptor setDataReleases(List dataReleases) { - this.dataReleases = dataReleases; - return this; + public MongoDBCollection getMongoDBCollection(String data, int release) throws CellBaseException { + return DataReleaseSingleton.getInstance().getMongoDBCollection(mongoDataStore.getDatabaseName(), data, release); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java index 1b203e3875..236dc4737f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java @@ -46,6 +46,7 @@ import java.util.function.Consumer; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; +import static org.opencb.cellbase.lib.EtlCommons.CLINICAL_VARIANTS_DATA; /** * Created by fjlopez on 06/12/16. @@ -65,14 +66,6 @@ public ClinicalMongoDBAdaptor(MongoDataStore mongoDataStore, GenomeManager genom super(mongoDataStore); this.genomeManager = genomeManager; - - init(); - } - - private void init() { - logger.debug("ClinicalMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("clinical_variants"); } public CellBaseDataResult next(Query query, QueryOptions options) { @@ -103,16 +96,14 @@ public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, public CellBaseDataResult count(Query query) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.count(bson)); } public CellBaseDataResult distinct(Query query, String field) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bson)); } @@ -128,8 +119,7 @@ public CellBaseDataResult get(Query query, QueryOptions options) throws logger.debug("query: {}", bson.toBsonDocument().toJson()); logger.debug("queryOptions: {}", options.toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.find(bson, null, Variant.class, parsedOptions)); } @@ -140,8 +130,7 @@ public CellBaseDataResult nativeGet(Query query, QueryOptions options) throws Ce logger.debug("query: {}", bson.toBsonDocument().toJson()); logger.debug("queryOptions: {}", options.toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return new CellBaseDataResult<>(mongoDBCollection.find(bson, parsedOptions)); } @@ -152,8 +141,7 @@ public Iterator iterator(Query query, QueryOptions options) { public Iterator nativeIterator(Query query, QueryOptions options) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, - (Integer) query.getOrDefault(DATA_RELEASE_PARAM, 0)); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getInt(DATA_RELEASE_PARAM)); return mongoDBCollection.nativeQuery().find(bson, options); } @@ -355,7 +343,7 @@ private CellBaseDataResult getClinvarPhenotypeGeneRelations(QueryOptions queryOp fields.put("associatedGenes", 1); pipeline.add(new Document("$project", fields)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, dataRelease); return executeAggregation2("", pipeline, queryOptions, mongoDBCollection); } @@ -377,7 +365,7 @@ private CellBaseDataResult getGwasPhenotypeGeneRelations(QueryOptions queryOptio fields.put("associatedGenes", 1); pipeline.add(new Document("$project", fields)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, dataRelease); return executeAggregation2("", pipeline, queryOptions, mongoDBCollection); } @@ -466,7 +454,7 @@ public CellBaseIterator iterator(ClinicalVariantQuery query) throws CellBaseExce Bson projection = getProjection(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Variant.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CLINICAL_VARIANTS_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java index 94a686310b..528012f418 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java @@ -32,6 +32,7 @@ import org.opencb.cellbase.core.api.query.ProjectionQueryOptions; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.lib.EtlCommons; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.cellbase.lib.iterator.CellBaseIterator; import org.opencb.cellbase.lib.iterator.CellBaseMongoDBIterator; @@ -43,13 +44,15 @@ import java.util.*; import java.util.regex.Pattern; +import static org.opencb.cellbase.lib.EtlCommons.GENE_DATA; + /** * Created by imedina on 25/11/15. */ public class GeneMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { private static final Set CONSTRAINT_NAMES = new HashSet<>(); - private Map refseqCollectionByRelease; +// private Map refseqCollectionByRelease; private static final GenericDocumentComplexConverter CONVERTER; @@ -66,15 +69,15 @@ public class GeneMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCor public GeneMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - this.init(); +// this.init(); } - private void init() { - mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); - refseqCollectionByRelease = buildCollectionByReleaseMap("refseq"); - - logger.debug("GeneMongoDBAdaptor initialised"); - } +// private void init() { +// mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); +// refseqCollectionByRelease = buildCollectionByReleaseMap("refseq"); +// +// logger.debug("GeneMongoDBAdaptor initialised"); +// } @Override public CellBaseDataResult aggregationStats(GeneQuery query) { @@ -97,10 +100,10 @@ public List> info(List ids, ProjectionQueryOpti orBsonList.add(Filters.eq("name", id)); Bson query = Filters.or(orBsonList); if (StringUtils.isEmpty(source) || ParamConstants.QueryParams.ENSEMBL.key().equalsIgnoreCase(source)) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); results.add(new CellBaseDataResult<>(mongoDBCollection.find(query, projection, CONVERTER, new QueryOptions()))); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(EtlCommons.REFSEQ_DATA, dataRelease); results.add(new CellBaseDataResult<>(mongoDBCollection.find(query, projection, CONVERTER, new QueryOptions()))); } } @@ -115,10 +118,10 @@ public CellBaseIterator iterator(GeneQuery query) throws CellBaseException MongoDBIterator iterator; if (query.getSource() != null && !query.getSource().isEmpty() && ParamConstants.QueryParams.REFSEQ.key() .equalsIgnoreCase(query.getSource().get(0))) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(EtlCommons.REFSEQ_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); } return new CellBaseMongoDBIterator<>(iterator); @@ -127,7 +130,7 @@ public CellBaseIterator iterator(GeneQuery query) throws CellBaseException @Override public CellBaseDataResult distinct(GeneQuery geneQuery) throws CellBaseException { Bson bsonDocument = parseQuery(geneQuery); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, geneQuery.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, geneQuery.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(geneQuery.getFacet(), bsonDocument, String.class)); } @@ -135,7 +138,7 @@ public CellBaseDataResult distinct(GeneQuery geneQuery) throws CellBaseE public CellBaseDataResult groupBy(GeneQuery geneQuery) throws CellBaseException { Bson bsonQuery = parseQuery(geneQuery); logger.info("geneQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, geneQuery.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, geneQuery.getDataRelease()); return groupBy(bsonQuery, geneQuery, "name", mongoDBCollection); } @@ -157,7 +160,7 @@ public CellBaseDataResult startsWith(String id, QueryOptions options, int projection = Projections.exclude("transcripts", "annotation"); } } - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(regex, projection, CONVERTER, options)); } @@ -355,7 +358,7 @@ public CellBaseDataResult getTfbs(String geneId, QueryOptions qu List pipeline = unwindAndMatchTranscripts(query, queryOptions); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(TranscriptTfbs.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); MongoDBIterator iterator = mongoDBCollection.iterator(pipeline, converter, queryOptions); List tfbs = new ArrayList<>(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java index e8d40c802e..7901e34e2e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java @@ -44,6 +44,7 @@ import java.util.*; +import static org.opencb.cellbase.lib.EtlCommons.*; import static org.opencb.cellbase.lib.MongoDBCollectionConfiguration.CONSERVATION_CHUNK_SIZE; import static org.opencb.cellbase.lib.MongoDBCollectionConfiguration.GENOME_SEQUENCE_CHUNK_SIZE; @@ -64,20 +65,10 @@ public class GenomeMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseC public GenomeMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("GenomeMongoDBAdaptor: in 'constructor'"); - - genomeInfoMongoDBCollectionByRelease = buildCollectionByReleaseMap("genome_info"); - mongoDBCollectionByRelease = buildCollectionByReleaseMap("genome_sequence"); - conservationMongoDBCollectionByRelease = buildCollectionByReleaseMap("conservation"); } public CellBaseDataResult getGenomeInfo(QueryOptions queryOptions, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(new Document(), queryOptions)); } @@ -88,7 +79,7 @@ public CellBaseDataResult getChromosomeInfo(String chromosomeId, QueryOptions qu queryOptions.addToListOption("include", "chromosomes.$"); } Document dbObject = new Document("chromosomes", new Document("$elemMatch", new Document("name", chromosomeId))); - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, dataRelease); return executeQuery(chromosomeId, dbObject, queryOptions, mongoDBCollection); } @@ -263,7 +254,7 @@ public List>> getConservation(List< ids.add(region.toString()); } - MongoDBCollection mongoDBCollection = getCollectionByRelease(conservationMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CONSERVATION_DATA, dataRelease); List cellBaseDataResults = executeQueryList2(ids, queries, options, mongoDBCollection); List>> conservationCellBaseDataResults = new ArrayList<>(); @@ -355,7 +346,7 @@ public List> getAllScoresByRegionList(List reg ids.add(region.toString()); } - MongoDBCollection mongoDBCollection = getCollectionByRelease(conservationMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CONSERVATION_DATA, dataRelease); List queryResults = executeQueryList2(ids, queries, options, mongoDBCollection); // List queryResults = executeQueryList(ids, queries, options); @@ -419,7 +410,7 @@ public List> getAllScoresByRegionList(List reg public CellBaseDataResult getGenomeSequenceRawData(List chunkIds, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_SEQUENCE_DATA, dataRelease); CellBaseQueryOptions queryOptions = new CellBaseQueryOptions(); queryOptions.setExcludes(Arrays.asList("_id", "_chunkIds")); @@ -438,7 +429,7 @@ public CellBaseDataResult getGenomeSequenceRawData(List(mongoDBCollection.find(bson, options)); } @@ -461,7 +452,7 @@ public CellBaseIterator iterator(GenomeQuery query) throws CellBaseE QueryOptions queryOptions = query.toQueryOptions(); List pipeline = unwind(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Chromosome.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(pipeline, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -511,14 +502,14 @@ public CellBaseDataResult aggregationStats(GenomeQuery query) { public CellBaseDataResult groupBy(GenomeQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("query: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_SEQUENCE_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @Override public CellBaseDataResult distinct(GenomeQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(genomeInfoMongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_INFO_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -526,7 +517,7 @@ public CellBaseDataResult distinct(GenomeQuery query) throws CellBaseExc public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) throws CellBaseException { List> results = new ArrayList<>(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENOME_SEQUENCE_DATA, dataRelease); for (String id : ids) { Bson projection = getProjection(queryOptions); List orBsonList = new ArrayList<>(ids.size()); @@ -580,7 +571,7 @@ public Collection getConservationScoreChunkIds(Region region) { public CellBaseDataResult getConservationScoreRegion(List chunkIds, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(conservationMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(CONSERVATION_DATA, dataRelease); Bson projection = getProjection(options); List orBsonList = new ArrayList<>(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java index e5cd4d38cc..4413f4bf48 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java @@ -50,11 +50,6 @@ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements CellBaseCoreDB public MetaMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - init(); - } - - - private void init() { logger.debug("MetaMongoDBAdaptor: in 'constructor'"); mongoDBCollection = mongoDataStore.getCollection("metadata"); apiKeyStatsMongoDBCollection = mongoDataStore.getCollection("apikey_stats", WriteConcern.ACKNOWLEDGED, ReadPreference.primary()); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java index bf649886af..f3968fb2df 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MissenseVariationFunctionalScoreMongoDBAdaptor.java @@ -33,19 +33,13 @@ import java.util.Collections; import java.util.List; +import static org.opencb.cellbase.lib.EtlCommons.MISSENSE_VARIATION_SCORE_DATA; + public class MissenseVariationFunctionalScoreMongoDBAdaptor extends CellBaseDBAdaptor { public MissenseVariationFunctionalScoreMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("MissenseVariationFunctionalScoreMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("missense_variation_functional_score"); } public CellBaseDataResult query(String chromosome, int position, String reference, int dataRelease) @@ -56,7 +50,7 @@ public CellBaseDataResult query(String chromosom andBsonList.add(Filters.eq("reference", reference)); Bson query = Filters.and(andBsonList); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(MISSENSE_VARIATION_SCORE_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(query, null, MissenseVariantFunctionalScore.class, new QueryOptions())); } @@ -73,7 +67,7 @@ public CellBaseDataResult getScores(St final String id = chromosome + ":" + position + ":" + reference + ":" + alternate; - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(MISSENSE_VARIATION_SCORE_DATA, dataRelease); DataResult missenseVariantFunctionalScoreDataResult = mongoDBCollection.find(query, null, MissenseVariantFunctionalScore.class, new QueryOptions()); @@ -97,7 +91,7 @@ public CellBaseDataResult getScores(St public CellBaseDataResult getScores(String chromosome, List positions, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(MISSENSE_VARIATION_SCORE_DATA, dataRelease); Bson projection = getProjection(options); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java index bcf9fcaa18..d0e55e360e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java @@ -715,4 +715,8 @@ protected List getIntervalFeatureFrequencies(Region re // public void setAssembly(String assembly) { // this.assembly = assembly; // } + + public MongoDataStore getMongoDataStore() { + return mongoDataStore; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java index f1e664a508..00b55506ca 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/OntologyMongoDBAdaptor.java @@ -38,6 +38,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.OBO_DATA; + public class OntologyMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { private static final GenericDocumentComplexConverter CONVERTER; @@ -48,14 +50,6 @@ public class OntologyMongoDBAdaptor extends CellBaseDBAdaptor implements CellBas public OntologyMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("OntologyMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("ontology"); } @Override @@ -64,7 +58,7 @@ public CellBaseIterator iterator(OntologyQuery query) throws CellB Bson projection = getProjection(query); QueryOptions queryOptions = query.toQueryOptions(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -74,7 +68,7 @@ public List> info(List ids, ProjectionQ String apiKey) throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, dataRelease); for (String id : ids) { List orBsonList = new ArrayList<>(ids.size()); orBsonList.add(Filters.eq("id", id)); @@ -88,7 +82,7 @@ public List> info(List ids, ProjectionQ @Override public CellBaseDataResult distinct(OntologyQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -101,7 +95,7 @@ public CellBaseDataResult aggregationStats(OntologyQuery query) { public CellBaseDataResult groupBy(OntologyQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("geneQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(OBO_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java index f0a601a13f..10a05a11f2 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PharmacogenomicsMongoDBAdaptor.java @@ -39,6 +39,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.PHARMACOGENOMICS_DATA; + /** * Created by jtarraga on 9/4/23. */ @@ -53,14 +55,6 @@ public class PharmacogenomicsMongoDBAdaptor extends CellBaseDBAdaptor public PharmacogenomicsMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - mongoDBCollectionByRelease = buildCollectionByReleaseMap("pharmacogenomics"); - - logger.debug("PharmacogenomicsMongoDBAdaptor initialised"); } @Override @@ -79,7 +73,7 @@ public List> info(List ids, Projectio orBsonList.add(Filters.eq("id", id)); orBsonList.add(Filters.eq("name", id)); Bson query = Filters.or(orBsonList); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PHARMACOGENOMICS_DATA, dataRelease); results.add(new CellBaseDataResult<>(mongoDBCollection.find(query, projection, CONVERTER, new QueryOptions()))); } return results; @@ -91,7 +85,7 @@ public CellBaseIterator iterator(PharmaChemicalQuery query) thro QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); MongoDBIterator iterator; - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PHARMACOGENOMICS_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -99,7 +93,7 @@ public CellBaseIterator iterator(PharmaChemicalQuery query) thro @Override public CellBaseDataResult distinct(PharmaChemicalQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PHARMACOGENOMICS_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java index 353b4042c4..c1646f2db0 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java @@ -42,6 +42,9 @@ import java.util.*; +import static org.opencb.cellbase.lib.EtlCommons.PROTEIN_DATA; +import static org.opencb.cellbase.lib.EtlCommons.PROTEIN_FUNCTIONAL_PREDICTION_DATA; + /** * Created by imedina on 01/12/15. */ @@ -79,18 +82,8 @@ public class ProteinMongoDBAdaptor extends CellBaseDBAdaptor implements CellBase aaShortNameMap.put("VAL", "V"); } - public ProteinMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("ProteinMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("protein"); - proteinSubstitutionMongoDBCollectionByRelease = buildCollectionByReleaseMap("protein_functional_prediction"); } public CellBaseDataResult getSubstitutionScores(TranscriptQuery query, Integer position, String aa) throws CellBaseException { @@ -100,8 +93,7 @@ public CellBaseDataResult getSubstitutionScores(TranscriptQuery query, In if (query.getTranscriptsId() != null && query.getTranscriptsId().get(0) != null) { String transcriptId = query.getTranscriptsId().get(0).split("\\.")[0]; Bson transcript = Filters.eq("transcriptId", transcriptId); - MongoDBCollection mongoDBCollection = getCollectionByRelease(proteinSubstitutionMongoDBCollectionByRelease, - query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_FUNCTIONAL_PREDICTION_DATA, query.getDataRelease()); String aaShortName = null; // If position and aa change are provided we create a 'projection' to return only the required data from the database @@ -272,7 +264,7 @@ public CellBaseDataResult getVariantAnnotation(String groupFields.put("feature", new Document("$addToSet", "$feature")); pipeline.add(new Document("$group", groupFields)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, dataRelease); proteinVariantData = executeAggregation2(ensemblTranscriptId + "_" + String.valueOf(position) + "_" + aaAlternate, pipeline, new QueryOptions(), mongoDBCollection); if (proteinVariantData.getNumResults() > 0) { @@ -294,7 +286,7 @@ public CellBaseIterator iterator(ProteinQuery query) throws CellBaseExcep QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Entry.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -303,7 +295,7 @@ public CellBaseIterator iterator(ProteinQuery query) throws CellBaseExcep public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) throws CellBaseException { List> results = new ArrayList<>(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, dataRelease); for (String id : ids) { Bson projection = getProjection(queryOptions); List orBsonList = new ArrayList<>(ids.size()); @@ -324,14 +316,14 @@ public CellBaseDataResult aggregationStats(ProteinQuery query) { public CellBaseDataResult groupBy(ProteinQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("proteinQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @Override public CellBaseDataResult distinct(ProteinQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -490,7 +482,7 @@ private ProteinVariantAnnotation processProteinVariantData(ProteinVariantAnnotat public CellBaseDataResult getProteinSubstitutionRawData(List transcriptIds, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(proteinSubstitutionMongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PROTEIN_FUNCTIONAL_PREDICTION_DATA, dataRelease); // Be sure to exclude the internal field "_id" Bson projection; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java index 5c8fcb571f..91087b66c1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/PublicationMongoDBAdaptor.java @@ -40,6 +40,8 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.opencb.cellbase.lib.EtlCommons.PUBMED_DATA; + public class PublicationMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { private static final GenericDocumentComplexConverter CONVERTER; @@ -50,14 +52,6 @@ public class PublicationMongoDBAdaptor extends CellBaseDBAdaptor implements Cell public PublicationMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("PublicationMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("pubmed"); } @Override @@ -66,7 +60,7 @@ public CellBaseIterator iterator(PublicationQuery query) throws C Bson projection = getProjection(query); QueryOptions queryOptions = query.toQueryOptions(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -76,7 +70,7 @@ public List> info(List ids, Projection String apiKey) throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, dataRelease); for (String id : ids) { List orBsonList = new ArrayList<>(ids.size()); orBsonList.add(Filters.eq("id", id)); @@ -89,7 +83,7 @@ public List> info(List ids, Projection @Override public CellBaseDataResult distinct(PublicationQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @@ -102,7 +96,7 @@ public CellBaseDataResult aggregationStats(PublicationQuery query public CellBaseDataResult groupBy(PublicationQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("geneQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(PUBMED_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java index e7373b57be..098b5fa4f1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java @@ -38,6 +38,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.REGULATORY_REGION_DATA; + /** * Created by imedina on 07/12/15. */ @@ -51,14 +53,6 @@ public class RegulationMongoDBAdaptor extends CellBaseDBAdaptor implements CellB public RegulationMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("RegulationMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("regulatory_region"); } public Bson parseQuery(RegulationQuery query) { @@ -98,7 +92,7 @@ public CellBaseIterator iterator(RegulationQuery query) throw Bson bson = parseQuery(query); QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REGULATORY_REGION_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -112,7 +106,7 @@ public List> info(List ids, Projec @Override public CellBaseDataResult distinct(RegulationQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REGULATORY_REGION_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java index 4098f9c535..747e821c0d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ReleaseMongoDBAdaptor.java @@ -47,7 +47,7 @@ public class ReleaseMongoDBAdaptor extends MongoDBAdaptor implements CellBaseCor private MongoDBCollection mongoDBCollection; - private final String DATA_RELEASE_COLLECTION_NAME = "data_release"; + public static final String DATA_RELEASE_COLLECTION_NAME = "data_release"; public ReleaseMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java index 7105be25bc..04bb2ea5d1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java @@ -38,23 +38,15 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.REPEATS_DATA; + /** * Created by fjlopez on 10/05/17. */ public class RepeatsMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { - private static final String REPEAT_COLLECTION = "repeats"; - public RepeatsMongoDBAdaptor(MongoDataStore mongoDatastore) { super(mongoDatastore); - - init(); - } - - private void init() { - logger.debug("RepeatsMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap(REPEAT_COLLECTION); } public Bson parseQuery(RepeatsQuery query) { @@ -95,7 +87,7 @@ public CellBaseIterator iterator(RepeatsQuery query) throws CellBaseException { QueryOptions queryOptions = query.toQueryOptions(); Bson projection = getProjection(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Repeat.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REPEATS_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java index 6b3d78ce83..ee69e98184 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SnpMongoDBAdaptor.java @@ -41,6 +41,7 @@ import static org.opencb.cellbase.core.ParamConstants.API_KEY_PARAM; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; +import static org.opencb.cellbase.lib.EtlCommons.SNP_DATA; public class SnpMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { @@ -52,14 +53,6 @@ public class SnpMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCore public SnpMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - logger.debug("SnpMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("snp"); } @Override @@ -68,7 +61,7 @@ public CellBaseIterator iterator(SnpQuery query) throws CellBaseException { Bson projection = getProjection(query); QueryOptions queryOptions = query.toQueryOptions(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, CONVERTER, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -78,7 +71,7 @@ public List> info(List ids, ProjectionQueryOptio throws CellBaseException { List> results = new ArrayList<>(); Bson projection = getProjection(queryOptions); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, dataRelease); for (String id : ids) { List orBsonList = new ArrayList<>(); orBsonList.add(Filters.eq("id", id)); @@ -92,7 +85,7 @@ public List> info(List ids, ProjectionQueryOptio public CellBaseDataResult distinct(SnpQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("snpQuery distinct: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonQuery, String.class)); } @@ -105,7 +98,7 @@ public CellBaseDataResult aggregationStats(SnpQuery query) { public CellBaseDataResult groupBy(SnpQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("snpQuery groupBy: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @@ -118,7 +111,7 @@ public CellBaseDataResult startsWith(String id, QueryOptions options, int d projection = Projections.exclude(options.getAsStringList(QueryOptions.EXCLUDE)); } - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(regex, projection, CONVERTER, options)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java index c4dd2d4656..553f579ff3 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/SpliceScoreMongoDBAdaptor.java @@ -23,7 +23,6 @@ import org.opencb.biodata.models.core.SpliceScoreAlternate; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.result.CellBaseDataResult; -import org.opencb.cellbase.lib.EtlCommons; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDBCollection; @@ -33,18 +32,12 @@ import java.util.Collections; import java.util.List; +import static org.opencb.cellbase.lib.EtlCommons.SPLICE_SCORE_DATA; + public class SpliceScoreMongoDBAdaptor extends CellBaseDBAdaptor { public SpliceScoreMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("SpliceScoreMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap(EtlCommons.SPLICE_SCORE_DATA); } public CellBaseDataResult getScores(String chromosome, int position, String reference, String alternate) @@ -67,7 +60,7 @@ public CellBaseDataResult getScores(String chromosome, int position final String id = chromosome + ":" + position + ":" + ref + ":" + alt; - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SPLICE_SCORE_DATA, dataRelease); DataResult spliceScoreDataResult = mongoDBCollection.find(query, null, SpliceScore.class, new QueryOptions()); List results = new ArrayList<>(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java index 9fc26d1c57..0bbf49e09e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java @@ -43,13 +43,14 @@ import java.util.*; +import static org.opencb.cellbase.lib.EtlCommons.GENE_DATA; +import static org.opencb.cellbase.lib.EtlCommons.REFSEQ_DATA; + /** * Created by swaathi on 27/11/15. */ public class TranscriptMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCoreDBAdaptor { - private Map refseqCollectionByRelease = null; - private static final GenericDocumentComplexConverter CONVERTER; static { @@ -58,13 +59,6 @@ public class TranscriptMongoDBAdaptor extends CellBaseDBAdaptor implements CellB public TranscriptMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - this.init(); - } - - private void init() { - mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); - refseqCollectionByRelease = buildCollectionByReleaseMap("refseq"); } @Override @@ -73,10 +67,10 @@ public CellBaseIterator iterator(TranscriptQuery query) throws CellB List pipeline = unwindAndMatchTranscripts(query, queryOptions); MongoDBIterator iterator; if (query.getSource() != null && !query.getSource().isEmpty() && "RefSeq".equalsIgnoreCase(query.getSource().get(0))) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REFSEQ_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } return new CellBaseMongoDBIterator<>(iterator); @@ -110,10 +104,10 @@ public List> info(List ids, ProjectionQue List pipeline = unwindAndMatchTranscripts(bson, queryOptions); MongoDBIterator iterator; if (StringUtils.isNotEmpty(source) && ParamConstants.QueryParams.REFSEQ.key().equalsIgnoreCase(source)) { - MongoDBCollection mongoDBCollection = getCollectionByRelease(refseqCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(REFSEQ_DATA, dataRelease); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } else { - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, dataRelease); iterator = mongoDBCollection.iterator(pipeline, CONVERTER, queryOptions); } @@ -160,7 +154,7 @@ public CellBaseDataResult count(TranscriptQuery query) throws CellBaseExce List projections = unwind(query); Bson group = Aggregates.group("transcripts", Accumulators.sum("count", 1)); projections.add(group); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); CellBaseDataResult cellBaseDataResult = new CellBaseDataResult(mongoDBCollection.aggregate(projections, null)); Number number = (Number) cellBaseDataResult.first().get("count"); Long count = number.longValue(); @@ -177,7 +171,7 @@ public CellBaseDataResult aggregationStats(TranscriptQuery query) { public CellBaseDataResult groupBy(TranscriptQuery query) throws CellBaseException { Bson bsonQuery = parseQuery(query); logger.info("transcriptQuery: {}", bsonQuery.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); return groupBy(bsonQuery, query, "name", mongoDBCollection); } @@ -185,7 +179,7 @@ public CellBaseDataResult groupBy(TranscriptQuery query) throws Cell public CellBaseDataResult distinct(TranscriptQuery query) throws CellBaseException { Bson bsonDocument = parseQuery(query); logger.info("transcriptQuery: {}", bsonDocument.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); return new CellBaseDataResult<>(mongoDBCollection.distinct(query.getFacet(), bsonDocument, String.class)); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java index 3c33266f7e..85e1c08781 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java @@ -58,6 +58,7 @@ import static org.opencb.cellbase.core.ParamConstants.API_KEY_PARAM; import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; +import static org.opencb.cellbase.lib.EtlCommons.*; import static org.opencb.cellbase.lib.MongoDBCollectionConfiguration.VARIATION_FUNCTIONAL_SCORE_CHUNK_SIZE; /** @@ -79,16 +80,6 @@ public class VariantMongoDBAdaptor extends CellBaseDBAdaptor implements CellBase public VariantMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("VariationMongoDBAdaptor: in 'constructor'"); - - mongoDBCollectionByRelease = buildCollectionByReleaseMap("variation"); - caddDBCollectionByRelease = buildCollectionByReleaseMap("variation_functional_score"); - snpDBCollectionByRelease = buildCollectionByReleaseMap("snp"); } public CellBaseDataResult next(Query query, QueryOptions options) { @@ -104,7 +95,7 @@ public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, if (query.getString(ParamConstants.QueryParams.REGION.key()) != null) { Region region = Region.parseRegion(query.getString(ParamConstants.QueryParams.REGION.key())); Bson bsonDocument = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return getIntervalFrequencies(bsonDocument, region, intervalSize, options, mongoDBCollection); } return null; @@ -129,13 +120,13 @@ public CellBaseDataResult update(List objectList, String field, String[] i public CellBaseDataResult count(Query query, int dataRelease) throws CellBaseException { Bson document = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult(mongoDBCollection.count(document)); } public CellBaseDataResult distinct(Query query, String field, int dataRelease) throws CellBaseException { Bson document = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult(mongoDBCollection.distinct(field, document)); } @@ -150,7 +141,7 @@ public CellBaseDataResult get(Query query, QueryOptions inputOptions, i // options = addPrivateExcludeOptions(options); logger.debug("query: {}", bson.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult<>(mongoDBCollection.find(bson, null, Variant.class, options)); } @@ -178,20 +169,20 @@ public CellBaseDataResult nativeGet(Query query, QueryOptions options, int dataR Bson bson = parseQuery(query); // options.put(MongoDBCollection.SKIP_COUNT, true); logger.debug("query: {}", bson.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new CellBaseDataResult(mongoDBCollection.find(bson, options)); } public Iterator iterator(Query query, QueryOptions inputOptions, int dataRelease) throws CellBaseException { Bson bson = parseQuery(query); QueryOptions options = addPrivateExcludeOptions(new QueryOptions(inputOptions)); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return new VariantMongoDBIterator(mongoDBCollection.nativeQuery().find(bson, options)); } public Iterator nativeIterator(Query query, QueryOptions options, int dataRelease) throws CellBaseException { Bson bson = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return mongoDBCollection.nativeQuery().find(bson, options); } @@ -205,13 +196,13 @@ public void forEach(Query query, Consumer action, QueryOptions o public CellBaseDataResult groupBy(Query query, String field, QueryOptions options, int dataRelease) throws CellBaseException { Bson bsonQuery = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return groupBy(bsonQuery, field, "name", options, mongoDBCollection); } public CellBaseDataResult groupBy(Query query, List fields, QueryOptions options, int dataRelease) throws CellBaseException { Bson bsonQuery = parseQuery(query); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); return groupBy(bsonQuery, fields, "name", options, mongoDBCollection); } @@ -481,7 +472,7 @@ private CellBaseDataResult updateAnnotation(List variantDocument QueryOptions options = new QueryOptions("upsert", false); options.put("multi", false); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); bulkWriteResult = new CellBaseDataResult<>(mongoDBCollection.update(queries, updates, options)); logger.info("{} object updated", bulkWriteResult.first().getModifiedCount()); @@ -534,7 +525,7 @@ private CellBaseDataResult updatePopulationFrequencies(List vari QueryOptions options = new QueryOptions("upsert", true); options.put("multi", false); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); bulkWriteResult = new CellBaseDataResult<>(mongoDBCollection.update(queries, updates, options)); logger.info("{} object updated", bulkWriteResult.first().getUpserts().size() + bulkWriteResult.first().getModifiedCount()); @@ -576,7 +567,7 @@ public CellBaseDataResult getFunctionalScoreVariant(Variant variant, Quer // QueryBuilder builder = QueryBuilder.start("_chunkIds").is(chunkId); Document query = new Document("_chunkIds", chunkId); - MongoDBCollection mongoDBCollection = getCollectionByRelease(caddDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_FUNCTIONAL_SCORE_DATA, dataRelease); CellBaseDataResult result = executeQuery(chromosome + "_" + position + "_" + reference + "_" + alternate, query, queryOptions, mongoDBCollection); @@ -731,7 +722,7 @@ public CellBaseIterator iterator(VariantQuery query) throws CellBaseExc Bson projection = getProjection(query); VariantConverter converter = new VariantConverter(); logger.info("query: {}", bson.toBsonDocument().toJson()); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(null, bson, projection, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } @@ -755,7 +746,7 @@ public CellBaseDataResult distinct(VariantQuery query) { public List> info(List ids, ProjectionQueryOptions queryOptions, int dataRelease, String apiKey) throws CellBaseException { List> results = new ArrayList<>(); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_DATA, dataRelease); Bson projection = getProjection(queryOptions); List variantIds = getVariantIds(ids, dataRelease); List orBsonList = new ArrayList<>(variantIds.size()); @@ -777,7 +768,7 @@ public List getFunctionalScoreChunkIds(Region region) { public CellBaseDataResult getFunctionalScoreRegion(List chunkIds, CellBaseQueryOptions options, int dataRelease) throws CellBaseException { - MongoDBCollection mongoDBCollection = getCollectionByRelease(caddDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(VARIATION_FUNCTIONAL_SCORE_DATA, dataRelease); Bson projection = getProjection(options); List orBsonList = new ArrayList<>(); @@ -811,7 +802,7 @@ private List getVariantIds(List ids, int dataRelease) throws Cel Bson query = Filters.or(orBsonList); // 2. We must exclude as much information as possible to improve performance - MongoDBCollection mongoDBCollection = getCollectionByRelease(snpDBCollectionByRelease, dataRelease); + MongoDBCollection mongoDBCollection = getMongoDBCollection(SNP_DATA, dataRelease); DataResult snpDataResult = mongoDBCollection.find(query, Projections.exclude(ANNOTATION_FIELD), Snp.class, new QueryOptions()); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java index 42889fb02f..6c578eabf6 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java @@ -40,6 +40,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.cellbase.lib.EtlCommons.GENE_DATA; + /** * Created by imedina on 07/12/15. */ @@ -47,13 +49,6 @@ public class XRefMongoDBAdaptor extends CellBaseDBAdaptor implements CellBaseCor public XRefMongoDBAdaptor(MongoDataStore mongoDataStore) { super(mongoDataStore); - - init(); - } - - private void init() { - logger.debug("XRefMongoDBAdaptor: in 'constructor'"); - mongoDBCollectionByRelease = buildCollectionByReleaseMap("gene"); } @Override @@ -61,7 +56,7 @@ public CellBaseIterator iterator(XrefQuery query) throws CellBaseException QueryOptions queryOptions = query.toQueryOptions(); List pipeline = unwind(query); GenericDocumentComplexConverter converter = new GenericDocumentComplexConverter<>(Xref.class); - MongoDBCollection mongoDBCollection = getCollectionByRelease(mongoDBCollectionByRelease, query.getDataRelease()); + MongoDBCollection mongoDBCollection = getMongoDBCollection(GENE_DATA, query.getDataRelease()); MongoDBIterator iterator = mongoDBCollection.iterator(pipeline, converter, queryOptions); return new CellBaseMongoDBIterator<>(iterator); } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java new file mode 100644 index 0000000000..99280240f5 --- /dev/null +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DataReleaseSingleton.java @@ -0,0 +1,237 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.cellbase.lib.impl.core.singleton; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import com.mongodb.client.model.changestream.FullDocument; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.config.SpeciesConfiguration; +import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.core.models.DataRelease; +import org.opencb.cellbase.lib.db.MongoDBManager; +import org.opencb.cellbase.lib.impl.core.ReleaseMongoDBAdaptor; +import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public final class DataReleaseSingleton { + + // Map where the key is dbname and the value is the DatabaseInfo + private Map dbInfoMap = new HashMap<>(); + + private ExecutorService executorService; + private CellBaseManagerFactory managerFactory; + + private static DataReleaseSingleton instance; + + private static final Logger LOGGER = LoggerFactory.getLogger(DataReleaseSingleton.class); + + public static final String UNKOWN_DATABASE_MSG_PREFIX = "Unknown database "; + public static final String INVALID_RELEASE_MSG_PREFIX = "Invalid release "; + public static final String INVALID_DATA_MSG_PREFIX = "Invalid data "; + + // Private constructor to prevent instantiation + private DataReleaseSingleton(CellBaseManagerFactory managerFactory) throws CellBaseException { + // Create the executor service with a 4-thread pool and add shutdown hook to terminate thread pool + this.executorService = Executors.newFixedThreadPool(4); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LOGGER.info("Shutting down thread pool..."); + executorService.shutdownNow(); + try { + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.info("Thread pool did not terminate in the specified time."); + } + } catch (InterruptedException e) { + LOGGER.info("Thread pool termination interrupted."); + Thread.currentThread().interrupt(); + } + })); + + this.managerFactory = managerFactory; + + // Support multi species and assemblies + CellBaseConfiguration configuration = managerFactory.getConfiguration(); + for (SpeciesConfiguration vertebrate : configuration.getSpecies().getVertebrates()) { + for (SpeciesConfiguration.Assembly assembly : vertebrate.getAssemblies()) { + String databaseName = MongoDBManager.getDatabaseName(vertebrate.getId(), assembly.getName(), configuration.getVersion()); + // This is necessary, before creating the database name the assembly is "cleaned", and we need to get the data release + // manager from the species and the assembly + DatabaseInfo dbInfo = new DatabaseInfo(databaseName, vertebrate.getId(), assembly.getName(), new ReentrantLock(), + new HashMap<>()); + dbInfoMap.put(databaseName, dbInfo); + + MongoClient mongoClient = managerFactory.getDataReleaseManager(vertebrate.getId(), assembly.getName()).getMongoDatastore() + .getMongoClient(); + MongoDatabase database = mongoClient.getDatabase(databaseName); + MongoCollection collection = database.getCollection(ReleaseMongoDBAdaptor.DATA_RELEASE_COLLECTION_NAME); + LOGGER.info("Setting listener for database {} and collection {}", database.getName(), collection.getNamespace() + .getCollectionName()); + // Set up the change stream for the collection + executorService.submit(() -> { + while (!Thread.currentThread().isInterrupted()) { + try { + collection.watch().fullDocument(FullDocument.UPDATE_LOOKUP).forEach(changeStreamDocument -> { + try { + handleDocumentChange(changeStreamDocument); + } catch (CellBaseException e) { + LOGGER.warn("Exception from handle document change function (database = {}, collection = {}): {}", + collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), e); + } + }); + } catch (Exception e) { + LOGGER.error("Failed to watch collection (database = {}, collection = {}): {}", + collection.getNamespace().getDatabaseName(), collection.getNamespace().getCollectionName(), e); + try { + // Sleep 5 sec before retrying to avoid tight loop + Thread.sleep(5000); + } catch (InterruptedException ie) { + // Restore interrupt status, and exit loop if interrupted + Thread.currentThread().interrupt(); + break; + } + } + } + }); + } + } + } + + // Initialization method to set up the instance with parameters + public static synchronized void initialize(CellBaseManagerFactory managerFactory) throws CellBaseException { + if (instance == null) { + instance = new DataReleaseSingleton(managerFactory); + } + } + + // Method to get the single instance of the class + public static DataReleaseSingleton getInstance() { + if (instance == null) { + throw new IllegalStateException("Singleton not initialized. Call the function 'initialize' first."); + } + return instance; + } + + // Method to load data from MongoDB and cache it + private void loadData(String dbname) throws CellBaseException { + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + + String species = dbInfo.getSpecies(); + String assembly = dbInfo.getAssembly(); + ReleaseMongoDBAdaptor releaseMongoDBAdaptor = managerFactory.getDataReleaseManager(species, assembly).getReleaseDBAdaptor(); + List dataReleases = releaseMongoDBAdaptor.getAll().getResults(); + if (CollectionUtils.isNotEmpty(dataReleases)) { + for (DataRelease dataRelease : dataReleases) { + Map collectionMap = new HashMap<>(); + for (Map.Entry entry : dataRelease.getCollections().entrySet()) { + collectionMap.put(entry.getKey(), releaseMongoDBAdaptor.getMongoDataStore().getCollection(entry.getValue())); + } + dbInfo.getCacheData().put(dataRelease.getRelease(), collectionMap); + } + } + } + + public void checkDataRelease(String species, String assembly, String version, int release) throws CellBaseException { + checkDataRelease(MongoDBManager.getDatabaseName(species, assembly, version), release); + } + + public void checkDataRelease(String dbname, int release) throws CellBaseException { + checkDataRelease(dbname, release, null); + } + + public void checkDataRelease(String dbname, int release, String data) throws CellBaseException { + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + + if (!dbInfo.getCacheData().containsKey(release) + || (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data))) { + try { + // Lock and load data if necessary + dbInfo.getLock().lock(); + if (!dbInfo.getCacheData().containsKey(release) + || (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data))) { + // Load the data releases from the MongoDB collection for that database name + loadData(dbname); + } + } finally { + dbInfo.getLock().unlock(); + } + + // Check after loading + if (!dbInfo.getCacheData().containsKey(release)) { + // If the release is invalid, throw an exception + String msg = INVALID_RELEASE_MSG_PREFIX + release + ". The available data releases are: " + + dbInfo.getCacheData().keySet(); + throw new CellBaseException(msg); + } + if (StringUtils.isNotEmpty(data) && !dbInfo.getCacheData().get(release).containsKey(data)) { + // If the data is invalid, throw an exception + String msg = INVALID_DATA_MSG_PREFIX + " '" + data + "', it's not present in release " + release + + ". The available data are: " + dbInfo.getCacheData().get(release).keySet(); + throw new CellBaseException(msg); + } + } + } + + // Method to get collection name based on the data and the release + public MongoDBCollection getMongoDBCollection(String dbname, String data, int release) throws CellBaseException { + checkDataRelease(dbname, release, data); + + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + return dbInfo.getCacheData().get(release).get(data); + } + + private void handleDocumentChange(ChangeStreamDocument changeStreamDocument) throws CellBaseException { + // Get database name + String dbname = changeStreamDocument.getNamespace().getDatabaseName(); + String collectionName = changeStreamDocument.getNamespace().getCollectionName(); + LOGGER.info("Collection {} of database {} has been updated", collectionName, dbname); + + DatabaseInfo dbInfo = getDatabaseInfo(dbname); + + // Handle the change event + dbInfo.getLock().lock(); + try { + loadData(dbname); + } finally { + dbInfo.getLock().unlock(); + } + } + + private DatabaseInfo getDatabaseInfo(String dbname) throws CellBaseException { + if (!dbInfoMap.containsKey(dbname)) { + // Unknown database + String msg = UNKOWN_DATABASE_MSG_PREFIX + dbname; + throw new CellBaseException(msg); + } + + return dbInfoMap.get(dbname); + } +} diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java new file mode 100644 index 0000000000..d953f278d1 --- /dev/null +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/singleton/DatabaseInfo.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.cellbase.lib.impl.core.singleton; + +import org.opencb.commons.datastore.mongodb.MongoDBCollection; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +public class DatabaseInfo { + private String dbName; + private String species; + private String assembly; + private ReentrantLock lock; + private Map> cacheData; + + public DatabaseInfo() { + this.lock = new ReentrantLock(); + this.cacheData = new HashMap<>(); + } + + public DatabaseInfo(String dbName, String species, String assembly, ReentrantLock lock, + Map> cacheData) { + this.dbName = dbName; + this.species = species; + this.assembly = assembly; + this.lock = lock; + this.cacheData = cacheData; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("DatabaseInfo{"); + sb.append("dbName='").append(dbName).append('\''); + sb.append(", species='").append(species).append('\''); + sb.append(", assembly='").append(assembly).append('\''); + sb.append(", lock=").append(lock); + sb.append(", cacheData=").append(cacheData); + sb.append('}'); + return sb.toString(); + } + + public String getDbName() { + return dbName; + } + + public DatabaseInfo setDbName(String dbName) { + this.dbName = dbName; + return this; + } + + public String getSpecies() { + return species; + } + + public DatabaseInfo setSpecies(String species) { + this.species = species; + return this; + } + + public String getAssembly() { + return assembly; + } + + public DatabaseInfo setAssembly(String assembly) { + this.assembly = assembly; + return this; + } + + public ReentrantLock getLock() { + return lock; + } + + public DatabaseInfo setRwLock(ReentrantLock lock) { + this.lock = lock; + return this; + } + + public Map> getCacheData() { + return cacheData; + } + + public DatabaseInfo setCacheData(Map> cacheData) { + this.cacheData = cacheData; + return this; + } +} diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java index 07c5a0f863..3ee8e79590 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java @@ -107,4 +107,8 @@ protected List createQueries(Query query, String csvField, String queryKe public void close() { mongoDBManager.close(); } + + public MongoDataStore getMongoDatastore() { + return mongoDatastore; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java index ba6e90e150..9752b03004 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java @@ -20,6 +20,7 @@ import org.opencb.cellbase.core.config.SpeciesConfiguration; import org.opencb.cellbase.core.exception.CellBaseException; import org.opencb.cellbase.core.utils.SpeciesUtils; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +51,7 @@ public class CellBaseManagerFactory { private Logger logger; - public CellBaseManagerFactory(CellBaseConfiguration configuration) { + public CellBaseManagerFactory(CellBaseConfiguration configuration) throws CellBaseException { this.configuration = configuration; logger = LoggerFactory.getLogger(this.getClass()); @@ -67,6 +68,8 @@ public CellBaseManagerFactory(CellBaseConfiguration configuration) { ontologyManagers = new HashMap<>(); dataReleaseManagers = new HashMap<>(); pharmacogenomicsManagers = new HashMap<>(); + + DataReleaseSingleton.initialize(this); } private String getMultiKey(String species, String assembly) { @@ -323,7 +326,7 @@ public OntologyManager getOntologyManager(String species, String assembly) throw return ontologyManagers.get(multiKey); } - public DataReleaseManager getDataRelesaseManager(String species) throws CellBaseException { + public DataReleaseManager getDataReleaseManager(String species) throws CellBaseException { if (species == null) { throw new CellBaseException("Species is required."); } @@ -374,4 +377,8 @@ public PharmacogenomicsManager getPharmacogenomicsManager(String species, String } return pharmacogenomicsManagers.get(multiKey); } + + public CellBaseConfiguration getConfiguration() { + return configuration; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java index c768cb15dc..ac5c27865a 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/DataReleaseManager.java @@ -240,4 +240,13 @@ public DataRelease checkDataRelease(int inRelease) throws CellBaseException { + ". Valid data releases are: " + StringUtils.join(dataReleases.stream().map(dr -> dr.getRelease()) .collect(Collectors.toList()), ",")); } + + public ReleaseMongoDBAdaptor getReleaseDBAdaptor() { + return releaseDBAdaptor; + } + + public DataReleaseManager setReleaseDBAdaptor(ReleaseMongoDBAdaptor releaseDBAdaptor) { + this.releaseDBAdaptor = releaseDBAdaptor; + return this; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java index 1b86b49367..cc64afe6bf 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/variant/annotation/VariantAnnotationCalculator.java @@ -469,7 +469,7 @@ private List runAnnotationProcess(List normalizedVar FutureSnpAnnotator futureSnpAnnotator = null; Future>> snpFuture = null; - if (annotatorSet.contains("xrefs") && dataRelease.getCollections().containsKey(EtlCommons.SNP_COLLECTION_NAME)) { + if (annotatorSet.contains("xrefs") && dataRelease.getCollections().containsKey(EtlCommons.SNP_DATA)) { futureSnpAnnotator = new FutureSnpAnnotator(normalizedVariantList, dataRelease.getRelease(), variantManager, logger); snpFuture = CACHED_THREAD_POOL.submit(futureSnpAnnotator); } diff --git a/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java new file mode 100644 index 0000000000..ce48737958 --- /dev/null +++ b/cellbase-lib/src/test/java/org/opencb/cellbase/lib/impl/core/DataReleaseSingletonTest.java @@ -0,0 +1,98 @@ +package org.opencb.cellbase.lib.impl.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.hamcrest.CoreMatchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opencb.biodata.models.core.Gene; +import org.opencb.cellbase.core.api.GeneQuery; +import org.opencb.cellbase.core.api.query.QueryException; +import org.opencb.cellbase.core.exception.CellBaseException; +import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.lib.GenericMongoDBAdaptorTest; +import org.opencb.cellbase.lib.db.MongoDBManager; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; +import org.opencb.cellbase.lib.managers.GeneManager; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDataStore; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.opencb.cellbase.core.ParamConstants.DATA_RELEASE_PARAM; + +class DataReleaseSingletonTest extends GenericMongoDBAdaptorTest { + + private String dbname; + + @BeforeEach + public void init() throws CellBaseException { + dbname = MongoDBManager.getDatabaseName(SPECIES, ASSEMBLY, cellBaseConfiguration.getVersion()); + } + + @Test + public void testChangeCollectionMap() throws CellBaseException, QueryException, IllegalAccessException, JsonProcessingException { + MongoDBManager mongoDBManager = new MongoDBManager(cellBaseConfiguration); + MongoDataStore mongoDatastore = mongoDBManager.createMongoDBDatastore(SPECIES, ASSEMBLY); + + GeneManager geneManager = cellBaseManagerFactory.getGeneManager(SPECIES, ASSEMBLY); + +// DataReleaseManager dataReleaseManager = new DataReleaseManager(SPECIES, ASSEMBLY, cellBaseConfiguration); +// DataRelease newDataRelease = dataReleaseManager.createRelease(); +// assertEquals(5, newDataRelease.getRelease()); + + Map paramMap = new HashMap<>(); + paramMap.put("id", "ENSG00000248746"); + paramMap.put("include", "id,name,start,end"); + paramMap.put(DATA_RELEASE_PARAM, "1"); + + GeneQuery geneQuery = new GeneQuery(paramMap); + geneQuery.setCount(Boolean.TRUE); + + CellBaseDataResult cellBaseDataResult = geneManager.search(geneQuery); + // WARNING: these values below may slightly change from one data version to another + assertEquals(1, cellBaseDataResult.getNumMatches()); + assertThat(cellBaseDataResult.getResults().stream().map(gene -> gene.getName()).collect(Collectors.toList()), + CoreMatchers.hasItems("ACTN3")); + } + + @Test + public void testDataReleaseSingleton() throws CellBaseException { + MongoDBCollection collection = DataReleaseSingleton.getInstance().getMongoDBCollection(dbname, "gene", 1); + assertTrue(collection != null); + } + + @Test + public void testCheckDatabaseFail() throws CellBaseException { + CellBaseException exception = assertThrows(CellBaseException.class, () -> DataReleaseSingleton.getInstance() + .checkDataRelease("toto", 10)); + assertTrue(exception.getMessage().startsWith(DataReleaseSingleton.UNKOWN_DATABASE_MSG_PREFIX)); + } + + @Test + public void testCheckReleaseFail() throws CellBaseException { + CellBaseException exception = assertThrows(CellBaseException.class, () -> DataReleaseSingleton.getInstance() + .checkDataRelease(dbname, 10)); + assertTrue(exception.getMessage().startsWith(DataReleaseSingleton.INVALID_RELEASE_MSG_PREFIX)); + } + + @Test + public void testCheckReleasePass() throws CellBaseException { + DataReleaseSingleton.getInstance().checkDataRelease(dbname, 1); + } + + @Test + public void testCheckDataFail() throws CellBaseException { + CellBaseException exception = assertThrows(CellBaseException.class, () -> DataReleaseSingleton.getInstance() + .checkDataRelease(dbname, 1, "toto")); + assertTrue(exception.getMessage().startsWith(DataReleaseSingleton.INVALID_DATA_MSG_PREFIX)); + } + + @Test + public void testCheckDataPass() throws CellBaseException { + DataReleaseSingleton.getInstance().checkDataRelease(dbname, 1, "gene"); + } +} \ No newline at end of file diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index fc961bb9ae..775cdd8d39 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -36,6 +36,7 @@ import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.core.utils.SpeciesUtils; +import org.opencb.cellbase.lib.impl.core.singleton.DataReleaseSingleton; import org.opencb.cellbase.lib.managers.CellBaseManagerFactory; import org.opencb.cellbase.lib.managers.DataReleaseManager; import org.opencb.cellbase.lib.managers.MetaManager; @@ -226,6 +227,7 @@ protected int getDataRelease() throws CellBaseException { + "' for CellBase version '" + version + "'"); return defaultDataRelease.getRelease(); } else { + DataReleaseSingleton.getInstance().checkDataRelease(species, assembly, version, dataRelease); return dataRelease; } } catch (NumberFormatException e) {