Intelligent avoid of files re-download

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "cellbase-web/lib/jsorolla"]
 	path = cellbase-web/lib/jsorolla
-	url = https://github.com/opencb/jsorolla.git
+	url = https://github.com/Lucioric2000/jsorolla.git

cellbase-app/app/bin/obsolete/genome-fetcher/genome-fetcher.py

@@ -101,15 +101,15 @@
         logging.debug(sp_obj['assembly'])
         outfile = sequence_folder+"/"+sp_short.capitalize() + ".{0}".format(sp_obj['assembly'])+".fa.gz"
         logging.debug(outfile)
-        command = "wget --tries=10 " + url_seq +" -O '"+outfile+"' -o "+outfile+".log"
+        command = "wget -N --tries=10 " + url_seq +" -O '"+outfile+"' -o "+outfile+".log"
         logging.debug(command)
         os.system(command)
         cmd = "./genome_info.pl --species '{0}' -o {1}/genome_info.json".format(sp, sequence_folder)
         logging.debug(cmd)
         os.system(cmd)
         # for i in sp_obj['chromosomes']:
         #     outfile = seq_folder+"/chrom_"+i+".fa.gz"
-        #     command = "wget --tries=10 " + url_seq+"/*.dna.chromosome."+i+".fa.gz -O '"+outfile+"' -o "+outfile+".log"
+        #     command = "wget -N --tries=10 " + url_seq+"/*.dna.chromosome."+i+".fa.gz -O '"+outfile+"' -o "+outfile+".log"
         #     logging.debug(command)
         #     os.system(command)
 
@@ -121,7 +121,7 @@
         url_gtf = sp_obj['sequence_url'].replace("fasta", "gtf")+"{0}".format(sp_short)
         logging.debug(url_gtf)
         outfile = gene_folder+"/"+sp_short+".gtf.gz"
-        command = "wget --tries=10 " + url_gtf+"/*.gtf.gz -O '"+outfile+"' -o "+outfile+".log"
+        command = "wget -N --tries=10 " + url_gtf+"/*.gtf.gz -O '"+outfile+"' -o "+outfile+".log"
         logging.debug(command)
         os.system(command)
         if sp_obj['phylo'] is not 'Bacteria':
@@ -159,7 +159,7 @@
             variation_url = sp_obj['variation_url']
             for file in variation_files:
                 outfile = variation_folder+"/"+file
-                command = "wget --tries=10 " + variation_url+"/"+file+" -O '"+outfile+"' -o "+outfile+".log"
+                command = "wget -N --tries=10 " + variation_url+"/"+file+" -O '"+outfile+"' -o "+outfile+".log"
                 logging.debug(command)
                 os.system(command)
 
@@ -172,7 +172,7 @@
             regulation_url = sp_obj['regulation_url']+sp_short
             for file in regulation_files:
                 outfile = regulation_folder+"/"+file
-                command = "wget --tries=10 " + regulation_url+"/"+file+" -O '"+outfile+"' -o "+outfile+".log"
+                command = "wget -N --tries=10 " + regulation_url+"/"+file+" -O '"+outfile+"' -o "+outfile+".log"
                 logging.debug(command)
                 os.system(command)
 

cellbase-app/pom.xml

@@ -47,6 +47,11 @@
             <artifactId>commons-io</artifactId>
             <version>2.4</version>
         </dependency>
+        <dependency>
+            <groupId>commons-net</groupId>
+            <artifactId>commons-net</artifactId>
+            <version>${commons-net.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.spockframework</groupId>
             <artifactId>spock-core</artifactId>

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/DownloadCommandExecutor.java

@@ -24,6 +24,8 @@
 import org.apache.commons.collections.map.HashedMap;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPFile;
 import org.apache.commons.lang.StringUtils;
 import org.opencb.cellbase.core.config.Species;
 import org.opencb.commons.utils.FileUtils;
@@ -1165,6 +1167,25 @@ private void downloadFile(String url, String outputFileName) throws IOException,
     private void downloadFile(String url, String outputFileName, List<String> wgetAdditionalArgs)
             throws IOException, InterruptedException {
 
+        File file = new File(outputFileName);
+        if (file.exists()) {
+            FTPClient ftpClient = new FTPClient();
+            // code to connect and login....
+
+            FTPFile remotefile = ftpClient.mlistFile(outputFileName);
+            long remotesize = remotefile.getSize();
+            long localsize = file.length();
+            long locallastmodified = file.lastModified();
+            logger.info("File " + outputFileName + " exists, with size " + localsize + " vs "
+                + remotesize + " in remote and will not be downloaded again");
+            if (remotesize == localsize) {
+                logger.info("Download avoided because the files have the same size");
+                return;
+            }
+            //If the filestamps differ, regardless of the timestamps downloads the file
+            Calendar remotets = remotefile.getTimestamp();
+            logger.info("local ts " + locallastmodified + "wheras remote ts is " + remotets.toString());
+        }
         List<String> wgetArgs = new ArrayList<>(Arrays.asList("--tries=10", url, "-O", outputFileName, "-o",
                 outputFileName + ".log"));
         if (wgetAdditionalArgs != null && !wgetAdditionalArgs.isEmpty()) {

cellbase-app/src/main/java/org/opencb/cellbase/app/transform/clinical/variant/DOCMIndexer.java

@@ -30,7 +30,7 @@ public class DOCMIndexer extends ClinicalIndexer {
     private static final String DISEASE = "disease";
     private static final String SOURCE_PUBMED_ID = "source_pubmed_id";
     private static final String PMID = "PMID:";
-    private static final String URL_PREFIX = "http://docm.genome.wustl.edu/variants/";
+    private static final String URL_PREFIX = "http://docm.info/variants/";
     private static final String HGVS = "hgvs";
     private static final String GENE = "gene";
     private static final String TRANSCRIPT = "transcript";