[KNOWAGE-4803] Added support for scheduled XLS export

Author: Marco Balestri

knowagecockpitengine/src/main/java/it/eng/knowage/engine/cockpit/api/export/excel/ExcelExporter.java

@@ -18,14 +18,22 @@
 package it.eng.knowage.engine.cockpit.api.export.excel;
 
 import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import javax.ws.rs.core.UriBuilder;
+
+import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.LogMF;
 import org.apache.log4j.Logger;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
@@ -49,7 +57,9 @@
 import it.eng.knowage.engine.cockpit.api.export.excel.crosstab.CrosstabXLSXExporter;
 import it.eng.qbe.serializer.SerializationException;
 import it.eng.spago.error.EMFAbstractError;
+import it.eng.spago.error.EMFUserError;
 import it.eng.spagobi.analiticalmodel.document.bo.ObjTemplate;
+import it.eng.spagobi.commons.SingletonConfig;
 import it.eng.spagobi.commons.constants.SpagoBIConstants;
 import it.eng.spagobi.commons.dao.DAOFactory;
 import it.eng.spagobi.tools.dataset.bo.IDataSet;
@@ -74,14 +84,18 @@ public class ExcelExporter {
 	private final JSONObject body;
 	private Locale locale;
 	private int uniqueId = 0;
+	private String requestURL = "";
 
 	private static final String[] WIDGETS_TO_IGNORE = { "image", "text", "python", "r" };
+	private static final String SCRIPT_NAME = "cockpit-export-xls.js";
+	private static final String CONFIG_NAME_FOR_EXPORT_SCRIPT_PATH = "internal.nodejs.chromium.export.path";
 
 	// Old implementation with parameterMap
-	public ExcelExporter(String outputType, String userUniqueIdentifier, Map<String, String[]> parameterMap) {
+	public ExcelExporter(String outputType, String userUniqueIdentifier, Map<String, String[]> parameterMap, String requestURL) {
 		this.outputType = outputType;
 		this.userUniqueIdentifier = userUniqueIdentifier;
 		this.exportWidget = false;
+		this.requestURL = requestURL;
 		this.body = new JSONObject();
 
 		Locale locale = getLocale(parameterMap);
@@ -159,6 +173,61 @@ public String getMimeType() {
 		return mimeType;
 	}
 
+	// used only for scheduled exports
+	// leverages on an external script that uses chromium to open the cockpit and click on the export button
+	public byte[] getBinaryData(String documentLabel) throws IOException, InterruptedException, EMFUserError {
+
+		final Path outputDir = Files.createTempDirectory("knowage-xls-exporter-");
+
+		String encodedUserId = Base64.encodeBase64String(userUniqueIdentifier.getBytes("UTF-8"));
+
+		// Script
+		String cockpitExportScriptPath = SingletonConfig.getInstance().getConfigValue(CONFIG_NAME_FOR_EXPORT_SCRIPT_PATH);
+		Path exportScriptFullPath = Paths.get(cockpitExportScriptPath, SCRIPT_NAME);
+
+		if (!Files.isRegularFile(exportScriptFullPath)) {
+			String msg = String.format("Cannot find export script at \"%s\": did you set the correct value for %s configuration?", exportScriptFullPath,
+					CONFIG_NAME_FOR_EXPORT_SCRIPT_PATH);
+			IllegalStateException ex = new IllegalStateException(msg);
+			logger.error(msg, ex);
+			throw ex;
+		}
+
+		URI url = UriBuilder.fromUri(requestURL).replaceQueryParam("outputType_description", "HTML").replaceQueryParam("outputType", "HTML").build();
+
+		ProcessBuilder processBuilder = new ProcessBuilder("node", exportScriptFullPath.toString(), encodedUserId, outputDir.toString(), url.toString());
+		Process exec = processBuilder.start();
+		exec.waitFor();
+		// the script creates the resulting xls and saves it to outputFile
+		Path outputFile = outputDir.resolve(documentLabel + "." + outputType.toLowerCase());
+		return getByteArrayFromFile(outputFile, outputDir);
+	}
+
+	private byte[] getByteArrayFromFile(Path excelFile, Path outputDir) {
+		try {
+			FileInputStream fis = new FileInputStream(excelFile.toString());
+			ByteArrayOutputStream bos = new ByteArrayOutputStream();
+			byte[] buf = new byte[1024];
+			for (int readNum; (readNum = fis.read(buf)) != -1;) {
+				// Writes len bytes from the specified byte array starting at offset off to this byte array output stream
+				bos.write(buf, 0, readNum); // no doubt here is 0
+			}
+			fis.close();
+			return bos.toByteArray();
+		} catch (Exception e) {
+			logger.error("Cannot serialize excel file", e);
+			throw new SpagoBIRuntimeException("Cannot serialize excel file", e);
+		} finally {
+			try {
+				if (Files.isRegularFile(excelFile))
+					Files.delete(excelFile);
+				Files.delete(outputDir);
+			} catch (Exception e) {
+				logger.error("Cannot delete temp file", e);
+			}
+		}
+	}
+
 	public byte[] getBinaryData(Integer documentId, String documentLabel, String templateString, String options) throws JSONException, SerializationException {
 		if (templateString == null) {
 			ObjTemplate template = null;

knowagecockpitengine/src/main/java/it/eng/knowage/engine/cockpit/api/page/PageResource.java

@@ -161,6 +161,8 @@ private void openPageInternal(String pageName) {
 				String outputType = request.getParameter(OUTPUT_TYPE);
 				if ("xls".equalsIgnoreCase(outputType) || "xlsx".equalsIgnoreCase(outputType)) {
 					request.setAttribute("template", getIOManager().getTemplateAsString());
+					String requestURL = getRequestUrlForExcelExport(request);
+					request.setAttribute("requestURL", requestURL);
 					dispatchUrl = "/WEB-INF/jsp/ngCockpitExportExcel.jsp";
 					response.setContentType(MediaType.APPLICATION_OCTET_STREAM);
 				} else if ("pdf".equalsIgnoreCase(outputType)) {
@@ -267,6 +269,35 @@ private String getRequestUrlForPdfExport(HttpServletRequest request) throws Unsu
 		return sb.toString();
 	}
 
+	private String getRequestUrlForExcelExport(HttpServletRequest request) throws UnsupportedEncodingException {
+
+		String documentLabel = request.getParameter("DOCUMENT_LABEL");
+		BIObject biObject = null;
+		try {
+			biObject = DAOFactory.getBIObjectDAO().loadBIObjectByLabel(documentLabel);
+		} catch (EMFUserError e) {
+			throw new SpagoBIRuntimeException("Error retrieving document with label " + documentLabel, e);
+		}
+		Engine eng = biObject.getEngine();
+		String externalUrl = GeneralUtilities.getExternalEngineUrl(eng);
+
+		StringBuilder sb = new StringBuilder(externalUrl);
+		String sep = "?";
+		Map<String, String[]> parameterMap = request.getParameterMap();
+		for (String parameter : parameterMap.keySet()) {
+			String[] values = parameterMap.get(parameter);
+			if (values != null && values.length > 0) {
+				sb.append(sep);
+				sb.append(URLEncoder.encode(parameter, "UTF-8"));
+				sb.append("=");
+				sb.append(URLEncoder.encode(values[0], "UTF-8"));
+				sep = "&";
+			}
+		}
+		sb.append("&scheduledexport=true");
+		return sb.toString();
+	}
+
 	public String getServiceHostUrl() {
 		String serviceURL = SpagoBIUtilities.readJndiResource(SingletonConfig.getInstance().getConfigValue("SPAGOBI.SPAGOBI_SERVICE_JNDI"));
 		serviceURL = serviceURL.substring(0, serviceURL.lastIndexOf('/'));

knowagecockpitengine/src/main/webapp/WEB-INF/jsp/ngCockpitExportExcel.jsp

@@ -24,18 +24,19 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 <%@ page contentType="applicaton/octet-stream" %>
 <%
+
 String outputType = request.getParameter("outputType");
 String userId = request.getParameter("user_id");
 Map<String,String[]> parameterMap = request.getParameterMap();
 
 Integer documentId = Integer.valueOf(request.getParameter("document"));
 String documentLabel = request.getParameter("DOCUMENT_LABEL");
-String template = (String) request.getAttribute("template");
+String requestURL = (String) request.getAttribute("requestURL");
 
-ExcelExporter excelExporter = new ExcelExporter(outputType, userId, parameterMap);
+ExcelExporter excelExporter = new ExcelExporter(outputType, userId, parameterMap, requestURL);
 String mimeType = excelExporter.getMimeType();
 if(mimeType != null){
-	byte[] data = excelExporter.getBinaryData(documentId, documentLabel, template, "");
+	byte[] data = excelExporter.getBinaryData(documentLabel);
 	
 	response.setHeader("Content-length", Integer.toString(data.length));
 	response.setHeader("Content-Type", mimeType);

knowagecockpitengine/src/main/webapp/js/src/angular_1.4/cockpit/directives/cockpit-toolbar/cockpitToolbar.js

@@ -37,12 +37,17 @@ angular.module('cockpitModule')
 	}
 });
 
-function cockpitToolbarControllerFunction($scope,$timeout,$q,windowCommunicationService,cockpitModule_datasetServices,cockpitModule_widgetServices,cockpitModule_templateServices,cockpitModule_properties,cockpitModule_template,$mdDialog,sbiModule_translate,sbiModule_restServices,sbiModule_messaging,sbiModule_download,sbiModule_user,sbiModule_cockpitDocument,sbiModule_config,cockpitModule_gridsterOptions,$mdPanel,cockpitModule_widgetConfigurator,$mdToast,cockpitModule_generalServices,cockpitModule_widgetSelection,$rootScope){
+function cockpitToolbarControllerFunction($scope,$timeout,$q,$location,windowCommunicationService,cockpitModule_datasetServices,cockpitModule_widgetServices,cockpitModule_templateServices,cockpitModule_properties,cockpitModule_template,$mdDialog,sbiModule_translate,sbiModule_restServices,sbiModule_messaging,sbiModule_download,sbiModule_user,sbiModule_cockpitDocument,sbiModule_config,cockpitModule_gridsterOptions,$mdPanel,cockpitModule_widgetConfigurator,$mdToast,cockpitModule_generalServices,cockpitModule_widgetSelection,$rootScope){
 	$scope.translate = sbiModule_translate;
 	$scope.cockpitModule_properties=cockpitModule_properties;
 	$scope.cockpitModule_template=cockpitModule_template;
 	$scope.cockpitModule_widgetServices=cockpitModule_widgetServices;
 
+	if ($location.search()['scheduledexport'])
+		$scope.isScheduledExcelExport = true;
+	else
+		$scope.isScheduledExcelExport = false;
+
 	$scope.openGeneralConfigurationDialog=function(){
 		cockpitModule_generalServices.openGeneralConfiguration();
 	}

knowagecockpitengine/src/main/webapp/js/src/angular_1.4/cockpit/directives/cockpit-toolbar/templates/cockpitToolbar.html

@@ -57,10 +57,14 @@
 				</md-button>
       		</md-fab-actions>
       	</md-fab-speed-dial>
-      
+          		
 		<md-button aria-label="data" class="md-fab md-raised md-mini fixedSelectionButton animation shrink-down" ng-click="openSelections()" title = "Selections" ng-if="cockpitModule_template.configuration.showSelectionButton && cockpitModule_properties.HAVE_SELECTIONS_OR_FILTERS==true">
 			<md-icon md-font-icon="fa fa-check-square-o"></md-icon>
 			<md-tooltip md-direction="bottom">Selections</md-tooltip>
 		</md-button>
+		
+		<!-- This button is used only for scheduled excel export with chromium -->
+		<md-button id="scheduledExcelExportButton" ng-hide="!isScheduledExcelExport" aria-label="menu" class="md-fab md-warn" ng-click="exportExcel('xlsExport')" ><md-button>
+		
     </div>
 </div>