LDEV-5879 switch update provider to use maven

https://luceeserver.atlassian.net/browse/LDEV-5879

Author: michaeloffner

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
     name: Java build
     runs-on: ubuntu-latest
     env:
-      luceeVersion: 6.2.0.321
+      luceeVersion: 6.2.2.91
       #luceeVersionQuery: 6.0.3/all/jar
     steps:
     - uses: actions/checkout@v4
@@ -59,6 +59,7 @@ jobs:
         testAdditional: ${{ github.workspace }}/tests
         testSevices: mysql
         LUCEE_ADMIN_ENABLED: false
+        DEBUG: true
     - name: Run Lucee Test Suite (testLabels="data-provider-integration")
       uses: lucee/script-runner@main
       #continue-on-error: true
@@ -68,11 +69,18 @@ jobs:
         luceeVersion: ${{ env.luceeVersion }}
         #luceeVersionQuery: ${{ env.luceeVersionQuery }}
         extensionDir: ${{ github.workspace }}/
+        luceeCFConfig: ${{ github.workspace }}/devops/.CFconfig-update.json5
       env:
         testLabels: data-provider-integration
         testAdditional: ${{ github.workspace }}/tests
         testSevices: mysql
         LUCEE_ADMIN_ENABLED: false
+        DEBUG: true
+    - name: debug failure
+      if: ${{ failure() }}
+      run: |
+          pwd
+          ls -lR ${{ github.workspace }}
 
   build:
     name: Docker build & publish

.gitignore

@@ -27,3 +27,4 @@ update/WEB-INF
 update/log-maven-download.log
 update/missing-bundles.txt
 /tests/artifacts
+/apps/updateserver/services/legacy/artifacts

README.md

@@ -7,3 +7,31 @@ This repo contains the application code for the following services used by Lucee
 * https://extension.lucee.org
 
 https://luceeserver.atlassian.net/issues/?jql=labels%20%3D%20%22updates%22
+
+# Local Development
+
+By default, the data provider is configured to work in production.
+
+Create an `.env file` with the following settings to work locally.
+
+```env
+ALLOW_RELOAD=true
+S3_CORE_ROOT=/var/local_s3/lucee_downloads/
+S3_EXTENSIONS_ROOT=/var/local_s3/lucee_ext/
+S3_BUNDLES_ROOT=/var/local_s3/lucee_bundles/
+UPDATE_PROVIDER=http://127.0.0.1:8889/rest/update/provider/
+UPDATE_PROVIDER_INT=http://update:8888/rest/update/provider/ # internal docker networking
+EXTENSION_PROVIDER=http://127.0.0.1:8889/rest/extension/provider/
+EXTENSION_PROVIDER_INT=http://update:8888/rest/extension/provider/ # internal docker networking
+DOWNLOADS_URL=http://download:8888/
+```
+
+Create folders under `local_s3`
+
+- in `lucee_downloads` place a few sample Lucee full jars
+- in `lucee_ext` place some sample extension lex files
+
+The run `docker compose up`
+
+Running locally, the downloads page is at http://127.0.0.1:8888/
+and the update provider is http://127.0.0.1:8889/rest/update/provider/list?extended=true

apps/download/Application.cfc

@@ -7,34 +7,46 @@ component {
 		disallowDoctypeDecl: true
 	};
 
-	//this.s3.accessKeyId = server.system.environment.S3_DOWNLOAD_ACCESS_KEY_ID;
-	//this.s3.awsSecretKey = server.system.environment.S3_DOWNLOAD_SECRET_KEY;
-
 	request.s3Root="s3:///extension-downloads/";
 	request.s3URL="https://s3-eu-west-1.amazonaws.com/extension-downloads/";
 
 	function onApplicationStart() {
-		lock name="configure-sentry" type="exclusive" timeout=5{
-			// workaround for LDEV-5371
-			// if LUCEE_LOGGING_FORCE_APPENDER=console is set, sentry is bypassed
-			var deploy = expandPath( '{lucee-server}/../deploy/' );
-			var sentry_json = deploy & "/.CFconfig-sentry.json"; 
-			if ( FileExists( sentry_json ) ){
-				configImport( sentry_json, "server", "admin" );
-				systemOutput("sentry.json loaded via configImport - LDEV-5371", true);
-				fileDelete( sentry_json );
+		application.extensionMeta = deserializeJson( fileRead( "extensionMeta.json" ) );
+		application.sentryDsn = server.system.environment.SENTRY_DSN ?: "";
+
+		var sentryLogger = new services.SentryLogger(
+			config = {
+				dsn = application.sentryDsn,
+				environment = server.system.environment.SENTRY_ENVIRONMENT ?: "production",
+				release = server.system.environment.SENTRY_RELEASE ?: "",
+				serverName = server.system.environment.SENTRY_SERVER_NAME ?: cgi.server_name
 			}
-		}
-		application.extensionMeta = deserializeJson(fileRead("extensionMeta.json"));
+		);
+
+		application.sentryLogger = sentryLogger;
 	}
 
-	function onError(e){
- 		if (cgi.script_name contains "admin" or cgi.script_name contains "lucee"){
+	function onError( e ){
+		if ( cgi.script_name contains "admin" or cgi.script_name contains "lucee" ){
 			header statuscode="418" statustext="nice try, you're a teapot";
 			return;
 		}
+
+		// Log to Sentry
+		try {
+			if ( structKeyExists( application, "sentryLogger" ) ) {
+				application.sentryLogger.logException(
+					exception = arguments.e,
+					level = "error"
+				);
+			}
+		} catch ( any err ) {
+			// Don't let Sentry failures break error handling
+			systemOutput( "Failed to log error to Sentry: #err.message#", true );
+		}
+
 		header statuscode="500" statustext="Server error";
-		echo("Sorry, Server error");
+		echo( "Sorry, Server error" );
 	}
 
 	function onRequest( string requestedPath ) output=true {

apps/download/download.cfc

@@ -1,29 +1,28 @@
 component {
 
-	variables.EXTENSION_PROVIDER="https://extension.lucee.org/rest/extension/provider/info?withLogo=true&type=all";
-	//variables.EXTENSION_PROVIDER="http://127.0.0.1:8889/rest/extension/provider/info?withLogo=true&type=all";
-
-	variables.EXTENSION_DOWNLOAD="https://extension.lucee.org/rest/extension/provider/{type}/{id}";
-
-	variables.UPDATE_PROVIDER = "https://update.lucee.org/rest/update/provider";
-
-	//variables.UPDATE_PROVIDER = "http://127.0.0.1:8889/rest/update/provider";
+	// provider urls are to communicate between the docker instances, so they need different urls
 
+	// set EXTENSION_PROVIDER_INT=http://update:8888/rest/extension/provider in .env for local testing
+	variables.EXTENSION_PROVIDER = server.system.environment.EXTENSION_PROVIDER_INT ?: "https://extension.lucee.org/rest/extension/provider";
+	
+	// set DOWNLOAD_UPDATE_PROVIDER_INT=http://update:8888/rest/update/provider in .env for local testing
+	variables.UPDATE_PROVIDER = server.system.environment.UPDATE_PROVIDER_INT ?: "https://update.lucee.org/rest/update/provider";
 
 	function getExtensions(flush=false) localmode=true {
+		var extUrl = EXTENSION_PROVIDER &"/info?withLogo=true&type=all";
 		if ( arguments.flush || isNull( application.extInfo ) ) {
-			http url=EXTENSION_PROVIDER&"&flush="&arguments.flush result="http";
+			http url=extUrl result="http";
 
 			if ( isNull( http.status_code ) || http.status_code != 200 )
-				throw "could not connect to extension provider (#EXTENSION_PROVIDER#)";
+				throw "could not connect to extension provider (#extUrl#)";
 
 			var data = deSerializeJson( http.fileContent, false );
-			if (!structKeyExists( data, "meta" ) ) {
+			if ( !structKeyExists( data, "meta" ) ) {
 				systemOutput( "error fetching extensions, falling back on cache", true);
-				http url=EXTENSION_PROVIDER result="http";
+				http url=extUrl result="http";
 
 				if ( isNull( http.status_code ) || http.status_code != 200 )
-					throw "could not connect to extension provider (#EXTENSION_PROVIDER#)";
+					throw "could not connect to extension provider (#extUrl#)";
 
 				data = deSerializeJson( http.fileContent, false );
 				application.extInfo = data.extensions;
@@ -240,8 +239,12 @@ component {
 			application[ "changelogLastUpdated" ] = lastUpdated;
 			// application.mavenInfo = {};  // not currently used
 			// maven dates are static, only purge if unknown
-			loop collection="#application.mavenDates#" key="local.version" value="local.date" {
-				if ( len(date) eq 0 ) structDelete( application.mavenDates, version );
+			if ( structKeyExists( application, "mavenDates" ) ){
+				loop collection="#application.mavenDates#" key="local.version" value="local.date" {
+					if ( len(date) eq 0 ) structDelete( application.mavenDates, version );
+				}
+			} else {
+				application.mavenDates = {};
 			}
 
 		}