From 08246f741d1067f45da2436a6cc3a2a43e809b61 Mon Sep 17 00:00:00 2001 From: Alex Deparvu Date: Fri, 16 Jun 2023 13:53:08 -0700 Subject: [PATCH] [solr] Solr 9 support --- bin/bindings.properties | 3 +- pom.xml | 1 + solr9/README.md | 118 ++++++ solr9/pom.xml | 84 ++++ .../java/site/ycsb/db/solr9/SolrClient.java | 364 ++++++++++++++++++ .../java/site/ycsb/db/solr9/package-info.java | 23 ++ solr9/src/main/resources/log4j.properties | 22 ++ .../ycsb/db/solr9/SolrClientBaseTest.java | 169 ++++++++ .../ycsb/db/solr9/SolrClientCloudTest.java | 56 +++ .../site/ycsb/db/solr9/SolrClientTest.java | 63 +++ solr9/src/test/resources/log4j.properties | 29 ++ .../src/test/resources/solr_config/schema.xml | 66 ++++ .../test/resources/solr_config/solrconfig.xml | 63 +++ 13 files changed, 1060 insertions(+), 1 deletion(-) create mode 100644 solr9/README.md create mode 100644 solr9/pom.xml create mode 100644 solr9/src/main/java/site/ycsb/db/solr9/SolrClient.java create mode 100644 solr9/src/main/java/site/ycsb/db/solr9/package-info.java create mode 100644 solr9/src/main/resources/log4j.properties create mode 100644 solr9/src/test/java/site/ycsb/db/solr9/SolrClientBaseTest.java create mode 100644 solr9/src/test/java/site/ycsb/db/solr9/SolrClientCloudTest.java create mode 100644 solr9/src/test/java/site/ycsb/db/solr9/SolrClientTest.java create mode 100644 solr9/src/test/resources/log4j.properties create mode 100644 solr9/src/test/resources/solr_config/schema.xml create mode 100644 solr9/src/test/resources/solr_config/solrconfig.xml diff --git a/bin/bindings.properties b/bin/bindings.properties index 5c767c7599..54ebdc68c3 100755 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -70,7 +70,8 @@ rocksdb:site.ycsb.db.rocksdb.RocksDBClient s3:site.ycsb.db.S3Client scylla:site.ycsb.db.scylla.ScyllaCQLClient solr7:site.ycsb.db.solr7.SolrClient +solr9:site.ycsb.db.solr9.SolrClient tarantool:site.ycsb.db.TarantoolClient tablestore:site.ycsb.db.tablestore.TableStoreClient voltdb:site.ycsb.db.voltdb.VoltClient4 -zookeeper:site.ycsb.db.zookeeper.ZKClient \ No newline at end of file +zookeeper:site.ycsb.db.zookeeper.ZKClient diff --git a/pom.xml b/pom.xml index de2d5eea23..9eddb8f1ed 100644 --- a/pom.xml +++ b/pom.xml @@ -199,6 +199,7 @@ LICENSE file. seaweedfs scylla solr7 + solr9 tarantool tablestore voltdb diff --git a/solr9/README.md b/solr9/README.md new file mode 100644 index 0000000000..f1119a657a --- /dev/null +++ b/solr9/README.md @@ -0,0 +1,118 @@ + + +## Quick Start + +This section describes how to run YCSB on Solr running locally. + +### 1. Set Up YCSB + +Clone the YCSB git repository and compile: + + git clone git://github.com/brianfrankcooper/YCSB.git + cd YCSB + mvn -pl site.ycsb:solr9-binding -am clean package + +### 2. Set Up Solr + +There must be a running Solr instance with a core/collection pre-defined and configured. +- See this [API](https://solr.apache.org/guide/solr/9_2/configuration-guide/coreadmin-api.html#coreadmin-create) reference on how to create a core. +- See this [API](https://solr.apache.org/guide/solr/9_2/deployment-guide/collection-management.html#create) reference on how to create a collection in SolrCloud mode. + +The `conf/schema.xml` configuration file present in the core/collection just created must be configured to handle the expected field names during benchmarking. +Below illustrates a sample from a schema config file that matches the default field names used by the YCSB client: + + + + + + + + + + + + + +If running in [SolrCloud](https://solr.apache.org/guide/solr/9_2/deployment-guide/cluster-types.html#solrcloud-mode) mode ensure there is an external Zookeeper cluster running. +- See [here](https://solr.apache.org/guide/solr/9_2/deployment-guide/zookeeper-ensemble.html) for details on how to set up an external Zookeeper cluster. +- See [here](https://solr.apache.org/guide/solr/9_2/deployment-guide/zookeeper-file-management.html) for instructions on how to use Zookeeper to manage your core/collection configuration files. + +### 3. Run YCSB + +Now you are ready to run! First, load the data: + + ./bin/ycsb load solr9 -s -P workloads/workloada -p table= + +Then, run the workload: + + ./bin/ycsb run solr9 -s -P workloads/workloada -p table= + +For further configuration see below: + +### Default Configuration Parameters +The default settings for the Solr node that is created is as follows: + +- `solr.cloud` + - A Boolean value indicating if Solr is running in SolrCloud mode. If so there must be an external Zookeeper cluster running also. + - Default value is `true` and therefore expects Solr to be running in cloud mode. + +- `solr.base.url` + - The base URL in which to interface with a running Solr instance in stand-alone mode + - Default value is `http://localhost:8983/solr + +- `solr.commit.within.time` + - The max time in ms to wait for a commit when in batch mode, ignored otherwise + - Default value is `1000ms` + +- `solr.batch.mode` + - Indicates if inserts/updates/deletes should be committed in batches (frequency controlled by the `solr.commit.within.time` parameter) or commit 1 document at a time. + - Default value is `false` + +- `solr.zookeeper.hosts` + - A list of comma separated host:port pairs of Zookeeper nodes used to manage SolrCloud configurations. + - Must be passed when in [SolrCloud](https://solr.apache.org/guide/solr/9_2/deployment-guide/cluster-types.html#solrcloud-mode) mode. + - Default value is `localhost:2181` + +- `solr.basicAuth` + - Enables [basic authentication](https://solr.apache.org/guide/solr/9_2/deployment-guide/basic-authentication-plugin.html) for Solr client + - Example `-p solr.basicAuth=solr:SolrRocks` + +- `solr.httpClientConfig` + - Enables [basic authentication via a config file](https://solr.apache.org/guide/solr/9_2/deployment-guide/basic-authentication-plugin.html) for Solr client + - Example `-p solr.httpClientConfig=path/to/config` + +### Custom Configuration +If you wish to customize the settings used to create the Solr node +you can created a new property file that contains your desired Solr +node settings and pass it in via the parameter to 'bin/ycsb' script. Note that +the default properties will be kept if you don't explicitly overwrite them. + +Assuming that we have a properties file named "myproperties.data" that contains +custom Solr node configuration you can execute the following to +pass it into the Solr client: + + ./bin/ycsb run solr9 -P workloads/workloada -P myproperties.data -s + +If you wish to use SolrCloud mode ensure a Solr cluster is running with an +external Zookeeper cluster and an appropriate collection has been created. +Make sure to pass the following properties as parameters to 'bin/ycsb' script. + + solr.cloud=true + solr.zookeeper.hosts=:,...,: + + diff --git a/solr9/pom.xml b/solr9/pom.xml new file mode 100644 index 0000000000..3fc2f0f413 --- /dev/null +++ b/solr9/pom.xml @@ -0,0 +1,84 @@ + + + + + 4.0.0 + + site.ycsb + binding-parent + 0.18.0-SNAPSHOT + ../binding-parent + + + solr9-binding + Solr 9 Binding + jar + + + 9.2.0 + + + + + site.ycsb + core + ${project.version} + provided + + + org.apache.solr + solr-solrj + ${solr9.version} + + + org.apache.solr + solr-solrj-zookeeper + ${solr9.version} + + + + commons-codec + commons-codec + 1.14 + + + org.slf4j + slf4j-api + 2.0.6 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.20.0 + + + org.apache.solr + solr-test-framework + ${solr9.version} + test + + + jdk.tools + jdk.tools + + + + + diff --git a/solr9/src/main/java/site/ycsb/db/solr9/SolrClient.java b/solr9/src/main/java/site/ycsb/db/solr9/SolrClient.java new file mode 100644 index 0000000000..8c8a64297d --- /dev/null +++ b/solr9/src/main/java/site/ycsb/db/solr9/SolrClient.java @@ -0,0 +1,364 @@ +/** + * Copyright (c) 2023 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package site.ycsb.db.solr9; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; + +import site.ycsb.ByteIterator; +import site.ycsb.DB; +import site.ycsb.DBException; +import site.ycsb.Status; +import site.ycsb.StringByteIterator; + +/** + * Solr client for YCSB framework. + * + *

+ * Default properties to set: + *

+ *
    + * See README.md + *
+ * + */ +public class SolrClient extends DB { + + public static final String DEFAULT_CLOUD_MODE = "true"; + public static final String DEFAULT_BATCH_MODE = "false"; + public static final String DEFAULT_ZOOKEEPER_HOSTS = "localhost:2181"; + public static final String DEFAULT_SOLR_BASE_URL = "http://localhost:8983/solr"; + public static final String DEFAULT_COMMIT_WITHIN_TIME = "1000"; + + private org.apache.solr.client.solrj.SolrClient client; + private Integer commitTime; + private Boolean batchMode; + + /** + * Initialize any state for this DB. Called once per DB instance; there is one DB instance per + * client thread. + */ + @Override + public void init() throws DBException { + Properties props = getProperties(); + commitTime = Integer + .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); + batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); + + String jaasConfPath = props.getProperty("solr.jaas.conf.path"); + if(jaasConfPath != null) { + System.setProperty("java.security.auth.login.config", jaasConfPath); + HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder()); + } + + // https://solr.apache.org/guide/solr/latest/deployment-guide/enabling-ssl.html + Set sslProperties = Set.of("keyStore", "keyStorePassword", "keyStoreType", "trustStore", + "trustStorePassword", "trustStoreType"); + for (String p : sslProperties) { + String v = props.getProperty("solr." + p); + if (v != null) { + System.setProperty("javax.net.ssl." + p, v); + } + } + + // https://solr.apache.org/guide/solr/latest/deployment-guide/basic-authentication-plugin.html + String basicAuth = props.getProperty("solr.basicAuth"); + if (basicAuth != null) { + System.setProperty("solr.httpclient.builder.factory", + "org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"); + System.setProperty("basicauth", basicAuth); + } + String basicAuthFile = props.getProperty("solr.httpClientConfig"); + if (basicAuthFile != null) { + System.setProperty("solr.httpclient.builder.factory", + "org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"); + System.setProperty("solr.httpclient.config", basicAuthFile); + } + + // Check if Solr cluster is running in SolrCloud or Stand-alone mode + Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); + System.err.println("Solr Cloud Mode = " + cloudMode); + if (cloudMode) { + System.err.println("Solr Zookeeper Remote Hosts = " + + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); + List zkHosts = Arrays + .asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(",")); + client = new CloudHttp2SolrClient.Builder(zkHosts, Optional.empty()).build(); + } else { + client = new Http2SolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build(); + } + } + + @Override + public void cleanup() throws DBException { + try { + client.close(); + } catch (IOException e) { + throw new DBException(e); + } + } + + /** + * Insert a record in the database. Any field/value pairs in the specified values HashMap will be + * written into the record with the specified record key. + * + * @param table + * The name of the table + * @param key + * The record key of the record to insert. + * @param values + * A HashMap of field/value pairs to insert in the record + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status insert(String table, String key, Map values) { + try { + SolrInputDocument doc = new SolrInputDocument(); + + doc.addField("id", key); + for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { + doc.addField(entry.getKey(), entry.getValue()); + } + UpdateResponse response; + if (batchMode) { + response = client.add(table, doc, commitTime); + } else { + response = client.add(table, doc); + client.commit(table); + } + return checkStatus(response.getStatus()); + + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Delete a record from the database. + * + * @param table + * The name of the table + * @param key + * The record key of the record to delete. + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status delete(String table, String key) { + try { + UpdateResponse response; + if (batchMode) { + response = client.deleteById(table, key, commitTime); + } else { + response = client.deleteById(table, key); + client.commit(table); + } + return checkStatus(response.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Read a record from the database. Each field/value pair from the result will be stored in a + * HashMap. + * + * @param table + * The name of the table + * @param key + * The record key of the record to read. + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A HashMap of field/value pairs for the result + * @return Zero on success, a non-zero error code on error or "not found". + */ + @Override + public Status read(String table, String key, Set fields, + Map result) { + try { + Boolean returnFields = false; + String[] fieldList = null; + if (fields != null) { + returnFields = true; + fieldList = fields.toArray(new String[fields.size()]); + } + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + key); + if (returnFields) { + query.setFields(fieldList); + } + final QueryResponse response = client.query(table, query); + SolrDocumentList results = response.getResults(); + if ((results != null) && (results.getNumFound() > 0)) { + for (String field : results.get(0).getFieldNames()) { + result.put(field, + new StringByteIterator(String.valueOf(results.get(0).getFirstValue(field)))); + } + } + return checkStatus(response.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Update a record in the database. Any field/value pairs in the specified values HashMap will be + * written into the record with the specified record key, overwriting any existing values with the + * same field name. + * + * @param table + * The name of the table + * @param key + * The record key of the record to write. + * @param values + * A HashMap of field/value pairs to update in the record + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status update(String table, String key, Map values) { + try { + SolrInputDocument updatedDoc = new SolrInputDocument(); + updatedDoc.addField("id", key); + + for (Entry entry : StringByteIterator.getStringMap(values).entrySet()) { + updatedDoc.addField(entry.getKey(), Collections.singletonMap("set", entry.getValue())); + } + + UpdateResponse writeResponse; + if (batchMode) { + writeResponse = client.add(table, updatedDoc, commitTime); + } else { + writeResponse = client.add(table, updatedDoc); + client.commit(table); + } + return checkStatus(writeResponse.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + /** + * Perform a range scan for a set of records in the database. Each field/value pair from the + * result will be stored in a HashMap. + * + * @param table + * The name of the table + * @param startkey + * The record key of the first record to read. + * @param recordcount + * The number of records to read + * @param fields + * The list of fields to read, or null for all of them + * @param result + * A Vector of HashMaps, where each HashMap is a set field/value pairs for one record + * @return Zero on success, a non-zero error code on error. See this class's description for a + * discussion of error codes. + */ + @Override + public Status scan(String table, String startkey, int recordcount, Set fields, + Vector> result) { + try { + Boolean returnFields = false; + String[] fieldList = null; + if (fields != null) { + returnFields = true; + fieldList = fields.toArray(new String[fields.size()]); + } + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setParam("fq", "id:[ " + startkey + " TO * ]"); + if (returnFields) { + query.setFields(fieldList); + } + query.setRows(recordcount); + final QueryResponse response = client.query(table, query); + SolrDocumentList results = response.getResults(); + + HashMap entry; + + for (SolrDocument hit : results) { + entry = new HashMap<>((int) results.getNumFound()); + for (String field : hit.getFieldNames()) { + entry.put(field, new StringByteIterator(String.valueOf(hit.getFirstValue(field)))); + } + result.add(entry); + } + return checkStatus(response.getStatus()); + } catch (IOException | SolrServerException e) { + e.printStackTrace(); + } + return Status.ERROR; + } + + private Status checkStatus(int status) { + Status responseStatus; + switch (status) { + case 0: + responseStatus = Status.OK; + break; + case 400: + responseStatus = Status.BAD_REQUEST; + break; + case 403: + responseStatus = Status.FORBIDDEN; + break; + case 404: + responseStatus = Status.NOT_FOUND; + break; + case 500: + responseStatus = Status.ERROR; + break; + case 503: + responseStatus = Status.SERVICE_UNAVAILABLE; + break; + default: + responseStatus = Status.UNEXPECTED_STATE; + break; + } + return responseStatus; + } + +} diff --git a/solr9/src/main/java/site/ycsb/db/solr9/package-info.java b/solr9/src/main/java/site/ycsb/db/solr9/package-info.java new file mode 100644 index 0000000000..a1a8473aa6 --- /dev/null +++ b/solr9/src/main/java/site/ycsb/db/solr9/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2023 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for + * Solr. + */ +package site.ycsb.db.solr9; + diff --git a/solr9/src/main/resources/log4j.properties b/solr9/src/main/resources/log4j.properties new file mode 100644 index 0000000000..fea7ce0879 --- /dev/null +++ b/solr9/src/main/resources/log4j.properties @@ -0,0 +1,22 @@ +# Copyright (c) 2020 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. +# Root logger option +log4j.rootLogger=INFO, stderr + +# Direct log messages to stderr +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/solr9/src/test/java/site/ycsb/db/solr9/SolrClientBaseTest.java b/solr9/src/test/java/site/ycsb/db/solr9/SolrClientBaseTest.java new file mode 100644 index 0000000000..1f01a7a21c --- /dev/null +++ b/solr9/src/test/java/site/ycsb/db/solr9/SolrClientBaseTest.java @@ -0,0 +1,169 @@ +/** + * Copyright (c) 2023 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package site.ycsb.db.solr9; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Properties; +import java.util.Set; +import java.util.Vector; + +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.embedded.JettyConfig; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import site.ycsb.ByteIterator; +import site.ycsb.DB; +import site.ycsb.Status; +import site.ycsb.StringByteIterator; +import site.ycsb.workloads.CoreWorkload; + +public abstract class SolrClientBaseTest { + + protected static MiniSolrCloudCluster miniSolrCloudCluster; + private DB instance; + private final static HashMap MOCK_DATA; + protected final static String MOCK_TABLE = "ycsb"; + private final static String MOCK_KEY0 = "0"; + private final static String MOCK_KEY1 = "1"; + private final static int NUM_RECORDS = 10; + private final static String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT; + + static { + MOCK_DATA = new HashMap<>(NUM_RECORDS); + for (int i = 0; i < NUM_RECORDS; i++) { + MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i)); + } + } + + @BeforeClass + public static void onlyOnce() throws Exception { + Path miniSolrCloudClusterTempDirectory = Files.createTempDirectory("miniSolrCloudCluster"); + miniSolrCloudClusterTempDirectory.toFile().deleteOnExit(); + miniSolrCloudCluster = new MiniSolrCloudCluster(1, miniSolrCloudClusterTempDirectory, JettyConfig.builder().build()); + + // Upload Solr configuration + URL configDir = SolrClientBaseTest.class.getClassLoader().getResource("solr_config"); + assertNotNull(configDir); + miniSolrCloudCluster.uploadConfigSet(Paths.get(configDir.toURI()), MOCK_TABLE); + } + + @AfterClass + public static void destroy() throws Exception { + if(miniSolrCloudCluster != null) { + miniSolrCloudCluster.shutdown(); + } + } + + @Before + public void setup() throws Exception { + CollectionAdminRequest.createCollection(MOCK_TABLE, MOCK_TABLE, 1, 1) + .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory") + .process(miniSolrCloudCluster.getSolrClient()); + miniSolrCloudCluster.waitForActiveCollection(MOCK_TABLE, 1, 1); + Thread.sleep(1000); + + instance = getDB(); + } + + @After + public void tearDown() throws Exception { + if(miniSolrCloudCluster != null) { + CollectionAdminRequest.deleteCollection(MOCK_TABLE) + .processAndWait(miniSolrCloudCluster.getSolrClient(), 60); + Thread.sleep(1000); + } + } + + @Test + public void testInsert() throws Exception { + Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA); + assertEquals(Status.OK, result); + } + + @Test + public void testDelete() throws Exception { + Status result = instance.delete(MOCK_TABLE, MOCK_KEY1); + assertEquals(Status.OK, result); + } + + @Test + public void testRead() throws Exception { + Set fields = MOCK_DATA.keySet(); + HashMap resultParam = new HashMap<>(NUM_RECORDS); + Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam); + assertEquals(Status.OK, result); + } + + @Test + public void testUpdate() throws Exception { + HashMap newValues = new HashMap<>(NUM_RECORDS); + + for (int i = 0; i < NUM_RECORDS; i++) { + newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i)); + } + + Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues); + assertEquals(Status.OK, result); + + //validate that the values changed + HashMap resultParam = new HashMap<>(NUM_RECORDS); + instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam); + + for (int i = 0; i < NUM_RECORDS; i++) { + assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString()); + } + } + + @Test + public void testScan() throws Exception { + Set fields = MOCK_DATA.keySet(); + Vector> resultParam = new Vector<>(NUM_RECORDS); + Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, NUM_RECORDS, fields, resultParam); + assertEquals(Status.OK, result); + } + + /** + * Gets the test DB. + * + * @return The test DB. + */ + protected DB getDB() { + return getDB(new Properties()); + } + + /** + * Gets the test DB. + * + * @param props + * Properties to pass to the client. + * @return The test DB. + */ + protected abstract DB getDB(Properties props); +} diff --git a/solr9/src/test/java/site/ycsb/db/solr9/SolrClientCloudTest.java b/solr9/src/test/java/site/ycsb/db/solr9/SolrClientCloudTest.java new file mode 100644 index 0000000000..cc874fe8f3 --- /dev/null +++ b/solr9/src/test/java/site/ycsb/db/solr9/SolrClientCloudTest.java @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2023 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db.solr9; + +import site.ycsb.DB; +import org.junit.After; + +import java.util.Properties; + +import static org.junit.Assume.assumeNoException; + +public class SolrClientCloudTest extends SolrClientBaseTest { + + private SolrClient instance; + + @After + public void tearDown() throws Exception { + try { + if(instance != null) { + instance.cleanup(); + } + } finally { + super.tearDown(); + } + } + + @Override + protected DB getDB(Properties props) { + instance = new SolrClient(); + + props.setProperty("solr.cloud", "true"); + props.setProperty("solr.zookeeper.hosts", miniSolrCloudCluster.getZkServer().getZkAddress()); + + instance.setProperties(props); + try { + instance.init(); + } catch (Exception error) { + assumeNoException(error); + } + return instance; + } +} diff --git a/solr9/src/test/java/site/ycsb/db/solr9/SolrClientTest.java b/solr9/src/test/java/site/ycsb/db/solr9/SolrClientTest.java new file mode 100644 index 0000000000..eafce8b3fe --- /dev/null +++ b/solr9/src/test/java/site/ycsb/db/solr9/SolrClientTest.java @@ -0,0 +1,63 @@ +/** + * Copyright (c) 2023 YCSB contributors. All rights reserved. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package site.ycsb.db.solr9; + +import static org.junit.Assume.assumeNoException; + +import java.util.Properties; + +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.After; + +import site.ycsb.DB; + +public class SolrClientTest extends SolrClientBaseTest { + + private SolrClient instance; + + @After + public void tearDown() throws Exception { + try { + if(instance != null) { + instance.cleanup(); + } + } finally { + super.tearDown(); + } + } + + @Override + protected DB getDB(Properties props) { + instance = new SolrClient(); + + // Use the first Solr server in the cluster. + // Doesn't matter if there are more since requests will be forwarded properly by Solr. + JettySolrRunner jettySolrRunner = miniSolrCloudCluster.getJettySolrRunners().get(0); + String solrBaseUrl = jettySolrRunner.getBaseUrl().toString(); + + props.setProperty("solr.cloud", "false"); + props.setProperty("solr.base.url", solrBaseUrl); + instance.setProperties(props); + + try { + instance.init(); + } catch (Exception error) { + assumeNoException(error); + } + return instance; + } +} diff --git a/solr9/src/test/resources/log4j.properties b/solr9/src/test/resources/log4j.properties new file mode 100644 index 0000000000..456e1de073 --- /dev/null +++ b/solr9/src/test/resources/log4j.properties @@ -0,0 +1,29 @@ +# Copyright (c) 2020 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. +# + +# Root logger option +log4j.rootLogger=INFO, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n + +# Suppress messages from ZooKeeper +log4j.logger.org.apache.zookeeper=ERROR +# Solr classes are too chatty in test at INFO +log4j.logger.org.apache.solr=ERROR +log4j.logger.org.eclipse.jetty=ERROR diff --git a/solr9/src/test/resources/solr_config/schema.xml b/solr9/src/test/resources/solr_config/schema.xml new file mode 100644 index 0000000000..8e00b78007 --- /dev/null +++ b/solr9/src/test/resources/solr_config/schema.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/solr9/src/test/resources/solr_config/solrconfig.xml b/solr9/src/test/resources/solr_config/solrconfig.xml new file mode 100644 index 0000000000..7c58457dcd --- /dev/null +++ b/solr9/src/test/resources/solr_config/solrconfig.xml @@ -0,0 +1,63 @@ + + + + + + + + ${tests.luceneMatchVersion:LATEST} + + ${useCompoundFile:false} + + ${solr.data.dir:} + + + + + ${solr.data.dir:} + + + + + + + + + + + + *:* + + + all + + server-enabled.txt + + + + + solr + + + +